From 8aff1e8f5ab24328b97671f6dcc0528d09f9365e Mon Sep 17 00:00:00 2001 From: PascalNun Date: Sun, 1 Nov 2026 23:25:13 +0100 Subject: [PATCH 1/2] feat: broaden bounded source discovery and downloads --- .github/workflows/validate.yml | 3 + CHANGELOG.md | 10 + RhinoSpatial.Core/FeatureCollectionPager.cs | 210 ++++++++++++++ RhinoSpatial.Core/GeoJsonReader.cs | 3 + RhinoSpatial.Core/Lod2AtomClient.cs | 149 +++------- RhinoSpatial.Core/Lod2AtomFeedReader.cs | 18 +- RhinoSpatial.Core/OgcApiFeaturesClient.cs | 252 +++++++++------- .../RemoteReferenceMetadataReader.cs | 45 +++ RhinoSpatial.Core/ShapefileFeatureReader.cs | 19 +- RhinoSpatial.Core/SourceDownloadCache.cs | 274 ++++++++++++++++++ RhinoSpatial.Core/SourceHttp.cs | 39 +++ RhinoSpatial.Core/TerrainAtomLoader.cs | 131 +++++++++ RhinoSpatial.Core/TerrainTextGridReader.cs | 14 +- RhinoSpatial.Core/VectorSourceResolver.cs | 9 +- RhinoSpatial.Core/WfsClient.cs | 67 +++-- RhinoSpatial.Core/WfsRequestOptions.cs | 2 + RhinoSpatial/LoadTerrainComponent.cs | 27 +- RhinoSpatial/Lod2LoadComponent.cs | 69 ++++- RhinoSpatial/SpatialContextComponent.cs | 25 ++ RhinoSpatial/WfsListLayersComponent.cs | 6 +- RhinoSpatial/WfsLoadComponent.cs | 149 +++++++--- docs/COMPATIBILITY_IMPLEMENTATION_PLAN.md | 54 ++++ docs/COMPATIBILITY_RESEARCH.md | 42 +++ docs/COMPONENT_REFERENCE.md | 36 ++- docs/ROADMAP.md | 10 + docs/TEST_SOURCES.md | 24 +- examples/COMPATIBILITY_WORKFLOWS.md | 140 +++++++++ examples/README.md | 7 + examples/VALIDATION.md | 16 + examples/sources.json | 128 +++++++- sandbox/Program.cs | 15 +- sandbox/SourceCompatibilityChecks.cs | 260 +++++++++++++++++ sandbox/SourceCompatibilityProbe.cs | 64 ++++ 33 files changed, 1983 insertions(+), 334 deletions(-) create mode 100644 RhinoSpatial.Core/FeatureCollectionPager.cs create mode 100644 RhinoSpatial.Core/RemoteReferenceMetadataReader.cs create mode 100644 RhinoSpatial.Core/SourceDownloadCache.cs create mode 100644 RhinoSpatial.Core/SourceHttp.cs create mode 100644 RhinoSpatial.Core/TerrainAtomLoader.cs create mode 100644 docs/COMPATIBILITY_IMPLEMENTATION_PLAN.md create mode 100644 docs/COMPATIBILITY_RESEARCH.md create mode 100644 examples/COMPATIBILITY_WORKFLOWS.md create mode 100644 sandbox/SourceCompatibilityChecks.cs create mode 100644 sandbox/SourceCompatibilityProbe.cs diff --git a/.github/workflows/validate.yml b/.github/workflows/validate.yml index 2a60b61..edb5998 100644 --- a/.github/workflows/validate.yml +++ b/.github/workflows/validate.yml @@ -46,6 +46,9 @@ jobs: - name: Check ATOM downloads and cache run: dotnet run --project RhinoSpatial.Sandbox.csproj -c Release --no-build -- lod2-atom-check + - name: Check source compatibility + run: dotnet run --project RhinoSpatial.Sandbox.csproj -c Release --no-build -- source-compatibility-check + - name: Check 3D Tiles selection run: dotnet run --project RhinoSpatial.Sandbox.csproj -c Release --no-build -- tiles-selection-check diff --git a/CHANGELOG.md b/CHANGELOG.md index 3f257ae..07095da 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,16 @@ RhinoSpatial improves its established source categories through real project use. Earlier releases remain documented on [GitHub Releases](https://github.com/PascalNun/RhinoSpatial/releases). +## Unreleased + +- Follow OGC API next links and supported WFS paging with explicit user caps, deduplication, repeated-page protection and completeness diagnostics. Preserve raw GML for the building loader. +- Discover feature collections from OGC API landing pages, collection lists and advertised links using the existing layer-list/vector components. Layer selection is optional for a single collection and direct files. +- Load bounded, validated direct GeoJSON and Shapefile ZIP downloads, plus remote CityJSON/CityGML building files. Check companion files, CRS metadata, archive paths and expanded sizes; reuse validated downloads. +- Load terrain ATOM feeds through `Load Terrain`, using declared placement metadata and an aligned raster mosaic. Preserve source sample centers, NoData holes and the shared elevation baseline; reject incompatible grids or conflicting heights. +- Use ATOM and OGC API metadata as Spatial Context references without intentionally downloading feature data or archives. Existing saved selections retain priority. +- Correct projected Shapefile WKT authority detection so nested datum/ellipsoid EPSG codes cannot override the actual source CRS. Unsupported declared CRS metadata fails explicitly. +- Add deterministic compatibility checks to CI and save [copyable example workflows](examples/COMPATIBILITY_WORKFLOWS.md), source URLs and live probe results. New Rhino/Grasshopper visual checks remain pending; component GUIDs and input/output layouts are retained. No public release or installed-plugin replacement is included in this batch. + ## 0.4.0 — 2026-11-01 Version 0.4.0 is the recommended public release and is published as GitHub Latest. Component names, GUIDs and normal input/output connections are retained. Provider-specific compatibility continues to evolve. diff --git a/RhinoSpatial.Core/FeatureCollectionPager.cs b/RhinoSpatial.Core/FeatureCollectionPager.cs new file mode 100644 index 0000000..8105345 --- /dev/null +++ b/RhinoSpatial.Core/FeatureCollectionPager.cs @@ -0,0 +1,210 @@ +using System.Globalization; +using System.Security.Cryptography; +using System.Text; +using System.Text.Json; +using System.Xml.Linq; + +namespace RhinoSpatial.Core; + +public record PagedFeatureResponse(string Text, string StatusNote, int PageCount); +public record FeaturePageContent(string Text, Uri Url); + +/// Keeps source geometries intact while assembling bounded JSON or GML pages. +public static class FeatureCollectionPager +{ + public const int PageSize = 1000; + public const int MaximumFeatures = 100000; + public const int MaximumPages = 100; + + public static Task LoadAsync(string firstUrl, + Func> fetch, int maxFeatures, + Func? offsetUrl = null, CancellationToken cancellationToken = default) => + LoadHttpAsync(firstUrl, async (url, token) => new FeaturePageContent(await fetch(url, token).ConfigureAwait(false), new Uri(url)), + maxFeatures, offsetUrl, cancellationToken); + + public static async Task LoadHttpAsync(string firstUrl, + Func> fetch, int maxFeatures, + Func? offsetUrl = null, CancellationToken cancellationToken = default) + { + if (maxFeatures < 0) throw new ArgumentOutOfRangeException(nameof(maxFeatures)); + using var timeout = CancellationTokenSource.CreateLinkedTokenSource(cancellationToken); + timeout.CancelAfter(TimeSpan.FromMinutes(2)); + var cap = maxFeatures > 0 ? Math.Min(maxFeatures, MaximumFeatures) : MaximumFeatures; + var visited = new HashSet(StringComparer.Ordinal); + var pageHashes = new HashSet(StringComparer.Ordinal); + var ids = new HashSet(StringComparer.Ordinal); + var records = new List(); + Page? first = null; + var url = firstUrl; + var pages = 0; + var rawCount = 0; + long bytes = 0; + long? matchedTotal = null; + var note = ""; + string? firstJsonSrs = null; + var anonymous = false; + try + { + while (true) + { + timeout.Token.ThrowIfCancellationRequested(); + if (++pages > MaximumPages || !visited.Add(url)) + throw new InvalidDataException("Feature pagination repeated a URL or exceeded 100 pages. No partial result was returned; reduce the area or set Max Features."); + var response = await fetch(url, timeout.Token).ConfigureAwait(false); + var body = response.Text; + url = response.Url.AbsoluteUri; + bytes += Encoding.UTF8.GetByteCount(body); + if (bytes > 128L * 1024 * 1024) + throw new InvalidDataException("Feature responses exceeded 128 MiB. Reduce the area or set Max Features."); + var page = Page.Read(body); + first ??= page; + if (matchedTotal.HasValue && page.Total.HasValue && matchedTotal != page.Total) + throw new InvalidDataException("The provider changed its matching feature total during pagination; retry the selection."); + matchedTotal ??= page.Total; + if (page.IsJson != first.IsJson) + throw new InvalidDataException("The provider changed feature encoding between pages."); + var fingerprint = Convert.ToHexString(SHA256.HashData(Encoding.UTF8.GetBytes(string.Join("\n", page.Records)))); + if (page.Records.Count > 0 && !pageHashes.Add(fingerprint)) + throw new InvalidDataException("The provider repeated a feature page instead of advancing. No partial result was returned."); + if (page.IsJson) + { + var srs = SpatialReferenceTransform.NormalizeSrs(GeoJsonReader.TryReadSourceSrs(body) ?? "EPSG:4326"); + if (firstJsonSrs is not null && firstJsonSrs != srs) + throw new InvalidDataException("The provider changed GeoJSON coordinate systems between pages."); + firstJsonSrs ??= srs; + } + var before = records.Count; + for (var i = 0; i < page.Records.Count && records.Count < cap; i++) + { + if (page.Keys[i].Length == 0) anonymous = true; + if (page.Keys[i].Length == 0 || ids.Add(page.Keys[i])) records.Add(page.Records[i]); + } + rawCount += page.Records.Count; + if (matchedTotal.HasValue && records.Count > matchedTotal.Value) + throw new InvalidDataException("The provider returned more feature records than its advertised matching total; retry the selection."); + var more = page.Next is not null || matchedTotal > rawCount || + (offsetUrl is not null && matchedTotal is null && page.Records.Count >= PageSize); + if (records.Count >= cap) + { + if (maxFeatures <= 0 || maxFeatures > MaximumFeatures) + { + if (more || page.Records.Count > records.Count - before) + throw new InvalidDataException("Feature selection reached the 100,000-feature safety limit. Reduce the area or set Max Features."); + } + note = maxFeatures > 0 && maxFeatures <= MaximumFeatures + ? $"Stopped at Max Features ({maxFeatures}); further matching features may exist." : ""; + break; + } + if (page.Records.Count == 0) + { + if (page.Next is not null || matchedTotal > rawCount) + throw new InvalidDataException("The provider returned an empty page before completing its advertised feature result."); + break; + } + if (records.Count == before) + throw new InvalidDataException("Feature pagination made no progress; all returned IDs were duplicates."); + if (!more) + { + if (matchedTotal > records.Count) + throw new InvalidDataException("The provider ended paging before returning its advertised unique feature total."); + if (matchedTotal is null) note = "Provider supplied no matching total; paging ended at its last advertised page."; + break; + } + if (page.Next is not null) + { + var next = new Uri(new Uri(url), page.Next); + if (next.Scheme is not ("http" or "https") || !string.IsNullOrEmpty(next.UserInfo)) + throw new InvalidDataException("Feature pagination requires an HTTP(S) next link without embedded credentials."); + url = next.AbsoluteUri; + } + else if (offsetUrl is not null) + url = offsetUrl(rawCount, Math.Min(PageSize, cap - records.Count)); + else + throw new InvalidDataException("The provider advertised more features but supplied no supported paging mechanism. No partial result was returned."); + } + } + catch (OperationCanceledException) when (!cancellationToken.IsCancellationRequested) + { throw new TimeoutException("Feature pagination timed out after two minutes. Reduce the selected area or set Max Features."); } + var identityNote = anonymous ? " Features without stable IDs are retained; identity-based deduplication is unavailable for those records." : ""; + return new PagedFeatureResponse(first!.Merge(records), $"Read {records.Count} {(anonymous ? "" : "unique ")}feature(s) in {pages} page(s). {note}{identityNote}".Trim(), pages); + } + + private sealed record Page(bool IsJson, string Original, List Records, List Keys, string? Next, long? Total) + { + public static Page Read(string body) + { + var records = new List(); + var keys = new List(); + if (body.TrimStart().StartsWith("{", StringComparison.Ordinal)) + { + using var document = JsonDocument.Parse(body); + var root = document.RootElement; + if (!root.TryGetProperty("features", out var features) || features.ValueKind != JsonValueKind.Array) + throw new InvalidDataException("Expected a GeoJSON FeatureCollection, not service metadata or an error document."); + foreach (var feature in features.EnumerateArray()) + { + var text = JsonSerializer.Serialize(feature); + records.Add(text); + keys.Add(feature.TryGetProperty("id", out var id) && id.ValueKind is JsonValueKind.String or JsonValueKind.Number + ? "id:" + id.ValueKind + ":" + id.GetRawText() : ""); + } + string? next = null; + if (root.TryGetProperty("links", out var links) && links.ValueKind == JsonValueKind.Array) + foreach (var link in links.EnumerateArray()) + if (link.TryGetProperty("rel", out var rel) && rel.GetString() == "next" && link.TryGetProperty("href", out var href)) + next = href.GetString(); + long? total = null; + if ((root.TryGetProperty("numberMatched", out var count) || root.TryGetProperty("totalFeatures", out count)) && count.TryGetInt64Safe(out var n) && n >= 0) total = n; + return new Page(true, body, records, keys, next, total); + } + var xml = XDocument.Parse(body); + var element = xml.Root ?? throw new InvalidDataException("Empty feature response."); + if (element.Name.LocalName != "FeatureCollection") + throw new InvalidDataException("Expected a WFS FeatureCollection: " + element.Value[..Math.Min(250, element.Value.Length)]); + foreach (var member in element.Elements().Where(e => e.Name.LocalName is "member" or "featureMember" or "featureMembers")) + foreach (var feature in member.Elements()) + { + var wrapper = new XElement(XName.Get("member", "http://www.opengis.net/wfs/2.0"), new XElement(feature)); + // Preserve inherited coordinate-system declarations and namespace bindings. + foreach (var attribute in element.Attributes().Where(a => a.IsNamespaceDeclaration || a.Name.LocalName is "srsName" or "srsDimension")) + if (wrapper.Attribute(attribute.Name) is null) wrapper.Add(new XAttribute(attribute)); + var text = wrapper.ToString(SaveOptions.DisableFormatting); + records.Add(text); + var id = feature.Attributes().FirstOrDefault(a => a.Name.LocalName is "id" or "fid")?.Value; + keys.Add(string.IsNullOrEmpty(id) ? "" : "id:" + id); + } + var totalText = (string?)element.Attribute("numberMatched"); + return new Page(false, body, records, keys, (string?)element.Attribute("next"), + long.TryParse(totalText, NumberStyles.Integer, CultureInfo.InvariantCulture, out var totalValue) && totalValue >= 0 ? totalValue : null); + } + + public string Merge(List records) + { + if (IsJson) + { + using var document = JsonDocument.Parse(Original); + using var stream = new MemoryStream(); + using (var writer = new Utf8JsonWriter(stream)) + { + writer.WriteStartObject(); + foreach (var property in document.RootElement.EnumerateObject()) + if (property.Name is not ("features" or "links" or "numberReturned" or "numberMatched" or "totalFeatures")) property.WriteTo(writer); + writer.WritePropertyName("features"); writer.WriteStartArray(); + foreach (var record in records) { using var value = JsonDocument.Parse(record); value.RootElement.WriteTo(writer); } + writer.WriteEndArray(); writer.WriteNumber("numberReturned", records.Count); writer.WriteEndObject(); + } + return Encoding.UTF8.GetString(stream.ToArray()); + } + var xml = XDocument.Parse(Original); + var root = xml.Root!; + root.Elements().Where(e => e.Name.LocalName is "member" or "featureMember" or "featureMembers").Remove(); + root.Attribute("next")?.Remove(); root.Attribute("previous")?.Remove(); + root.SetAttributeValue("numberReturned", records.Count); + foreach (var record in records) root.Add(XElement.Parse(record)); + return xml.ToString(SaveOptions.DisableFormatting); + } + } + + private static bool TryGetInt64Safe(this JsonElement element, out long value) + { value = 0; return element.ValueKind == JsonValueKind.Number && element.TryGetInt64(out value); } +} diff --git a/RhinoSpatial.Core/GeoJsonReader.cs b/RhinoSpatial.Core/GeoJsonReader.cs index a091dcc..12d3e68 100644 --- a/RhinoSpatial.Core/GeoJsonReader.cs +++ b/RhinoSpatial.Core/GeoJsonReader.cs @@ -52,6 +52,9 @@ public static List ReadFeatures(string geoJson, string sourceLayerNa using var document = JsonDocument.Parse(geoJson); var root = document.RootElement; + if (root.TryGetProperty("type", out var rootType) && rootType.GetString() == "Feature") + return new List { ReadFeature(root, sourceLayerName) }; + if (!root.TryGetProperty("features", out var featuresElement) || featuresElement.ValueKind != JsonValueKind.Array) { throw new InvalidOperationException("The GeoJSON response does not contain a valid features array."); diff --git a/RhinoSpatial.Core/Lod2AtomClient.cs b/RhinoSpatial.Core/Lod2AtomClient.cs index e8bd3c4..186ae4b 100644 --- a/RhinoSpatial.Core/Lod2AtomClient.cs +++ b/RhinoSpatial.Core/Lod2AtomClient.cs @@ -12,7 +12,7 @@ namespace RhinoSpatial.Core; public record Lod2AtomSelection(IReadOnlyList Tiles, int FeedCount, int DownloadCount); public record Lod2AtomCachedTile(Lod2AtomLink Tile, string FilePath, bool UsedCache); -public sealed class Lod2AtomClient +public class AtomDownloadClient { private static readonly HttpClient SharedHttpClient = new() { Timeout = Timeout.InfiniteTimeSpan }; private static readonly ConcurrentDictionary FeedCache = new(StringComparer.Ordinal); @@ -23,10 +23,15 @@ public sealed class Lod2AtomClient private const long MaximumDownloadBytes = 512L * 1024 * 1024; private const long MaximumExpandedBytes = 1024L * 1024 * 1024; - public Lod2AtomClient(HttpClient? httpClient = null, string? cacheDirectory = null) + private readonly AtomDownloadKind _kind; + private readonly SourceDownloadCache _downloads; + + public AtomDownloadClient(HttpClient? httpClient = null, string? cacheDirectory = null, AtomDownloadKind kind = AtomDownloadKind.Buildings) { _http = httpClient ?? SharedHttpClient; - _cacheDirectory = cacheDirectory ?? DefaultCacheDirectory; + _kind = kind; + _cacheDirectory = cacheDirectory ?? (kind == AtomDownloadKind.Buildings ? DefaultCacheDirectory : Path.Combine(Environment.GetFolderPath(Environment.SpecialFolder.LocalApplicationData), "RhinoSpatial", "TerrainAtomCache")); + _downloads = new SourceDownloadCache(kind, _http, _cacheDirectory); } public async Task TrySelectAsync(string source, SpatialContext2D context, CancellationToken cancellationToken = default) @@ -67,7 +72,7 @@ bool Intersects(AtomSpatialExtent extent) cancellationToken.ThrowIfCancellationRequested(); if (++feedCount > 64 || current.Depth > 8) throw new InvalidOperationException("ATOM feed traversal limit reached. Select a more specific dataset feed."); - var feed = Lod2AtomFeedReader.Read(current.Document, current.Url, current.Extent); + var feed = Lod2AtomFeedReader.Read(current.Document, current.Url, current.Extent, _kind); downloadCount += feed.Downloads.Count; if (downloadCount > 50000) throw new InvalidOperationException("The ATOM catalogue exceeds the supported download-entry limit."); @@ -93,7 +98,7 @@ bool Intersects(AtomSpatialExtent extent) } } if (missing.Count > 0) - throw new InvalidOperationException($"The ATOM feed does not provide sufficient per-tile spatial metadata to select downloads safely. Example entries: {string.Join(", ", missing)}. No building archives were downloaded. Use a spatially indexed feed or local tiles."); + throw new InvalidOperationException($"The ATOM feed does not provide sufficient per-tile spatial metadata to select downloads safely. Example entries: {string.Join(", ", missing)}. No data archives were downloaded. Use a spatially indexed feed or local tiles."); if (tiles.Count > 32) throw new InvalidOperationException($"The area selects {tiles.Count} ATOM tiles; the limit is 32 per solution. Select a smaller study area."); return new Lod2AtomSelection(tiles.Values.OrderBy(tile => tile.Url.AbsoluteUri, StringComparer.Ordinal).ToList(), feedCount, downloadCount); @@ -168,129 +173,41 @@ private async Task> DownloadCoreAsync(Lod2Atom try { var prefix = $"ATOM {result.Count + 1}/{selection.Tiles.Count}"; - var cachedTile = await DownloadTileAsync(tile, note => progress?.Invoke($"{prefix}: {note}"), cancellationToken).ConfigureAwait(false); + var cachedTile = await _downloads.DownloadAsync(tile, note => progress?.Invoke($"{prefix}: {note}"), cancellationToken).ConfigureAwait(false); result.Add(cachedTile); progress?.Invoke($"{prefix}: {(cachedTile.UsedCache ? "reused cache" : "download complete")}"); } catch (Exception ex) when (ex is not OperationCanceledException) { - throw new InvalidOperationException($"Could not load ATOM tile '{tile.Title}': {ex.Message} Completed {result.Count}/{selection.Tiles.Count} tile(s); successful downloads remain cached. No partial building result was returned.", ex); + throw new InvalidOperationException($"Could not load ATOM tile '{tile.Title}': {ex.Message} Completed {result.Count}/{selection.Tiles.Count} tile(s); successful downloads remain cached. No partial source result was returned.", ex); } } return result; } - private sealed record CacheStamp(long Length, string Hash, DateTime DownloadedUtc); - - private async Task DownloadTileAsync(Lod2AtomLink tile, Action progress, CancellationToken cancellationToken) - { - Directory.CreateDirectory(_cacheDirectory); - var key = Convert.ToHexString(SHA256.HashData(Encoding.UTF8.GetBytes(tile.Url.AbsoluteUri + "\n" + tile.Revision))); - var target = Path.Combine(_cacheDirectory, key + ".zip"); - using var timeout = CancellationTokenSource.CreateLinkedTokenSource(cancellationToken); - timeout.CancelAfter(TimeSpan.FromMinutes(3)); - var temporary = target + "." + Guid.NewGuid().ToString("N") + ".partial"; - try - { - // A filesystem lock also serializes writers from separate Rhino processes. - await using var cacheLock = await AcquireLockAsync(target + ".lock", timeout.Token).ConfigureAwait(false); - if (await IsValidCacheAsync(target, tile.Revision, timeout.Token).ConfigureAwait(false)) - return new Lod2AtomCachedTile(tile, target, true); - using var response = await _http.GetAsync(tile.Url, HttpCompletionOption.ResponseHeadersRead, timeout.Token).ConfigureAwait(false); - response.EnsureSuccessStatusCode(); - var expectedLength = response.Content.Headers.ContentLength; - if (expectedLength > MaximumDownloadBytes) - throw new InvalidOperationException("Archive exceeds the 512 MiB download limit."); - await using (var input = await response.Content.ReadAsStreamAsync(timeout.Token).ConfigureAwait(false)) - await using (var output = new FileStream(temporary, FileMode.CreateNew, FileAccess.Write, FileShare.None, 81920, useAsync: true)) - { - var buffer = new byte[81920]; - long count = 0; - long lastProgressBytes = 0; - int read; - while ((read = await input.ReadAsync(buffer, timeout.Token).ConfigureAwait(false)) != 0) - { - count += read; - if (count > MaximumDownloadBytes) throw new InvalidOperationException("Archive exceeds the 512 MiB download limit."); - await output.WriteAsync(buffer.AsMemory(0, read), timeout.Token).ConfigureAwait(false); - if (count - lastProgressBytes >= 1024 * 1024) - { - progress($"downloaded {count / (1024 * 1024)} MiB" + (expectedLength.HasValue ? $" / {expectedLength.Value / (1024 * 1024)} MiB" : "")); - lastProgressBytes = count; - } - } - if (expectedLength.HasValue && count != expectedLength.Value) - throw new InvalidDataException("The archive download was incomplete."); - } - await ValidateArchiveAsync(temporary, timeout.Token).ConfigureAwait(false); - await using var validated = File.OpenRead(temporary); - var stamp = new CacheStamp(validated.Length, Convert.ToHexString(await SHA256.HashDataAsync(validated, timeout.Token).ConfigureAwait(false)), DateTime.UtcNow); - validated.Close(); - File.Move(temporary, target, overwrite: true); - await File.WriteAllTextAsync(temporary, JsonSerializer.Serialize(stamp), timeout.Token).ConfigureAwait(false); - File.Move(temporary, target + ".json", overwrite: true); - return new Lod2AtomCachedTile(tile, target, false); - } - catch (OperationCanceledException) when (!cancellationToken.IsCancellationRequested) - { - throw new TimeoutException("The ATOM archive download timed out after three minutes."); - } - finally - { - if (File.Exists(temporary)) File.Delete(temporary); - } - } - - private static async Task AcquireLockAsync(string path, CancellationToken cancellationToken) - { - while (true) - { - cancellationToken.ThrowIfCancellationRequested(); - try { return new FileStream(path, FileMode.OpenOrCreate, FileAccess.ReadWrite, FileShare.None); } - catch (IOException) { await Task.Delay(150, cancellationToken).ConfigureAwait(false); } - } - } + public Task DownloadFileAsync(Lod2AtomLink tile, CancellationToken cancellationToken = default) => + _downloads.DownloadAsync(tile, cancellationToken: cancellationToken); - private static async Task IsValidCacheAsync(string path, string revision, CancellationToken cancellationToken) + public async Task TryReadReferenceAsync(string source, CancellationToken cancellationToken = default) { - try - { - if (!File.Exists(path) || !File.Exists(path + ".json")) return false; - var stamp = JsonSerializer.Deserialize(await File.ReadAllTextAsync(path + ".json", cancellationToken).ConfigureAwait(false)); - if (stamp is null || new FileInfo(path).Length != stamp.Length || - (string.IsNullOrWhiteSpace(revision) && DateTime.UtcNow - stamp.DownloadedUtc > TimeSpan.FromDays(1))) return false; - await using var stream = File.OpenRead(path); - return Convert.ToHexString(await SHA256.HashDataAsync(stream, cancellationToken).ConfigureAwait(false)) == stamp.Hash; - } - catch (Exception ex) when (ex is IOException or JsonException or InvalidDataException) { return false; } + var feed = await LoadFeedAsync(new Uri(source), true, cancellationToken).ConfigureAwait(false); + if (feed is null) return null; + var root = feed.Value.Document.Root!; + var extents = root.Elements(Lod2AtomFeedReader.Atom + "entry").Prepend(root) + .Select(Lod2AtomFeedReader.ReadExtent).Where(e => e is not null).Cast().ToList(); + if (extents.Count == 0) throw new InvalidDataException("The ATOM feed has no readable coverage extent; use its service-level feed or select an area manually."); + var boxes = extents.Select(e => Lod2AtomFeedReader.TransformBounds(e, "EPSG:4326")).ToList(); + var wgs = new BoundingBox2D(boxes.Min(b => b.MinX), boxes.Min(b => b.MinY), boxes.Max(b => b.MaxX), boxes.Max(b => b.MaxY)); + var codes = root.Elements(Lod2AtomFeedReader.Atom + "entry").Prepend(root) + .Select(Lod2AtomFeedReader.ReadCategorySrs).Where(s => s is not null).Distinct().ToList(); + var srs = codes.Count == 1 ? codes[0]! : "EPSG:4326"; + var native = Lod2AtomFeedReader.TransformBounds(new AtomSpatialExtent("EPSG:4326", wgs), srs); + return new ReferenceSourceMetadata("ATOM", srs, native, wgs, extents.Count); } +} - private static async Task ValidateArchiveAsync(string path, CancellationToken cancellationToken) - { - using var archive = ZipFile.OpenRead(path); - if (archive.Entries.Count > 10000) throw new InvalidDataException("ZIP contains too many entries."); - long expandedBytes = 0; - var buildingFiles = 0; - foreach (var entry in archive.Entries) - { - if (entry.Length > 256L * 1024 * 1024 || (expandedBytes += entry.Length) > MaximumExpandedBytes) - throw new InvalidDataException("ZIP exceeds the supported expanded-size limit (256 MiB per file, 1 GiB total)."); - var extension = Path.GetExtension(entry.FullName).ToLowerInvariant(); - if (extension is not (".gml" or ".citygml" or ".xml")) continue; - await using var stream = entry.Open(); - using var reader = XmlReader.Create(stream, new XmlReaderSettings - { - Async = true, DtdProcessing = DtdProcessing.Prohibit, XmlResolver = null, MaxCharactersInDocument = 256L * 1024 * 1024 - }); - await reader.MoveToContentAsync().ConfigureAwait(false); - if (reader.LocalName == "CityModel" && reader.NamespaceURI.StartsWith("http://www.opengis.net/citygml", StringComparison.Ordinal)) - { - buildingFiles++; - // Verify XML and decompression through EOF before publishing a completed cache file. - while (await reader.ReadAsync().ConfigureAwait(false)) cancellationToken.ThrowIfCancellationRequested(); - } - } - if (buildingFiles == 0) - throw new InvalidDataException("Unsupported or invalid archive: expected a ZIP containing CityGML building data. Other archive formats and remote CityJSON downloads are not supported."); - } +// Keep the original building client API available for existing callers. +public sealed class Lod2AtomClient : AtomDownloadClient +{ + public Lod2AtomClient(HttpClient? httpClient = null, string? cacheDirectory = null) : base(httpClient, cacheDirectory) { } } diff --git a/RhinoSpatial.Core/Lod2AtomFeedReader.cs b/RhinoSpatial.Core/Lod2AtomFeedReader.cs index 126aa87..bd6375d 100644 --- a/RhinoSpatial.Core/Lod2AtomFeedReader.cs +++ b/RhinoSpatial.Core/Lod2AtomFeedReader.cs @@ -6,7 +6,7 @@ namespace RhinoSpatial.Core; public record AtomSpatialExtent(string SrsName, BoundingBox2D Bounds); -public record Lod2AtomLink(Uri Url, string Title, string Revision, AtomSpatialExtent? Extent); +public record Lod2AtomLink(Uri Url, string Title, string Revision, AtomSpatialExtent? Extent, string NativeSrs = ""); public record Lod2AtomFeed(IReadOnlyList Feeds, IReadOnlyList Downloads); /// Reads download metadata only; never opens building archives to locate tiles. @@ -15,7 +15,7 @@ public static class Lod2AtomFeedReader public static readonly XNamespace Atom = "http://www.w3.org/2005/Atom"; private static readonly XNamespace GeoRss = "http://www.georss.org/georss"; - public static Lod2AtomFeed Read(XDocument document, Uri source, AtomSpatialExtent? inheritedExtent = null) + public static Lod2AtomFeed Read(XDocument document, Uri source, AtomSpatialExtent? inheritedExtent = null, AtomDownloadKind kind = AtomDownloadKind.Buildings) { var root = document.Root; if (root?.Name != Atom + "feed") @@ -38,16 +38,16 @@ public static Lod2AtomFeed Read(XDocument document, Uri source, AtomSpatialExten foreach (var link in links) { var contentType = ((string?)link.Attribute("type") ?? "").Split(';')[0].Trim(); - if (contentType.Equals("text/csv", StringComparison.OrdinalIgnoreCase) || - contentType.StartsWith("image/", StringComparison.OrdinalIgnoreCase)) - throw new InvalidOperationException($"This ATOM download advertises '{contentType}', not CityGML building data. Load LoD2 Buildings requires a CityGML download feed. Terrain ATOM feeds are not supported; use local terrain files or a WCS source with Load Terrain."); + if (kind == AtomDownloadKind.Buildings && (contentType.Equals("text/csv", StringComparison.OrdinalIgnoreCase) || + contentType.StartsWith("image/", StringComparison.OrdinalIgnoreCase))) + throw new InvalidOperationException($"This ATOM download advertises '{contentType}', not CityGML building data. Load LoD2 Buildings requires a CityGML download feed. Use the terrain download feed with Load Terrain instead."); var url = ResolveUrl(link, source); var tileExtent = ReadExtent(link); if (tileExtent is null && links.Count == 1) tileExtent = extent; if (tileExtent is null && gridSize.HasValue && srs is not null) tileExtent = ReadDeclaredGridTile(url, srs, gridSize.Value, coverage); - downloads.Add(CreateLink(link, entry, url, tileExtent)); + downloads.Add(CreateLink(link, entry, url, tileExtent) with { NativeSrs = srs ?? "" }); } } @@ -83,7 +83,7 @@ private static bool IsDownloadLink(XElement link) var type = ((string?)link.Attribute("type") ?? "").ToLowerInvariant(); var href = ((string?)link.Attribute("href") ?? "").Split('?', '#')[0]; return (relation == "enclosure" && !type.Contains("atom+xml")) || type.Contains("zip") || type.Contains("gmz") || type.Contains("7z") || - new[] { ".zip", ".gmz", ".7z", ".gz", ".tar", ".rar" }.Any(extension => href.EndsWith(extension, StringComparison.OrdinalIgnoreCase)); + new[] { ".zip", ".gmz", ".7z", ".gz", ".tar", ".rar", ".cityjson", ".json", ".geojson", ".gml", ".citygml", ".asc", ".xyz", ".csv", ".tif", ".tiff" }.Any(extension => href.EndsWith(extension, StringComparison.OrdinalIgnoreCase)); } public static Uri ResolveUrl(XElement link, Uri source) @@ -100,7 +100,7 @@ public static Uri ResolveUrl(XElement link, Uri source) return new UriBuilder(uri) { Fragment = "" }.Uri; } - private static string? ReadCategorySrs(XElement owner) + public static string? ReadCategorySrs(XElement owner) { var codes = owner.Elements(Atom + "category") .Select(category => Regex.Match((string?)category.Attribute("term") ?? "", @"EPSG(?::|/)(?:[^:/]*[:/])?(\d+)$", RegexOptions.IgnoreCase)) @@ -135,7 +135,7 @@ public static Uri ResolveUrl(XElement link, Uri source) var bounds = new BoundingBox2D(x, y, x + size, y + size); var extent = new AtomSpatialExtent(srs, bounds); var coverageBounds = TransformBounds(coverage, srs); - if (!Intersects(bounds, coverageBounds)) + if (!Intersects(bounds, new BoundingBox2D(coverageBounds.MinX - size, coverageBounds.MinY - size, coverageBounds.MaxX + size, coverageBounds.MaxY + size))) throw new InvalidOperationException($"The declared ATOM grid and coverage disagree for '{url.LocalPath}'. Tile placement cannot be determined safely."); return extent; } diff --git a/RhinoSpatial.Core/OgcApiFeaturesClient.cs b/RhinoSpatial.Core/OgcApiFeaturesClient.cs index 928ae8d..c2f0c92 100644 --- a/RhinoSpatial.Core/OgcApiFeaturesClient.cs +++ b/RhinoSpatial.Core/OgcApiFeaturesClient.cs @@ -1,138 +1,186 @@ -using System; -using System.Collections.Generic; -using System.Net.Http; +using System.Collections.Concurrent; +using System.Globalization; using System.Net.Http.Headers; -using System.Text; -using System.Threading.Tasks; +using System.Text.Json; -namespace RhinoSpatial.Core +namespace RhinoSpatial.Core; + +public record OgcFeatureCollection(string Id, string Title, string ItemsUrl, BoundingBox2D? Wgs84Bounds); +public record OgcFeatureResult(List Features, string StatusNote); + +public sealed class OgcApiFeaturesClient { - public class OgcApiFeaturesClient + private static readonly HttpClient SharedHttpClient = new() { Timeout = TimeSpan.FromSeconds(30) }; + private static readonly ConcurrentDictionary? Collections)> DiscoveryCache = new(); + private readonly HttpClient _http; + + static OgcApiFeaturesClient() { - private static readonly HttpClient SharedHttpClient = new() - { - Timeout = TimeSpan.FromSeconds(30) - }; + SharedHttpClient.DefaultRequestHeaders.UserAgent.Add(new ProductInfoHeaderValue("RhinoSpatial", "1.0")); + SharedHttpClient.DefaultRequestHeaders.Accept.Add(new MediaTypeWithQualityHeaderValue("application/json")); + SharedHttpClient.DefaultRequestHeaders.Accept.Add(new MediaTypeWithQualityHeaderValue("application/geo+json")); + } - static OgcApiFeaturesClient() - { - SharedHttpClient.DefaultRequestHeaders.UserAgent.Add(new ProductInfoHeaderValue("RhinoSpatial", "1.0")); - SharedHttpClient.DefaultRequestHeaders.Accept.Add(new MediaTypeWithQualityHeaderValue("application/geo+json")); - SharedHttpClient.DefaultRequestHeaders.Accept.Add(new MediaTypeWithQualityHeaderValue("application/json")); - SharedHttpClient.DefaultRequestHeaders.Accept.Add(new MediaTypeWithQualityHeaderValue("*/*")); - } + public OgcApiFeaturesClient() : this(null) { } + public OgcApiFeaturesClient(HttpClient? httpClient) => _http = httpClient ?? SharedHttpClient; - public async Task> LoadFeaturesAsync( - string sourceUrl, - string sourceLayerName, - BoundingBox2D? wgs84BoundingBox, - int maxFeatures) + public async Task?> TryDiscoverAsync(string sourceUrl, CancellationToken cancellationToken = default) + { + if (!Uri.TryCreate(sourceUrl, UriKind.Absolute, out var source) || source.Scheme is not ("http" or "https")) return null; + if (System.Text.RegularExpressions.Regex.IsMatch(sourceUrl, @"[?&]service=(?:wfs|wms|wcs)(?:&|$)", System.Text.RegularExpressions.RegexOptions.IgnoreCase)) return null; + if (ReferenceEquals(_http, SharedHttpClient) && DiscoveryCache.TryGetValue(sourceUrl, out var cached) && cached.Expires > DateTime.UtcNow) + return cached.Collections; + if (source.AbsolutePath.TrimEnd('/').EndsWith("/items", StringComparison.OrdinalIgnoreCase)) + return new[] { new OgcFeatureCollection(CollectionName(source), CollectionName(source), sourceUrl, null) }; + using var timeout = CancellationTokenSource.CreateLinkedTokenSource(cancellationToken); + timeout.CancelAfter(TimeSpan.FromSeconds(45)); + var queue = new Queue(); queue.Enqueue(sourceUrl); + var visited = new HashSet(StringComparer.Ordinal); + var collections = new Dictionary(StringComparer.Ordinal); + var recognized = false; + while (queue.TryDequeue(out var url)) { - var requestUrl = BuildItemsRequestUrl(sourceUrl, wgs84BoundingBox, maxFeatures); - var response = await SharedHttpClient.GetAsync(requestUrl).ConfigureAwait(false); - var body = await response.Content.ReadAsStringAsync().ConfigureAwait(false); - - if (!response.IsSuccessStatusCode) + if (!visited.Add(url)) continue; + if (visited.Count > 32) throw new InvalidDataException("OGC API discovery exceeded 32 metadata pages."); + (string Text, Uri Url, string MediaType) response; + try { response = await SourceHttp.ReadAsync(_http, url, timeout.Token).ConfigureAwait(false); } + catch (HttpRequestException) when (!recognized && !LooksLikeOgcApiFeaturesUrl(sourceUrl)) { return Store(null); } + if (!response.Text.TrimStart().StartsWith("{", StringComparison.Ordinal)) { - throw new InvalidOperationException( - $"The OGC API Features endpoint returned {(int)response.StatusCode} {response.ReasonPhrase}. URL: {requestUrl}"); + if (recognized) throw new InvalidDataException("An OGC API metadata link did not return JSON."); + return Store(null); } - - return GeoJsonReader.ReadFeatures(body, sourceLayerName); - } - - public static bool LooksLikeOgcApiFeaturesUrl(string? sourceUrl) - { - if (string.IsNullOrWhiteSpace(sourceUrl) || - !Uri.TryCreate(sourceUrl.Trim(), UriKind.Absolute, out var uri)) + using var document = JsonDocument.Parse(response.Text); + var root = document.RootElement; + if (root.TryGetProperty("type", out var type) && type.GetString() == "FeatureCollection") { - return false; + if (!LooksLikeOgcApiFeaturesUrl(sourceUrl)) return Store(null); + return Store(new[] { new OgcFeatureCollection(CollectionName(source), CollectionName(source), sourceUrl, null) }); } - - var path = uri.AbsolutePath; - return path.Contains("/collections/", StringComparison.OrdinalIgnoreCase) || - path.Contains("/items", StringComparison.OrdinalIgnoreCase); - } - - private static string BuildItemsRequestUrl( - string sourceUrl, - BoundingBox2D? wgs84BoundingBox, - int maxFeatures) - { - var builder = new UriBuilder(sourceUrl.Trim()); - if (!builder.Path.Contains("/items", StringComparison.OrdinalIgnoreCase) && - builder.Path.Contains("/collections/", StringComparison.OrdinalIgnoreCase)) + if (root.TryGetProperty("collections", out var entries) && entries.ValueKind == JsonValueKind.Array) { - builder.Path = builder.Path.TrimEnd('/') + "/items"; + recognized = true; + foreach (var collection in entries.EnumerateArray()) AddCollection(collection, response.Url, false); + var next = Link(root, "next", response.Url); + if (next is not null) queue.Enqueue(next); } - - var query = ParseQuery(builder.Query); - query["f"] = "json"; - if (wgs84BoundingBox is not null) + else if (root.TryGetProperty("id", out _) && + (Link(root, "items", response.Url) is not null || LooksLikeOgcApiFeaturesUrl(sourceUrl))) { - query["bbox"] = string.Join( - ",", - FormatNumber(wgs84BoundingBox.MinX), - FormatNumber(wgs84BoundingBox.MinY), - FormatNumber(wgs84BoundingBox.MaxX), - FormatNumber(wgs84BoundingBox.MaxY)); + recognized = true; AddCollection(root, response.Url, true); } + else if (Link(root, "data", response.Url) is { } data) + { recognized = true; queue.Enqueue(data); } + else if (!recognized) return Store(null); + if (collections.Count > 1000) throw new InvalidDataException("OGC API discovery exceeded 1,000 collections; use a specific collection URL."); + } + return Store(recognized ? collections.Values.ToList() : null); - if (maxFeatures > 0) + IReadOnlyList? Store(IReadOnlyList? value) + { + if (ReferenceEquals(_http, SharedHttpClient)) { - query["limit"] = maxFeatures.ToString(System.Globalization.CultureInfo.InvariantCulture); + if (DiscoveryCache.Count > 128) DiscoveryCache.Clear(); + DiscoveryCache[sourceUrl] = (DateTime.UtcNow.AddMinutes(5), value); } - - builder.Query = BuildQuery(query); - return builder.Uri.ToString(); + return value; } - private static Dictionary ParseQuery(string query) + void AddCollection(JsonElement value, Uri baseUrl, bool isSingle) { - var values = new Dictionary(StringComparer.OrdinalIgnoreCase); - var trimmed = query.TrimStart('?'); - if (string.IsNullOrWhiteSpace(trimmed)) + if (!value.TryGetProperty("id", out var idElement) || idElement.ValueKind != JsonValueKind.String) return; + if (value.TryGetProperty("itemType", out var itemType) && itemType.GetString() != "feature") return; + var id = idElement.GetString()!; + var self = Link(value, "self", baseUrl); + var items = Link(value, "items", baseUrl); + if (items is null) { - return values; + var builder = new UriBuilder(self ?? baseUrl.AbsoluteUri); + if (self is null && !isSingle) builder.Path = builder.Path.TrimEnd('/') + "/" + Uri.EscapeDataString(id); + builder.Path = builder.Path.TrimEnd('/') + "/items"; + items = builder.Uri.AbsoluteUri; } - - foreach (var part in trimmed.Split('&', StringSplitOptions.RemoveEmptyEntries)) + BoundingBox2D? bounds = null; + if (value.TryGetProperty("extent", out var extent) && extent.TryGetProperty("spatial", out var spatial) && + (!spatial.TryGetProperty("crs", out var crs) || crs.GetString()?.Contains("CRS84", StringComparison.OrdinalIgnoreCase) == true) && + spatial.TryGetProperty("bbox", out var boxes) && boxes.ValueKind == JsonValueKind.Array) { - var separatorIndex = part.IndexOf('=', StringComparison.Ordinal); - if (separatorIndex < 0) + foreach (var box in boxes.EnumerateArray()) { - values[Uri.UnescapeDataString(part)] = string.Empty; - continue; + if (box.ValueKind != JsonValueKind.Array || box.GetArrayLength() is not (4 or 6)) continue; + if (box.EnumerateArray().Any(v => v.ValueKind != JsonValueKind.Number)) continue; + var n = box.GetArrayLength() / 2; + var b = new BoundingBox2D(box[0].GetDouble(), box[1].GetDouble(), box[n].GetDouble(), box[n + 1].GetDouble()); + if (!double.IsFinite(b.MinX) || !double.IsFinite(b.MaxX) || !double.IsFinite(b.MinY) || !double.IsFinite(b.MaxY) || + b.MinX < -180 || b.MinX > 180 || b.MaxX < -180 || b.MaxX > 180 || b.MinY < -90 || b.MaxY > 90 || b.MinY > b.MaxY) continue; + // A wrapped collection extent needs a conservative world-width map view. + if (b.MinX > b.MaxX) b = new BoundingBox2D(-180, b.MinY, 180, b.MaxY); + bounds = bounds is null ? b : new BoundingBox2D(Math.Min(bounds.MinX, b.MinX), Math.Min(bounds.MinY, b.MinY), Math.Max(bounds.MaxX, b.MaxX), Math.Max(bounds.MaxY, b.MaxY)); } - - values[Uri.UnescapeDataString(part[..separatorIndex])] = - Uri.UnescapeDataString(part[(separatorIndex + 1)..]); } - - return values; + collections[id] = new OgcFeatureCollection(id, value.TryGetProperty("title", out var title) ? title.GetString() ?? id : id, items, bounds); } + } - private static string BuildQuery(IReadOnlyDictionary values) - { - var builder = new StringBuilder(); - foreach (var entry in values) - { - if (builder.Length > 0) - { - builder.Append('&'); - } + public async Task> LoadFeaturesAsync(string sourceUrl, string sourceLayerName, BoundingBox2D? wgs84BoundingBox, int maxFeatures) => + (await LoadFeaturesWithStatusAsync(sourceUrl, sourceLayerName, wgs84BoundingBox, maxFeatures).ConfigureAwait(false)).Features; - builder.Append(Uri.EscapeDataString(entry.Key)); - builder.Append('='); - builder.Append(Uri.EscapeDataString(entry.Value)); - } + public async Task LoadFeaturesWithStatusAsync(string sourceUrl, string sourceLayerName, + BoundingBox2D? wgs84BoundingBox, int maxFeatures, CancellationToken cancellationToken = default) + { + var collections = await TryDiscoverAsync(sourceUrl, cancellationToken).ConfigureAwait(false) + ?? throw new InvalidDataException("The URL is not an OGC API Features source."); + var collection = collections.FirstOrDefault(c => c.Id == sourceLayerName) ?? + (collections.Count == 1 ? collections[0] : throw new InvalidOperationException("Select an OGC API collection using List WFS Layers and the Layer input.")); + var url = BuildItemsRequestUrl(collection.ItemsUrl, wgs84BoundingBox, maxFeatures); + var result = await FeatureCollectionPager.LoadHttpAsync(url, + async (pageUrl, token) => { var page = await SourceHttp.ReadAsync(_http, pageUrl, token).ConfigureAwait(false); return new FeaturePageContent(page.Text, page.Url); }, + maxFeatures, cancellationToken: cancellationToken).ConfigureAwait(false); + var returnedSrs = GeoJsonReader.TryReadSourceSrs(result.Text); + if (returnedSrs is not null && SpatialReferenceTransform.NormalizeSrs(returnedSrs) != "EPSG:4326") + throw new InvalidDataException("The OGC API returned a non-default GeoJSON CRS despite a longitude/latitude request."); + return new OgcFeatureResult(GeoJsonReader.ReadFeatures(result.Text, collection.Id), result.StatusNote); + } - return builder.ToString(); - } + public static bool LooksLikeOgcApiFeaturesUrl(string? sourceUrl) => + Uri.TryCreate(sourceUrl, UriKind.Absolute, out var uri) && + (uri.AbsolutePath.Contains("/collections/", StringComparison.OrdinalIgnoreCase) || + uri.AbsolutePath.TrimEnd('/').EndsWith("/collections", StringComparison.OrdinalIgnoreCase) || + uri.AbsolutePath.Contains("/items", StringComparison.OrdinalIgnoreCase)); + + private static string CollectionName(Uri uri) + { + var parts = uri.AbsolutePath.Split('/', StringSplitOptions.RemoveEmptyEntries); + var index = Array.FindIndex(parts, p => p == "collections"); + return index >= 0 && index + 1 < parts.Length ? Uri.UnescapeDataString(parts[index + 1]) : "features"; + } - private static string FormatNumber(double value) + private static string? Link(JsonElement owner, string relation, Uri source) + { + if (!owner.TryGetProperty("links", out var links) || links.ValueKind != JsonValueKind.Array) return null; + var matches = links.EnumerateArray().Where(l => l.TryGetProperty("rel", out var rel) && rel.GetString() == relation && l.TryGetProperty("href", out _)) + .OrderByDescending(l => l.TryGetProperty("type", out var type) && type.GetString()?.Contains("json", StringComparison.OrdinalIgnoreCase) == true); + foreach (var link in matches) { - return value.ToString("G17", System.Globalization.CultureInfo.InvariantCulture); + var result = new Uri(source, link.GetProperty("href").GetString()!); + if (result.Scheme is not ("http" or "https") || !string.IsNullOrEmpty(result.UserInfo)) throw new InvalidDataException("Unsupported OGC API link URL."); + return result.AbsoluteUri; } + return null; + } + + private static string BuildItemsRequestUrl(string sourceUrl, BoundingBox2D? bounds, int maxFeatures) + { + var builder = new UriBuilder(sourceUrl); + var query = builder.Query.TrimStart('?').Split('&', StringSplitOptions.RemoveEmptyEntries) + .Select(p => p.Split('=', 2)).ToDictionary(p => Uri.UnescapeDataString(p[0]), p => p.Length == 2 ? Uri.UnescapeDataString(p[1]) : "", StringComparer.OrdinalIgnoreCase); + var pageSize = query.TryGetValue("limit", out var limit) && int.TryParse(limit, out var n) && n > 0 ? Math.Min(n, FeatureCollectionPager.PageSize) : FeatureCollectionPager.PageSize; + query["f"] = "json"; + query["limit"] = (maxFeatures > 0 ? Math.Min(maxFeatures, pageSize) : pageSize).ToString(CultureInfo.InvariantCulture); + // This client requests GeoJSON in its default longitude/latitude CRS. + query.Remove("crs"); query.Remove("bbox-crs"); + if (bounds is not null) query["bbox"] = string.Join(",", new[] { bounds.MinX, bounds.MinY, bounds.MaxX, bounds.MaxY }.Select(x => x.ToString("G17", CultureInfo.InvariantCulture))); + builder.Query = string.Join("&", query.Select(p => Uri.EscapeDataString(p.Key) + "=" + Uri.EscapeDataString(p.Value))); + return builder.Uri.AbsoluteUri; } } diff --git a/RhinoSpatial.Core/RemoteReferenceMetadataReader.cs b/RhinoSpatial.Core/RemoteReferenceMetadataReader.cs new file mode 100644 index 0000000..928b8c0 --- /dev/null +++ b/RhinoSpatial.Core/RemoteReferenceMetadataReader.cs @@ -0,0 +1,45 @@ +using System.Collections.Concurrent; + +namespace RhinoSpatial.Core; + +/// Uses discovery metadata endpoints and rejects known data-file paths. +public sealed class RemoteReferenceMetadataReader +{ + private static readonly ConcurrentDictionary Cache = new(); + private readonly AtomDownloadClient _atom; + private readonly OgcApiFeaturesClient _ogc; + private readonly bool _cache; + public RemoteReferenceMetadataReader(HttpClient? http = null) + { _atom = new AtomDownloadClient(http); _ogc = new OgcApiFeaturesClient(http); _cache = http is null; } + + public async Task TryReadAsync(string source, string? layer = null, CancellationToken cancellationToken = default) + { + if (!Uri.TryCreate(source, UriKind.Absolute, out var uri) || uri.Scheme is not ("http" or "https")) return null; + if (System.Text.RegularExpressions.Regex.IsMatch(source, @"[?&]service=(wfs|wms|wcs)(?:&|$)", System.Text.RegularExpressions.RegexOptions.IgnoreCase)) return null; + if (new[] { ".zip", ".geojson", ".cityjson", ".tif", ".tiff", ".asc", ".xyz", ".csv", ".gml", ".citygml" } + .Contains(Path.GetExtension(uri.AbsolutePath), StringComparer.OrdinalIgnoreCase)) + throw new InvalidDataException("Use the provider's service/feed metadata URL as Reference Source, or a supported local file. Remote data files are not downloaded to orient the map."); + var key = source + "\n" + layer; + if (_cache && Cache.TryGetValue(key, out var cached) && cached.Expires > DateTime.UtcNow) return cached.Value; + // The parent collection holds metadata; /items would download feature data. + var metadataUrl = source; + if (uri.AbsolutePath.TrimEnd('/').EndsWith("/items", StringComparison.OrdinalIgnoreCase)) + { + var parent = new UriBuilder(uri); parent.Path = parent.Path.TrimEnd('/')[..^6]; parent.Query = "f=json"; + metadataUrl = parent.Uri.AbsoluteUri; + } + var collections = await _ogc.TryDiscoverAsync(metadataUrl, cancellationToken).ConfigureAwait(false); + ReferenceSourceMetadata? result; + if (collections is not null) + { + var selected = string.IsNullOrWhiteSpace(layer) ? collections : collections.Where(c => c.Id == layer).ToList(); + if (selected.Count == 0) throw new InvalidDataException("The selected OGC API reference collection was not found."); + var bounds = selected.Select(c => c.Wgs84Bounds).Where(b => b is not null).Cast().ToList(); + var union = bounds.Count == 0 ? null : new BoundingBox2D(bounds.Min(b => b.MinX), bounds.Min(b => b.MinY), bounds.Max(b => b.MaxX), bounds.Max(b => b.MaxY)); + result = new ReferenceSourceMetadata("OGC API Features", "EPSG:4326", union, union, selected.Count); + } + else result = await _atom.TryReadReferenceAsync(source, cancellationToken).ConfigureAwait(false); + if (_cache) { if (Cache.Count > 128) Cache.Clear(); Cache[key] = (DateTime.UtcNow.AddMinutes(5), result); } + return result; + } +} diff --git a/RhinoSpatial.Core/ShapefileFeatureReader.cs b/RhinoSpatial.Core/ShapefileFeatureReader.cs index c0e407a..650524a 100644 --- a/RhinoSpatial.Core/ShapefileFeatureReader.cs +++ b/RhinoSpatial.Core/ShapefileFeatureReader.cs @@ -133,20 +133,20 @@ private static string ResolveSourceSrs(string shapefilePath, string fallbackSrs) var projectionText = File.ReadAllText(prjPath); var normalized = NormalizeKnownProjection(projectionText); - return string.IsNullOrWhiteSpace(normalized) ? fallbackSrs : normalized; + return string.IsNullOrWhiteSpace(normalized) + ? throw new InvalidOperationException("The Shapefile .prj contains an unsupported CRS; no context-CRS assumption was used.") : normalized; } private static string NormalizeKnownProjection(string projectionText) { var text = projectionText ?? string.Empty; - var authorityMatch = Regex.Match( - text, - "AUTHORITY\\[\\s*\"EPSG\"\\s*,\\s*\"(?\\d+)\"\\s*\\]", - RegexOptions.IgnoreCase); - if (authorityMatch.Success) + try { - return $"EPSG:{authorityMatch.Groups["epsg"].Value}"; + var coordinateSystem = new ProjNet.CoordinateSystems.CoordinateSystemFactory().CreateFromWkt(text); + if (coordinateSystem.Authority?.Equals("EPSG", StringComparison.OrdinalIgnoreCase) == true && coordinateSystem.AuthorityCode > 0) + return "EPSG:" + coordinateSystem.AuthorityCode.ToString(CultureInfo.InvariantCulture); } + catch { /* Unsupported WKT dialects may still have a known, explicit CRS name below. */ } if (text.Contains("ETRS_1989_UTM_Zone_32N", StringComparison.OrdinalIgnoreCase) || text.Contains("ETRS89 / UTM zone 32N", StringComparison.OrdinalIgnoreCase)) @@ -166,8 +166,9 @@ private static string NormalizeKnownProjection(string projectionText) return "EPSG:3857"; } - if (text.Contains("WGS_1984", StringComparison.OrdinalIgnoreCase) || - text.Contains("WGS 84", StringComparison.OrdinalIgnoreCase)) + if ((text.TrimStart().StartsWith("GEOGCS[", StringComparison.OrdinalIgnoreCase) || + text.TrimStart().StartsWith("GEOGCRS[", StringComparison.OrdinalIgnoreCase)) && + (text.Contains("WGS_1984", StringComparison.OrdinalIgnoreCase) || text.Contains("WGS 84", StringComparison.OrdinalIgnoreCase))) { return "EPSG:4326"; } diff --git a/RhinoSpatial.Core/SourceDownloadCache.cs b/RhinoSpatial.Core/SourceDownloadCache.cs new file mode 100644 index 0000000..2a178e4 --- /dev/null +++ b/RhinoSpatial.Core/SourceDownloadCache.cs @@ -0,0 +1,274 @@ +using System.Globalization; +using System.IO.Compression; +using System.Net.Http; +using System.Security.Cryptography; +using System.Text; +using System.Text.Json; +using System.Xml; + +namespace RhinoSpatial.Core; + +public enum AtomDownloadKind { Buildings, Terrain, Vector } + +internal sealed class LimitedReadStream : Stream +{ + private readonly Stream inner; + private readonly long maximum; + public LimitedReadStream(Stream stream, long limit) { inner = stream; maximum = limit; } + private long _read; + private int Count(int count) { _read += count; if (_read > maximum) throw new InvalidDataException("ZIP expanded beyond its declared entry size."); return count; } + public override int Read(byte[] buffer, int offset, int count) => Count(inner.Read(buffer, offset, count)); + public override int Read(Span buffer) => Count(inner.Read(buffer)); + public override async ValueTask ReadAsync(Memory buffer, CancellationToken token = default) => Count(await inner.ReadAsync(buffer, token).ConfigureAwait(false)); + public override Task ReadAsync(byte[] buffer, int offset, int count, CancellationToken token) => ReadAsync(buffer.AsMemory(offset, count), token).AsTask(); + public override bool CanRead => true; + public override bool CanSeek => false; + public override bool CanWrite => false; + public override long Length => throw new NotSupportedException(); + public override long Position { get => _read; set => throw new NotSupportedException(); } + public override void Flush() => throw new NotSupportedException(); + public override long Seek(long offset, SeekOrigin origin) => throw new NotSupportedException(); + public override void SetLength(long value) => throw new NotSupportedException(); + public override void Write(byte[] buffer, int offset, int count) => throw new NotSupportedException(); + protected override void Dispose(bool disposing) { if (disposing) inner.Dispose(); base.Dispose(disposing); } +} + +/// Validated, atomic payload cache shared by direct downloads and ATOM services. +public sealed class SourceDownloadCache +{ + private static readonly HttpClient SharedHttp = new() { Timeout = Timeout.InfiniteTimeSpan }; + private static readonly string[] Extensions = { ".zip", ".geojson", ".cityjson", ".gml", ".tif", ".asc", ".csv" }; + private readonly HttpClient _http; + private readonly string _directory; + private readonly AtomDownloadKind _kind; + private sealed record Stamp(long Length, string Hash, DateTime DownloadedUtc); + public SourceDownloadCache(AtomDownloadKind kind, HttpClient? http = null, string? directory = null) + { + _kind = kind; _http = http ?? SharedHttp; + _directory = directory ?? Path.Combine(Environment.GetFolderPath(Environment.SpecialFolder.LocalApplicationData), + "RhinoSpatial", kind == AtomDownloadKind.Buildings ? "Lod2AtomCache" : kind == AtomDownloadKind.Terrain ? "TerrainAtomCache" : "VectorDownloadCache"); + } + + public async Task DownloadAsync(Lod2AtomLink tile, Action? progress = null, CancellationToken cancellationToken = default) + { + Directory.CreateDirectory(_directory); + var key = Convert.ToHexString(SHA256.HashData(Encoding.UTF8.GetBytes(tile.Url.AbsoluteUri + "\n" + tile.Revision))); + var stem = Path.Combine(_directory, key); + var temporary = stem + "." + Guid.NewGuid().ToString("N") + ".partial"; + using var timeout = CancellationTokenSource.CreateLinkedTokenSource(cancellationToken); + timeout.CancelAfter(TimeSpan.FromMinutes(3)); + try + { + await using var cacheLock = await LockAsync(stem + ".lock", timeout.Token).ConfigureAwait(false); + foreach (var suffix in Extensions) + if (await ValidCacheAsync(stem + suffix, tile.Revision, timeout.Token).ConfigureAwait(false)) + return new Lod2AtomCachedTile(tile, stem + suffix, true); + using var response = await _http.GetAsync(tile.Url, HttpCompletionOption.ResponseHeadersRead, timeout.Token).ConfigureAwait(false); + response.EnsureSuccessStatusCode(); + var maximum = _kind == AtomDownloadKind.Vector ? 128L * 1024 * 1024 : 512L * 1024 * 1024; + var expected = response.Content.Headers.ContentLength; + if (expected > maximum) throw new InvalidDataException("Download exceeds the supported payload size limit."); + await using (var input = await response.Content.ReadAsStreamAsync(timeout.Token).ConfigureAwait(false)) + await using (var output = new FileStream(temporary, FileMode.CreateNew, FileAccess.Write, FileShare.None, 81920, true)) + { + var buffer = new byte[81920]; long count = 0; long previous = 0; int read; + while ((read = await input.ReadAsync(buffer, timeout.Token).ConfigureAwait(false)) > 0) + { + count += read; + if (count > maximum) throw new InvalidDataException("Download exceeds the supported payload size limit."); + await output.WriteAsync(buffer.AsMemory(0, read), timeout.Token).ConfigureAwait(false); + if (count - previous > 1024 * 1024) { progress?.Invoke($"downloaded {count / (1024 * 1024)} MiB"); previous = count; } + } + if (expected.HasValue && count != expected) throw new InvalidDataException("The source download was incomplete."); + } + var extension = await InspectPayloadAsync(temporary, _kind, timeout.Token).ConfigureAwait(false); + var target = stem + extension; + Stamp stamp; + await using (var input = File.OpenRead(temporary)) + stamp = new Stamp(input.Length, Convert.ToHexString(await SHA256.HashDataAsync(input, timeout.Token).ConfigureAwait(false)), DateTime.UtcNow); + File.Move(temporary, target, true); + await File.WriteAllTextAsync(temporary, JsonSerializer.Serialize(stamp), timeout.Token).ConfigureAwait(false); + File.Move(temporary, target + ".json", true); + return new Lod2AtomCachedTile(tile, target, false); + } + catch (OperationCanceledException) when (!cancellationToken.IsCancellationRequested) + { throw new TimeoutException("The source download timed out after three minutes."); } + finally { if (File.Exists(temporary)) File.Delete(temporary); } + } + + private static async Task LockAsync(string path, CancellationToken token) + { + while (true) + { + token.ThrowIfCancellationRequested(); + try { return new FileStream(path, FileMode.OpenOrCreate, FileAccess.ReadWrite, FileShare.None); } + catch (IOException) { await Task.Delay(150, token).ConfigureAwait(false); } + } + } + private static async Task ValidCacheAsync(string path, string revision, CancellationToken token) + { + try + { + if (!File.Exists(path) || !File.Exists(path + ".json")) return false; + var stamp = JsonSerializer.Deserialize(await File.ReadAllTextAsync(path + ".json", token).ConfigureAwait(false)); + if (stamp is null || new FileInfo(path).Length != stamp.Length || + (string.IsNullOrWhiteSpace(revision) && DateTime.UtcNow - stamp.DownloadedUtc > TimeSpan.FromDays(1))) return false; + await using var input = File.OpenRead(path); + return Convert.ToHexString(await SHA256.HashDataAsync(input, token).ConfigureAwait(false)) == stamp.Hash; + } + catch (Exception ex) when (ex is IOException or JsonException) { return false; } + } + + private static async Task InspectPayloadAsync(string path, AtomDownloadKind kind, CancellationToken token) + { + var header = new byte[512]; int length; + await using (var input = File.OpenRead(path)) length = await input.ReadAsync(header, token).ConfigureAwait(false); + if (length >= 4 && header[0] == 'P' && header[1] == 'K') + { + using var archive = ZipFile.OpenRead(path); + ValidateEntries(archive); + var supported = 0; + var names = archive.Entries.Select(e => e.FullName).ToHashSet(StringComparer.OrdinalIgnoreCase); + foreach (var entry in archive.Entries) + { + token.ThrowIfCancellationRequested(); + var extension = Path.GetExtension(entry.FullName).ToLowerInvariant(); + if (kind == AtomDownloadKind.Vector && extension == ".shp") + { + foreach (var companion in new[] { ".shx", ".dbf", ".prj" }) + if (!names.Contains(Path.ChangeExtension(entry.FullName, companion))) throw new InvalidDataException("Shapefile ZIP is missing a matching " + companion + " companion file."); + supported++; continue; + } + var candidate = kind switch + { + AtomDownloadKind.Buildings => extension is ".gml" or ".citygml" or ".xml" or ".json" or ".cityjson" or ".geojson", + AtomDownloadKind.Terrain => extension is ".tif" or ".tiff" or ".asc" or ".xyz" or ".csv", + _ => extension is ".json" or ".geojson" + }; + if (!candidate) continue; + await using var stream = new LimitedReadStream(entry.Open(), entry.Length); + if (kind == AtomDownloadKind.Buildings && extension is ".gml" or ".citygml" or ".xml") + { if (await IsBuildingXmlAsync(stream, token).ConfigureAwait(false)) supported++; } + else if (extension is ".json" or ".geojson" or ".cityjson") + { if (await IsSupportedJsonAsync(stream, kind, token).ConfigureAwait(false)) supported++; } + else + { + // Read through EOF to verify decompression before promoting a completed cache entry. + var buffer = new byte[81920]; while (await stream.ReadAsync(buffer, token).ConfigureAwait(false) > 0) { } + supported++; + } + } + if (supported == 0) throw new InvalidDataException($"Archive contains no supported {kind} data. Expected a ZIP with CityGML/CityJSON buildings, terrain grids, or vector files for the selected loader."); + return ".zip"; + } + if (new FileInfo(path).Length > 256L * 1024 * 1024) throw new InvalidDataException("Direct data file exceeds 256 MiB; use a smaller dataset."); + var text = Encoding.UTF8.GetString(header, 0, length).TrimStart('\uFEFF', ' ', '\r', '\n', '\t'); + if (text.StartsWith("{", StringComparison.Ordinal)) + { + await using var input = File.OpenRead(path); + if (await IsSupportedJsonAsync(input, kind, token).ConfigureAwait(false)) + return kind == AtomDownloadKind.Buildings ? ".cityjson" : ".geojson"; + } + if (kind == AtomDownloadKind.Buildings && text.StartsWith("<", StringComparison.Ordinal)) + { + await using var input = File.OpenRead(path); + if (await IsBuildingXmlAsync(input, token).ConfigureAwait(false)) return ".gml"; + } + if (kind == AtomDownloadKind.Terrain) + { + if (length >= 4 && ((header[0] == 'I' && header[1] == 'I') || (header[0] == 'M' && header[1] == 'M'))) + { GeoTiffReader.ReadImageInfo(path); return ".tif"; } + if (text.StartsWith("ncols", StringComparison.OrdinalIgnoreCase)) return ".asc"; + foreach (var line in text.Split('\n')) + { + var parts = line.Split(new[] { ' ', '\t', ',', ';' }, StringSplitOptions.RemoveEmptyEntries); + if (parts.Length >= 3 && parts.Take(3).All(p => double.TryParse(p, NumberStyles.Float, CultureInfo.InvariantCulture, out _))) return ".csv"; + } + } + throw new InvalidDataException($"Unsupported {kind} download; its content does not match a supported data file or ZIP archive."); + } + + private static async Task IsSupportedJsonAsync(Stream input, AtomDownloadKind kind, CancellationToken token) + { + using var document = await JsonDocument.ParseAsync(input, cancellationToken: token).ConfigureAwait(false); + var root = document.RootElement; + if (!root.TryGetProperty("type", out var type) || type.ValueKind != JsonValueKind.String) return false; + if (kind == AtomDownloadKind.Buildings && type.GetString() == "CityJSON") + { + if (!root.TryGetProperty("metadata", out var meta) || !meta.TryGetProperty("referenceSystem", out _)) + throw new InvalidDataException("Downloaded CityJSON must declare its source CRS in metadata.referenceSystem."); + return root.TryGetProperty("CityObjects", out _) && root.TryGetProperty("vertices", out _); + } + return kind == AtomDownloadKind.Vector && type.GetString() is "FeatureCollection" or "Feature"; + } + private static async Task IsBuildingXmlAsync(Stream input, CancellationToken token) + { + using var reader = XmlReader.Create(input, new XmlReaderSettings { Async = true, DtdProcessing = DtdProcessing.Prohibit, XmlResolver = null, MaxCharactersInDocument = 256L * 1024 * 1024 }); + await reader.MoveToContentAsync().ConfigureAwait(false); + if (reader.LocalName != "CityModel" || !reader.NamespaceURI.StartsWith("http://www.opengis.net/citygml", StringComparison.Ordinal)) return false; + while (await reader.ReadAsync().ConfigureAwait(false)) token.ThrowIfCancellationRequested(); + return true; + } + internal static void ValidateEntries(ZipArchive archive) + { + if (archive.Entries.Count > 10000) throw new InvalidDataException("ZIP contains too many entries."); + long size = 0; var names = new HashSet(StringComparer.OrdinalIgnoreCase); + foreach (var entry in archive.Entries) + { + var name = entry.FullName.Replace('\\', '/'); + if (name.StartsWith('/') || name.Contains(':') || name.Split('/').Any(p => p is ".." or ".") || + ((entry.ExternalAttributes >> 16) & 0xF000) == 0xA000 || !names.Add(name)) + throw new InvalidDataException("ZIP contains an unsafe, symbolic-link or duplicate entry path."); + if (entry.Length > 256L * 1024 * 1024 || (size += entry.Length) > 1024L * 1024 * 1024) + throw new InvalidDataException("ZIP exceeds the expanded-size limit (256 MiB per file, 1 GiB total)."); + } + } +} + +public sealed class SourceFilesLease : IDisposable +{ + private readonly string? _temporary; + public IReadOnlyList Files { get; } + private SourceFilesLease(IReadOnlyList files, string? temporary) { Files = files; _temporary = temporary; } + public static SourceFilesLease Open(string path, CancellationToken cancellationToken = default) + { + if (!Path.GetExtension(path).Equals(".zip", StringComparison.OrdinalIgnoreCase)) return new SourceFilesLease(new[] { path }, null); + var directory = Path.Combine(Path.GetTempPath(), "rhinospatial-source-" + Guid.NewGuid().ToString("N")); + try + { + using var archive = ZipFile.OpenRead(path); SourceDownloadCache.ValidateEntries(archive); + Directory.CreateDirectory(directory); + foreach (var entry in archive.Entries) + { + if (string.IsNullOrEmpty(entry.Name)) continue; + var target = Path.Combine(directory, entry.FullName.Replace('\\', '/')); + Directory.CreateDirectory(Path.GetDirectoryName(target)!); + cancellationToken.ThrowIfCancellationRequested(); + using var input = entry.Open(); using var output = File.Create(target); + var buffer = new byte[81920]; long count = 0; int read; + while ((read = input.Read(buffer)) > 0) + { + cancellationToken.ThrowIfCancellationRequested(); count += read; + if (count > entry.Length || count > 256L * 1024 * 1024) throw new InvalidDataException("ZIP expanded beyond its declared entry size."); + output.Write(buffer, 0, read); + } + if (count != entry.Length) throw new InvalidDataException("ZIP entry is truncated."); + } + var extracted = Directory.GetFiles(directory, "*", SearchOption.AllDirectories); + foreach (var shape in extracted.Where(f => Path.GetExtension(f).Equals(".shp", StringComparison.OrdinalIgnoreCase))) + foreach (var extension in new[] { ".dbf", ".shx", ".prj", ".cpg" }) + { + var expected = Path.ChangeExtension(shape, extension); + var actual = extracted.FirstOrDefault(f => f.Equals(expected, StringComparison.OrdinalIgnoreCase)); + if (actual is not null && actual != expected) + { + var intermediate = actual + ".normalize-" + Guid.NewGuid().ToString("N"); + File.Move(actual, intermediate); File.Move(intermediate, expected); + } + } + return new SourceFilesLease(Directory.GetFiles(directory, "*", SearchOption.AllDirectories), directory); + } + catch { if (Directory.Exists(directory)) Directory.Delete(directory, true); throw; } + } + public void Dispose() { if (_temporary is not null && Directory.Exists(_temporary)) Directory.Delete(_temporary, true); } +} diff --git a/RhinoSpatial.Core/SourceHttp.cs b/RhinoSpatial.Core/SourceHttp.cs new file mode 100644 index 0000000..703ddbe --- /dev/null +++ b/RhinoSpatial.Core/SourceHttp.cs @@ -0,0 +1,39 @@ +using System.Net.Http; +using System.Text; + +namespace RhinoSpatial.Core; + +internal static class SourceHttp +{ + public static async Task<(string Text, Uri Url, string MediaType)> ReadAsync(HttpClient http, string url, + CancellationToken cancellationToken = default, int maximumBytes = 16 * 1024 * 1024) + { + using var timeout = CancellationTokenSource.CreateLinkedTokenSource(cancellationToken); + timeout.CancelAfter(TimeSpan.FromSeconds(30)); + using var response = await http.GetAsync(url, HttpCompletionOption.ResponseHeadersRead, timeout.Token).ConfigureAwait(false); + response.EnsureSuccessStatusCode(); + if (response.Content.Headers.ContentLength > maximumBytes) + throw new InvalidDataException("Source response exceeds the supported size limit."); + await using var input = await response.Content.ReadAsStreamAsync(timeout.Token).ConfigureAwait(false); + using var output = new MemoryStream(); + var buffer = new byte[81920]; + int count; + while ((count = await input.ReadAsync(buffer, timeout.Token).ConfigureAwait(false)) > 0) + { + if (output.Length + count > maximumBytes) throw new InvalidDataException("Source response exceeds the supported size limit."); + output.Write(buffer, 0, count); + } + var data = output.ToArray(); + var charset = response.Content.Headers.ContentType?.CharSet?.Trim('"'); + if (string.IsNullOrWhiteSpace(charset)) + { + var declaration = Encoding.ASCII.GetString(data, 0, Math.Min(data.Length, 256)); + var match = System.Text.RegularExpressions.Regex.Match(declaration, "^\\s*<\\?xml[^>]*encoding=['\"]([^'\"]+)['\"]", System.Text.RegularExpressions.RegexOptions.IgnoreCase); + if (match.Success) charset = match.Groups[1].Value; + } + var encoding = string.IsNullOrWhiteSpace(charset) ? Encoding.UTF8 : Encoding.GetEncoding(charset); + using var decoded = new StreamReader(new MemoryStream(data), encoding, true); + return (await decoded.ReadToEndAsync().ConfigureAwait(false), response.RequestMessage?.RequestUri ?? new Uri(url), + response.Content.Headers.ContentType?.MediaType ?? ""); + } +} diff --git a/RhinoSpatial.Core/TerrainAtomLoader.cs b/RhinoSpatial.Core/TerrainAtomLoader.cs new file mode 100644 index 0000000..f8cc81b --- /dev/null +++ b/RhinoSpatial.Core/TerrainAtomLoader.cs @@ -0,0 +1,131 @@ +namespace RhinoSpatial.Core; + +public record TerrainAtomResult(TerrainRasterData Raster, BoundingBox2D Bounds, string Status); + +public sealed class TerrainAtomLoader +{ + private readonly AtomDownloadClient _client; + public TerrainAtomLoader(HttpClient? http = null, string? cacheDirectory = null) => _client = new AtomDownloadClient(http, cacheDirectory, AtomDownloadKind.Terrain); + + public async Task TryLoadAsync(string source, SpatialContext2D context, + Action? progress = null, CancellationToken cancellationToken = default) + { + var selection = await _client.TrySelectAsync(source, context, cancellationToken).ConfigureAwait(false); + if (selection is null) return null; + if (selection.Tiles.Count == 0) throw new InvalidOperationException("No terrain ATOM tiles intersect the selected Spatial Context."); + var downloads = await _client.DownloadAsync(selection, progress, cancellationToken).ConfigureAwait(false); + var rasters = new List(); + long samples = 0; + foreach (var download in downloads) + { + cancellationToken.ThrowIfCancellationRequested(); + using var files = SourceFilesLease.Open(download.FilePath, cancellationToken); + foreach (var file in files.Files) + { + var extension = Path.GetExtension(file).ToLowerInvariant(); + if (extension is not (".tif" or ".tiff" or ".asc" or ".csv" or ".xyz")) continue; + progress?.Invoke("Reading terrain " + Path.GetFileName(download.Tile.Url.LocalPath)); + var srs = download.Tile.NativeSrs; + TerrainRasterData raster; + if (extension is ".tif" or ".tiff") + { + var metadata = GeoTiffReader.ReadImageInfo(file); + srs = metadata.SrsName; + raster = TerrainRasterReader.ReadRaster(file, Path.GetFileName(file), srs); + } + else + { + if (string.IsNullOrWhiteSpace(srs)) + throw new InvalidDataException("Terrain text downloads require an unambiguous source CRS in the ATOM category metadata."); + var window = Lod2AtomFeedReader.TransformBounds(new AtomSpatialExtent(context.ResolvedSrs, context.RequestBoundingBox), srs); + raster = TerrainTextGridReader.ReadWindow(file, Path.GetFileName(file), srs, window, cancellationToken).Raster; + } + var bounds = Lod2AtomFeedReader.TransformBounds(new AtomSpatialExtent(context.ResolvedSrs, context.RequestBoundingBox), srs); + raster = TerrainRasterMosaic.Crop(raster, bounds); + samples += (long)raster.Width * raster.Height; + if (samples > 8388608) throw new InvalidOperationException("Terrain ATOM windows exceed eight million source samples; select a smaller area."); + rasters.Add(raster); + } + } + if (rasters.Count == 0) throw new InvalidDataException("Selected terrain archives contained no readable terrain grids."); + var mosaic = TerrainRasterMosaic.Merge(rasters, cancellationToken); + var finalBounds = TerrainRasterMosaic.Bounds(mosaic); + return new TerrainAtomResult(mosaic, finalBounds, + $"ATOM terrain: {downloads.Count} selected tile(s), {downloads.Count(t => t.UsedCache)} cached, {mosaic.Width}x{mosaic.Height} samples, {mosaic.SrsName}. Tiles: {string.Join(", ", downloads.Select(t => Path.GetFileName(t.Tile.Url.LocalPath)))}."); + } +} + +public static class TerrainRasterMosaic +{ + private const int MaximumCells = 4194304; + private static void Validate(TerrainRasterData raster) + { + if (raster.Width < 2 || raster.Height < 2 || raster.OffsetVectorX.X <= 0 || raster.OffsetVectorY.Y >= 0 || + Math.Abs(raster.OffsetVectorX.Y) > 1e-9 || Math.Abs(raster.OffsetVectorY.X) > 1e-9) + throw new InvalidDataException("Terrain ATOM mosaics require regular north-up grids with at least 2x2 samples."); + } + public static BoundingBox2D Bounds(TerrainRasterData raster) => new(raster.Origin.X, + raster.Origin.Y + (raster.Height - 1) * raster.OffsetVectorY.Y, + raster.Origin.X + (raster.Width - 1) * raster.OffsetVectorX.X, raster.Origin.Y); + + public static TerrainRasterData Crop(TerrainRasterData raster, BoundingBox2D window) + { + Validate(raster); + var x0 = Math.Clamp((int)Math.Floor((window.MinX - raster.Origin.X) / raster.OffsetVectorX.X), 0, raster.Width - 1); + var x1 = Math.Clamp((int)Math.Ceiling((window.MaxX - raster.Origin.X) / raster.OffsetVectorX.X), 0, raster.Width - 1); + var y0 = Math.Clamp((int)Math.Floor((raster.Origin.Y - window.MaxY) / -raster.OffsetVectorY.Y), 0, raster.Height - 1); + var y1 = Math.Clamp((int)Math.Ceiling((raster.Origin.Y - window.MinY) / -raster.OffsetVectorY.Y), 0, raster.Height - 1); + var width = x1 - x0 + 1; var height = y1 - y0 + 1; + if (width < 2 || height < 2 || (long)width * height > MaximumCells) + throw new InvalidDataException("Terrain window has fewer than 2x2 or more than four million samples; adjust the selected area."); + var samples = new float[width * height]; + for (var row = 0; row < height; row++) Array.Copy(raster.Elevations, (row + y0) * raster.Width + x0, samples, row * width, width); + return raster with { Width = width, Height = height, Origin = new Coordinate2D(raster.Origin.X + x0 * raster.OffsetVectorX.X, raster.Origin.Y + y0 * raster.OffsetVectorY.Y), Elevations = samples }; + } + + public static TerrainRasterData Merge(IReadOnlyList rasters, CancellationToken cancellationToken = default) + { + if (rasters.Count == 0) throw new ArgumentException("No terrain rasters."); + var first = rasters[0]; + foreach (var raster in rasters) + { + Validate(raster); + if (SpatialReferenceTransform.NormalizeSrs(raster.SrsName) != SpatialReferenceTransform.NormalizeSrs(first.SrsName) || + Math.Abs(raster.OffsetVectorX.X - first.OffsetVectorX.X) > 1e-6 || Math.Abs(raster.OffsetVectorY.Y - first.OffsetVectorY.Y) > 1e-6) + throw new InvalidDataException("Terrain tiles have different coordinate systems or grid spacing. They cannot be mosaicked without resampling."); + } + var boxes = rasters.Select(Bounds).ToList(); + var minX = boxes.Min(b => b.MinX); var maxY = boxes.Max(b => b.MaxY); + var dx = first.OffsetVectorX.X; var dy = -first.OffsetVectorY.Y; + int Index(double value) + { + var rounded = Math.Round(value); + if (!double.IsFinite(value) || Math.Abs(value - rounded) > 1e-5 || rounded < 0 || rounded > int.MaxValue) + throw new InvalidDataException("Terrain tile origins are not aligned to one regular grid."); + return (int)rounded; + } + var width = checked(Index((boxes.Max(b => b.MaxX) - minX) / dx) + 1); + var height = checked(Index((maxY - boxes.Min(b => b.MinY)) / dy) + 1); + if ((long)width * height > MaximumCells) throw new InvalidDataException("Terrain mosaic exceeds four million cells; select a smaller area."); + var values = Enumerable.Repeat(float.NaN, width * height).ToArray(); + foreach (var raster in rasters) + { + var x0 = Index((raster.Origin.X - minX) / dx); var y0 = Index((maxY - raster.Origin.Y) / dy); + for (var row = 0; row < raster.Height; row++) + { + cancellationToken.ThrowIfCancellationRequested(); + for (var column = 0; column < raster.Width; column++) + { + var value = raster.Elevations[row * raster.Width + column]; + if (TerrainMeshBuilder.IsNoData(raster, value)) continue; + var index = (row + y0) * width + column + x0; + if (float.IsFinite(values[index]) && Math.Abs(values[index] - value) > 0.01f) + throw new InvalidDataException("Overlapping terrain tiles disagree on elevation; no artificial seam was created."); + values[index] = value; + } + } + } + return first with { CoverageId = "ATOM terrain mosaic", Width = width, Height = height, + Origin = new Coordinate2D(minX, maxY), NoDataValue = double.NaN, Elevations = values }; + } +} diff --git a/RhinoSpatial.Core/TerrainTextGridReader.cs b/RhinoSpatial.Core/TerrainTextGridReader.cs index 18c708a..b112549 100644 --- a/RhinoSpatial.Core/TerrainTextGridReader.cs +++ b/RhinoSpatial.Core/TerrainTextGridReader.cs @@ -222,11 +222,21 @@ private static TerrainTextGridSourceMetadata ReadXyzGridMetadata( "XYZ/CSV terrain grid"); } - private static TerrainTextGridReadResult ReadXyzGrid(string filePath, string coverageId, string sourceSrs) + public static TerrainTextGridReadResult ReadWindow(string filePath, string coverageId, string sourceSrs, + BoundingBox2D bounds, System.Threading.CancellationToken cancellationToken = default) + { + if (Path.GetExtension(filePath).Equals(".asc", StringComparison.OrdinalIgnoreCase)) + return ReadRaster(filePath, coverageId, sourceSrs); + return ReadXyzGrid(filePath, coverageId, sourceSrs, bounds, cancellationToken); + } + + private static TerrainTextGridReadResult ReadXyzGrid(string filePath, string coverageId, string sourceSrs, + BoundingBox2D? bounds = null, System.Threading.CancellationToken cancellationToken = default) { var samples = new List<(double X, double Y, float Z)>(); foreach (var line in File.ReadLines(filePath)) { + cancellationToken.ThrowIfCancellationRequested(); var parts = SplitFields(line); if (parts.Length < 3 || !double.TryParse(parts[0], NumberStyles.Float, CultureInfo.InvariantCulture, out var x) || @@ -237,6 +247,8 @@ private static TerrainTextGridReadResult ReadXyzGrid(string filePath, string cov } RequireFinitePosition(x, y); + if (bounds is not null && (x < bounds.MinX || x > bounds.MaxX || y < bounds.MinY || y > bounds.MaxY)) continue; + if (bounds is not null && samples.Count >= 4194304) throw new InvalidOperationException("Terrain window exceeds four million samples; select a smaller area."); samples.Add((x, y, z)); } diff --git a/RhinoSpatial.Core/VectorSourceResolver.cs b/RhinoSpatial.Core/VectorSourceResolver.cs index e83c38d..3694662 100644 --- a/RhinoSpatial.Core/VectorSourceResolver.cs +++ b/RhinoSpatial.Core/VectorSourceResolver.cs @@ -10,7 +10,8 @@ public enum VectorSourceKind Wfs, OgcApiFeatures, LocalShapefile, - LocalGeoJson + LocalGeoJson, + RemoteDownload } public record VectorSourceInfo( @@ -58,6 +59,10 @@ public static VectorSourceInfo Resolve(string? source) ResolveOgcApiFeaturesLayerName(trimmed)); } + if (IsHttpUrl(trimmed) && new[] { ".geojson", ".json", ".zip" }.Contains(Path.GetExtension(new Uri(trimmed).AbsolutePath), StringComparer.OrdinalIgnoreCase) && + !System.Text.RegularExpressions.Regex.IsMatch(trimmed, @"[?&]service=wfs(?:&|$)", System.Text.RegularExpressions.RegexOptions.IgnoreCase)) + return new VectorSourceInfo(trimmed, VectorSourceKind.RemoteDownload, Path.GetFileNameWithoutExtension(new Uri(trimmed).AbsolutePath)); + if (IsHttpUrl(trimmed)) { return new VectorSourceInfo(trimmed, VectorSourceKind.Wfs, string.Empty); @@ -95,6 +100,8 @@ private static string ResolveOgcApiFeaturesLayerName(string sourceUrl) return Uri.UnescapeDataString(pathParts[collectionIndex + 1]); } + if (pathParts.LastOrDefault()?.Equals("collections", StringComparison.OrdinalIgnoreCase) == true) return string.Empty; + return string.IsNullOrWhiteSpace(pathParts.LastOrDefault()) ? "OGC API Features" : Uri.UnescapeDataString(pathParts.Last()); diff --git a/RhinoSpatial.Core/WfsClient.cs b/RhinoSpatial.Core/WfsClient.cs index 529bd03..324ce6b 100644 --- a/RhinoSpatial.Core/WfsClient.cs +++ b/RhinoSpatial.Core/WfsClient.cs @@ -5,6 +5,7 @@ using System.Net.Http.Headers; using System.Text; using System.Threading.Tasks; +using System.Threading; namespace RhinoSpatial.Core { @@ -31,6 +32,7 @@ public class WfsClient "MAXFEATURES", "COUNT", "OUTPUTFORMAT", + "STARTINDEX", "BBOX" }; @@ -40,17 +42,25 @@ static WfsClient() SharedHttpClient.DefaultRequestHeaders.Accept.Add(new MediaTypeWithQualityHeaderValue("*/*")); } + private readonly HttpClient _http; + public WfsClient() : this(null) { } + public WfsClient(HttpClient? httpClient) => _http = httpClient ?? SharedHttpClient; + private sealed record FeatureResponseCacheEntry(WfsFeatureResponse Response, DateTimeOffset StoredAtUtc); - public async Task> LoadFeaturesAsync(WfsRequestOptions options) + public Task> LoadFeaturesAsync(WfsRequestOptions options) => LoadFeaturesAsync(options, CancellationToken.None); + + public async Task> LoadFeaturesAsync(WfsRequestOptions options, CancellationToken cancellationToken) { - var result = await LoadFeaturesWithStatusAsync(options); + var result = await LoadFeaturesWithStatusAsync(options, cancellationToken); return result.Features; } - public async Task LoadFeaturesWithStatusAsync(WfsRequestOptions options) + public Task LoadFeaturesWithStatusAsync(WfsRequestOptions options) => LoadFeaturesWithStatusAsync(options, CancellationToken.None); + + public async Task LoadFeaturesWithStatusAsync(WfsRequestOptions options, CancellationToken cancellationToken) { - var featureResponse = await LoadFeatureResponseAsync(options); + var featureResponse = await LoadFeatureResponseAsync(options, cancellationToken); if (TryReadFeatures(featureResponse.ResponseText, featureResponse.AppliedOptions.TypeName, out var features)) { @@ -60,35 +70,40 @@ public async Task LoadFeaturesWithStatusAsync(WfsRequestOp throw CreateUnsupportedFeatureResponseException(featureResponse.ResponseText); } - public async Task LoadFeatureResponseAsync(WfsRequestOptions options) + public Task LoadFeatureResponseAsync(WfsRequestOptions options) => LoadFeatureResponseAsync(options, CancellationToken.None); + + public async Task LoadFeatureResponseAsync(WfsRequestOptions options, CancellationToken cancellationToken) { - options = await PrepareRequestOptionsAsync(options); + options = await PrepareRequestOptionsAsync(options).WaitAsync(cancellationToken); var cacheKey = BuildFeatureResponseCacheKey(options); - if (TryGetCachedFeatureResponse(cacheKey, allowStale: false, out var cachedResponse)) + if (ReferenceEquals(_http, SharedHttpClient) && TryGetCachedFeatureResponse(cacheKey, allowStale: false, out var cachedResponse)) { return cachedResponse; } - var loadTask = FeatureResponseTaskCache.GetOrAdd(cacheKey, _ => LoadFeatureResponseUncachedAsync(options)); + var useSharedTask = !cancellationToken.CanBeCanceled && ReferenceEquals(_http, SharedHttpClient); + var loadTask = useSharedTask + ? FeatureResponseTaskCache.GetOrAdd(cacheKey, _ => LoadFeatureResponseUncachedAsync(options)) + : LoadFeatureResponseUncachedAsync(options, cancellationToken); try { var response = await loadTask; - FeatureResponseCache[cacheKey] = new FeatureResponseCacheEntry(response, DateTimeOffset.UtcNow); + if (ReferenceEquals(_http, SharedHttpClient)) FeatureResponseCache[cacheKey] = new FeatureResponseCacheEntry(response, DateTimeOffset.UtcNow); return response; } - catch (Exception ex) when (IsTransientFailure(ex) && + catch (Exception ex) when (!cancellationToken.IsCancellationRequested && ReferenceEquals(_http, SharedHttpClient) && IsTransientFailure(ex) && TryGetCachedFeatureResponse(cacheKey, allowStale: true, out var staleCachedResponse)) { return staleCachedResponse with { - StatusNote = "Using cached WFS result because the live WFS service was temporarily unavailable." + StatusNote = staleCachedResponse.StatusNote + " Using cached WFS result because the live WFS service was temporarily unavailable." }; } finally { - if (loadTask.IsCompleted) + if (useSharedTask && loadTask.IsCompleted) { FeatureResponseTaskCache.TryRemove(cacheKey, out _); } @@ -107,6 +122,7 @@ public async Task LoadCapabilitiesAsync(string baseUrl) var cacheKey = string.IsNullOrWhiteSpace(requestedVersion) ? normalizedBaseUrl : $"{normalizedBaseUrl}|VERSION={requestedVersion}"; + if (!ReferenceEquals(_http, SharedHttpClient)) return await LoadCapabilitiesUncachedAsync(normalizedBaseUrl, requestedVersion); var loadTask = CapabilitiesCache.GetOrAdd(cacheKey, _ => LoadCapabilitiesUncachedAsync(normalizedBaseUrl, requestedVersion)); try @@ -115,7 +131,7 @@ public async Task LoadCapabilitiesAsync(string baseUrl) } catch { - CapabilitiesCache.TryRemove(normalizedBaseUrl, out _); + CapabilitiesCache.TryRemove(cacheKey, out _); throw; } } @@ -168,6 +184,8 @@ public static string BuildGetFeatureRequestUrl(WfsRequestOptions options) builder.Append(Uri.EscapeDataString(options.OutputFormat)); } + if (options.StartIndex > 0) builder.Append("&STARTINDEX=" + options.StartIndex); + if (options.BoundingBox is not null) { builder.Append("&BBOX="); @@ -249,7 +267,7 @@ private async Task LoadCapabilitiesUncachedAsync(string nor throw lastException ?? new InvalidOperationException("The WFS capabilities request sequence did not produce a capabilities response."); } - private async Task LoadFeatureResponseUncachedAsync(WfsRequestOptions options) + private async Task LoadFeatureResponseUncachedAsync(WfsRequestOptions options, CancellationToken cancellationToken = default) { Exception? lastRequestException = null; @@ -259,12 +277,23 @@ private async Task LoadFeatureResponseUncachedAsync(WfsReque try { - var response = await GetStringAsync(requestUrl); + var pageOptions = CloneOptions(candidateOptions); + var supportsOffset = pageOptions.Version.StartsWith("2.", StringComparison.Ordinal); + if (supportsOffset) pageOptions.MaxFeatures = options.MaxFeatures > 0 + ? Math.Min(options.MaxFeatures, FeatureCollectionPager.PageSize) : FeatureCollectionPager.PageSize; + var paged = await FeatureCollectionPager.LoadHttpAsync(BuildGetFeatureRequestUrl(pageOptions), + async (url, token) => { var page = await SourceHttp.ReadAsync(_http, url, token).ConfigureAwait(false); return new FeaturePageContent(page.Text, page.Url); }, + options.MaxFeatures, + supportsOffset ? (offset, count) => { + var next = CloneOptions(pageOptions); next.StartIndex = candidateOptions.StartIndex + offset; next.MaxFeatures = count; + return BuildGetFeatureRequestUrl(next); + } : null, cancellationToken).ConfigureAwait(false); + var response = paged.Text; var statusNote = lastRequestException is null ? string.Empty : $"Retried WFS request with output format '{candidateOptions.OutputFormat}' after the provider rejected an earlier request."; - return new WfsFeatureResponse(response, candidateOptions, statusNote); + return new WfsFeatureResponse(response, candidateOptions, (statusNote + " " + paged.StatusNote).Trim()); } catch (Exception ex) when (ShouldTryNextRequestCandidate(ex)) { @@ -330,6 +359,7 @@ private static void AppendIfMissing(List sequence, WfsRequest GetFeatureBaseUrl = source.GetFeatureBaseUrl, TypeName = source.TypeName, MaxFeatures = source.MaxFeatures, + StartIndex = source.StartIndex, Version = version, SrsName = source.SrsName, OutputFormat = outputFormat, @@ -345,6 +375,7 @@ private static WfsRequestOptions CloneOptions(WfsRequestOptions options) GetFeatureBaseUrl = options.GetFeatureBaseUrl, TypeName = options.TypeName, MaxFeatures = options.MaxFeatures, + StartIndex = options.StartIndex, Version = options.Version, SrsName = options.SrsName, OutputFormat = options.OutputFormat, @@ -429,6 +460,8 @@ private static string BuildFeatureResponseCacheKey(WfsRequestOptions options) builder.Append('|'); builder.Append(options.MaxFeatures); builder.Append('|'); + builder.Append(options.StartIndex); + builder.Append('|'); builder.Append(options.Version); builder.Append('|'); builder.Append(options.SrsName); @@ -548,7 +581,7 @@ private async Task GetStringAsync(string requestUrl) try { using var request = new HttpRequestMessage(HttpMethod.Get, requestUrl); - using var response = await SharedHttpClient.SendAsync(request); + using var response = await _http.SendAsync(request); var responseText = await response.Content.ReadAsStringAsync(); if (!response.IsSuccessStatusCode) diff --git a/RhinoSpatial.Core/WfsRequestOptions.cs b/RhinoSpatial.Core/WfsRequestOptions.cs index 30cbf7c..601cc5a 100644 --- a/RhinoSpatial.Core/WfsRequestOptions.cs +++ b/RhinoSpatial.Core/WfsRequestOptions.cs @@ -10,6 +10,8 @@ public class WfsRequestOptions public int MaxFeatures { get; set; } + public int StartIndex { get; set; } + public string Version { get; set; } = "1.1.0"; public string SrsName { get; set; } = string.Empty; diff --git a/RhinoSpatial/LoadTerrainComponent.cs b/RhinoSpatial/LoadTerrainComponent.cs index ea62d5f..c7a731a 100644 --- a/RhinoSpatial/LoadTerrainComponent.cs +++ b/RhinoSpatial/LoadTerrainComponent.cs @@ -17,6 +17,7 @@ public class LoadTerrainComponent : GH_TaskCapableComponent ComputeForSourceAsync( return ComputeLocalTextGridTerrain(source, spatialContext); } + if (!System.Text.RegularExpressions.Regex.IsMatch(source.BaseUrl, @"[?&]service=wcs(?:&|$)", System.Text.RegularExpressions.RegexOptions.IgnoreCase)) + { + var progressActive = 1; + TerrainAtomResult? atom; + try + { + atom = await _atom.TryLoadAsync(source.BaseUrl, spatialContext, progress => + Eto.Forms.Application.Instance.AsyncInvoke(() => + { + if (System.Threading.Volatile.Read(ref progressActive) != 0 && !cancellationToken.IsCancellationRequested && OnPingDocument() is not null) + { Message = progress; OnDisplayExpired(false); } + }), cancellationToken).ConfigureAwait(false); + } + finally { System.Threading.Volatile.Write(ref progressActive, 0); } + if (atom is not null) + { + var terrain = BuildTerrainMesh(atom.Raster, atom.Bounds, spatialContext, out var baseline); + SpatialTerrainCache.Store(spatialContext, atom.Raster.SrsName, atom.Bounds, atom.Raster, baseline); + return new SolveResults { TerrainMeshes = new List { terrain }, + Status = atom.Status + " " + BuildTerrainAlignmentNote(spatialContext.UseAbsoluteCoordinates) + BuildMissingElevationNote(atom.Raster), MessageLevel = GH_RuntimeMessageLevel.Remark }; + } + } + try { var capabilities = await _wcsClient.LoadCapabilitiesAsync(source.BaseUrl); diff --git a/RhinoSpatial/Lod2LoadComponent.cs b/RhinoSpatial/Lod2LoadComponent.cs index 7e139d6..c050887 100644 --- a/RhinoSpatial/Lod2LoadComponent.cs +++ b/RhinoSpatial/Lod2LoadComponent.cs @@ -241,6 +241,15 @@ private SolveResults Compute(RequestData requestData, CancellationToken cancella return ComputeLocalCityGmlSource(requestData); } + if (new[] { ".zip", ".gmz", ".cityjson", ".json", ".gml", ".citygml", ".xml" } + .Contains(Path.GetExtension(new Uri(requestData.Source).AbsolutePath), StringComparer.OrdinalIgnoreCase) && + !System.Text.RegularExpressions.Regex.IsMatch(requestData.Source, @"[?&]service=wfs(?:&|$)", System.Text.RegularExpressions.RegexOptions.IgnoreCase)) + { + var tile = new Lod2AtomLink(new Uri(requestData.Source), "Building download", "", null); + var downloaded = _atomClient.DownloadFileAsync(tile, cancellationToken).GetAwaiter().GetResult(); + return ComputeDownloadedBuildings(requestData, new[] { downloaded }); + } + // Explicit WFS URLs retain their existing request path without an extra discovery GET. if (!System.Text.RegularExpressions.Regex.IsMatch(requestData.Source, @"[?&]service=wfs(?:&|$)", System.Text.RegularExpressions.RegexOptions.IgnoreCase)) { @@ -350,15 +359,7 @@ private SolveResults ComputeAtomSource(RequestData requestData, Lod2AtomSelectio Volatile.Write(ref progressActive, 0); } cancellationToken.ThrowIfCancellationRequested(); - var localRequest = new RequestData - { - Source = downloaded[0].FilePath, - SourceKind = Lod2SourceKind.Zip, - LocalSourcePaths = downloaded.Select(tile => tile.FilePath).ToList(), - SpatialContext = requestData.SpatialContext, - LayerName = requestData.LayerName - }; - var result = ComputeLocalCityGmlSource(localRequest); + var result = ComputeDownloadedBuildings(requestData, downloaded); var cachedCount = downloaded.Count(tile => tile.UsedCache); return new SolveResults { @@ -369,6 +370,47 @@ private SolveResults ComputeAtomSource(RequestData requestData, Lod2AtomSelectio }; } + private SolveResults ComputeDownloadedBuildings(RequestData requestData, IReadOnlyList downloads) + { + var sources = new Dictionary<(bool Json, Lod2SourceKind Kind), List>(); + foreach (var download in downloads) + { + var kind = Path.GetExtension(download.FilePath).Equals(".zip", StringComparison.OrdinalIgnoreCase) ? Lod2SourceKind.Zip : Lod2SourceKind.File; + if (EnumerateLocalSourceFiles(download.FilePath, kind).Any(file => + { using var stream = file.OpenRead(); return Lod2GmlReader.IsCityGmlDocument(stream); })) Add(false); + if (EnumerateLocalCityJsonSourceFiles(download.FilePath, kind).Any(file => IsCityJsonDocument(file.ReadText()))) Add(true); + void Add(bool json) + { + if (!sources.TryGetValue((json, kind), out var files)) sources[(json, kind)] = files = new List(); + files.Add(download.FilePath); + } + } + var groups = new List(); + foreach (var group in sources) + { + var local = new RequestData { Source = group.Value[0], SourceKind = group.Key.Kind, + LocalSourcePaths = group.Value, SpatialContext = requestData.SpatialContext, LayerName = requestData.LayerName }; + var result = group.Key.Json ? ComputeLocalCityJsonSource(local) : ComputeLocalCityGmlSource(local); + if (result.MessageLevel == GH_RuntimeMessageLevel.Error) throw new InvalidDataException(result.Status); + groups.Add(result); + } + if (groups.Count == 0) throw new InvalidDataException("Downloaded source contained no CityGML or CityJSON building documents."); + if (groups.Count == 1) return groups[0]; + var tree = new GH_Structure(); + for (var group = 0; group < groups.Count; group++) + for (var branch = 0; branch < groups[group].BrepTree.PathCount; branch++) + tree.AppendRange(groups[group].BrepTree.Branches[branch], + new GH_Path(new[] { group }.Concat(groups[group].BrepTree.Paths[branch].Indices).ToArray())); + return new SolveResults { BrepTree = tree, BuildingCount = groups.Sum(g => g.BuildingCount), + Status = string.Join(" ", groups.Select(g => g.Status)), MessageLevel = GH_RuntimeMessageLevel.Remark }; + } + + private static bool IsCityJsonDocument(string text) + { + using var document = System.Text.Json.JsonDocument.Parse(text); + return document.RootElement.TryGetProperty("type", out var type) && type.GetString() == "CityJSON"; + } + private SolveResults ComputeLocalCityGmlSource(RequestData requestData) { var timings = new LocalCityGmlTimings(); @@ -516,7 +558,9 @@ private SolveResults ComputeSafe(RequestData requestData, CancellationToken canc private SolveResults ComputeLocalCityJsonSource(RequestData requestData) { - var sourceFiles = EnumerateLocalCityJsonSourceFiles(requestData.Source, requestData.SourceKind).ToList(); + var sourceFiles = (requestData.LocalSourcePaths ?? new[] { requestData.Source }) + .SelectMany(source => EnumerateLocalCityJsonSourceFiles(source, requestData.SourceKind)) + .Where(file => requestData.LocalSourcePaths is null || IsCityJsonDocument(file.ReadText())).ToList(); if (sourceFiles.Count == 0) { return new SolveResults @@ -538,7 +582,7 @@ private SolveResults ComputeLocalCityJsonSource(RequestData requestData) var jsonText = sourceFile.ReadText(); metadata = CityJsonReader.ReadSourceMetadata(jsonText); } - catch + catch when (requestData.LocalSourcePaths is null) { if (failedFileNames.Count < 5) { @@ -548,6 +592,9 @@ private SolveResults ComputeLocalCityJsonSource(RequestData requestData) continue; } + if (requestData.LocalSourcePaths is not null && (string.IsNullOrWhiteSpace(metadata.SrsName) || + !SpatialReferenceTransform.TryTransformXY(metadata.SrsName, requestData.SpatialContext.ResolvedSrs, 0, 0, out _, out _))) + throw new InvalidDataException("Downloaded CityJSON has missing or unsupported source CRS metadata."); var candidateSrs = ResolveLocalCityGmlSrs(metadata.SrsName, requestData.SpatialContext); if (metadata.BoundingBox is not null && diff --git a/RhinoSpatial/SpatialContextComponent.cs b/RhinoSpatial/SpatialContextComponent.cs index 9d283cf..568668d 100644 --- a/RhinoSpatial/SpatialContextComponent.cs +++ b/RhinoSpatial/SpatialContextComponent.cs @@ -20,6 +20,7 @@ public class SpatialContextComponent : GH_TaskCapableComponent target, GH_RuntimeMessageLevel.Warning); } + try + { + var remote = _remoteReference.TryReadAsync(requestData.BaseUrl, + RhinoSpatialInputParser.ParseLayerName(requestData.LayerSelection)).GetAwaiter().GetResult(); + if (remote is not null) + { + var supported = NormalizeSupportedMapSrs(remote.SrsName); + return string.IsNullOrEmpty(supported) + ? ("EPSG:3857", "Reference CRS is not supported by the map helper; using EPSG:3857.", GH_RuntimeMessageLevel.Warning) + : (supported, string.Empty, null); + } + } + catch (Exception ex) + { return ("EPSG:3857", "Could not read reference metadata: " + ex.Message, GH_RuntimeMessageLevel.Warning); } + if (ReferenceSourceMetadataReader.IsLocalSource(requestData.BaseUrl)) { try @@ -746,6 +762,15 @@ private async Task OpenMapAsync(string? baseUrl, string? layerSelection, string? return (null, normalizedRequestedSrs); } + try + { + var remote = await _remoteReference.TryReadAsync(baseUrl, RhinoSpatialInputParser.ParseLayerName(layerSelection ?? string.Empty)); + if (remote is not null) + return (remote.Wgs84BoundingBox, string.IsNullOrWhiteSpace(normalizedRequestedSrs) + ? NormalizeSupportedMapSrs(remote.SrsName) : normalizedRequestedSrs); + } + catch { return (null, normalizedRequestedSrs); } + if (ReferenceSourceMetadataReader.IsLocalSource(baseUrl)) { try diff --git a/RhinoSpatial/WfsListLayersComponent.cs b/RhinoSpatial/WfsListLayersComponent.cs index 6e6b97a..2cc5b19 100644 --- a/RhinoSpatial/WfsListLayersComponent.cs +++ b/RhinoSpatial/WfsListLayersComponent.cs @@ -26,7 +26,7 @@ private class RequestData public WfsListLayersComponent() : base("List WFS Layers", "WFS Layers", - "List available layers from a WFS GetCapabilities response.", + "List WFS layers or OGC API Features collections from a service root or metadata URL.", "RhinoSpatial", "Layers") { } @@ -117,7 +117,9 @@ private bool TryGetRequestData(IGH_DataAccess dataAccess, out RequestData reques private SolveResults Compute(RequestData requestData) { - var layers = _wfsClient.LoadLayersAsync(requestData.BaseUrl).GetAwaiter().GetResult(); + var collections = new OgcApiFeaturesClient().TryDiscoverAsync(requestData.BaseUrl).GetAwaiter().GetResult(); + var layers = collections is null ? _wfsClient.LoadLayersAsync(requestData.BaseUrl).GetAwaiter().GetResult() + : System.Linq.Enumerable.ToList(System.Linq.Enumerable.Select(collections, c => new WfsLayerInfo(c.Id, c.Title, "EPSG:4326", new List(), c.Wgs84Bounds))); return new SolveResults { diff --git a/RhinoSpatial/WfsLoadComponent.cs b/RhinoSpatial/WfsLoadComponent.cs index dfa38d3..ffae36a 100644 --- a/RhinoSpatial/WfsLoadComponent.cs +++ b/RhinoSpatial/WfsLoadComponent.cs @@ -15,6 +15,7 @@ public class WfsLoadComponent : GH_TaskCapableComponent RequestedLayerNames { get; init; } = new(); public int MaxFeatures { get; init; } @@ -63,10 +66,11 @@ public WfsLoadComponent() protected override void RegisterInputParams(GH_InputParamManager pManager) { - pManager.AddTextParameter("WFS Service URL", "WFS URL", "Base URL of the WFS service, OGC API Features collection/items URL, or path to a local Shapefile (.shp) or GeoJSON (.geojson/.json). If left empty, RhinoSpatial will try to inherit it from the connected Layer input.", GH_ParamAccess.item); - pManager.AddTextParameter("Layer", "Layer", "One or more layer names or layer entries. Use List Item to choose one layer, or merge explicit selections if you want to load several layers. Optional for local Shapefile, local GeoJSON, and OGC API Features sources.", GH_ParamAccess.list); - pManager.AddIntegerParameter("Max Features", "Max Features", "Maximum number of features to request. Use 0 to request all available features.", GH_ParamAccess.item, 0); + pManager.AddTextParameter("WFS Service URL", "WFS URL", "WFS service, OGC API Features root/collection/items URL, direct GeoJSON or vector ZIP URL, or local Shapefile/GeoJSON path. If left empty, RhinoSpatial will try to inherit it from the connected Layer input.", GH_ParamAccess.item); + pManager.AddTextParameter("Layer", "Layer", "One or more layer names or layer entries. Use List Item to choose one layer, or merge explicit selections if you want to load several layers. Optional for files and a single OGC API collection; choose a collection explicitly when an API root exposes several.", GH_ParamAccess.list); + pManager.AddIntegerParameter("Max Features", "Max Features", "Per-layer feature cap. Use 0 to follow available pages within the selected area and safety limits; read Status for completeness diagnostics.", GH_ParamAccess.item, 0); pManager.AddTextParameter("Spatial Context", "Spatial Context", "Shared RhinoSpatial spatial context. This is required so WFS, WMS, LoD2, terrain, GeoTIFF, and OSM outputs stay aligned.", GH_ParamAccess.item); + pManager[1].Optional = true; } protected override void RegisterOutputParams(GH_OutputParamManager pManager) @@ -128,10 +132,7 @@ private bool TryGetRequestData(IGH_DataAccess dataAccess, out RequestData reques dataAccess.GetData(0, ref baseUrl); - if (!dataAccess.GetDataList(1, layerSelections)) - { - return false; - } + dataAccess.GetDataList(1, layerSelections); dataAccess.GetData(2, ref maxFeatures); dataAccess.GetData(3, ref spatialContextText); @@ -155,7 +156,7 @@ private bool TryGetRequestData(IGH_DataAccess dataAccess, out RequestData reques var sourceInfo = VectorSourceResolver.Resolve(baseUrl); if (!sourceInfo.IsSupported) { - AddRuntimeMessage(GH_RuntimeMessageLevel.Error, $"WFS URL must be a WFS service URL, an OGC API Features collection/items URL, a local .shp file path, or a local .geojson/.json file path. Source was not found or is unsupported: {baseUrl}"); + AddRuntimeMessage(GH_RuntimeMessageLevel.Error, $"WFS URL must be a WFS service URL, an OGC API Features URL, a supported vector download URL, or a local Shapefile/GeoJSON path. Source was not found or is unsupported: {baseUrl}"); return false; } @@ -166,14 +167,14 @@ private bool TryGetRequestData(IGH_DataAccess dataAccess, out RequestData reques .Distinct(StringComparer.OrdinalIgnoreCase) .ToList(); - if (requestedLayerNames.Count == 0 && sourceInfo.CanUseDefaultLayerName) + if (requestedLayerNames.Count == 0 && sourceInfo.CanUseDefaultLayerName && !string.IsNullOrWhiteSpace(sourceInfo.DefaultLayerName)) { requestedLayerNames.Add(sourceInfo.DefaultLayerName); } - if (requestedLayerNames.Count == 0) + if (maxFeatures < 0) { - AddRuntimeMessage(GH_RuntimeMessageLevel.Error, "At least one valid layer selection is required."); + AddRuntimeMessage(GH_RuntimeMessageLevel.Error, "Max Features must be zero or a positive number."); return false; } @@ -200,6 +201,7 @@ private bool TryGetRequestData(IGH_DataAccess dataAccess, out RequestData reques RequestedLayerNames = requestedLayerNames, MaxFeatures = maxFeatures, SpatialContext = spatialContext, + CancellationToken = CancelToken, SourceKind = sourceInfo.Kind }; @@ -212,13 +214,14 @@ private SolveResults Compute(RequestData requestData) { VectorSourceKind.LocalShapefile => ComputeLocalShapefile(requestData), VectorSourceKind.LocalGeoJson => ComputeLocalGeoJson(requestData), - VectorSourceKind.OgcApiFeatures => ComputeOgcApiFeatures(requestData), - _ => ComputeWfs(requestData) + VectorSourceKind.RemoteDownload => ComputeRemoteVector(requestData), + _ => ComputeRemoteService(requestData) }; } private SolveResults ComputeWfs(RequestData requestData) { + if (requestData.RequestedLayerNames.Count == 0) throw new InvalidOperationException("Select a WFS layer with List WFS Layers."); var resolvedSrsName = requestData.SpatialContext.ResolvedSrs; var features = new List(); var statusNotes = new List(); @@ -234,7 +237,7 @@ private SolveResults ComputeWfs(RequestData requestData) BoundingBox = requestData.SpatialContext.RequestBoundingBox }; - var layerResult = _wfsClient.LoadFeaturesWithStatusAsync(layerRequestOptions).GetAwaiter().GetResult(); + var layerResult = _wfsClient.LoadFeaturesWithStatusAsync(layerRequestOptions, requestData.CancellationToken).GetAwaiter().GetResult(); var filterResult = SpatialFeatureFilter.FilterToBounds( layerResult.Features, requestData.SpatialContext.RequestBoundingBox); @@ -265,42 +268,96 @@ private SolveResults ComputeWfs(RequestData requestData) maxFeaturesText, resolvedSrsText, statusNotes), - ResolveMessageLevel(features.Count, statusNotes.Count > 0), - statusNotes.Count > 0); + ResolveMessageLevel(features.Count, statusNotes.Any(note => note.Contains("Using cached WFS", StringComparison.Ordinal) || note.StartsWith("Filtered ", StringComparison.Ordinal))), + statusNotes.Any(note => note.Contains("Using cached WFS", StringComparison.Ordinal))); } - private SolveResults ComputeOgcApiFeatures(RequestData requestData) + private SolveResults ComputeRemoteService(RequestData requestData) { - var layerName = requestData.RequestedLayerNames[0]; - var sourceFeatures = _ogcApiFeaturesClient.LoadFeaturesAsync( - requestData.BaseUrl, - layerName, - requestData.SpatialContext.Wgs84BoundingBox, - requestData.MaxFeatures) - .GetAwaiter() - .GetResult(); - var projectedFeatures = WfsFeatureGeometryTransformer.TransformFeatures( - sourceFeatures, - "EPSG:4326", - requestData.SpatialContext.ResolvedSrs); - var filterResult = SpatialFeatureFilter.FilterToBounds( - projectedFeatures, - requestData.SpatialContext.RequestBoundingBox); - var features = filterResult.Features; - var outputData = BuildVectorOutput(requestData, features); - var maxFeaturesText = FormatMaxFeatures(requestData.MaxFeatures); - var coordinateText = FormatCoordinateMode(requestData.SpatialContext); - var filterNote = BuildOversizedFilterNote(filterResult, "feature"); - var filterSuffix = FormatOptionalStatusNote(filterNote); - var status = features.Count == 0 - ? $"No features were found for OGC API Features source '{Path.GetFileName(requestData.BaseUrl)}' inside the current Spatial Context.{filterSuffix}" - : $"Loaded {features.Count} feature(s) and {outputData.GeometryItemCount} geometry item(s) from OGC API Features source '{layerName}' with {maxFeaturesText} features, assumed GeoJSON CRS EPSG:4326, {coordinateText}{filterSuffix}"; + var collections = _ogcApiFeaturesClient.TryDiscoverAsync(requestData.BaseUrl, requestData.CancellationToken).GetAwaiter().GetResult(); + if (collections is null) + { + if (requestData.RequestedLayerNames.Count == 0) return ComputeRemoteVector(requestData); + return ComputeWfs(requestData); + } + if (collections.Count == 0) throw new InvalidDataException("This OGC API exposes no feature collections."); + var names = requestData.RequestedLayerNames; + if (names.Count == 0 || (collections.Count == 1 && !names.Contains(collections[0].Id))) + names = collections.Count == 1 ? new List { collections[0].Id } + : throw new InvalidOperationException("This OGC API has several collections. Use List WFS Layers and select the collection(s) to load."); + if (names.Any(name => !collections.Any(c => c.Id == name))) + throw new InvalidOperationException("An OGC API collection selection was not found. Refresh List WFS Layers."); + var resolved = CopyRequest(requestData, requestData.BaseUrl, VectorSourceKind.OgcApiFeatures, names); + var features = new List(); + var notes = new List(); + foreach (var name in names) + { + var result = _ogcApiFeaturesClient.LoadFeaturesWithStatusAsync(requestData.BaseUrl, name, + requestData.SpatialContext.Wgs84BoundingBox, requestData.MaxFeatures, requestData.CancellationToken).GetAwaiter().GetResult(); + var projected = WfsFeatureGeometryTransformer.TransformFeatures(result.Features, "EPSG:4326", requestData.SpatialContext.ResolvedSrs); + var filtered = SpatialFeatureFilter.FilterToBounds(projected, requestData.SpatialContext.RequestBoundingBox); + features.AddRange(filtered.Features); + notes.Add(name + ": " + result.StatusNote); + if (filtered.OversizedEnclosingFeatureCount > 0) notes.Add(BuildOversizedFilterNote(filtered, "feature")); + } + var output = BuildVectorOutput(resolved, features); + return BuildSolveResults(output, features.Count, + $"Loaded {features.Count} OGC API feature(s), {output.GeometryItemCount} geometry item(s), {FormatCoordinateMode(requestData.SpatialContext)}. {string.Join(" ", notes)}", + features.Count == 0 ? GH_RuntimeMessageLevel.Warning : GH_RuntimeMessageLevel.Remark); + } - return BuildSolveResults( - outputData, - features.Count, - status, - ResolveMessageLevel(features.Count, filterResult.OversizedEnclosingFeatureCount > 0)); + private static RequestData CopyRequest(RequestData source, string path, VectorSourceKind kind, List layers) => new() + { + BaseUrl = path, SourceKind = kind, RequestedLayerNames = layers, + MaxFeatures = source.MaxFeatures, SpatialContext = source.SpatialContext, CancellationToken = source.CancellationToken + }; + + private SolveResults ComputeRemoteVector(RequestData requestData) + { + var tile = new Lod2AtomLink(new Uri(requestData.BaseUrl), "Vector download", "", null); + var downloaded = _downloads.DownloadAsync(tile, cancellationToken: requestData.CancellationToken).GetAwaiter().GetResult(); + using var files = SourceFilesLease.Open(downloaded.FilePath, requestData.CancellationToken); + var candidates = files.Files.Where(path => Path.GetExtension(path).Equals(".shp", StringComparison.OrdinalIgnoreCase) || + Path.GetExtension(path).Equals(".geojson", StringComparison.OrdinalIgnoreCase) || Path.GetExtension(path).Equals(".json", StringComparison.OrdinalIgnoreCase)).ToList(); + var features = new List(); + var names = new List(); + var notes = new List(); + var failed = false; + foreach (var file in candidates) + { + requestData.CancellationToken.ThrowIfCancellationRequested(); + var name = candidates.Count == 1 && requestData.RequestedLayerNames.Count == 1 + ? requestData.RequestedLayerNames[0] : Path.GetFileNameWithoutExtension(file); + if (requestData.RequestedLayerNames.Count > 0 && candidates.Count > 1 && !requestData.RequestedLayerNames.Contains(name)) continue; + if (Path.GetExtension(file).Equals(".shp", StringComparison.OrdinalIgnoreCase)) + { + var metadata = ShapefileFeatureReader.ReadSourceMetadata(file); + if (string.IsNullOrWhiteSpace(metadata.SourceSrs)) throw new InvalidDataException("Downloaded Shapefile must have a resolvable source CRS in its .prj file."); + names.Add(name); + var read = ShapefileFeatureReader.ReadFeatures(file, name, requestData.SpatialContext, requestData.MaxFeatures); + features.AddRange(read.Features); + failed |= read.FailedFeatureCount > 0; + notes.Add($"{name}: scanned {read.SourceFeatureCount}, outside {read.SkippedOutsideContextCount}, failed {read.FailedFeatureCount}."); + } + else + { + var json = File.ReadAllText(file); + using var document = System.Text.Json.JsonDocument.Parse(json); + if (!document.RootElement.TryGetProperty("type", out var type) || type.GetString() is not ("Feature" or "FeatureCollection")) continue; + names.Add(name); + var projected = WfsFeatureGeometryTransformer.TransformFeatures(GeoJsonReader.ReadFeatures(json, name), + GeoJsonReader.TryReadSourceSrs(json) ?? "EPSG:4326", requestData.SpatialContext.ResolvedSrs); + var filtered = SpatialFeatureFilter.FilterToBounds(projected, requestData.SpatialContext.RequestBoundingBox); + features.AddRange(requestData.MaxFeatures > 0 ? filtered.Features.Take(requestData.MaxFeatures) : filtered.Features); + if (filtered.OversizedEnclosingFeatureCount > 0) notes.Add(BuildOversizedFilterNote(filtered, "feature")); + } + } + if (names.Count == 0) throw new InvalidDataException("The vector download contains no matching Shapefile or GeoJSON layer."); + var resolved = CopyRequest(requestData, downloaded.FilePath, VectorSourceKind.RemoteDownload, names); + var output = BuildVectorOutput(resolved, features); + return BuildSolveResults(output, features.Count, + $"Loaded {features.Count} feature(s) from a validated vector download ({(downloaded.UsedCache ? "cached" : "downloaded")}), {FormatCoordinateMode(requestData.SpatialContext)}. {string.Join(" ", notes)}", + features.Count == 0 || failed ? GH_RuntimeMessageLevel.Warning : GH_RuntimeMessageLevel.Remark); } private SolveResults ComputeLocalShapefile(RequestData requestData) diff --git a/docs/COMPATIBILITY_IMPLEMENTATION_PLAN.md b/docs/COMPATIBILITY_IMPLEMENTATION_PLAN.md new file mode 100644 index 0000000..a1cfbd2 --- /dev/null +++ b/docs/COMPATIBILITY_IMPLEMENTATION_PLAN.md @@ -0,0 +1,54 @@ +# Source compatibility implementation plan + +Started 2026-11-01 from 0.4.0 on `codex/source-compatibility`. +The [research audit](COMPATIBILITY_RESEARCH.md) records the full backlog. + +## Authorized scope + +1. **Feature completeness:** bounded OGC API and WFS pagination, explicit user + limits, duplicate/repeated-page protection and honest completeness diagnostics. +2. **Dataset discovery:** follow OGC API landing-page and collection links; allow + the existing layer list and vector loader to select collections without adding + component parameters. +3. **Direct downloads:** recognize remote GeoJSON and supported vector archives, + and extend building downloads to CityJSON. Validate formats, companion files, + coordinate metadata and archive paths before use. +4. **Terrain ATOM:** share feed selection and validated download infrastructure + with the building workflow; load intersecting terrain tiles, retain NoData and + one elevation baseline, and reject unsupported or ambiguous placement. +5. **Reference metadata:** use ATOM feed coverage/CRS to orient Spatial Context + without downloading geometry. Saved selections retain priority. + +## Validation and examples + +- Add deterministic HTTP fixtures for discovery, pagination, download failures, + cache reuse, limits and cancellation, plus terrain tile placement checks. +- Run the existing six suites and the new compatibility suite, a Release build, + JSON validation and a final diff review. +- Exercise small live provider requests where available. Record URLs, selected + layers/collections, bounds, CRS and actual results in the source manifest and + a copyable example guide; distinguish live results from fixture-only cases. +- Record remaining Rhino visual checks explicitly. Preserve component GUIDs, + input/output layouts, localization and elevation rules. No Google export, + extraction, persistent tile caching or key storage. +- Update component documentation and an Unreleased changelog. This task does not + publish another version or change the installed plugin automatically. + +## Progress + +- [x] Research recorded and implementation branch created. +- [x] Feature pagination and completeness checks. +- [x] OGC API discovery and existing component integration. +- [x] Direct vector/building downloads. +- [x] Terrain ATOM loading and shared placement. +- [x] ATOM reference metadata. +- [x] Saved example URLs, documentation and final validation. + +## Recorded results + +- Release solution build passed with zero warnings and errors; all seven suites passed, including 26 new compatibility checks. Component GUIDs and parameter names/order were compared against the 0.4.0 baseline; source manifest JSON and unique ids were checked. +- Live results and exact reproduction inputs are saved in [the example guide](../examples/COMPATIBILITY_WORKFLOWS.md): API-root discovery, five-page capped loading, single- and two-tile DGM1 meshes, direct GeoJSON/Shapefile ZIPs and a declared-CRS CityJSON source. +- The Delft-hosted older sample failed TLS on this machine; the cjio maintainer fixture provided a successful alternative. Certificate validation was retained. +- New Rhino/Grasshopper visual checks and Windows runtime checks remain on [the manual checklist](../examples/VALIDATION.md#unreleased-source-compatibility), including progress/cancellation, saved definitions and cross-layer XY/Z alignment. +- Priorities 6–10 remain backlog items. No multi-LoD selection, CityJSON sequence/template, affine/world-file, WMTS, COG or GeoPackage implementation is claimed. +- Public release 0.4.0 and the installed plugin were not changed. diff --git a/docs/COMPATIBILITY_RESEARCH.md b/docs/COMPATIBILITY_RESEARCH.md new file mode 100644 index 0000000..e128e8e --- /dev/null +++ b/docs/COMPATIBILITY_RESEARCH.md @@ -0,0 +1,42 @@ +# Compatibility research — 2026-11-01 + +Reviewed RhinoSpatial 0.4.0 at `2bd239ee409fdb06f2aa49aa81c2cffbcc1d2be5` against official provider and format documentation. This is a code/documentation audit, not a live reproduction of every provider failure. Priorities are recommendations; no runtime behavior was changed. + +Implementation of priorities 1–5 is tracked in the [implementation plan](COMPATIBILITY_IMPLEMENTATION_PLAN.md) and [example guide](../examples/COMPATIBILITY_WORKFLOWS.md). The evidence below describes the audited 0.4.0 baseline, before those changes. + +## Highest-value improvements + +| Priority | Gap | Current evidence | Proposed behavior and acceptance check | +| --- | --- | --- | --- | +| 1 | Bounded feature pagination and completeness diagnostics | `RhinoSpatial.Core/OgcApiFeaturesClient.cs:25` performs one request, then parses features without following collection links. `WfsClient.cs:157` sends feature limits but has no paging loop. | Follow OGC API next links and supported WFS paging until the user cap or end of results. Detect repeated pages, deduplicate IDs, preserve bounds, and report truncation. Test a server whose page limit is smaller than the requested feature count. Existing MaxFeatures remains a deliberate safety limit. | +| 2 | Terrain ATOM downloads | `docs/COMPONENT_REFERENCE.md:275` explicitly excludes terrain ATOM. `LoadTerrainComponent.cs` routes WCS and local files. Berlin officially publishes DGM1 through ATOM. | Reuse bounded feed discovery/download infrastructure with terrain-specific archive validation, tile placement and mosaicking. Verify adjacent DGM tiles, CRS, seams, overlapping edges and NoData. Do not reuse the building filename convention without validating terrain metadata. | +| 3 | OGC API landing-page and collection discovery | `OgcApiFeaturesClient.cs:44` recognizes paths containing /collections/ or /items. `VectorSourceResolver.cs:61` classifies other HTTP URLs as WFS. | Accept the API landing page, follow its data/collection links and expose collection titles/extents. Test a root URL with two collections and relative links; avoid guessing URL paths. | +| 4 | ATOM metadata as Spatial Context Reference Source | `docs/COMPONENT_REFERENCE.md:281` explicitly says this is unsupported. | Use feed extent/CRS metadata to orient the map without downloading geometry. Test missing and ambiguous metadata, geographic axis order and preservation of saved-selection priority. | +| 5 | Direct data-download URLs and packaged vector files | `VectorSourceResolver.cs:37–63` supports local SHP/GeoJSON; a remote .geojson URL falls through to WFS. `Lod2AtomClient.cs:294` explicitly excludes remote CityJSON. | Detect response type/content before service routing; safely download bounded GeoJSON and appropriate archives, including Shapefile companion files. Extend building downloads to validated CityJSON separately. Test redirects, misleading extensions, archive limits and missing CRS. | +| 6 | CityJSON LoD selection and sequence support | `CityJsonReader.cs:24` parses a single JSON document. Lines 44–51 traverse all geometry entries without selecting their lod. There is no CityJSONFeature sequence or GeometryInstance template handling. | First select one suitable LoD per object, rather than combining all supplied LoDs. Then add .city.jsonl streaming and template resolution where building datasets require them. Multi-LoD duplication is a code-derived risk, not a reproduced visual bug. Test a two-LoD building, a sequence header with shared transform, and a template instance. | + +## Useful follow-on work + +| Priority | Gap | Current evidence | Proposed behavior and acceptance check | +| --- | --- | --- | --- | +| 7 | Broader raster georeferencing | `GeoTiffReader.cs:32` requires an embedded EPSG code, and line 84 explicitly rejects rotated/sheared transforms. No world-file reader was found. | Add supported affine transforms and sidecar georeferencing (.tfw/.jgw/.pgw with a resolvable CRS). Test rotated imagery and pixel-center conventions. A world file alone does not identify the CRS; fail clearly instead of assuming one. | +| 8 | WMTS imagery | Existing imagery clients implement WMS; no WMTS or tile-matrix-set client was found in the reviewed C# code. | Support an additional delivery mechanism within imagery: read capabilities, choose a suitable tile matrix/resolution, request only intersecting tiles and align the mosaic. Test a projected tile matrix, not just Web Mercator. This is not a request to add caching to Google 3D Tiles. | +| 9 | Remote Cloud Optimized GeoTIFF access | GeoTIFF readers open local paths with Tiff.Open; no HTTP range reader was found. | Read appropriate overviews and spatial windows from a remote COG using bounded byte-range requests. A compatible local COG may already work as an ordinary TIFF; the missing capability is efficient remote subset access. Test transferred byte budgets and a server without range support. | +| 10 | GeoPackage vector input | VectorSourceKind and resolver contain WFS, OGC API Features, local Shapefile and local GeoJSON, but no .gpkg route or reader. | Add read-only feature-layer discovery and bounded spatial reads from a GeoPackage. Start with vector features, not every optional GeoPackage extension. Test multiple layers, CRS metadata and polygon holes. This is a larger optional format extension; defer until a real project supplies a useful test dataset. | + +## Official references + +- [OGC API Features Core: landing pages, collections and paging](https://docs.ogc.org/is/17-069r3/17-069r3.html) +- [Berlin DGM1 official ATOM service metadata](https://gdi.berlin.de/geonetwork/srv/api/records/fa02f9e1-a0df-3be1-b0aa-bc624c0c7ff5) +- [Berlin DGM1 provider documentation](https://gdi.berlin.de/data/dgm1/docs/dgm1.pdf) +- [CityJSON specification: LoD and geometry templates](https://www.cityjson.org/specs/) +- [CityJSON sequences and CityJSONFeature](https://www.cityjson.org/cityjsonseq/) +- [OGC GeoTIFF georeferencing standard](https://docs.ogc.org/is/19-008r4/19-008r4.html) +- [Esri world-file documentation](https://pro.arcgis.com/en/pro-app/3.6/help/data/imagery/world-files-for-raster-datasets.htm) +- [OGC WMTS standard](https://www.ogc.org/standards/wmts/) +- [OGC Cloud Optimized GeoTIFF standard](https://docs.ogc.org/is/21-026/21-026.html) +- [OGC GeoPackage specification](https://www.geopackage.org/spec/) + +## Suggested order + +Start with pagination/completeness and terrain ATOM. Add landing-page discovery and ATOM reference metadata next. Keep these inside the current components where practical, preserve their GUIDs and ordinary connections, and verify with bounded provider requests plus deterministic fixtures. Choose subsequent format work from actual user datasets, rather than treating this list as a mandatory expansion roadmap. diff --git a/docs/COMPONENT_REFERENCE.md b/docs/COMPONENT_REFERENCE.md index ff082ba..6cc03c2 100644 --- a/docs/COMPONENT_REFERENCE.md +++ b/docs/COMPONENT_REFERENCE.md @@ -8,6 +8,8 @@ RhinoSpatial is organized around one rule: Every loader consumes the same `Spatial Context`, so maps, vector data, terrain, buildings, rasters, OSM context, and visual reference layers can share one Rhino and Grasshopper study space. +> The source-compatibility additions described here are **Unreleased**. The public 0.4.0 package and installed plugin are unchanged. See the [example recipes](../examples/COMPATIBILITY_WORKFLOWS.md) for tested inputs. + ## Component groups The Grasshopper tab is organized as: @@ -63,6 +65,7 @@ The component returns one `Spatial Context` output for all aligned source and vi Supported reference metadata includes: - WFS, WMS, and WCS service URLs +- OGC API Features landing/collection metadata and ATOM feed coverage (Unreleased) - GeoTIFF - Shapefile - GeoJSON @@ -93,7 +96,7 @@ Terrain, LoD2 buildings, and localized Google 3D Tiles participate in one shared ### List WFS Layers -Connect a WFS service URL to inspect the layers advertised by its `GetCapabilities` response. Each output entry combines the service layer name and title. Use `List Item` to select one layer, or explicitly merge several entries for a multi-layer WFS request. +Connect a WFS service URL to inspect the layers advertised by its `GetCapabilities` response. OGC API Features roots and collection metadata URLs are also accepted; the component follows advertised data/items links and returns collection ids and titles. Each output entry combines the service layer name and title. Use `List Item` to select one layer, or explicitly merge several entries for a multi-layer WFS request. Public interface: @@ -118,14 +121,15 @@ Use `Load WFS` for parcels, planning layers, boundaries, roads, building footpri The public component name is retained for Grasshopper compatibility, but the accepted source category is broader than WFS: - WFS service URLs -- OGC API Features collection or items URLs +- OGC API Features landing-page, collection-list, collection or items URLs +- direct GeoJSON URLs and ZIPs containing GeoJSON or complete Shapefile companion sets - local Shapefiles (`.shp`) - local GeoJSON (`.geojson` or GeoJSON `.json`) Public inputs: - `WFS URL` — service URL, OGC API Features URL, or local vector-file path -- `Layer` — one or more WFS layer entries; optional for local files and OGC API Features +- `Layer` — one or more WFS layer entries; optional for files and a single OGC API collection; choose explicitly when a root exposes multiple collections - `Max Features` — request limit; the default `0` requests all available features within the bounded query - `Spatial Context` — the shared selected area and placement context @@ -140,7 +144,11 @@ The `Geometry` output is a tree grouped by layer and feature. For local vector files, connect the file path directly to `WFS URL`. `Layer` can remain empty and is used only as a label when provided. OGC API Features URLs can also be connected directly; RhinoSpatial requests GeoJSON features for the context bounding box. -`Max Features` limits the request when a provider supports it. Use `0` to request all features available within the bounded query, but keep broad or dense study areas conservative. +`Max Features` is a deliberate per-layer cap. `0` follows available pages within the bounded query. OGC API next links and WFS next links are supported; WFS 2 can also advance `STARTINDEX` by the actual returned count. Duplicate ids are removed. Identical records without stable ids remain separate, with a deduplication caveat; declared GeoJSON CRS changes between pages are rejected. Repeated pages, failed required pages and known incomplete responses fail explicitly instead of returning partial selections. Without a matching total, the status states that completeness cannot be independently verified. Legacy WFS without next links cannot be assumed pageable. + +Pagination has safety limits of 100 pages, 100,000 unique features, 128 MiB total text and two minutes; a single response is limited to 16 MiB. Hitting a safety limit while more data is advertised returns an error; hitting the user's cap is reported as intentional. These bounds apply to service paging, not a promise that every server implements filtering correctly. + +Direct vector downloads are limited to 128 MiB compressed, 256 MiB per expanded file and 1 GiB expanded per archive. ZIPs require `.shp`, `.shx`, `.dbf` and a resolvable `.prj` for each imported Shapefile. Unsafe paths, symlinks, duplicate paths and incomplete transfers are rejected. Validated payloads persist under `RhinoSpatial/VectorDownloadCache`; revisionless URLs refresh after one day. Source JSON is inspected by content, and GeoJSON may contain a FeatureCollection or a single Feature. Unknown declared CRS metadata fails rather than assuming the context. ### Outputs and parsing @@ -214,6 +222,7 @@ Use `Load Terrain` to create ground geometry aligned with the same study area as Supported sources include: - WCS services +- ATOM feeds containing supported GeoTIFF, ASC or regular XYZ/CSV terrain tiles - local GeoTIFF DEM (`.tif` or `.tiff`) - Esri ASCII Grid (`.asc`) - regular XYZ or CSV point grids with x, y, z columns @@ -221,7 +230,7 @@ Supported sources include: Public inputs: -- `Terrain URL` — WCS URL or local DEM/grid path; leave empty for the quick global fallback +- `Terrain URL` — WCS URL, terrain ATOM feed or local DEM/grid path; leave empty for the quick global fallback - `Coverage` — optional WCS coverage id or EPSG override for local text grids - `Spatial Context` — the shared selected area and placement context @@ -235,6 +244,8 @@ Esri ASCII Grid samples are placed at cell centers, while reference metadata des XYZ/CSV terrain requires finite, evenly spaced X and Y coordinates. Sparse regular grids retain missing cells as NoData; irregularly spaced samples return an error instead of being stretched onto an artificial grid. +For ATOM, connect the feed directly and leave `Coverage` empty. Text grids require a declared native feed CRS; GeoTIFFs use embedded metadata. Tiles are selected from supported spatial metadata or explicitly declared UTM grids. Compatible north-up grids are cropped to the study area and merged before mesh construction, so adjacent samples and the elevation baseline remain consistent. Incompatible spacing/CRS/alignment and conflicting overlap heights fail explicitly. The mosaic is limited to 4,194,304 cells and eight million aggregate cropped source samples. See the [Berlin DGM1 example](../examples/COMPATIBILITY_WORKFLOWS.md#berlin-dgm1-terrain-from-atom). Downloads use the shared ATOM limits below and persist under `RhinoSpatial/TerrainAtomCache`. + The built-in fallback is intended for small-area orientation and short request times. It fails quickly with a useful status when the area is too large, no usable elevation samples are available, or the request cannot complete promptly. Project work should prefer official or project-specific terrain. The WCS path supports ordinary single-part GeoTIFF responses and multipart GeoTIFF responses. Where capabilities omit a WGS84 extent, RhinoSpatial can use `DescribeCoverage` metadata to orient the context. @@ -246,7 +257,8 @@ Use `Load LoD2 Buildings` for official building massing, roof forms, and 3D urba One `LoD2 Source` input accepts: - a LoD2 WFS service URL -- an ATOM download-service URL with selectable CityGML ZIP tiles +- an ATOM download-service URL with selectable CityGML/CityJSON tiles +- a direct supported building-file or ZIP/GMZ download URL - local CityGML/GML/XML - local CityJSON - a folder containing supported building files @@ -256,7 +268,7 @@ For WFS sources, select a building layer when needed. For ATOM and local data, ` Public inputs: -- `LoD2 Source` — WFS or ATOM URL, supported local file, folder, or ZIP path +- `LoD2 Source` — WFS/ATOM URL, direct supported download URL, local file, folder, or ZIP path - `Layer` — optional WFS layer name or local output label - `Spatial Context` — the shared selected area and placement context @@ -268,17 +280,17 @@ Connect the service URL and a `Spatial Context`; no tile naming, manual download Tile selection supports GeoRSS Simple boxes/polygons and GeoRSS GML envelopes, including native projected bounds. GeoRSS Simple coordinates are latitude/longitude independently of a dataset's CRS category. GML uses its declared CRS and authority axis order; an omitted CRS defaults to WGS84 latitude/longitude. Relative links and `xml:base` are resolved against the response URL. -Some feeds, including Berlin's, publish a single dataset extent and many download links. These require additional tile information. The supported grid convention is a declared square grid size, an EPSG:25832 or EPSG:25833 category, an enclosing geographic extent, and filenames ending in `_EEE_NNNN` with southwest UTM coordinates in kilometres. Berlin explicitly links a `1x1 Km` tile index. This rule depends on metadata and the supported naming convention, not the service hostname. Arbitrary numeric filenames or whole-dataset bounds are never sufficient on their own. A feed with missing, ambiguous or unsupported spatial metadata returns a limitation before downloading building archives. Start from the service feed so its enclosing extent remains available. +Some feeds, including Berlin's, publish a single dataset extent and many download links. These require additional tile information. The supported grid convention is a declared square grid size, an EPSG:25832 or EPSG:25833 category, an enclosing geographic extent, and filenames ending in `_EEE_NNNN` with southwest UTM coordinates in kilometres. Berlin buildings explicitly link a `1x1 Km` tile index; terrain uses a declared `2x2 Km` index. Coarse enclosing coverage allows a one-declared-tile margin for edge/buffer tiles; distant contradictory grids are rejected. This rule depends on metadata and the supported naming convention, not the service hostname. Arbitrary numeric filenames or whole-dataset bounds are never sufficient on their own. A feed with missing, ambiguous or unsupported spatial metadata returns a limitation before downloading building archives. Start from the service feed so its enclosing extent remains available. -ATOM currently imports ZIP/GMZ containers holding CityGML (`.gml`, `.citygml`, `.xml`). Other archive formats and remote CityJSON downloads return an explicit unsupported-format error. Local CityJSON/file/folder/ZIP workflows remain supported. Downloaded CityGML must declare a source CRS that can transform into the context; no guessed CRS or approximate bounding-box placement is used. Standard AdV names such as `urn:adv:crs:ETRS89_UTM33*DE_DHHN2016_NH` resolve to UTM XY; published heights are retained without a vertical-datum conversion. +ATOM and direct building download URLs support CityGML (`.gml`, `.citygml`, `.xml`) and CityJSON (`.json`, `.cityjson`), including ZIP/GMZ containers. Downloaded CityJSON must declare `metadata.referenceSystem`; sequences and geometry templates are not added by this batch. Local CityJSON/file/folder/ZIP workflows remain supported. Downloaded CityGML must declare a source CRS that can transform into the context; no guessed CRS or approximate bounding-box placement is used. Standard AdV names such as `urn:adv:crs:ETRS89_UTM33*DE_DHHN2016_NH` resolve to UTM XY; published heights are retained without a vertical-datum conversion. -Use a building dataset, not a terrain feed: Berlin's `/data/a_lod2/atom/` supplies LoD2 buildings, while `/data/dgm1/atom/` supplies CSV terrain grids. Explicit CSV/raster download MIME types are rejected before grid selection with a source-format explanation. `Load Terrain` supports local grids and WCS, but does not yet read ATOM feeds. +Use a building dataset, not a terrain feed: Berlin's `/data/a_lod2/atom/` supplies LoD2 buildings, while `/data/dgm1/atom/` supplies CSV terrain grids. Explicit CSV/raster download MIME types are rejected before grid selection with a source-format explanation. Use the DGM1 feed directly with `Load Terrain`; its terrain-specific parser and mosaic path are separate from the building reader. -Completed ZIPs persist in the user's local application-data folder under `RhinoSpatial/Lod2AtomCache`. Repeated solutions reuse them, including after restarting Rhino. The cache key includes the download URL and feed entry's `updated` value. With no revision value, archives refresh after one day. Cached length and SHA-256 are checked before reuse. Downloads are written to temporary files, checked for complete transfer and readable CityGML, and promoted only on success. Failures can be retried; successfully completed tiles remain cached. Delete this cache directory while RhinoSpatial is idle to reclaim space or force fresh downloads. Feed metadata is refreshed after five minutes within the running process. +Completed ZIPs persist in the user's local application-data folder under `RhinoSpatial/Lod2AtomCache`. Repeated solutions reuse them, including after restarting Rhino. The cache key includes the download URL and feed entry's `updated` value. With no revision value, archives refresh after one day. Cached length and SHA-256 are checked before reuse. Downloads are written to temporary files, checked for complete transfer and supported readable payloads, and promoted only on success. Failures can be retried; successfully completed tiles remain cached. Delete this cache directory while RhinoSpatial is idle to reclaim space or force fresh downloads. Feed metadata is refreshed after five minutes within the running process. The component displays tile/download progress while working. `Status` reports selected tile filenames, downloaded and cached counts, cache location, parsed buildings, source CRS and conversion diagnostics. A failed required download or CityGML parse returns an error rather than presenting a partial area as complete. -Limits are 64 feeds, 8 subfeed levels, 50,000 download entries and 32 selected tiles per solution; 512 MiB compressed per archive, 256 MiB per expanded file and 1 GiB total expanded per archive. Feed requests time out after 30 seconds, discovery after two minutes, individual downloads after three minutes and the download batch after five minutes. Reduce the study area or use a more specific spatially indexed feed if a limit is reached. ATOM metadata is not yet supported by `Spatial Context → Reference Source`; define the area with the map or use an existing supported reference. +Limits are 64 feeds, 8 subfeed levels, 50,000 download entries and 32 selected tiles per solution; 512 MiB compressed per archive, 256 MiB per expanded file and 1 GiB total expanded per archive. Feed requests time out after 30 seconds, discovery after two minutes, individual downloads after three minutes and the download batch after five minutes. Reduce the study area or use a more specific spatially indexed feed if a limit is reached. ATOM feed coverage and native CRS are supported by `Spatial Context → Reference Source`. Use a service feed containing coverage; a dataset feed lacking coverage reports that limitation. Reference processing reads feed metadata without downloading archives. Saved map selections retain priority. ### Filtering and performance diff --git a/docs/ROADMAP.md b/docs/ROADMAP.md index 58046ec..22f0862 100644 --- a/docs/ROADMAP.md +++ b/docs/ROADMAP.md @@ -43,6 +43,16 @@ The current release already provides: These are the starting conditions for future work, not unfinished roadmap promises. +## Unreleased compatibility batch + +The [research backlog](COMPATIBILITY_RESEARCH.md) is recorded and priorities 1–5 +are implemented on the source-compatibility branch: bounded feature paging, API-root +collection discovery, direct vector/building downloads, terrain ATOM and reference +metadata. [The implementation plan](COMPATIBILITY_IMPLEMENTATION_PLAN.md) records +validation and remaining work; [example recipes](../examples/COMPATIBILITY_WORKFLOWS.md) +save tested URLs and bounds. Interactive Rhino/Windows checks remain before release. +Priorities 6–10 remain follow-on work. + ## Now: real-use stabilization The most useful next improvements should come from real project use rather than speculative expansion. diff --git a/docs/TEST_SOURCES.md b/docs/TEST_SOURCES.md index d0af1e0..d4bf2eb 100644 --- a/docs/TEST_SOURCES.md +++ b/docs/TEST_SOURCES.md @@ -44,6 +44,28 @@ Use these together with: `dotnet run --project RhinoSpatial.Sandbox.csproj -- lod2atom ` Add `download` to download/reuse selected archives and parse buildings through the existing CityGML reader. +## Unreleased source-compatibility validation (2026-11-01) + +The [implementation plan](COMPATIBILITY_IMPLEMENTATION_PLAN.md) records scope and +status; [example workflows](../examples/COMPATIBILITY_WORKFLOWS.md) save copyable +URLs, context bounds, layer choices and reproduction commands. The source manifest +includes verification dates and distinguishes development-only behavior. + +`dotnet run --project RhinoSpatial.Sandbox.csproj -c Release -- source-compatibility-check` +exercises deterministic HTTP fixtures for pagination, caps, totals, duplicate and +repeated pages, redirects, discovery, cancellation, validated downloads, unsafe or +incomplete archives, projected WKT authority, terrain mosaics and metadata-only +references. It is included in CI and makes no external requests. + +Live probes found 16 collections at the pygeoapi root and loaded five lakes across +five pages with an intentional cap. Berlin DGM1 yielded a 400 × 400 single-tile +raster and a 200 × 200 two-tile mosaic, both built into Core meshes. Direct Natural +Earth GeoJSON parsed 177 features, its Shapefile ZIP returned three sample features, +and the cjio Delft CityJSON fixture parsed 160 buildings. These are download/parser/ +Core-placement checks; new interactive Rhino/Grasshopper and Windows validation +remains pending. The older Delft-hosted CityJSON URL failed TLS in this batch; +historical local-file results below remain historical results. + ## Deterministic correctness regressions The source-reader checks generate temporary, non-private fixtures for ASCII corner/center origins, reference extents, optional NoData, wrapped/truncated/excess cell values, invalid headers, sparse and irregular XYZ grids, non-finite positions, GeoJSON null attributes, empty geometries, malformed points, and versioned EPSG names. They cover format behavior documented by [Esri](https://help.arcgis.com/en/arcgisdesktop/10.0/help/009t/009t0000000z000000.htm) and [GeoJSON RFC 7946](https://www.rfc-editor.org/rfc/rfc7946.html). @@ -73,7 +95,7 @@ Bounded comparison requests for the same Museum Island EPSG:25833 bounds also re - WFS: `https://gdi.berlin.de/services/wfs/alkis?SERVICE=WFS&REQUEST=GetCapabilities`, layer `alkis:gebaeudeflaechen` - WMS: `https://gdi.berlin.de/services/wms/alkis_gebaeude?SERVICE=WMS&REQUEST=GetCapabilities`, layer `gebaeude` -These confirm bounded source loading and compatible native coordinate metadata. They do **not** prove visual alignment, Brep quality, progress display or saved-definition compatibility in Rhino. Complete the ATOM entries in `examples/VALIDATION.md` before release. ATOM reference-map preloading remains deferred; use the map helper or another supported reference source. +These confirm bounded source loading and compatible native coordinate metadata. They do **not** prove visual alignment, Brep quality, progress display or saved-definition compatibility in Rhino. Complete the ATOM entries in `examples/VALIDATION.md` before release. The unreleased compatibility branch adds ATOM reference-map preloading from feed metadata; interactive validation of that addition remains pending. ## Current Tested Sources diff --git a/examples/COMPATIBILITY_WORKFLOWS.md b/examples/COMPATIBILITY_WORKFLOWS.md new file mode 100644 index 0000000..d2b50e4 --- /dev/null +++ b/examples/COMPATIBILITY_WORKFLOWS.md @@ -0,0 +1,140 @@ +# Source compatibility examples + +These additions are **Unreleased** and require a build from the source-compatibility +branch. The public 0.4.0 package has not been replaced. URLs and results below were +checked on 2026-11-01; public servers can change. + +Use the existing Grasshopper examples as starting points: `01-wms-wfs-basics.gh` +for vectors, `02-geotiff-terrain.gh` for terrain, and `03-lod2-buildings.gh` for +buildings. These recipes do not add new binary Grasshopper definitions. Keep one +shared `Spatial Context` for every layer in a study. The machine-readable inputs +are also saved in [sources.json](sources.json). + +## Discover collections from an API root + +Connect this URL to `List WFS Layers`, then select `lakes | Large Lakes`: + +```text +https://demo.pygeoapi.io/master/ +``` + +Connect the same URL and selected entry to `Load WFS`. Set `Max Features` to `5` +for a small check. Define a context in North America; the bounded probe used +EPSG:4326 bounds `-100,30,-50,60`. The root exposed 16 collections during testing. +For a root containing several collections, choose a layer explicitly; a sole +collection can be loaded with `Layer` empty. + +To exercise pagination, use this URL directly in `Load WFS`, with `Layer` empty: + +```text +https://demo.pygeoapi.io/master/collections/lakes/items?limit=1 +``` + +The bounded probe returned five unique features across five pages and reported +the deliberate `Max Features = 5` stop. `0` follows available pages subject to +the documented safety limits. A provider without a matching total receives an +explicit completeness caveat. + +An API root or collection metadata URL can also be used as `Reference Source`. +`Reference Layer = lakes` selects its extent. This only orients the map; draw a +smaller study area before loading. A saved selection still takes priority. + +## Berlin DGM1 terrain from ATOM + +Connect this URL to `Load Terrain → Terrain URL`; leave `Coverage` empty: + +```text +https://gdi.berlin.de/data/dgm1/atom/ +``` + +The same URL can be connected to `Spatial Context → Reference Source` to infer +EPSG:25833 and the feed coverage. No terrain archives are fetched for reference +metadata. Draw a small area near Museum Island, Berlin. + +Repeatable native bounds and observed results: + +| EPSG:25833 bounds (minX,minY,maxX,maxY) | Selected archives | Result | +| --- | --- | --- | +| `391300,5820100,391700,5820500` | `DGM1_390_5820.zip` | 400 × 400 samples; 160,000 mesh vertices and 159,201 quad faces; heights 30.75–36.85 m | +| `391900,5820100,392100,5820300` | `DGM1_390_5820.zip`, `DGM1_392_5820.zip` | 200 × 200 joined samples; 40,000 vertices and 39,601 quad faces; heights 31.17–42.21 m | + +These are sandbox/Core mesh results, not a completed visual Rhino alignment +check. The map helper selects rectangles interactively; the precise bounds above +can be reproduced with the sandbox commands below. Each source ZIP is about +16–17 MiB, even for a small window. Subsequent requests reuse validated downloads. +Berlin publishes a declared 2 km UTM grid; sample centers retain their `.5 m` +coordinates. Neighboring grids must agree in CRS, spacing and alignment. Missing +samples remain holes; conflicting overlaps fail explicitly. + +To compare buildings and terrain in Rhino, connect the **same** context to +`Load LoD2 Buildings` using: + +```text +https://gdi.berlin.de/data/a_lod2/atom/ +``` + +Leave its `Layer` empty. The building feed uses a different declared 1 km grid; +the terrain feed belongs in `Load Terrain`. + +## Direct GeoJSON and Shapefile ZIP downloads + +Connect either URL directly to `Load WFS → WFS URL`, with `Layer` empty: + +```text +https://raw.githubusercontent.com/nvkelso/natural-earth-vector/master/geojson/ne_110m_admin_0_countries.geojson +https://naturalearth.s3.amazonaws.com/110m_cultural/ne_110m_admin_0_countries.zip +``` + +Both contain EPSG:4326 country boundaries. The GeoJSON download parsed 177 +features; the ZIP passed validation and its Shapefile reader returned three +bounded sample features. Use these coarse world datasets for compatibility +checks. For a small site, use an appropriately detailed local dataset: the +existing filter may exclude country outlines that surround the whole site. + +ZIPs must contain complete `.shp`, `.shx`, `.dbf` and resolvable `.prj` sets. +Downloads are validated and cached; failed or unsafe archives are not promoted. +Remote data files are loader inputs. Download a file first if you want to use +its local metadata as a `Reference Source`. + +## Direct CityJSON buildings + +Connect this maintainer test dataset to `Load LoD2 Buildings → LoD2 Source`, +with `Layer` empty: + +```text +https://raw.githubusercontent.com/cityjson/cjio/master/tests/data/delft.json +``` + +The live download parsed 160 buildings. Its declared CRS is EPSG:7415 and its +native XY extent is `84616.468,447422.999,85140.839,447750.636`. It contains LoD1 +geometry, which the existing building reader accepts. It does not validate +multi-LoD selection, CityJSON sequences or geometry templates. + +For an accurately oriented study area, save a local copy and connect that path +to `Reference Source`; use a supported map SRS such as EPSG:3857, then select +the building extent in Delft. Load the original remote URL through the building +loader. EPSG:7415 uses the supported RD New horizontal transform; source Z +values are retained without vertical-datum conversion. + +The older [Delfshaven CityJSON example](https://3d.bk.tudelft.nl/opendata/cityjson/3dcities/v2.0/3-20-DELFSHAVEN.city.json) +failed TLS validation from this machine during this batch. Historical local-file +parser results remain in the catalogue; it is not a verified direct-download +example for this batch. Certificate checks were retained. + +## Reproduce the live probes + +Run from the repository root after a Release build: + +```sh +dotnet run --project RhinoSpatial.Sandbox.csproj -c Release --no-build -- compatibility-probe discover 'https://demo.pygeoapi.io/master/' +dotnet run --project RhinoSpatial.Sandbox.csproj -c Release --no-build -- ogcapi 'https://demo.pygeoapi.io/master/collections/lakes/items?limit=1' '-100,30,-50,60' 5 +dotnet run --project RhinoSpatial.Sandbox.csproj -c Release --no-build -- compatibility-probe reference 'https://gdi.berlin.de/data/dgm1/atom/' +dotnet run --project RhinoSpatial.Sandbox.csproj -c Release --no-build -- compatibility-probe terrain 'https://gdi.berlin.de/data/dgm1/atom/' EPSG:25833 391300,5820100,391700,5820500 +dotnet run --project RhinoSpatial.Sandbox.csproj -c Release --no-build -- compatibility-probe terrain 'https://gdi.berlin.de/data/dgm1/atom/' EPSG:25833 391900,5820100,392100,5820300 +dotnet run --project RhinoSpatial.Sandbox.csproj -c Release --no-build -- compatibility-probe vector 'https://raw.githubusercontent.com/nvkelso/natural-earth-vector/master/geojson/ne_110m_admin_0_countries.geojson' +dotnet run --project RhinoSpatial.Sandbox.csproj -c Release --no-build -- compatibility-probe vector 'https://naturalearth.s3.amazonaws.com/110m_cultural/ne_110m_admin_0_countries.zip' +dotnet run --project RhinoSpatial.Sandbox.csproj -c Release --no-build -- compatibility-probe buildings 'https://raw.githubusercontent.com/cityjson/cjio/master/tests/data/delft.json' +``` + +The deterministic `source-compatibility-check` command uses temporary fixtures +and makes no external requests. Live probes are deliberately separate from CI. diff --git a/examples/README.md b/examples/README.md index 72f91f8..71ad0d9 100644 --- a/examples/README.md +++ b/examples/README.md @@ -54,6 +54,13 @@ Public example services can change or become temporarily unavailable. For produc If an older saved definition shows Grasshopper IO archive warnings after a component layout change, replace the affected RhinoSpatial component once and reconnect it. The definitions in this repository should represent the current component layout and open without those warnings. +## Compatibility workflows (Unreleased) + +The [source compatibility recipes](COMPATIBILITY_WORKFLOWS.md) save tested URLs, +collection choices, bounds and observed results for API-root discovery, paging, +Berlin DGM1 ATOM terrain, direct GeoJSON/Shapefile ZIPs and CityJSON downloads. +These require the development build; the public 0.4.0 package is unchanged. + ## Source and testing references - [Curated example source list](sources.json) diff --git a/examples/VALIDATION.md b/examples/VALIDATION.md index ca185ed..f39f9d6 100644 --- a/examples/VALIDATION.md +++ b/examples/VALIDATION.md @@ -145,3 +145,19 @@ It is intentionally small and practical. The point is to confirm that the core d - run `scripts/build-yak.sh` on a machine with Rhino's Yak CLI available - confirm the generated Yak package includes the same user-facing docs, including `INSTALL.md` - confirm ignored staging/package artifacts are not committed + + +## Unreleased source compatibility + +Use [COMPATIBILITY_WORKFLOWS.md](COMPATIBILITY_WORKFLOWS.md) and record the Rhino +version/platform and actual results. These checks are pending for this batch: + +- [ ] Existing saved definitions open without changed GUIDs or parameter-layout warnings. +- [ ] API-root layer listing, explicit multi-collection choice and optional single-collection Layer work in Grasshopper. +- [ ] Paging status distinguishes the user cap, unknown totals and an incomplete/failed page. +- [ ] Berlin DGM1 single- and two-tile windows load with responsive asynchronous progress; cancellation and retry work. +- [ ] DGM1 mesh, building roofs and WMS agree in XY and Z under one localized context, regardless of solve order. +- [ ] NoData holes stay open and no seam appears at the terrain tile boundary. +- [ ] ATOM/API references orient a fresh map without geometry downloads, while saved selections retain priority. +- [ ] Direct GeoJSON, Shapefile ZIP and CityJSON URLs load; invalid CRS/companions return clear messages. +- [ ] Repeat the relevant checks on Windows before a public release. diff --git a/examples/sources.json b/examples/sources.json index 51d6bff..250421b 100644 --- a/examples/sources.json +++ b/examples/sources.json @@ -230,8 +230,14 @@ "notes": [ "Download and unzip before using with Load WFS local Shapefile mode", "Tested with sandbox shapefile command", - "EPSG:4326, 177 country features" - ] + "EPSG:4326, 177 country features", + "Unreleased: connect the direct_download_url ZIP directly to Load WFS, Layer empty.", + "2026-11-01: validated 214976-byte direct payload, EPSG:4326 metadata and three bounded sample features." + ], + "last_verified": "2026-11-01", + "availability": "Unreleased source-compatibility branch; not the public 0.4.0 package", + "example_guide": "COMPATIBILITY_WORKFLOWS.md", + "direct_download_url": "https://naturalearth.s3.amazonaws.com/110m_cultural/ne_110m_admin_0_countries.zip" }, { "id": "shapefile-naturalearth-populated-places", @@ -296,8 +302,23 @@ "notes": [ "Tested with sandbox ogcapi command", "Use directly in the WFS URL input; RhinoSpatial treats it as OGC API Features, not WFS", - "Good regression source for bbox-filtered GeoJSON item endpoints" - ] + "Good regression source for bbox-filtered GeoJSON item endpoints", + "2026-11-01: limit=1 probe read 5 unique features across 5 pages, then explicitly stopped at Max Features 5." + ], + "last_verified": "2026-11-01", + "availability": "Unreleased source-compatibility branch; not the public 0.4.0 package", + "example_guide": "COMPATIBILITY_WORKFLOWS.md", + "pagination_url": "https://demo.pygeoapi.io/master/collections/lakes/items?limit=1", + "max_features": 5, + "context": { + "srs": "EPSG:4326", + "bbox": [ + -100, + 30, + -50, + 60 + ] + } }, { "id": "wfs-geoserver-demo-naturalearth", @@ -623,7 +644,8 @@ "notes": [ "Downloaded and tested with sandbox lod2file command", "Parsed 853 buildings and 15482 surfaces", - "Source SRS EPSG:28992 is read and transformed explicitly for shared Spatial Context alignment" + "Source SRS EPSG:28992 is read and transformed explicitly for shared Spatial Context alignment", + "2026-11-01 direct-download retry failed TLS verification on this machine; historical local parser results above are unchanged. Use cityjson-cjio-delft-download for the verified direct-download recipe." ] }, { @@ -857,6 +879,102 @@ "Use only for bounded contextual preview aligned to Spatial Context", "Not a substitute for official editable project data" ] + }, + { + "id": "terrain-berlin-dgm1-atom", + "type": "Terrain ATOM", + "purpose": "Official terrain feed discovery, downloads and adjacent-grid mosaic", + "url": "https://gdi.berlin.de/data/dgm1/atom/", + "notes": [ + "Connect to Load Terrain, Coverage empty; the same feed is supported as Spatial Context Reference Source without archive downloads.", + "Declared 2 km EPSG:25833 grid; source heights and half-meter sample centers retained.", + "400 x 400 single-tile Core raster/mesh and 200 x 200 two-tile mosaic verified live; Rhino visual alignment pending.", + "16-17 MiB ZIP downloads per tile; subsequent validated cache reuse." + ], + "last_verified": "2026-11-01", + "availability": "Unreleased source-compatibility branch; not the public 0.4.0 package", + "example_guide": "COMPATIBILITY_WORKFLOWS.md", + "context": { + "srs": "EPSG:25833", + "bbox": [ + 391300, + 5820100, + 391700, + 5820500 + ] + }, + "boundary_context": { + "srs": "EPSG:25833", + "bbox": [ + 391900, + 5820100, + 392100, + 5820300 + ] + } + }, + { + "id": "ogcapi-pygeoapi-root", + "type": "OGC API Features", + "purpose": "Landing-page discovery and existing layer-list workflow", + "url": "https://demo.pygeoapi.io/master/", + "notes": [ + "16 feature collections discovered live; select lakes | Large Lakes using List WFS Layers.", + "Use the same root and layer with Load WFS. Multi-collection roots require an explicit collection choice.", + "Root/collection metadata supported as Reference Source; select a small context before loading." + ], + "last_verified": "2026-11-01", + "availability": "Unreleased source-compatibility branch; not the public 0.4.0 package", + "example_guide": "COMPATIBILITY_WORKFLOWS.md", + "layer": "lakes", + "max_features": 5, + "context": { + "srs": "EPSG:4326", + "bbox": [ + -100, + 30, + -50, + 60 + ] + } + }, + { + "id": "geojson-naturalearth-countries-download", + "type": "GeoJSON download", + "purpose": "Direct remote vector-file loading", + "url": "https://raw.githubusercontent.com/nvkelso/natural-earth-vector/master/geojson/ne_110m_admin_0_countries.geojson", + "notes": [ + "Connect directly to Load WFS with Layer empty; EPSG:4326.", + "Validated 838726-byte payload and parsed 177 features live.", + "Coarse world polygons are compatibility data; existing oversized-outline filtering can exclude them in small site contexts." + ], + "last_verified": "2026-11-01", + "availability": "Unreleased source-compatibility branch; not the public 0.4.0 package", + "example_guide": "COMPATIBILITY_WORKFLOWS.md" + }, + { + "id": "cityjson-cjio-delft-download", + "type": "CityJSON download", + "purpose": "Direct building-download and declared Dutch CRS check", + "url": "https://raw.githubusercontent.com/cityjson/cjio/master/tests/data/delft.json", + "notes": [ + "Connect to Load LoD2 Buildings with Layer empty; live parser returned 160 buildings.", + "CityJSON 2.0 maintainer fixture containing LoD1 geometry; not a multi-LoD, sequence or template test.", + "Source EPSG:7415; RD New XY transform supported, source heights retained.", + "For Reference Source use a downloaded local copy and a supported map SRS such as EPSG:3857. Rhino visual checks pending." + ], + "last_verified": "2026-11-01", + "availability": "Unreleased source-compatibility branch; not the public 0.4.0 package", + "example_guide": "COMPATIBILITY_WORKFLOWS.md", + "native_extent": { + "srs": "EPSG:7415", + "bbox": [ + 84616.468, + 447422.999, + 85140.839, + 447750.636 + ] + } } ] } diff --git a/sandbox/Program.cs b/sandbox/Program.cs index d8e9772..cd4def1 100644 --- a/sandbox/Program.cs +++ b/sandbox/Program.cs @@ -9,6 +9,17 @@ internal class Program { static async Task Main(string[] args) { + if (args.Length > 0 && args[0].Equals("compatibility-probe", StringComparison.OrdinalIgnoreCase)) + { + try { await SourceCompatibilityProbe.RunAsync(args); } + catch (Exception ex) { Console.Error.WriteLine(ex.Message); Environment.ExitCode = 1; } + return; + } + if (args.Length > 0 && args[0].Equals("source-compatibility-check", StringComparison.OrdinalIgnoreCase)) + { + await SourceCompatibilityChecks.RunAsync(); + return; + } if (args.Length > 0 && string.Equals(args[0], "lod2-atom-check", StringComparison.OrdinalIgnoreCase)) { await Lod2AtomChecks.RunAsync(); @@ -1206,7 +1217,9 @@ private static async Task RunOgcApiFeaturesAsync(string[] args) : 0; var startedAt = DateTime.UtcNow; var client = new OgcApiFeaturesClient(); - var features = await client.LoadFeaturesAsync(args[1], "ogcapi", bbox, maxFeatures); + var result = await client.LoadFeaturesWithStatusAsync(args[1], "ogcapi", bbox, maxFeatures); + var features = result.Features; + Console.WriteLine(result.StatusNote); var elapsed = DateTime.UtcNow - startedAt; Console.WriteLine("OGC API Features probe"); diff --git a/sandbox/SourceCompatibilityChecks.cs b/sandbox/SourceCompatibilityChecks.cs new file mode 100644 index 0000000..b3d210b --- /dev/null +++ b/sandbox/SourceCompatibilityChecks.cs @@ -0,0 +1,260 @@ +using System.Net; +using System.IO.Compression; +using System.Text; +using System.Text.Json; +using RhinoSpatial.Core; + +namespace RhinoSpatial.Sandbox; + +internal static class SourceCompatibilityChecks +{ + private static int _checks; + internal static void Require(bool condition, string message) + { if (!condition) throw new InvalidOperationException(message); } + private static async Task Check(string label, Func action) + { await action(); _checks++; Console.WriteLine("PASS " + label); } + internal sealed class Handler(Func response) : HttpMessageHandler + { + public List Requests { get; } = new(); + public Dictionary Payloads { get; } = new(); + protected override Task SendAsync(HttpRequestMessage request, CancellationToken cancellationToken) + { + cancellationToken.ThrowIfCancellationRequested(); + Requests.Add(request.RequestUri!); + return Task.FromResult(new HttpResponseMessage(HttpStatusCode.OK) + { Content = Payloads.TryGetValue(request.RequestUri!.AbsolutePath, out var bytes) ? new ByteArrayContent(bytes) : new StringContent(response(request.RequestUri!), Encoding.UTF8, "application/json"), RequestMessage = request }); + } + } + private static string Feature(int id) => "{\"type\":\"Feature\",\"id\":" + id + ",\"properties\":{},\"geometry\":{\"type\":\"Point\",\"coordinates\":[13,52]}}"; + private static string Page(int[] ids, string? next = null, int? total = null) => + "{\"type\":\"FeatureCollection\",\"features\":[" + string.Join(",", ids.Select(Feature)) + "]" + + (total.HasValue ? ",\"numberMatched\":" + total : "") + + (next is null ? "" : ",\"links\":[{\"rel\":\"next\",\"href\":" + JsonSerializer.Serialize(next) + "}]") + "}"; + + public static async Task RunAsync() + { + await Check("JSON pages resolve relative next links and deduplicate IDs", async () => + { + var pages = new Dictionary { ["https://fixture.test/items"] = Page(new[]{1,2},"?page=2"), ["https://fixture.test/items?page=2"] = Page(new[]{2,3}) }; + var result = await FeatureCollectionPager.LoadAsync("https://fixture.test/items", (url,_) => Task.FromResult(pages[url]),0); + Require(result.PageCount == 2 && GeoJsonReader.ReadFeatures(result.Text,"test").Count == 3,"Wrong merged page result."); + }); + await Check("user feature cap stops before the next request", async () => + { + var calls=0; + var result = await FeatureCollectionPager.LoadAsync("https://fixture.test/items",(_,_)=>{calls++;return Task.FromResult(Page(new[]{1,2},"?page=2"));},1); + Require(calls==1 && GeoJsonReader.ReadFeatures(result.Text,"test").Count==1 && result.StatusNote.Contains("Max Features"),"Cap was not applied."); + }); + await Check("repeated pages fail instead of returning a partial selection", async () => + { + try { await FeatureCollectionPager.LoadAsync("https://fixture.test/items",(_,_)=>Task.FromResult(Page(new[]{1},"?page=2")),0); } + catch(InvalidDataException){return;} throw new Exception("Repeated page accepted."); + }); + await Check("missing next link for a known incomplete collection fails", async () => + { + try { await FeatureCollectionPager.LoadAsync("https://fixture.test/items",(_,_)=>Task.FromResult(Page(new[]{1},total:2)),0); } + catch(InvalidDataException){return;} throw new Exception("Incomplete result accepted."); + }); + await Check("cancellation propagates", async () => + { + using var cancel=new CancellationTokenSource();cancel.Cancel(); + try {await FeatureCollectionPager.LoadAsync("https://fixture.test/items",(_,_)=>Task.FromResult(Page(new[]{1})),0,cancellationToken:cancel.Token);} + catch(OperationCanceledException){return;}throw new Exception("Cancellation ignored."); + }); + await Check("WFS 2 pages advance by the server's actual returned count", async () => + { + var handler=new Handler(url=>{ + var query=url.Query; + return query.Contains("STARTINDEX=2")?Page(new[]{3},total:3):query.Contains("STARTINDEX=1")?Page(new[]{2},total:3):Page(new[]{1},total:3); + }); + using var http=new HttpClient(handler); + var client=new WfsClient(http); + var result=await client.LoadFeaturesWithStatusAsync(new WfsRequestOptions{BaseUrl="https://fixture.test/wfs",GetFeatureBaseUrl="https://fixture.test/wfs",TypeName="buildings",Version="2.0.0"}); + Require(result.Features.Count==3 && handler.Requests.Count==3,"WFS paging failed."); + }); + await Check("GML pages preserve feature IDs, namespaces and geometry", async () => + { + string Gml(int id,string next="")=>"13 52"; + var result=await FeatureCollectionPager.LoadAsync("https://fixture.test/wfs",(url,_)=>Task.FromResult(url.Contains("page=2")?Gml(2):Gml(1,"next='?page=2'")),0); + var features=GmlReader.ReadFeatures(result.Text,"test"); + Require(features.Count==2 && features[1].Id=="f2" && features[1].Geometry.Points.Count==1,"GML merge lost geometry."); + }); + await Check("landing page follows data and items links without fetching features", async () => + { + var handler=new Handler(url=>url.AbsolutePath=="/api/"?"{\"links\":[{\"rel\":\"data\",\"href\":\"datasets\"}]}":"{\"collections\":[{\"id\":\"buildings\",\"title\":\"Buildings\",\"links\":[{\"rel\":\"items\",\"href\":\"features/buildings\",\"type\":\"application/geo+json\"}],\"extent\":{\"spatial\":{\"bbox\":[[13,52,14,53]]}}}]}"); + using var http=new HttpClient(handler);var client=new OgcApiFeaturesClient(http); + var result=await client.TryDiscoverAsync("https://fixture.test/api/"); + Require(result?.Count==1 && result[0].ItemsUrl=="https://fixture.test/api/features/buildings" && result[0].Wgs84Bounds?.MinX==13 && handler.Requests.Count==2,"Discovery followed wrong links."); + }); + await Check("OGC items URL is not requested until a bounding box is attached", async () => + { + var handler=new Handler(url=> { Require(url.Query.Contains("bbox="),"Unbounded items probe.");return Page(new[]{1}); }); + using var http=new HttpClient(handler);var client=new OgcApiFeaturesClient(http); + var result=await client.LoadFeaturesWithStatusAsync("https://fixture.test/collections/a/items","a",new BoundingBox2D(12,51,14,53),5); + Require(result.Features.Count==1 && handler.Requests.Count==1,"Unexpected items discovery request."); + }); + await Check("feature total is retained when later pages omit it", async () => + { + var calls = 0; + try { await FeatureCollectionPager.LoadAsync("https://fixture.test/items", (_,_) => Task.FromResult(++calls == 1 ? Page(new[]{1}, "?page=2", 3) : Page(new[]{2})), 0); } + catch (InvalidDataException) { return; } + throw new Exception("An incomplete later page hid the original matching total."); + }); + await Check("collection fallback paths retain query parameters correctly", async () => + { + using var http = new HttpClient(new Handler(_ => "{\"collections\":[{\"id\":\"a\"}]}")); + var collections = await new OgcApiFeaturesClient(http).TryDiscoverAsync("https://fixture.test/collections?f=json"); + Require(collections![0].ItemsUrl == "https://fixture.test/collections/a/items?f=json", "Collection path was appended to a query string."); + }); + await Check("single GeoJSON Feature downloads use the ordinary reader", () => + { + Require(GeoJsonReader.ReadFeatures(Feature(1), "download").Count == 1, "Single Feature unsupported."); + return Task.CompletedTask; + }); + await Check("relative paging links resolve against the final redirected response URL", async () => + { + var result = await FeatureCollectionPager.LoadHttpAsync("https://fixture.test/old", + (url,_) => Task.FromResult(url.Contains("page2") + ? new FeaturePageContent(Page(new[]{2}), new Uri(url)) + : new FeaturePageContent(Page(new[]{1},"page2"),new Uri("https://fixture.test/new/page1"))),0); + Require(result.PageCount == 2, "Redirected paging did not resolve."); + }); + await Check("point and wrapped collection extents do not break API discovery", async () => + { + using var http = new HttpClient(new Handler(_=>"{\"collections\":[{\"id\":\"point\",\"extent\":{\"spatial\":{\"bbox\":[[13,52,13,52]]}}},{\"id\":\"wrapped\",\"extent\":{\"spatial\":{\"bbox\":[[177,-20,-178,-15]]}}}]}")); + var collections = await new OgcApiFeaturesClient(http).TryDiscoverAsync("https://fixture.test/collections"); + Require(collections?.Count == 2 && collections[1].Wgs84Bounds?.MinX == -180,"Valid collection extent rejected."); + }); + await Check("identical anonymous records remain separate features", async () => + { + var body = Page(new[]{1,1},total:2).Replace("\"id\":1,", ""); + var result = await FeatureCollectionPager.LoadAsync("https://fixture.test/items", (_,_)=>Task.FromResult(body),0); + Require(GeoJsonReader.ReadFeatures(result.Text,"test").Count == 2 && result.StatusNote.Contains("without stable IDs"),"Anonymous records were silently collapsed."); + }); + await Check("GeoJSON pages cannot silently mix coordinate systems", async () => + { + var first = Page(new[]{1},"?page=2"); + var second = Page(new[]{2}).Replace("\"features\":", "\"crs\":{\"type\":\"name\",\"properties\":{\"name\":\"EPSG:3857\"}},\"features\":"); + try { await FeatureCollectionPager.LoadAsync("https://fixture.test/items", (url,_)=>Task.FromResult(url.Contains("page=2")?second:first),0); throw new Exception("Mixed CRS accepted."); } + catch (InvalidDataException) { } + }); + await RunDownloadAndTerrainChecksAsync(); + Console.WriteLine($"Source compatibility checks passed: {_checks}"); + } + private static byte[] Zip(params (string Name, string Text)[] files) + { + using var output = new MemoryStream(); + using (var archive = new ZipArchive(output, ZipArchiveMode.Create, true)) + foreach (var file in files) { using var writer = new StreamWriter(archive.CreateEntry(file.Name).Open()); writer.Write(file.Text); } + return output.ToArray(); + } + + private static async Task RunDownloadAndTerrainChecksAsync() + { + var directory = Path.Combine(Path.GetTempPath(), "rhinospatial-compatibility-" + Guid.NewGuid().ToString("N")); + try + { + var handler = new Handler(_ => throw new Exception("Unexpected network request.")); + using var http = new HttpClient(handler); + var vector = new SourceDownloadCache(AtomDownloadKind.Vector, http, directory); + var link = new Lod2AtomLink(new Uri("https://fixture.test/download"), "vector", "", null); + handler.Payloads["/download"] = Encoding.UTF8.GetBytes(Page(new[]{1,2})); + await Check("extensionless downloads detect GeoJSON content and reuse validated cache", async () => + { + var first = await vector.DownloadAsync(link); + var second = await new SourceDownloadCache(AtomDownloadKind.Vector, http, directory).DownloadAsync(link); + Require(first.FilePath.EndsWith(".geojson") && second.UsedCache && handler.Requests.Count == 1, "Direct-file cache failed."); + }); + await Check("projected Shapefile CRS uses outer WKT authority, not ellipsoid EPSG", () => + { + var shape = Path.Combine(directory,"projection.shp"); File.WriteAllBytes(shape,Array.Empty()); + const string wkt = "PROJCS[\"ETRS89 / UTM zone 33N\",GEOGCS[\"ETRS89\",DATUM[\"European_Terrestrial_Reference_System_1989\",SPHEROID[\"GRS 1980\",6378137,298.257222101,AUTHORITY[\"EPSG\",\"7019\"]],AUTHORITY[\"EPSG\",\"6258\"]],PRIMEM[\"Greenwich\",0],UNIT[\"degree\",0.0174532925199433],AUTHORITY[\"EPSG\",\"4258\"]],PROJECTION[\"Transverse_Mercator\"],PARAMETER[\"latitude_of_origin\",0],PARAMETER[\"central_meridian\",15],PARAMETER[\"scale_factor\",0.9996],PARAMETER[\"false_easting\",500000],PARAMETER[\"false_northing\",0],UNIT[\"metre\",1],AUTHORITY[\"EPSG\",\"25833\"]]"; + File.WriteAllText(Path.ChangeExtension(shape,".prj"),wkt); + Require(ShapefileFeatureReader.ReadSourceMetadata(shape).SourceSrs == "EPSG:25833","Nested EPSG authority used as the source CRS."); + return Task.CompletedTask; + }); + await Check("uppercase Shapefile companions normalize within temporary extraction", () => + { + var archive = Path.Combine(directory,"uppercase.zip"); + File.WriteAllBytes(archive,Zip(("A.SHP","shape"),("A.DBF","dbf"),("A.SHX","shx"),("A.PRJ","prj"))); + using var lease = SourceFilesLease.Open(archive); + var shape = lease.Files.Single(f=>f.EndsWith(".SHP")); + Require(lease.Files.Contains(Path.ChangeExtension(shape,".prj")),"PRJ case was not normalized."); + return Task.CompletedTask; + }); + await Check("archive traversal is rejected before extraction", async () => + { + handler.Payloads["/unsafe.zip"] = Zip(("../outside.geojson",Page(new[]{1}))); + try { await vector.DownloadAsync(link with { Url = new Uri("https://fixture.test/unsafe.zip") }); } + catch (InvalidDataException) { return; } throw new Exception("Unsafe ZIP accepted."); + }); + await Check("Shapefile download requires companion files", async () => + { + handler.Payloads["/shape.zip"] = Zip(("a.shp","not a shapefile"),("a.dbf","dbf")); + try { await vector.DownloadAsync(link with { Url = new Uri("https://fixture.test/shape.zip") }); } + catch (InvalidDataException) { return; } throw new Exception("Incomplete Shapefile accepted."); + }); + await Check("downloaded CityJSON declares CRS and preserves its source coordinates", async () => + { + const string city = "{\"type\":\"CityJSON\",\"version\":\"2.0\",\"metadata\":{\"referenceSystem\":\"https://www.opengis.net/def/crs/EPSG/0/25833\"},\"CityObjects\":{},\"vertices\":[[391000,5820000,42]]}"; + handler.Payloads["/city.json"] = Encoding.UTF8.GetBytes(city); + var building = new SourceDownloadCache(AtomDownloadKind.Buildings,http,Path.Combine(directory,"buildings")); + var result = await building.DownloadAsync(link with { Url = new Uri("https://fixture.test/city.json") }); + var metadata = CityJsonReader.ReadSourceMetadata(File.ReadAllText(result.FilePath)); + Require(metadata.SrsName == "EPSG:25833" && metadata.BoundingBox?.MinX == 391000, "CityJSON source metadata changed."); + handler.Payloads["/missing.json"] = Encoding.UTF8.GetBytes(city.Replace("\"metadata\":", "\"ignored\":")); + try { await building.DownloadAsync(link with { Url = new Uri("https://fixture.test/missing.json") }); } + catch (InvalidDataException) { return; } throw new Exception("Missing downloaded CRS accepted."); + }); + const string start = ""; + string Entry(string name,int x) => ""+name+""+x+" 5820000"+(x+3)+" 5820003"; + string Grid(int x) + { + var rows = new List(); + for(var y=0;y<3;y++) for(var column=0;column<3;column++) + rows.Add($"{x+column};{5820000+y};{(column == 1 && y == 1 ? "NaN" : "10")}"); + return string.Join("\n",rows); + } + var terrainHandler = new Handler(url => url.AbsolutePath == "/terrain" ? start + Entry("left",391000) + Entry("right",391003) + "" : throw new Exception("Unexpected terrain request.")); + terrainHandler.Payloads["/left.zip"] = Zip(("left.csv",Grid(391000))); + terrainHandler.Payloads["/right.zip"] = Zip(("right.csv",Grid(391003))); + using var terrainHttp = new HttpClient(terrainHandler); + var context = Lod2AtomChecks.Context("EPSG:25833",new BoundingBox2D(391000,5820000,391005,5820002)); + await Check("terrain ATOM joins adjacent CSV grids and preserves NoData holes", async () => + { + var terrain = await new TerrainAtomLoader(terrainHttp,Path.Combine(directory,"terrain")).TryLoadAsync("https://fixture.test/terrain", context); + Require(terrain?.Raster.Width == 6 && terrain.Raster.Height == 3 && float.IsNaN(terrain.Raster.Elevations[7]),"Terrain mosaic or NoData changed."); + var mesh = TerrainMeshBuilder.Build(terrain!.Raster,terrain.Bounds,context,512,0); + Require(mesh.Faces.Count > 0, "Terrain produced no connected surface."); + }); + await Check("terrain mosaics reject conflicting elevations and mismatched grids", () => + { + var raster = new TerrainRasterData("test","EPSG:25833",2,2,new Coordinate2D(0,1),new Coordinate2D(1,0),new Coordinate2D(0,-1),double.NaN,new[]{1f,1f,1f,1f}); + try { TerrainRasterMosaic.Merge(new[]{raster,raster with { Elevations = new[]{2f,2f,2f,2f} }}); } + catch (InvalidDataException) { + try { TerrainRasterMosaic.Merge(new[]{raster,raster with { OffsetVectorX = new Coordinate2D(2,0) }}); } + catch (InvalidDataException) {return Task.CompletedTask;} + } + throw new Exception("Incompatible terrain grids accepted."); + }); + await Check("ATOM reference metadata reads no archives and infers native CRS", async () => + { + var reference = await new RemoteReferenceMetadataReader(terrainHttp).TryReadAsync("https://fixture.test/terrain"); + Require(reference?.SourceKind == "ATOM" && reference.SrsName == "EPSG:25833" && reference.Wgs84BoundingBox is not null,"ATOM reference metadata missing."); + Require(terrainHandler.Requests.Count(r=>r.AbsolutePath.EndsWith(".zip")) == 2,"Reference metadata downloaded geometry."); + }); + await Check("OGC API items reference reads parent collection metadata only", async () => + { + var metadataHandler = new Handler(url => { + Require(url.AbsolutePath == "/collections/a", "Reference requested feature data."); + return "{\"id\":\"a\",\"extent\":{\"spatial\":{\"bbox\":[[13,52,14,53]]}}}"; + }); + using var metadataHttp = new HttpClient(metadataHandler); + var reference = await new RemoteReferenceMetadataReader(metadataHttp).TryReadAsync("https://fixture.test/collections/a/items"); + Require(reference?.Wgs84BoundingBox?.MinX == 13 && metadataHandler.Requests.Count == 1,"Reference did not use the collection extent."); + }); + } + finally { if(Directory.Exists(directory)) Directory.Delete(directory,true); } + } + +} diff --git a/sandbox/SourceCompatibilityProbe.cs b/sandbox/SourceCompatibilityProbe.cs new file mode 100644 index 0000000..ba86ec0 --- /dev/null +++ b/sandbox/SourceCompatibilityProbe.cs @@ -0,0 +1,64 @@ +using System.Globalization; +using RhinoSpatial.Core; + +namespace RhinoSpatial.Sandbox; + +internal static class SourceCompatibilityProbe +{ + public static async Task RunAsync(string[] args) + { + if(args.Length < 3) throw new ArgumentException("Usage: compatibility-probe discover|reference|vector|buildings|terrain [srs bbox]"); + var mode=args[1]; var url=args[2]; + if(mode=="discover") + { + var collections=await new OgcApiFeaturesClient().TryDiscoverAsync(url) ?? throw new Exception("Not an OGC API."); + Console.WriteLine($"Collections: {collections.Count}"); + foreach(var c in collections.Take(12)) Console.WriteLine($"{c.Id} | {c.Title} | {c.ItemsUrl} | {c.Wgs84Bounds}"); + } + else if(mode=="reference") + { + var metadata=await new RemoteReferenceMetadataReader().TryReadAsync(url) ?? throw new Exception("No remote metadata."); + Console.WriteLine(metadata); + } + else if(mode=="terrain") + { + if(args.Length<5) throw new Exception("Terrain probe requires source SRS and minX,minY,maxX,maxY."); + var values=args[4].Split(',').Select(x=>double.Parse(x,CultureInfo.InvariantCulture)).ToArray(); + var context=Lod2AtomChecks.Context(args[3],new BoundingBox2D(values[0],values[1],values[2],values[3])); + var result=await new TerrainAtomLoader().TryLoadAsync(url,context,Console.WriteLine) ?? throw new Exception("Not an ATOM feed."); + var mesh=TerrainMeshBuilder.Build(result.Raster,result.Bounds,context,512,0); + var finite=result.Raster.Elevations.Where(x=>!TerrainMeshBuilder.IsNoData(result.Raster,x)).ToArray(); + Console.WriteLine(result.Status); + Console.WriteLine($"Bounds: {result.Bounds}; mesh: {mesh.Vertices.Count} vertices, {mesh.Faces.Count} faces; heights {finite.Min()}..{finite.Max()}"); + } + else + { + var kind=mode=="buildings"?AtomDownloadKind.Buildings:AtomDownloadKind.Vector; + var result=await new SourceDownloadCache(kind).DownloadAsync(new Lod2AtomLink(new Uri(url),mode,"",null)); + Console.WriteLine($"Validated {(result.UsedCache?"cached":"downloaded")} payload: {new FileInfo(result.FilePath).Length} bytes"); + using var files=SourceFilesLease.Open(result.FilePath); + foreach(var file in files.Files) + { + var extension=Path.GetExtension(file).ToLowerInvariant(); + if(kind==AtomDownloadKind.Buildings && extension is ".cityjson" or ".json") + { + var text=File.ReadAllText(file); var metadata=CityJsonReader.ReadSourceMetadata(text); + Console.WriteLine($"CityJSON: {metadata.SrsName}, bounds {metadata.BoundingBox}, buildings {CityJsonReader.ReadBuildings(text,"probe").Count}"); + } + else if(kind==AtomDownloadKind.Vector && extension is ".geojson" or ".json") + { + var text=File.ReadAllText(file); + Console.WriteLine($"GeoJSON: {GeoJsonReader.ReadFeatures(text,"probe").Count} features, {GeoJsonReader.TryReadSourceSrs(text)??"EPSG:4326"}"); + } + else if(kind==AtomDownloadKind.Vector && extension==".shp") + { + var metadata=ShapefileFeatureReader.ReadSourceMetadata(file); + if(string.IsNullOrWhiteSpace(metadata.SourceSrs) || metadata.BoundingBox is null) throw new Exception("Shapefile metadata unresolved."); + var context=Lod2AtomChecks.Context(metadata.SourceSrs,metadata.BoundingBox); + var read=ShapefileFeatureReader.ReadFeatures(file,"probe",context,3); + Console.WriteLine($"Shapefile: {metadata.SourceSrs}, {read.Features.Count} bounded sample features, bounds {metadata.BoundingBox}"); + } + } + } + } +} From f20afe4cbd385f2bb30448d025c1e30d0289f28a Mon Sep 17 00:00:00 2001 From: PascalNun Date: Mon, 2 Nov 2026 00:24:23 +0100 Subject: [PATCH 2/2] feat: support CityJSON variants, affine rasters and WMTS --- .github/workflows/validate.yml | 3 + CHANGELOG.md | 5 + RhinoSpatial.Core/CityJsonReader.cs | 437 ++++++++---------- RhinoSpatial.Core/GeoTiffInfoCache.cs | 4 +- RhinoSpatial.Core/GeoTiffReader.cs | 150 ++---- RhinoSpatial.Core/Lod2AtomFeedReader.cs | 2 +- RhinoSpatial.Core/Models.cs | 4 +- RhinoSpatial.Core/RasterGeoTransform.cs | 67 +++ RhinoSpatial.Core/RasterImageCodec.cs | 37 ++ RhinoSpatial.Core/RasterImageMeshBuilder.cs | 70 +++ .../ReferenceSourceMetadataReader.cs | 15 +- .../RemoteReferenceMetadataReader.cs | 15 +- RhinoSpatial.Core/RhinoSpatial.Core.csproj | 1 + RhinoSpatial.Core/ShapefileFeatureReader.cs | 2 +- RhinoSpatial.Core/SourceDownloadCache.cs | 13 +- RhinoSpatial.Core/TerrainAtomLoader.cs | 4 +- RhinoSpatial.Core/TerrainMeshBuilder.cs | 5 +- RhinoSpatial.Core/TerrainRasterReader.cs | 13 +- RhinoSpatial.Core/WmtsClient.cs | 202 ++++++++ RhinoSpatial/LoadGeoTiffComponent.cs | 127 +---- RhinoSpatial/Lod2LoadComponent.cs | 25 +- RhinoSpatial/SpatialTerrainCache.cs | 8 +- RhinoSpatial/WmsListLayersComponent.cs | 9 +- RhinoSpatial/WmsLoadComponent.cs | 15 +- THIRD-PARTY-NOTICES.md | 16 + docs/COMPATIBILITY_IMPLEMENTATION_PLAN.md | 2 +- docs/COMPATIBILITY_RESEARCH.md | 2 +- docs/COMPONENT_REFERENCE.md | 26 +- docs/MEDIUM_COMPATIBILITY_PLAN.md | 36 ++ docs/ROADMAP.md | 2 +- docs/TEST_SOURCES.md | 6 + examples/MEDIUM_COMPATIBILITY_WORKFLOWS.md | 111 +++++ examples/README.md | 2 + examples/VALIDATION.md | 14 + .../fixtures/medium/building-lods.city.json | 228 +++++++++ .../medium/building-sequence.city.jsonl | 2 + .../medium/building-template.city.json | 156 +++++++ examples/fixtures/medium/rotated-raster.pgw | 6 + examples/fixtures/medium/rotated-raster.png | Bin 0 -> 99 bytes examples/fixtures/medium/rotated-raster.prj | 1 + examples/sources.json | 94 ++++ sandbox/MediumCompatibilityChecks.cs | 145 ++++++ sandbox/MediumCompatibilityProbe.cs | 71 +++ sandbox/Program.cs | 4 + 44 files changed, 1640 insertions(+), 517 deletions(-) create mode 100644 RhinoSpatial.Core/RasterGeoTransform.cs create mode 100644 RhinoSpatial.Core/RasterImageCodec.cs create mode 100644 RhinoSpatial.Core/RasterImageMeshBuilder.cs create mode 100644 RhinoSpatial.Core/WmtsClient.cs create mode 100644 docs/MEDIUM_COMPATIBILITY_PLAN.md create mode 100644 examples/MEDIUM_COMPATIBILITY_WORKFLOWS.md create mode 100644 examples/fixtures/medium/building-lods.city.json create mode 100644 examples/fixtures/medium/building-sequence.city.jsonl create mode 100644 examples/fixtures/medium/building-template.city.json create mode 100644 examples/fixtures/medium/rotated-raster.pgw create mode 100644 examples/fixtures/medium/rotated-raster.png create mode 100644 examples/fixtures/medium/rotated-raster.prj create mode 100644 sandbox/MediumCompatibilityChecks.cs create mode 100644 sandbox/MediumCompatibilityProbe.cs diff --git a/.github/workflows/validate.yml b/.github/workflows/validate.yml index edb5998..1a629e9 100644 --- a/.github/workflows/validate.yml +++ b/.github/workflows/validate.yml @@ -49,6 +49,9 @@ jobs: - name: Check source compatibility run: dotnet run --project RhinoSpatial.Sandbox.csproj -c Release --no-build -- source-compatibility-check + - name: Check medium-priority compatibility + run: dotnet run --project RhinoSpatial.Sandbox.csproj -c Release --no-build -- medium-compatibility-check + - name: Check 3D Tiles selection run: dotnet run --project RhinoSpatial.Sandbox.csproj -c Release --no-build -- tiles-selection-check diff --git a/CHANGELOG.md b/CHANGELOG.md index 07095da..d574cc1 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,11 @@ RhinoSpatial improves its established source categories through real project use ## Unreleased +- Select one suitable CityJSON LoD per object, retaining parts at that level; support streamed CityJSONSeq and affine geometry templates through existing local/archive/download paths. Reject malformed indices or ambiguous LoDs instead of returning partial rings. +- Support embedded affine GeoTIFF georeferencing and local world-file TIFF/PNG/JPEG with declared CRS, preserving rotated footprints, UV clipping, pixel centers and terrain sampling. Sidecar edits invalidate caches; projected CRS keys take priority over geographic keys. +- Support WMTS layer discovery, reference metadata, bounded native tile selection and PNG/JPEG mosaics through existing WMS components. Honor axis order, matrix limits, dimensions and advertised REST/KVP endpoints; validate completed caches and fail required tile errors explicitly. +- Add the managed `StbImageNext` 1.0.0 decoder (no native or further package dependencies), update third-party notices, and save [medium-priority recipes and synthetic examples](examples/MEDIUM_COMPATIBILITY_WORKFLOWS.md). New native Rhino/Windows checks remain pending. + - Follow OGC API next links and supported WFS paging with explicit user caps, deduplication, repeated-page protection and completeness diagnostics. Preserve raw GML for the building loader. - Discover feature collections from OGC API landing pages, collection lists and advertised links using the existing layer-list/vector components. Layer selection is optional for a single collection and direct files. - Load bounded, validated direct GeoJSON and Shapefile ZIP downloads, plus remote CityJSON/CityGML building files. Check companion files, CRS metadata, archive paths and expanded sizes; reuse validated downloads. diff --git a/RhinoSpatial.Core/CityJsonReader.cs b/RhinoSpatial.Core/CityJsonReader.cs index 0c77e82..dd5bab6 100644 --- a/RhinoSpatial.Core/CityJsonReader.cs +++ b/RhinoSpatial.Core/CityJsonReader.cs @@ -1,285 +1,228 @@ -using System; -using System.Collections.Generic; using System.Globalization; -using System.Linq; +using System.Text; using System.Text.Json; -namespace RhinoSpatial.Core +namespace RhinoSpatial.Core; + +public record CityJsonSourceMetadata(string SrsName, BoundingBox2D? BoundingBox); + +public static class CityJsonReader { - public record CityJsonSourceMetadata(string SrsName, BoundingBox2D? BoundingBox); + public static CityJsonSourceMetadata ReadSourceMetadata(string text) + { using var stream = new MemoryStream(Encoding.UTF8.GetBytes(text)); return ReadSourceMetadata(stream); } - public static class CityJsonReader + public static CityJsonSourceMetadata ReadSourceMetadata(Stream stream, CancellationToken cancellationToken = default) { - public static CityJsonSourceMetadata ReadSourceMetadata(string jsonText) + string srs = ""; BoundingBox2D? bounds = null; + foreach (var (root, header) in Records(stream, cancellationToken)) { - using var document = JsonDocument.Parse(jsonText); - var root = document.RootElement; - var vertices = ReadVertices(root); - var srsName = ReadSrsName(root); - return new CityJsonSourceMetadata(srsName, CalculateBounds(vertices)); + srs = ReadSrs(header); + if (header.TryGetProperty("metadata", out var metadata) && metadata.TryGetProperty("geographicalExtent", out var extent)) + { + var a = extent.EnumerateArray().Select(Number).ToArray(); + if (a.Length != 6 || a[0] > a[3] || a[1] > a[4]) throw new InvalidDataException("Invalid CityJSON geographicalExtent."); + return new(srs, new(a[0], a[1], a[3], a[4])); + } + var vertices = Vertices(root, header); + bounds = Union(bounds, Bounds(vertices)); + if (root.TryGetProperty("CityObjects", out var objects)) + foreach (var obj in objects.EnumerateObject()) + if (obj.Value.TryGetProperty("geometry", out var geometry)) + foreach (var g in geometry.EnumerateArray()) + if (Type(g) == "GeometryInstance") bounds = Union(bounds, Bounds(InstanceVertices(g, header, vertices))); } + return new(srs, bounds); + } - public static List ReadBuildings(string jsonText, string sourceLayerName, BoundingBox2D? filterBounds = null) - { - using var document = JsonDocument.Parse(jsonText); - var root = document.RootElement; - var vertices = ReadVertices(root); - var buildings = new List(); - - if (!root.TryGetProperty("CityObjects", out var cityObjects) || - cityObjects.ValueKind != JsonValueKind.Object) - { - return buildings; - } - - foreach (var cityObject in cityObjects.EnumerateObject()) - { - var value = cityObject.Value; - if (!IsBuildingObject(value)) - { - continue; - } + public static List ReadBuildings(string text, string layer, BoundingBox2D? filterBounds = null) + { using var stream = new MemoryStream(Encoding.UTF8.GetBytes(text)); return ReadBuildings(stream, layer, filterBounds); } + public static List ReadBuildings(Stream stream, string layer, BoundingBox2D? filterBounds = null, CancellationToken cancellationToken = default) + { + var result = new List(); var ids = new HashSet(StringComparer.Ordinal); + long pointCount = 0; + foreach (var (root, header) in Records(stream, cancellationToken)) + { + var vertices = Vertices(root, header); + if (!root.TryGetProperty("CityObjects", out var objects) || objects.ValueKind != JsonValueKind.Object) + throw new InvalidDataException("CityJSON requires a CityObjects object."); + foreach (var obj in objects.EnumerateObject()) + { + cancellationToken.ThrowIfCancellationRequested(); + if (Type(obj.Value) is not ("Building" or "BuildingPart" or "BuildingInstallation" or "BuildingConstructiveElement" or "BuildingRoom")) continue; + if (!ids.Add(obj.Name)) throw new InvalidDataException("Duplicate CityJSON building id: " + obj.Name); + if (!obj.Value.TryGetProperty("geometry", out var geometries)) continue; + var candidates = geometries.EnumerateArray().Select(g => (Geometry: g, Lod: ReadLod(g, header))).ToList(); + var lods = candidates.Where(g => g.Lod.HasValue).Select(g => g.Lod!.Value).Distinct().OrderBy(x => x).ToList(); + if (lods.Count > 0 && candidates.Any(c => c.Lod is null)) throw new InvalidDataException("CityJSON mixes declared and unspecified geometry LoDs; a complete representation cannot be chosen safely."); + decimal? selected = lods.Count == 0 ? null : lods.Where(l => l <= 2.2m).Select(l => (decimal?)l).LastOrDefault() ?? lods[0]; var surfaces = new List(); - if (value.TryGetProperty("geometry", out var geometries) && - geometries.ValueKind == JsonValueKind.Array) + foreach (var candidate in candidates.Where(c => c.Lod == selected)) { - foreach (var geometry in geometries.EnumerateArray()) + var geometry = candidate.Geometry; IReadOnlyList points = vertices; + if (Type(geometry) == "GeometryInstance") { - if (geometry.TryGetProperty("boundaries", out var boundaries)) - { - CollectSurfaces(boundaries, vertices, surfaces); - } + points = InstanceVertices(geometry, header, vertices); + geometry = Template(geometry, header); } + if (Type(geometry) is not ("MultiSurface" or "CompositeSurface" or "Solid" or "MultiSolid" or "CompositeSolid")) + throw new InvalidDataException("Unsupported CityJSON building primitive: " + Type(geometry)); + Collect(geometry.GetProperty("boundaries"), points, surfaces); } - - if (surfaces.Count == 0) - { - continue; - } - - var filteredSurfaces = filterBounds is null - ? surfaces - : surfaces.Where(surface => SurfaceIntersectsBounds(surface, filterBounds)).ToList(); - if (filteredSurfaces.Count == 0) - { - continue; - } - - buildings.Add(new Lod2Building( - cityObject.Name, - sourceLayerName, - filteredSurfaces, - new Dictionary - { - ["source_format"] = "CityJSON" - })); + pointCount += surfaces.Sum(s => (long)s.OuterPoints.Count + s.InnerRings.Sum(r => r.Count)); + if (pointCount > 4000000 || result.Count >= 100000) throw new InvalidDataException("CityJSON output exceeds bounded import limits; select a smaller dataset."); + if (filterBounds is not null) surfaces = surfaces.Where(s => Intersects(Bounds(s.OuterPoints), filterBounds)).ToList(); + if (surfaces.Count == 0) continue; + result.Add(new(obj.Name, layer, surfaces, new Dictionary { ["source_format"] = "CityJSON", ["source_lod"] = selected?.ToString(CultureInfo.InvariantCulture) ?? "unspecified" })); } - - return buildings; } + return result; + } - private static bool IsBuildingObject(JsonElement cityObject) + // Validation consumes every record, even when reference metadata can stop at a header extent. + public static bool Validate(Stream stream, bool requireCrs = false, CancellationToken cancellationToken = default) + { + var count = 0; + foreach (var (root, header) in Records(stream, cancellationToken, true)) { - if (!cityObject.TryGetProperty("type", out var typeElement) || - typeElement.ValueKind != JsonValueKind.String) - { - return false; - } - - var type = typeElement.GetString() ?? string.Empty; - return type.Contains("Building", StringComparison.OrdinalIgnoreCase); + count++; + if (requireCrs && string.IsNullOrWhiteSpace(ReadSrs(header))) throw new InvalidDataException("Downloaded CityJSON must declare metadata.referenceSystem."); + if (!root.TryGetProperty("CityObjects", out var objects) || objects.ValueKind != JsonValueKind.Object || !root.TryGetProperty("vertices", out _)) + throw new InvalidDataException("CityJSON requires CityObjects and vertices."); + _ = Vertices(root, header); } + return count > 0; + } - private static List ReadVertices(JsonElement root) - { - var scale = new[] { 1.0, 1.0, 1.0 }; - var translate = new[] { 0.0, 0.0, 0.0 }; - - if (root.TryGetProperty("transform", out var transform)) - { - if (transform.TryGetProperty("scale", out var scaleElement)) - { - ReadTriple(scaleElement, scale); - } - - if (transform.TryGetProperty("translate", out var translateElement)) - { - ReadTriple(translateElement, translate); - } - } - - var vertices = new List(); - if (!root.TryGetProperty("vertices", out var verticesElement) || - verticesElement.ValueKind != JsonValueKind.Array) - { - return vertices; - } - - foreach (var vertex in verticesElement.EnumerateArray()) - { - if (vertex.ValueKind != JsonValueKind.Array || vertex.GetArrayLength() < 3) - { - continue; - } - - vertices.Add(new Coordinate3D( - vertex[0].GetDouble() * scale[0] + translate[0], - vertex[1].GetDouble() * scale[1] + translate[1], - vertex[2].GetDouble() * scale[2] + translate[2])); - } - - return vertices; - } + public static bool IsCityJson(Stream stream) + { + try { using var e = Records(stream, default).GetEnumerator(); return e.MoveNext(); } + catch (JsonException) { return false; } + catch (InvalidDataException) { return false; } + } - private static void ReadTriple(JsonElement arrayElement, double[] target) + private static IEnumerable<(JsonElement Root, JsonElement Header)> Records(Stream stream, CancellationToken token, bool allowOtherRoot = false) + { + using var reader = new StreamReader(stream, new UTF8Encoding(false, true), true, 65536, leaveOpen: true); + JsonElement header = default; long total = 0; int count = 0; + while (true) { - if (arrayElement.ValueKind != JsonValueKind.Array) - { - return; - } - - for (var index = 0; index < Math.Min(3, arrayElement.GetArrayLength()); index++) - { - target[index] = arrayElement[index].GetDouble(); - } + int ch; do { ch = reader.Read(); } while (ch >= 0 && char.IsWhiteSpace((char)ch)); + if (ch < 0) yield break; + if (ch != '{') throw new InvalidDataException("Expected a CityJSON object or CityJSONSeq record."); + var text = new StringBuilder("{"); int depth = 1; bool quoted = false, escaped = false; + while (depth > 0) + { + ch = reader.Read(); if (ch < 0) throw new InvalidDataException("Truncated CityJSON record."); + var c = (char)ch; text.Append(c); + if (quoted) { if (escaped) escaped = false; else if (c == '\\') escaped = true; else if (c == '\"') quoted = false; } + else if (c == '\"') quoted = true; + else if (c is '{' or '[') depth++; + else if (c is '}' or ']') depth--; + if ((text.Length & 8191) == 0) token.ThrowIfCancellationRequested(); + if (text.Length > 64 * 1024 * 1024) throw new InvalidDataException("CityJSON record exceeds 64 MiB of text."); + } + total += text.Length; + if (total > 512L * 1024 * 1024 || ++count > 100001) throw new InvalidDataException("CityJSON stream exceeds import limits."); + using var doc = JsonDocument.Parse(text.ToString()); var root = doc.RootElement; + if (count == 1) + { + if (Type(root) != "CityJSON" && allowOtherRoot) yield break; + if (Type(root) != "CityJSON") throw new InvalidDataException("CityJSONSeq requires a CityJSON header before CityJSONFeature records."); + header = root.Clone(); + } + else + { + if (Type(root) != "CityJSONFeature") throw new InvalidDataException("Expected a CityJSONFeature sequence record."); + if (header.GetProperty("CityObjects").EnumerateObject().Any() || header.GetProperty("vertices").GetArrayLength() != 0) + throw new InvalidDataException("CityJSONSeq header CityObjects and vertices must be empty."); + if (root.TryGetProperty("transform", out _) || root.TryGetProperty("geometry-templates", out _) || root.TryGetProperty("metadata", out _)) + throw new InvalidDataException("CityJSONFeature inherits transform, templates and metadata from its header."); + } + token.ThrowIfCancellationRequested(); yield return (root, header); } + } - private static string ReadSrsName(JsonElement root) + private static string Type(JsonElement e) => e.TryGetProperty("type", out var t) && t.ValueKind == JsonValueKind.String ? t.GetString()! : ""; + private static double Number(JsonElement e) + { var n = e.GetDouble(); if (!double.IsFinite(n)) throw new InvalidDataException("CityJSON coordinates must be finite."); return n; } + private static string ReadSrs(JsonElement header) + { + if (!header.TryGetProperty("metadata", out var m) || !m.TryGetProperty("referenceSystem", out var r)) return ""; + var name = r.GetString() ?? ""; + var match = System.Text.RegularExpressions.Regex.Match(name, @"(?:EPSG[/,:]+(?:0[/,:]+)?)(\d+)$", System.Text.RegularExpressions.RegexOptions.IgnoreCase); + return match.Success ? "EPSG:" + match.Groups[1].Value : ""; + } + private static List Vertices(JsonElement root, JsonElement header) + { + var scale = new[] { 1d, 1d, 1d }; var translate = new[] { 0d, 0d, 0d }; + if (header.TryGetProperty("transform", out var t)) { - if (!root.TryGetProperty("metadata", out var metadata) || - metadata.ValueKind != JsonValueKind.Object || - !metadata.TryGetProperty("referenceSystem", out var referenceSystem) || - referenceSystem.ValueKind != JsonValueKind.String) - { - return string.Empty; - } - - var value = referenceSystem.GetString() ?? string.Empty; - var lastSlash = value.LastIndexOf('/'); - var epsgText = lastSlash >= 0 ? value[(lastSlash + 1)..] : value; - return int.TryParse(epsgText, NumberStyles.Integer, CultureInfo.InvariantCulture, out var epsg) - ? $"EPSG:{epsg}" - : string.Empty; + scale = t.GetProperty("scale").EnumerateArray().Select(Number).ToArray(); + translate = t.GetProperty("translate").EnumerateArray().Select(Number).ToArray(); + if (scale.Length != 3 || translate.Length != 3) throw new InvalidDataException("Invalid CityJSON transform."); } - - private static BoundingBox2D? CalculateBounds(IReadOnlyList vertices) + var result = new List(); + foreach (var v in root.GetProperty("vertices").EnumerateArray()) { - if (vertices.Count == 0) - { - return null; - } - - return new BoundingBox2D( - vertices.Min(point => point.X), - vertices.Min(point => point.Y), - vertices.Max(point => point.X), - vertices.Max(point => point.Y)); + var a = v.EnumerateArray().Select(Number).ToArray(); if (a.Length != 3) throw new InvalidDataException("CityJSON vertex must contain XYZ."); + var p = new Coordinate3D(a[0] * scale[0] + translate[0], a[1] * scale[1] + translate[1], a[2] * scale[2] + translate[2]); + if (!double.IsFinite(p.X) || !double.IsFinite(p.Y) || !double.IsFinite(p.Z)) throw new InvalidDataException("CityJSON transform overflow."); + result.Add(p); } - - private static void CollectSurfaces( - JsonElement element, - IReadOnlyList vertices, - List surfaces) + return result; + } + private static JsonElement Template(JsonElement g, JsonElement header) + { + var templates = header.GetProperty("geometry-templates").GetProperty("templates"); + var i = g.GetProperty("template").GetInt32(); + if (i < 0 || i >= templates.GetArrayLength()) throw new InvalidDataException("CityJSON template index out of range."); + return templates[i]; + } + private static decimal? ReadLod(JsonElement g, JsonElement header) + { + if (Type(g) == "GeometryInstance") g = Template(g, header); + if (!g.TryGetProperty("lod", out var lod)) return null; + var text = lod.ValueKind == JsonValueKind.String ? lod.GetString() : lod.GetRawText(); + if (!decimal.TryParse(text, NumberStyles.Float, CultureInfo.InvariantCulture, out var value) || value < 0 || value >= 5) + throw new InvalidDataException("Invalid CityJSON LoD: " + text); + return value; + } + private static List InstanceVertices(JsonElement g, JsonElement header, IReadOnlyList vertices) + { + var boundaries = g.GetProperty("boundaries"); if (boundaries.GetArrayLength() != 1) throw new InvalidDataException("GeometryInstance requires one anchor index."); + var index = boundaries[0].GetInt32(); if (index < 0 || index >= vertices.Count) throw new InvalidDataException("Invalid template anchor index."); + var anchor = vertices[index]; var m = g.GetProperty("transformationMatrix").EnumerateArray().Select(Number).ToArray(); + if (m.Length != 16 || m[12] != 0 || m[13] != 0 || m[14] != 0 || m[15] != 1) throw new InvalidDataException("GeometryInstance requires an affine row-major 4x4 matrix."); + var result = new List(); + foreach (var v in header.GetProperty("geometry-templates").GetProperty("vertices-templates").EnumerateArray()) { - if (TryReadSurface(element, vertices, out var surface)) - { - surfaces.Add(surface); - return; - } - - if (element.ValueKind != JsonValueKind.Array) - { - return; - } - - foreach (var child in element.EnumerateArray()) - { - CollectSurfaces(child, vertices, surfaces); - } + var a = v.EnumerateArray().Select(Number).ToArray(); if (a.Length != 3) throw new InvalidDataException("Invalid template vertex."); + var p = new Coordinate3D(m[0] * a[0] + m[1] * a[1] + m[2] * a[2] + m[3] + anchor.X, m[4] * a[0] + m[5] * a[1] + m[6] * a[2] + m[7] + anchor.Y, m[8] * a[0] + m[9] * a[1] + m[10] * a[2] + m[11] + anchor.Z); + if (!double.IsFinite(p.X) || !double.IsFinite(p.Y) || !double.IsFinite(p.Z)) throw new InvalidDataException("Template transform overflow."); + result.Add(p); } - - private static bool TryReadSurface( - JsonElement element, - IReadOnlyList vertices, - out SurfacePolygon3D surface) + return result; + } + private static void Collect(JsonElement e, IReadOnlyList vertices, List surfaces) + { + if (e.ValueKind != JsonValueKind.Array) throw new InvalidDataException("Invalid CityJSON boundaries."); + if (e.GetArrayLength() == 0) return; + if (e[0].ValueKind == JsonValueKind.Array && e[0].GetArrayLength() > 0 && e[0][0].ValueKind == JsonValueKind.Number) { - surface = new SurfacePolygon3D(new List(), new List>()); - if (element.ValueKind != JsonValueKind.Array || element.GetArrayLength() == 0) - { - return false; - } - - var firstRing = element[0]; - if (!IsIndexRing(firstRing)) - { - return false; - } - var rings = new List>(); - foreach (var ringElement in element.EnumerateArray()) - { - var ring = ReadRing(ringElement, vertices); - if (ring.Count >= 3) - { - rings.Add(ring); - } - } - - if (rings.Count == 0) + foreach (var ring in e.EnumerateArray()) { - return false; + if (ring.GetArrayLength() < 3) throw new InvalidDataException("CityJSON surface ring requires three vertices."); + var points = new List(); + foreach (var ix in ring.EnumerateArray()) { var i = ix.GetInt32(); if (i < 0 || i >= vertices.Count) throw new InvalidDataException("CityJSON vertex index out of range."); points.Add(vertices[i]); } + rings.Add(points); } - - surface = new SurfacePolygon3D( - rings[0], - rings.Skip(1).ToList()); - return true; - } - - private static bool IsIndexRing(JsonElement element) - { - return element.ValueKind == JsonValueKind.Array && - element.GetArrayLength() >= 3 && - element.EnumerateArray().All(item => item.ValueKind == JsonValueKind.Number); - } - - private static List ReadRing(JsonElement element, IReadOnlyList vertices) - { - var ring = new List(); - if (!IsIndexRing(element)) - { - return ring; - } - - foreach (var indexElement in element.EnumerateArray()) - { - var index = indexElement.GetInt32(); - if (index >= 0 && index < vertices.Count) - { - ring.Add(vertices[index]); - } - } - - return ring; - } - - private static bool SurfaceIntersectsBounds(SurfacePolygon3D surface, BoundingBox2D filterBounds) - { - if (surface.OuterPoints.Count == 0) - { - return false; - } - - var bounds = new BoundingBox2D( - surface.OuterPoints.Min(point => point.X), - surface.OuterPoints.Min(point => point.Y), - surface.OuterPoints.Max(point => point.X), - surface.OuterPoints.Max(point => point.Y)); - return bounds.MinX <= filterBounds.MaxX && - bounds.MaxX >= filterBounds.MinX && - bounds.MinY <= filterBounds.MaxY && - bounds.MaxY >= filterBounds.MinY; + surfaces.Add(new(rings[0], rings.Skip(1).ToList())); return; } + foreach (var child in e.EnumerateArray()) Collect(child, vertices, surfaces); } + private static BoundingBox2D? Bounds(IReadOnlyList p) => p.Count == 0 ? null : new(p.Min(v => v.X), p.Min(v => v.Y), p.Max(v => v.X), p.Max(v => v.Y)); + private static BoundingBox2D? Union(BoundingBox2D? a, BoundingBox2D? b) => a is null ? b : b is null ? a : new(Math.Min(a.MinX, b.MinX), Math.Min(a.MinY, b.MinY), Math.Max(a.MaxX, b.MaxX), Math.Max(a.MaxY, b.MaxY)); + private static bool Intersects(BoundingBox2D? a, BoundingBox2D b) => a is not null && a.MinX <= b.MaxX && a.MaxX >= b.MinX && a.MinY <= b.MaxY && a.MaxY >= b.MinY; } diff --git a/RhinoSpatial.Core/GeoTiffInfoCache.cs b/RhinoSpatial.Core/GeoTiffInfoCache.cs index 06505b3..e280a25 100644 --- a/RhinoSpatial.Core/GeoTiffInfoCache.cs +++ b/RhinoSpatial.Core/GeoTiffInfoCache.cs @@ -6,7 +6,7 @@ namespace RhinoSpatial.Core { public static class GeoTiffInfoCache { - private readonly record struct CacheKey(long FileSizeBytes, long LastWriteUtcTicks); + private readonly record struct CacheKey(long FileSizeBytes, long LastWriteUtcTicks, string Sidecars); private readonly record struct CacheEntry(CacheKey Key, GeoReferencedRasterInfo RasterInfo); @@ -15,7 +15,7 @@ public static class GeoTiffInfoCache public static GeoReferencedRasterInfo GetOrRead(string filePath) { var fileInfo = new FileInfo(filePath); - var key = new CacheKey(fileInfo.Length, fileInfo.LastWriteTimeUtc.Ticks); + var key = new CacheKey(fileInfo.Length, fileInfo.LastWriteTimeUtc.Ticks, RasterSidecars.Signature(filePath)); if (EntriesByPath.TryGetValue(filePath, out var cachedEntry) && cachedEntry.Key.Equals(key)) { diff --git a/RhinoSpatial.Core/GeoTiffReader.cs b/RhinoSpatial.Core/GeoTiffReader.cs index 6528584..93c8238 100644 --- a/RhinoSpatial.Core/GeoTiffReader.cs +++ b/RhinoSpatial.Core/GeoTiffReader.cs @@ -10,98 +10,55 @@ public static class GeoTiffReader private const ushort GeographicTypeGeoKey = 2048; private const ushort ProjectedCSTypeGeoKey = 3072; - public static GeoReferencedRasterInfo ReadImageInfo(string filePath) - { - TiffWarningSuppression.EnsureInstalled(); - using var tiff = Tiff.Open(filePath, "r"); - if (tiff is null) - { - throw new InvalidOperationException($"Failed to open GeoTIFF: {filePath}"); - } - - var width = tiff.GetField(TiffTag.IMAGEWIDTH)?[0].ToInt() ?? 0; - var height = tiff.GetField(TiffTag.IMAGELENGTH)?[0].ToInt() ?? 0; - if (width <= 0 || height <= 0) - { - throw new InvalidOperationException("The GeoTIFF image dimensions could not be read."); - } + public static GeoReferencedRasterInfo ReadImageInfo(string filePath) => ReadImageInfo(filePath,null); - var srsName = ReadSrsName(tiff); - if (string.IsNullOrWhiteSpace(srsName)) + public static GeoReferencedRasterInfo ReadImageInfo(string filePath,string? declaredFallbackSrs) + { + if (new[] { ".png", ".jpg", ".jpeg" }.Contains(Path.GetExtension(filePath), StringComparer.OrdinalIgnoreCase)) { - throw new InvalidOperationException("The GeoTIFF does not contain a readable EPSG code yet. The current RhinoSpatial GeoTIFF loader supports GeoTIFF files with embedded EPSG georeferencing."); + using var stream=File.OpenRead(filePath); + var info=StbImageNext.ImageInfo.FromStream(stream); + if(info.Width<=0 || info.Height<=0)throw new InvalidDataException("Invalid raster image dimensions."); + var sidecar=RasterSidecars.Read(filePath); + return new GeoReferencedRasterInfo(filePath,sidecar.Srs,info.Width,info.Height,sidecar.Transform.Bounds(info.Width,info.Height),new FileInfo(filePath).Length) { GeoTransform=sidecar.Transform }; } - - var boundingBox = ReadBoundingBox(tiff, width, height); - var fileSizeBytes = new FileInfo(filePath).Length; - return new GeoReferencedRasterInfo( - filePath, - srsName, - width, - height, - boundingBox, - fileSizeBytes); + TiffWarningSuppression.EnsureInstalled(); + using var tiff=Tiff.Open(filePath,"r"); + if(tiff is null) throw new InvalidOperationException("Failed to open raster: "+filePath); + var width=tiff.GetField(TiffTag.IMAGEWIDTH)?[0].ToInt()??0; + var height=tiff.GetField(TiffTag.IMAGELENGTH)?[0].ToInt()??0; + if(width<=0 || height<=0) throw new InvalidDataException("Invalid raster dimensions."); + var srs=ReadSrsName(tiff); + var transform=ReadTransform(tiff); + transform ??= RasterSidecars.ReadTransform(filePath); + if(string.IsNullOrWhiteSpace(srs))srs=string.IsNullOrWhiteSpace(declaredFallbackSrs)?RasterSidecars.ReadSrs(filePath):declaredFallbackSrs; + transform.Validate(); + return new GeoReferencedRasterInfo(filePath,srs,width,height,transform.Bounds(width,height),new FileInfo(filePath).Length) { GeoTransform=transform }; } - private static BoundingBox2D ReadBoundingBox(Tiff tiff, int width, int height) + private static RasterGeoTransform? ReadTransform(Tiff tiff) { - var pixelScale = TryReadDoubleArray(tiff, TiffTag.GEOTIFF_MODELPIXELSCALETAG); - var tiePoints = TryReadDoubleArray(tiff, TiffTag.GEOTIFF_MODELTIEPOINTTAG); - - if (pixelScale is { Length: >= 2 } && tiePoints is { Length: >= 6 }) + RasterGeoTransform? g=null; + var matrix=TryReadDoubleArray(tiff,TiffTag.GEOTIFF_MODELTRANSFORMATIONTAG); + if(matrix is { Length: >=16 }) { - var scaleX = Math.Abs(pixelScale[0]); - var scaleY = Math.Abs(pixelScale[1]); - if (scaleX <= 0.0 || scaleY <= 0.0) - { - throw new InvalidOperationException("The GeoTIFF pixel scale is invalid."); - } - - var rasterX = tiePoints[0]; - var rasterY = tiePoints[1]; - var modelX = tiePoints[3]; - var modelY = tiePoints[4]; - - var minX = modelX - rasterX * scaleX; - var maxY = modelY + rasterY * scaleY; - var maxX = minX + width * scaleX; - var minY = maxY - height * scaleY; - return new BoundingBox2D(minX, minY, maxX, maxY); + if(matrix.Any(v=>!double.IsFinite(v)) || matrix[12]!=0 || matrix[13]!=0 || matrix[15]!=1) + throw new InvalidDataException("Unsupported GeoTIFF projective transformation."); + g=new(matrix[0],matrix[1],matrix[3],matrix[4],matrix[5],matrix[7]); } - - var transform = TryReadDoubleArray(tiff, TiffTag.GEOTIFF_MODELTRANSFORMATIONTAG); - if (transform is { Length: >= 16 }) + else { - var scaleX = transform[0]; - var shearX = transform[1]; - var shearY = transform[4]; - var scaleY = transform[5]; - var translateX = transform[3]; - var translateY = transform[7]; - - if (Math.Abs(shearX) > 1e-9 || Math.Abs(shearY) > 1e-9) - { - throw new InvalidOperationException("The GeoTIFF uses rotated or sheared georeferencing, which the current RhinoSpatial GeoTIFF loader does not support yet."); - } - - if (Math.Abs(scaleX) <= 1e-9 || Math.Abs(scaleY) <= 1e-9) - { - throw new InvalidOperationException("The GeoTIFF transformation matrix is invalid."); - } - - var minX = translateX; - var maxY = translateY; - var maxX = minX + width * scaleX; - var minY = maxY + height * scaleY; - - return new BoundingBox2D( - Math.Min(minX, maxX), - Math.Min(minY, maxY), - Math.Max(minX, maxX), - Math.Max(minY, maxY)); + var scale=TryReadDoubleArray(tiff,TiffTag.GEOTIFF_MODELPIXELSCALETAG); + var tie=TryReadDoubleArray(tiff,TiffTag.GEOTIFF_MODELTIEPOINTTAG); + if(scale is {Length:>=2} && tie is {Length:>=6}) g=new(scale[0],0,tie[3]-tie[0]*scale[0],0,-scale[1],tie[4]+tie[1]*scale[1]); } - - throw new InvalidOperationException("The GeoTIFF does not expose supported georeferencing tags. The current RhinoSpatial GeoTIFF loader supports north-up GeoTIFF files with ModelTiepoint/ModelPixelScale or simple affine transform tags."); + if(g is null) return null; + var keys=TryReadUShortArray(tiff,TiffTag.GEOTIFF_GEOKEYDIRECTORYTAG); + if(keys is not null) + for(var i=4;i+3= geoKeys.Length) - { - break; - } - - var keyId = geoKeys[entryOffset]; - var tagLocation = geoKeys[entryOffset + 1]; - var valueOffset = geoKeys[entryOffset + 3]; - - if (tagLocation != 0) - { - continue; - } - - if (keyId == ProjectedCSTypeGeoKey || keyId == GeographicTypeGeoKey) - { - var normalizedSrs = $"EPSG:{valueOffset}"; - return normalizedSrs; - } - } - + foreach(var requested in new[]{ProjectedCSTypeGeoKey,GeographicTypeGeoKey}) + for(var offset=4;offset+30 && geoKeys[offset+3]!=32767) + return "EPSG:"+geoKeys[offset+3]; return string.Empty; } diff --git a/RhinoSpatial.Core/Lod2AtomFeedReader.cs b/RhinoSpatial.Core/Lod2AtomFeedReader.cs index bd6375d..0335c73 100644 --- a/RhinoSpatial.Core/Lod2AtomFeedReader.cs +++ b/RhinoSpatial.Core/Lod2AtomFeedReader.cs @@ -83,7 +83,7 @@ private static bool IsDownloadLink(XElement link) var type = ((string?)link.Attribute("type") ?? "").ToLowerInvariant(); var href = ((string?)link.Attribute("href") ?? "").Split('?', '#')[0]; return (relation == "enclosure" && !type.Contains("atom+xml")) || type.Contains("zip") || type.Contains("gmz") || type.Contains("7z") || - new[] { ".zip", ".gmz", ".7z", ".gz", ".tar", ".rar", ".cityjson", ".json", ".geojson", ".gml", ".citygml", ".asc", ".xyz", ".csv", ".tif", ".tiff" }.Any(extension => href.EndsWith(extension, StringComparison.OrdinalIgnoreCase)); + new[] { ".zip", ".gmz", ".7z", ".gz", ".tar", ".rar", ".cityjson", ".jsonl", ".json", ".geojson", ".gml", ".citygml", ".asc", ".xyz", ".csv", ".tif", ".tiff" }.Any(extension => href.EndsWith(extension, StringComparison.OrdinalIgnoreCase)); } public static Uri ResolveUrl(XElement link, Uri source) diff --git a/RhinoSpatial.Core/Models.cs b/RhinoSpatial.Core/Models.cs index 87ce868..7062234 100644 --- a/RhinoSpatial.Core/Models.cs +++ b/RhinoSpatial.Core/Models.cs @@ -125,7 +125,7 @@ public record TerrainRasterData( Coordinate2D OffsetVectorY, double? NoDataValue, float[] Elevations - ); + ) { public RasterGeoTransform? GeoTransform { get; init; } } public record WcsCoverageResult( string RequestUrl, @@ -142,7 +142,7 @@ public record GeoReferencedRasterInfo( int Height, BoundingBox2D BoundingBox, long FileSizeBytes - ); + ) { public RasterGeoTransform? GeoTransform { get; init; } } public record OsmAreaFeature( long Id, diff --git a/RhinoSpatial.Core/RasterGeoTransform.cs b/RhinoSpatial.Core/RasterGeoTransform.cs new file mode 100644 index 0000000..76566f8 --- /dev/null +++ b/RhinoSpatial.Core/RasterGeoTransform.cs @@ -0,0 +1,67 @@ +using System.Globalization; +using System.Security.Cryptography; + +namespace RhinoSpatial.Core; + +/// Maps pixel corners to source coordinates; sample centers are at (col+.5,row+.5). +public record RasterGeoTransform(double A, double B, double C, double D, double E, double F) +{ + public Coordinate2D At(double column, double row) => new(A * column + B * row + C, D * column + E * row + F); + public void Validate() + { + if (new[] { A, B, C, D, E, F }.Any(v => !double.IsFinite(v)) || !double.IsFinite(A * E - B * D) || A * E - B * D == 0 || Math.Abs(A * E - B * D) <= 1e-12 * (Math.Abs(A * E) + Math.Abs(B * D))) + throw new InvalidDataException("Raster georeferencing must be finite and invertible."); + } + public Coordinate2D Pixel(double x, double y) + { var det = A * E - B * D; return new((E * (x - C) - B * (y - F)) / det, (-D * (x - C) + A * (y - F)) / det); } + public BoundingBox2D Bounds(int width, int height) + { + var p = new[] { At(0, 0), At(width, 0), At(width, height), At(0, height) }; + return new(p.Min(p => p.X), p.Min(p => p.Y), p.Max(p => p.X), p.Max(p => p.Y)); + } + public static RasterGeoTransform FromWorldFile(string path) + { + var values = File.ReadAllLines(path).Where(l => !string.IsNullOrWhiteSpace(l)).Select(l => double.Parse(l, NumberStyles.Float, CultureInfo.InvariantCulture)).ToArray(); + if (values.Length != 6) throw new InvalidDataException("World file must contain six numeric lines."); + var g = new RasterGeoTransform(values[0], values[2], values[4] - (values[0] + values[2]) / 2, values[1], values[3], values[5] - (values[1] + values[3]) / 2); + g.Validate(); return g; + } +} + +public static class RasterSidecars +{ + public static IEnumerable Files(string raster) + { + var directory = Path.GetDirectoryName(Path.GetFullPath(raster))!; + var ext = Path.GetExtension(raster).TrimStart('.'); + var stem = Path.GetFileNameWithoutExtension(raster); + var names = ext.Length == 0 ? new[] { stem + ".wld", stem + ".prj" } : new[] { stem + "." + ext[0] + ext[^1] + "w", stem + "." + ext + "w", stem + ".wld", stem + ".prj" }; + var files = Directory.EnumerateFiles(directory, stem + ".*", new EnumerationOptions { MatchCasing = MatchCasing.CaseInsensitive }).ToList(); + foreach (var name in names) + { + var file = files.FirstOrDefault(f => Path.GetFileName(f).Equals(name, StringComparison.OrdinalIgnoreCase)); + if (file is not null) yield return file; + } + } + public static string Signature(string raster) => string.Join("|", Files(raster).Select(p => + { + if (new FileInfo(p).Length > 65536) return p + ":" + new FileInfo(p).Length + ":" + File.GetLastWriteTimeUtc(p).Ticks; + return p + ":" + Convert.ToHexString(SHA256.HashData(File.ReadAllBytes(p))); + })); + public static RasterGeoTransform ReadTransform(string raster) + { + var world = Files(raster).FirstOrDefault(p => !p.EndsWith(".prj", StringComparison.OrdinalIgnoreCase)); + if (world is null) throw new InvalidDataException("Raster requires readable embedded georeferencing or a world file."); + if (new FileInfo(world).Length > 65536) throw new InvalidDataException("World file exceeds 64 KiB."); + return RasterGeoTransform.FromWorldFile(world); + } + public static string ReadSrs(string raster) + { + var prj = Files(raster).FirstOrDefault(p => p.EndsWith(".prj", StringComparison.OrdinalIgnoreCase)); + if (prj is null) throw new InvalidDataException("Raster requires an embedded CRS or a resolvable .prj; no context CRS was assumed."); + if (new FileInfo(prj).Length > 65536) throw new InvalidDataException("Raster .prj exceeds 64 KiB."); + var srs = ShapefileFeatureReader.NormalizeKnownProjection(File.ReadAllText(prj)); + return string.IsNullOrWhiteSpace(srs) ? throw new InvalidDataException("Raster .prj CRS is unsupported.") : srs; + } + public static (RasterGeoTransform Transform, string Srs) Read(string raster) => (ReadTransform(raster), ReadSrs(raster)); +} diff --git a/RhinoSpatial.Core/RasterImageCodec.cs b/RhinoSpatial.Core/RasterImageCodec.cs new file mode 100644 index 0000000..5161cc4 --- /dev/null +++ b/RhinoSpatial.Core/RasterImageCodec.cs @@ -0,0 +1,37 @@ +using StbImageNext; +using System.Buffers.Binary; +using System.IO.Compression; +using System.Text; + +namespace RhinoSpatial.Core; + +public static class RasterImageCodec +{ + public static byte[] DecodeRgba(byte[] bytes, int width, int height) + { + using var input = new MemoryStream(bytes); + var info = ImageInfo.FromStream(input); + if (info.Width != width || info.Height != height || (long)width * height > 4194304) throw new InvalidDataException("Tile dimensions disagree with its declared matrix."); + return ImageResult.FromMemory(bytes, ColorComponents.RedGreenBlueAlpha).Data; + } + public static void WritePng(string path, int width, int height, byte[] rgba) + { + if (width <= 0 || height <= 0 || (long)width * height * 4 != rgba.Length) throw new InvalidDataException("Invalid RGBA image dimensions."); + using var output = File.Create(path); + output.Write(new byte[] { 137, 80, 78, 71, 13, 10, 26, 10 }); + var header = new byte[13]; BinaryPrimitives.WriteInt32BigEndian(header.AsSpan(0, 4), width); BinaryPrimitives.WriteInt32BigEndian(header.AsSpan(4, 4), height); header[8] = 8; header[9] = 6; + Chunk(output, "IHDR", header); + using var compressed = new MemoryStream(); + using (var zlib = new ZLibStream(compressed, CompressionLevel.Fastest, true)) + for (var row = 0; row < height; row++) { zlib.WriteByte(0); zlib.Write(rgba, row * width * 4, width * 4); } + Chunk(output, "IDAT", compressed.ToArray()); Chunk(output, "IEND", Array.Empty()); + } + private static void Chunk(Stream stream, string name, byte[] bytes) + { + Span length = stackalloc byte[4]; BinaryPrimitives.WriteInt32BigEndian(length, bytes.Length); stream.Write(length); + var type = Encoding.ASCII.GetBytes(name); stream.Write(type); stream.Write(bytes); + uint crc = 0xffffffff; + foreach (var b in type.Concat(bytes)) { crc ^= b; for (var bit = 0; bit < 8; bit++) crc = (crc >> 1) ^ ((crc & 1) != 0 ? 0xedb88320u : 0); } + BinaryPrimitives.WriteUInt32BigEndian(length, ~crc); stream.Write(length); + } +} diff --git a/RhinoSpatial.Core/RasterImageMeshBuilder.cs b/RhinoSpatial.Core/RasterImageMeshBuilder.cs new file mode 100644 index 0000000..8409973 --- /dev/null +++ b/RhinoSpatial.Core/RasterImageMeshBuilder.cs @@ -0,0 +1,70 @@ +namespace RhinoSpatial.Core; + +public record RasterImageVertex(double X, double Y, double U, double V); +public record RasterImageMeshData(List Vertices, List<(int A, int B, int C)> Faces) { public double CoverageRatio { get; init; } } + +public static class RasterImageMeshBuilder +{ + public static RasterImageMeshData Build(GeoReferencedRasterInfo info, SpatialContext2D context, int divisions = 16) + { + if (divisions < 1 || divisions > 128) throw new ArgumentOutOfRangeException(nameof(divisions)); + var g = info.GeoTransform ?? new RasterGeoTransform((info.BoundingBox.MaxX - info.BoundingBox.MinX) / info.Width, 0, info.BoundingBox.MinX, 0, -(info.BoundingBox.MaxY - info.BoundingBox.MinY) / info.Height, info.BoundingBox.MaxY); + g.Validate(); + var selected=Lod2AtomFeedReader.TransformBounds(new(context.ResolvedSrs,context.PlacementBoundingBox),info.SrsName); + var pixelCorners=new[]{g.Pixel(selected.MinX,selected.MinY),g.Pixel(selected.MaxX,selected.MinY),g.Pixel(selected.MaxX,selected.MaxY),g.Pixel(selected.MinX,selected.MaxY)}; + var u0=Math.Clamp(pixelCorners.Min(p=>p.X)/info.Width,0,1);var u1=Math.Clamp(pixelCorners.Max(p=>p.X)/info.Width,0,1); + var r0=Math.Clamp(pixelCorners.Min(p=>p.Y)/info.Height,0,1);var r1=Math.Clamp(pixelCorners.Max(p=>p.Y)/info.Height,0,1); + if(u1<=u0 || r1<=r0)return new(new(),new()); + var grid = new RasterImageVertex[(divisions + 1) * (divisions + 1)]; + for (var r = 0; r <= divisions; r++) for (var c = 0; c <= divisions; c++) + { + var uv = new Coordinate2D(u0+(u1-u0)*c/divisions,r0+(r1-r0)*r/divisions); var p = g.At(uv.X * info.Width, uv.Y * info.Height); + if (!SpatialReferenceTransform.TryTransformXY(info.SrsName, context.ResolvedSrs, p.X, p.Y, out var x, out var y) || !double.IsFinite(x) || !double.IsFinite(y)) + throw new InvalidDataException($"Cannot transform raster from '{info.SrsName}' into '{context.ResolvedSrs}'."); + grid[r * (divisions + 1) + c] = new(x, y, uv.X, 1 - uv.Y); + } + var vertices = new List(); var faces = new List<(int, int, int)>(); + var b = context.PlacementBoundingBox; double keptArea = 0; + for (var r = 0; r < divisions; r++) for (var c = 0; c < divisions; c++) + { + var i = r * (divisions + 1) + c; + // Triangles preserve an unambiguous UV interpolation when projection is nonlinear. + foreach (var triangle in new[] { new[] { grid[i], grid[i + 1], grid[i + divisions + 2] }, new[] { grid[i], grid[i + divisions + 2], grid[i + divisions + 1] } }) + { + var polygon = triangle.ToList(); + polygon = Clip(polygon, p => p.X - b.MinX); + polygon = Clip(polygon, p => b.MaxX - p.X); + polygon = Clip(polygon, p => p.Y - b.MinY); + polygon = Clip(polygon, p => b.MaxY - p.Y); + if (polygon.Count < 3) continue; + var start = vertices.Count; + vertices.AddRange(polygon.Select(p => p with { X = p.X - (context.UseAbsoluteCoordinates ? 0 : context.PlacementOrigin.X), Y = p.Y - (context.UseAbsoluteCoordinates ? 0 : context.PlacementOrigin.Y) })); + for (var n = 1; n < polygon.Count - 1; n++) + { + var a = polygon[0]; var d = polygon[n]; var e = polygon[n + 1]; + var cross = Cross(a, d, e); + if (Math.Abs(cross) > 1e-18) { keptArea += Math.Abs((d.U-a.U)*(e.V-a.V)-(d.V-a.V)*(e.U-a.U))/2; faces.Add(cross > 0 ? (start, start + n, start + n + 1) : (start, start + n + 1, start + n)); } + } + } + } + return new(vertices, faces) { CoverageRatio = Math.Clamp(keptArea,0,1) }; + } + private static double Cross(RasterImageVertex a, RasterImageVertex b, RasterImageVertex c) => (b.X - a.X) * (c.Y - a.Y) - (b.Y - a.Y) * (c.X - a.X); + private static List Clip(List input, Func distance) + { + var output = new List(); if (input.Count == 0) return output; + var previous = input[^1]; var a = distance(previous); + foreach (var point in input) + { + var b = distance(point); + if ((a >= 0) != (b >= 0)) + { + var t = a / (a - b); + output.Add(new(previous.X + t * (point.X - previous.X), previous.Y + t * (point.Y - previous.Y), previous.U + t * (point.U - previous.U), previous.V + t * (point.V - previous.V))); + } + if (b >= 0) output.Add(point); + previous = point; a = b; + } + return output; + } +} diff --git a/RhinoSpatial.Core/ReferenceSourceMetadataReader.cs b/RhinoSpatial.Core/ReferenceSourceMetadataReader.cs index 4f19cfd..3f0ff26 100644 --- a/RhinoSpatial.Core/ReferenceSourceMetadataReader.cs +++ b/RhinoSpatial.Core/ReferenceSourceMetadataReader.cs @@ -19,7 +19,7 @@ public static class ReferenceSourceMetadataReader private const int MaximumDirectoryFileCount = 256; private static readonly string[] SupportedFileExtensions = { - ".tif", ".tiff", ".shp", ".geojson", ".json", ".cityjson", + ".tif", ".tiff", ".shp", ".geojson", ".json", ".cityjson", ".jsonl", ".png", ".jpg", ".jpeg", ".gml", ".xml", ".asc", ".xyz", ".csv", ".zip" }; @@ -118,9 +118,10 @@ private static ReferenceSourceMetadata ReadFile( var extension = Path.GetExtension(filePath).ToLowerInvariant(); return extension switch { - ".tif" or ".tiff" => ReadGeoTiff(filePath), + ".tif" or ".tiff" or ".png" or ".jpg" or ".jpeg" => ReadGeoTiff(filePath), ".shp" => ReadShapefile(filePath, fallbackSrs), ".geojson" => ReadJson(File.ReadAllText(filePath), "GeoJSON", fallbackSrs), + ".jsonl" => ReadCityJsonSequence(filePath, fallbackSrs), ".json" or ".cityjson" => ReadJson(File.ReadAllText(filePath), "JSON", fallbackSrs), ".gml" or ".xml" => ReadGml(filePath, fallbackSrs), ".asc" or ".xyz" or ".csv" => ReadTerrainText(filePath, fallbackSrs), @@ -129,6 +130,13 @@ private static ReferenceSourceMetadata ReadFile( }; } + private static ReferenceSourceMetadata ReadCityJsonSequence(string filePath, string fallbackSrs) + { + using var stream = File.OpenRead(filePath); + var metadata = CityJsonReader.ReadSourceMetadata(stream); + return Create("CityJSONSeq", string.IsNullOrWhiteSpace(metadata.SrsName) ? fallbackSrs : metadata.SrsName, metadata.BoundingBox, 1); + } + private static ReferenceSourceMetadata ReadGeoTiff(string filePath) { var raster = GeoTiffInfoCache.GetOrRead(filePath); @@ -218,6 +226,7 @@ private static ReferenceSourceMetadata ReadZip( extension.Equals(".xml", StringComparison.OrdinalIgnoreCase) || extension.Equals(".json", StringComparison.OrdinalIgnoreCase) || extension.Equals(".cityjson", StringComparison.OrdinalIgnoreCase) || + extension.Equals(".jsonl", StringComparison.OrdinalIgnoreCase) || extension.Equals(".geojson", StringComparison.OrdinalIgnoreCase); }) .Take(MaximumDirectoryFileCount) @@ -243,6 +252,8 @@ private static ReferenceSourceMetadata ReadZip( continue; } + if(extension.Equals(".jsonl",StringComparison.OrdinalIgnoreCase)) + { var m=CityJsonReader.ReadSourceMetadata(stream);metadata.Add(Create("CityJSONSeq",string.IsNullOrWhiteSpace(m.SrsName)?fallbackSrs:m.SrsName,m.BoundingBox,1));continue; } using var reader = new StreamReader(stream); metadata.Add(ReadJson(reader.ReadToEnd(), "JSON", fallbackSrs)); } diff --git a/RhinoSpatial.Core/RemoteReferenceMetadataReader.cs b/RhinoSpatial.Core/RemoteReferenceMetadataReader.cs index 928b8c0..9bf1e5f 100644 --- a/RhinoSpatial.Core/RemoteReferenceMetadataReader.cs +++ b/RhinoSpatial.Core/RemoteReferenceMetadataReader.cs @@ -9,16 +9,27 @@ public sealed class RemoteReferenceMetadataReader private readonly AtomDownloadClient _atom; private readonly OgcApiFeaturesClient _ogc; private readonly bool _cache; + private readonly WmtsClient _wmts; public RemoteReferenceMetadataReader(HttpClient? http = null) - { _atom = new AtomDownloadClient(http); _ogc = new OgcApiFeaturesClient(http); _cache = http is null; } + { _atom = new AtomDownloadClient(http); _ogc = new OgcApiFeaturesClient(http); _cache = http is null; _wmts=new WmtsClient(http); } public async Task TryReadAsync(string source, string? layer = null, CancellationToken cancellationToken = default) { if (!Uri.TryCreate(source, UriKind.Absolute, out var uri) || uri.Scheme is not ("http" or "https")) return null; if (System.Text.RegularExpressions.Regex.IsMatch(source, @"[?&]service=(wfs|wms|wcs)(?:&|$)", System.Text.RegularExpressions.RegexOptions.IgnoreCase)) return null; - if (new[] { ".zip", ".geojson", ".cityjson", ".tif", ".tiff", ".asc", ".xyz", ".csv", ".gml", ".citygml" } + if (new[] { ".zip", ".geojson", ".cityjson", ".jsonl", ".png", ".jpg", ".jpeg", ".tif", ".tiff", ".asc", ".xyz", ".csv", ".gml", ".citygml" } .Contains(Path.GetExtension(uri.AbsolutePath), StringComparer.OrdinalIgnoreCase)) throw new InvalidDataException("Use the provider's service/feed metadata URL as Reference Source, or a supported local file. Remote data files are not downloaded to orient the map."); + if(WmtsClient.LooksLike(source)) + { + var wmts=await _wmts.TryLoadCapabilitiesAsync(source,cancellationToken).ConfigureAwait(false); + if(wmts is null) throw new InvalidDataException("WMTS reference metadata was not found."); + var selected=string.IsNullOrWhiteSpace(layer)?wmts.Layers:wmts.Layers.Where(l=>l.Id==layer).ToList(); + if(selected.Count==0)throw new InvalidDataException("WMTS reference layer was not found."); + var extents=selected.Select(l=>l.Wgs84Bounds).Where(b=>b is not null).Cast().ToList(); + var bounds=extents.Count==0?null:new BoundingBox2D(extents.Min(b=>b.MinX),extents.Min(b=>b.MinY),extents.Max(b=>b.MaxX),extents.Max(b=>b.MaxY)); + return new ReferenceSourceMetadata("WMTS","EPSG:4326",bounds,bounds,selected.Count); + } var key = source + "\n" + layer; if (_cache && Cache.TryGetValue(key, out var cached) && cached.Expires > DateTime.UtcNow) return cached.Value; // The parent collection holds metadata; /items would download feature data. diff --git a/RhinoSpatial.Core/RhinoSpatial.Core.csproj b/RhinoSpatial.Core/RhinoSpatial.Core.csproj index 0abf7ad..187d8c1 100644 --- a/RhinoSpatial.Core/RhinoSpatial.Core.csproj +++ b/RhinoSpatial.Core/RhinoSpatial.Core.csproj @@ -9,6 +9,7 @@ + diff --git a/RhinoSpatial.Core/ShapefileFeatureReader.cs b/RhinoSpatial.Core/ShapefileFeatureReader.cs index 650524a..5021a0b 100644 --- a/RhinoSpatial.Core/ShapefileFeatureReader.cs +++ b/RhinoSpatial.Core/ShapefileFeatureReader.cs @@ -137,7 +137,7 @@ private static string ResolveSourceSrs(string shapefilePath, string fallbackSrs) ? throw new InvalidOperationException("The Shapefile .prj contains an unsupported CRS; no context-CRS assumption was used.") : normalized; } - private static string NormalizeKnownProjection(string projectionText) + public static string NormalizeKnownProjection(string projectionText) { var text = projectionText ?? string.Empty; try diff --git a/RhinoSpatial.Core/SourceDownloadCache.cs b/RhinoSpatial.Core/SourceDownloadCache.cs index 2a178e4..32796a3 100644 --- a/RhinoSpatial.Core/SourceDownloadCache.cs +++ b/RhinoSpatial.Core/SourceDownloadCache.cs @@ -37,7 +37,7 @@ internal sealed class LimitedReadStream : Stream public sealed class SourceDownloadCache { private static readonly HttpClient SharedHttp = new() { Timeout = Timeout.InfiniteTimeSpan }; - private static readonly string[] Extensions = { ".zip", ".geojson", ".cityjson", ".gml", ".tif", ".asc", ".csv" }; + private static readonly string[] Extensions = { ".zip", ".geojson", ".cityjson", ".jsonl", ".gml", ".tif", ".asc", ".csv" }; private readonly HttpClient _http; private readonly string _directory; private readonly AtomDownloadKind _kind; @@ -141,7 +141,7 @@ private static async Task InspectPayloadAsync(string path, AtomDownloadK } var candidate = kind switch { - AtomDownloadKind.Buildings => extension is ".gml" or ".citygml" or ".xml" or ".json" or ".cityjson" or ".geojson", + AtomDownloadKind.Buildings => extension is ".gml" or ".citygml" or ".xml" or ".json" or ".cityjson" or ".jsonl" or ".geojson", AtomDownloadKind.Terrain => extension is ".tif" or ".tiff" or ".asc" or ".xyz" or ".csv", _ => extension is ".json" or ".geojson" }; @@ -149,7 +149,7 @@ private static async Task InspectPayloadAsync(string path, AtomDownloadK await using var stream = new LimitedReadStream(entry.Open(), entry.Length); if (kind == AtomDownloadKind.Buildings && extension is ".gml" or ".citygml" or ".xml") { if (await IsBuildingXmlAsync(stream, token).ConfigureAwait(false)) supported++; } - else if (extension is ".json" or ".geojson" or ".cityjson") + else if (extension is ".json" or ".geojson" or ".cityjson" or ".jsonl") { if (await IsSupportedJsonAsync(stream, kind, token).ConfigureAwait(false)) supported++; } else { @@ -190,15 +190,10 @@ private static async Task InspectPayloadAsync(string path, AtomDownloadK private static async Task IsSupportedJsonAsync(Stream input, AtomDownloadKind kind, CancellationToken token) { + if (kind == AtomDownloadKind.Buildings) return CityJsonReader.Validate(input, requireCrs: true, cancellationToken: token); using var document = await JsonDocument.ParseAsync(input, cancellationToken: token).ConfigureAwait(false); var root = document.RootElement; if (!root.TryGetProperty("type", out var type) || type.ValueKind != JsonValueKind.String) return false; - if (kind == AtomDownloadKind.Buildings && type.GetString() == "CityJSON") - { - if (!root.TryGetProperty("metadata", out var meta) || !meta.TryGetProperty("referenceSystem", out _)) - throw new InvalidDataException("Downloaded CityJSON must declare its source CRS in metadata.referenceSystem."); - return root.TryGetProperty("CityObjects", out _) && root.TryGetProperty("vertices", out _); - } return kind == AtomDownloadKind.Vector && type.GetString() is "FeatureCollection" or "Feature"; } private static async Task IsBuildingXmlAsync(Stream input, CancellationToken token) diff --git a/RhinoSpatial.Core/TerrainAtomLoader.cs b/RhinoSpatial.Core/TerrainAtomLoader.cs index f8cc81b..fa17284 100644 --- a/RhinoSpatial.Core/TerrainAtomLoader.cs +++ b/RhinoSpatial.Core/TerrainAtomLoader.cs @@ -80,7 +80,9 @@ public static TerrainRasterData Crop(TerrainRasterData raster, BoundingBox2D win throw new InvalidDataException("Terrain window has fewer than 2x2 or more than four million samples; adjust the selected area."); var samples = new float[width * height]; for (var row = 0; row < height; row++) Array.Copy(raster.Elevations, (row + y0) * raster.Width + x0, samples, row * width, width); - return raster with { Width = width, Height = height, Origin = new Coordinate2D(raster.Origin.X + x0 * raster.OffsetVectorX.X, raster.Origin.Y + y0 * raster.OffsetVectorY.Y), Elevations = samples }; + var geo = raster.GeoTransform; + if (geo is not null) { var corner=geo.At(x0,y0); geo=geo with {C=corner.X,F=corner.Y}; } + return raster with { GeoTransform=geo, Width = width, Height = height, Origin = new Coordinate2D(raster.Origin.X + x0 * raster.OffsetVectorX.X, raster.Origin.Y + y0 * raster.OffsetVectorY.Y), Elevations = samples }; } public static TerrainRasterData Merge(IReadOnlyList rasters, CancellationToken cancellationToken = default) diff --git a/RhinoSpatial.Core/TerrainMeshBuilder.cs b/RhinoSpatial.Core/TerrainMeshBuilder.cs index 0480136..0d71c1f 100644 --- a/RhinoSpatial.Core/TerrainMeshBuilder.cs +++ b/RhinoSpatial.Core/TerrainMeshBuilder.cs @@ -49,8 +49,9 @@ public static TerrainMeshData Build( continue; } - var sourceX = sampleBounds.MinX + xs[col] * stepX; - var sourceY = sampleBounds.MaxY - ys[row] * stepY; + var sourcePoint = raster.GeoTransform?.At(xs[col]+0.5,ys[row]+0.5); + var sourceX = sourcePoint?.X ?? sampleBounds.MinX + xs[col] * stepX; + var sourceY = sourcePoint?.Y ?? sampleBounds.MaxY - ys[row] * stepY; if (!SpatialReferenceTransform.TryTransformXY( raster.SrsName, context.ResolvedSrs, sourceX, sourceY, out var x, out var y) || !double.IsFinite(x) || !double.IsFinite(y)) diff --git a/RhinoSpatial.Core/TerrainRasterReader.cs b/RhinoSpatial.Core/TerrainRasterReader.cs index ecd17d6..9bdf56c 100644 --- a/RhinoSpatial.Core/TerrainRasterReader.cs +++ b/RhinoSpatial.Core/TerrainRasterReader.cs @@ -38,21 +38,24 @@ public static TerrainRasterData ReadRaster(string filePath, string coverageId, s ReadScanlineRaster(tiff, width, height, bitsPerSample, sampleFormat, elevations); } - var origin = new Coordinate2D(0, 0); - var offsetX = new Coordinate2D(1, 0); - var offsetY = new Coordinate2D(0, -1); + GeoReferencedRasterInfo? geo=null; + try { geo=GeoTiffReader.ReadImageInfo(filePath,srsName); } + catch (InvalidDataException) when (tiff.GetField(TiffTag.GEOTIFF_MODELPIXELSCALETAG) is null && tiff.GetField(TiffTag.GEOTIFF_MODELTRANSFORMATIONTAG) is null) { } + var origin = geo?.GeoTransform?.At(0.5,0.5) ?? new Coordinate2D(0, 0); + var offsetX = geo?.GeoTransform is { } gx ? new Coordinate2D(gx.A,gx.D) : new Coordinate2D(1,0); + var offsetY = geo?.GeoTransform is { } gy ? new Coordinate2D(gy.B,gy.E) : new Coordinate2D(0,-1); var noData = ReadNoDataValue(tiff); return new TerrainRasterData( coverageId, - srsName, + geo?.SrsName ?? srsName, width, height, origin, offsetX, offsetY, noData, - elevations); + elevations) { GeoTransform=geo?.GeoTransform }; } private static void ReadScanlineRaster( diff --git a/RhinoSpatial.Core/WmtsClient.cs b/RhinoSpatial.Core/WmtsClient.cs new file mode 100644 index 0000000..c4bf8fa --- /dev/null +++ b/RhinoSpatial.Core/WmtsClient.cs @@ -0,0 +1,202 @@ +using System.Collections.Concurrent; +using System.Globalization; +using System.Security.Cryptography; +using System.Text; +using System.Xml; +using System.Xml.Linq; + +namespace RhinoSpatial.Core; + +public record WmtsMatrix(string Id, double Resolution, Coordinate2D TopLeft, int TileWidth, int TileHeight, int Width, int Height); +public record WmtsMatrixSet(string Id, string Srs, List Matrices); +public record WmtsLimits(int MinRow, int MaxRow, int MinColumn, int MaxColumn); +public record WmtsLink(string Set, Dictionary Limits); +public record WmtsLayer(string Id, string Title, string Style, List Formats, List Links, Dictionary Dimensions, Dictionary Templates, BoundingBox2D? Wgs84Bounds); +public record WmtsCapabilities(List Layers, Dictionary Sets, string TileUrl); +public record WmtsTile(int Row, int Column, string Url); +public record WmtsPlan(WmtsLayer Layer, WmtsMatrixSet Set, WmtsMatrix Matrix, List Tiles, int MinRow, int MinColumn, int Width, int Height, RasterGeoTransform Transform, bool CoverageClipped); +public record WmtsImageResult(string LocalFilePath, GeoReferencedRasterInfo Raster, WmtsPlan Plan, bool UsedCache, string Status); + +public sealed class WmtsClient +{ + private static readonly HttpClient SharedHttp = new() { Timeout = Timeout.InfiniteTimeSpan }; + private static readonly ConcurrentDictionary Cache = new(); + private static readonly ConcurrentDictionary Gates = new(); + private readonly HttpClient _http; private readonly string _directory; + public WmtsClient(HttpClient? http = null, string? cacheDirectory = null) + { _http = http ?? SharedHttp; _directory = cacheDirectory ?? Path.Combine(Environment.GetFolderPath(Environment.SpecialFolder.LocalApplicationData), "RhinoSpatial", "WmtsCache"); } + public static bool LooksLike(string source) => source.Contains("wmts", StringComparison.OrdinalIgnoreCase); + public async Task TryLoadCapabilitiesAsync(string source, CancellationToken token = default) + { + if (System.Text.RegularExpressions.Regex.IsMatch(source, @"[?&]service=(?:wms|wfs|wcs)(?:&|$)", System.Text.RegularExpressions.RegexOptions.IgnoreCase)) return null; + if (ReferenceEquals(_http, SharedHttp) && Cache.TryGetValue(source, out var cached) && cached.Expires > DateTime.UtcNow) return cached.Value; + var uri = new Uri(source); var query = Query(uri.Query); + var url = source; + if (!uri.AbsolutePath.EndsWith(".xml", StringComparison.OrdinalIgnoreCase)) + url = WithQuery(source, new() { { "SERVICE", "WMTS" }, { "REQUEST", "GetCapabilities" }, { "VERSION", "1.0.0" } }); + WmtsCapabilities? result; + try + { + var response = await SourceHttp.ReadAsync(_http, url, token).ConfigureAwait(false); + using var reader = XmlReader.Create(new StringReader(response.Text), new XmlReaderSettings { DtdProcessing = DtdProcessing.Prohibit, XmlResolver = null }); + var doc = XDocument.Load(reader); + if (doc.Root?.Name.NamespaceName != "http://www.opengis.net/wmts/1.0") + { if (LooksLike(source)) throw new InvalidDataException("The source did not return WMTS capabilities."); result = null; } + else result = ReadCapabilities(doc, response.Url, query); + } + catch (Exception ex) when (!LooksLike(source) && ex is HttpRequestException or XmlException) { result = null; } + if (ReferenceEquals(_http, SharedHttp)) { if (Cache.Count > 128) Cache.Clear(); Cache[source] = (DateTime.UtcNow.AddMinutes(5), result); } + return result; + } + public static WmtsCapabilities ReadCapabilities(XDocument doc, Uri responseUrl, Dictionary? overrides = null) + { + XNamespace w = "http://www.opengis.net/wmts/1.0", o = "http://www.opengis.net/ows/1.1", x = "http://www.w3.org/1999/xlink"; + var contents = doc.Root?.Element(w + "Contents") ?? throw new InvalidDataException("WMTS capabilities has no Contents."); + var sets = new Dictionary(StringComparer.Ordinal); + foreach (var set in contents.Elements(w + "TileMatrixSet")) + { + var id = Text(set, o + "Identifier"); var declared = Text(set, o + "SupportedCRS"); + var srs = declared.Contains("CRS84", StringComparison.OrdinalIgnoreCase) || declared.Contains("CRS:84", StringComparison.OrdinalIgnoreCase) ? "EPSG:4326" : SpatialReferenceTransform.NormalizeSrs(declared); + if (!SpatialReferenceTransform.TryTransformXY(srs, "EPSG:4326", 0, 0, out _, out _)) continue; + var swap = OgcUrlUtilities.UsesLatitudeLongitudeAxisOrder(declared); + var degrees = srs is "EPSG:4326" or "EPSG:4258" or "EPSG:4283" or "EPSG:7844" or "EPSG:7423"; + var matrices = new List(); + foreach (var m in set.Elements(w + "TileMatrix")) + { + var top = Numbers(Text(m, w + "TopLeftCorner")); if (top.Length != 2) throw new InvalidDataException("Invalid WMTS top-left coordinate."); + var resolution = Double(Text(m, w + "ScaleDenominator")) * 0.00028 / (degrees ? 111319.49079327358 : 1); + var tw = Positive(Text(m, w + "TileWidth")); var th = Positive(Text(m, w + "TileHeight")); + if (resolution <= 0 || !double.IsFinite(resolution) || tw > 2048 || th > 2048) throw new InvalidDataException("Unsupported WMTS matrix resolution/tile size."); + matrices.Add(new(Text(m, o + "Identifier"), resolution, new(swap ? top[1] : top[0], swap ? top[0] : top[1]), tw, th, Positive(Text(m, w + "MatrixWidth")), Positive(Text(m, w + "MatrixHeight")))); + if (matrices.Count > 128) throw new InvalidDataException("WMTS exceeds 128 matrices per set."); + } + sets.Add(id, new(id, srs, matrices)); if (sets.Count > 128) throw new InvalidDataException("WMTS exceeds 128 matrix sets."); + } + var layers = new List(); + foreach (var l in contents.Elements(w + "Layer")) + { + var links = new List(); + foreach (var link in l.Elements(w + "TileMatrixSetLink")) + { + var limits = new Dictionary(StringComparer.Ordinal); + foreach (var limit in link.Element(w + "TileMatrixSetLimits")?.Elements(w + "TileMatrixLimits") ?? Enumerable.Empty()) + { + var minR = Nonnegative(Text(limit, w + "MinTileRow")); var maxR = Nonnegative(Text(limit, w + "MaxTileRow")); var minC = Nonnegative(Text(limit, w + "MinTileCol")); var maxC = Nonnegative(Text(limit, w + "MaxTileCol")); + if (minR > maxR || minC > maxC) throw new InvalidDataException("Invalid WMTS tile limits."); + limits.Add(Text(limit, w + "TileMatrix"), new(minR, maxR, minC, maxC)); + } + links.Add(new(Text(link, w + "TileMatrixSet"), limits)); + } + var dimensions = new Dictionary(StringComparer.OrdinalIgnoreCase); + foreach (var d in l.Elements(w + "Dimension")) + { + var name = Text(d, o + "Identifier"); var values = d.Elements(w + "Value").Select(e => e.Value).ToList(); + var value = overrides is not null && overrides.TryGetValue(name, out var supplied) ? supplied : (string?)d.Element(w + "Default") ?? (values.Count == 1 ? values[0] : ""); + dimensions.Add(name, value); + } + var templates = new Dictionary(StringComparer.OrdinalIgnoreCase); + foreach (var resource in l.Elements(w + "ResourceURL").Where(e => (string?)e.Attribute("resourceType") == "tile")) + templates[(string?)resource.Attribute("format") ?? ""] = Resolve(responseUrl, (string?)resource.Attribute("template") ?? "").Replace("%7B", "{", StringComparison.OrdinalIgnoreCase).Replace("%7D", "}", StringComparison.OrdinalIgnoreCase); + var bbox = l.Element(o + "WGS84BoundingBox"); BoundingBox2D? bounds = null; + if (bbox is not null) { var lo = Numbers(Text(bbox, o + "LowerCorner")); var hi = Numbers(Text(bbox, o + "UpperCorner")); if (lo.Length == 2 && hi.Length == 2) bounds = new(lo[0], lo[1], hi[0], hi[1]); } + var style = l.Elements(w + "Style").FirstOrDefault(e => (string?)e.Attribute("isDefault") is "true" or "1") ?? l.Element(w + "Style"); + layers.Add(new(Text(l, o + "Identifier"), (string?)l.Element(o + "Title") ?? Text(l, o + "Identifier"), style is null ? "" : Text(style, o + "Identifier"), l.Elements(w + "Format").Select(e => e.Value).ToList(), links, dimensions, templates, bounds)); + if (layers.Count > 2000) throw new InvalidDataException("WMTS exceeds 2000 layers."); + } + var operation = doc.Descendants(o + "Operation").FirstOrDefault(e => (string?)e.Attribute("name") == "GetTile"); + var get = operation?.Descendants(o + "Get").FirstOrDefault(e => !e.Descendants(o + "Value").Any() || e.Descendants(o + "Value").Any(v => v.Value == "KVP")); + var endpoint = (string?)get?.Attribute(x + "href"); + return new(layers, sets, string.IsNullOrWhiteSpace(endpoint) ? "" : OgcUrlUtilities.PreferSecureSameHostOperationUrl(responseUrl.AbsoluteUri, Resolve(responseUrl, endpoint))); + } + public static WmtsPlan Plan(WmtsCapabilities capabilities, string? layerName, SpatialContext2D context, string format = "image/png") + { + var layer = string.IsNullOrWhiteSpace(layerName) ? capabilities.Layers.FirstOrDefault() : capabilities.Layers.FirstOrDefault(l => l.Id == layerName); + if (layer is null) throw new InvalidDataException("The requested WMTS layer was not found."); + if (layer.Dimensions.Values.Any(string.IsNullOrWhiteSpace)) throw new InvalidDataException("WMTS dimension has no unambiguous default; set it in the source URL query."); + if (layer.Wgs84Bounds is { } coverage && coverage.MinX <= coverage.MaxX && coverage.MinY <= coverage.MaxY) + { + var requested = Lod2AtomFeedReader.TransformBounds(new(context.ResolvedSrs, context.PlacementBoundingBox), "EPSG:4326"); + if (requested.MinX > coverage.MaxX || requested.MaxX < coverage.MinX || requested.MinY > coverage.MaxY || requested.MaxY < coverage.MinY) + throw new InvalidDataException("WMTS layer declares no coverage for the selected Spatial Context."); + } + var selectedFormat = layer.Formats.FirstOrDefault(f => f == format && f is "image/png" or "image/jpeg") ?? layer.Formats.FirstOrDefault(f => f is "image/png" or "image/jpeg") ?? throw new InvalidDataException("WMTS layer has no supported PNG/JPEG format."); + foreach (var link in layer.Links.OrderByDescending(l => capabilities.Sets.TryGetValue(l.Set, out var s) && s.Srs == context.ResolvedSrs)) + { + if (!capabilities.Sets.TryGetValue(link.Set, out var set)) continue; + var bounds = Lod2AtomFeedReader.TransformBounds(new(context.ResolvedSrs, context.PlacementBoundingBox), set.Srs); + var desired = Math.Max(bounds.MaxX - bounds.MinX, bounds.MaxY - bounds.MinY) / 1024; + if (desired <= 0) throw new InvalidDataException("WMTS requires a nonempty selected area."); + foreach (var m in set.Matrices.OrderBy(m => Math.Abs(Math.Log(m.Resolution / desired)))) + { + if (link.Limits.Count > 0 && !link.Limits.ContainsKey(m.Id)) continue; + var sx = m.Resolution * m.TileWidth; var sy = m.Resolution * m.TileHeight; + var c0 = Math.Floor((bounds.MinX - m.TopLeft.X) / sx); var c1 = Math.Ceiling((bounds.MaxX - m.TopLeft.X) / sx) - 1; + var r0 = Math.Floor((m.TopLeft.Y - bounds.MaxY) / sy); var r1 = Math.Ceiling((m.TopLeft.Y - bounds.MinY) / sy) - 1; + var limits = link.Limits.GetValueOrDefault(m.Id) ?? new(0, m.Height - 1, 0, m.Width - 1); + var loC = Math.Max(Math.Max(c0, 0), limits.MinColumn); var hiC = Math.Min(Math.Min(c1, m.Width - 1), limits.MaxColumn); + var loR = Math.Max(Math.Max(r0, 0), limits.MinRow); var hiR = Math.Min(Math.Min(r1, m.Height - 1), limits.MaxRow); + if (hiC < loC || hiR < loR) continue; + var minC = (int)loC; var maxC = (int)hiC; var minR = (int)loR; var maxR = (int)hiR; + var count = (long)(maxC - minC + 1) * (maxR - minR + 1); var width = (long)(maxC - minC + 1) * m.TileWidth; var height = (long)(maxR - minR + 1) * m.TileHeight; + if (count > 64 || width * height > 4194304 || width > 8192 || height > 8192) continue; + var tiles = new List(); + for (var r = minR; r <= maxR; r++) for (var c = minC; c <= maxC; c++) + tiles.Add(new(r, c, TileUrl(capabilities, layer, set, m, r, c, selectedFormat))); + return new(layer, set, m, tiles, minR, minC, (int)width, (int)height, new(m.Resolution, 0, m.TopLeft.X + minC * sx, 0, -m.Resolution, m.TopLeft.Y - minR * sy), c0 < minC || c1 > maxC || r0 < minR || r1 > maxR); + } + } + throw new InvalidDataException("No supported WMTS matrix covers this area within 64 tiles/four million pixels; reduce the area or select another layer."); + } + public static string TileUrl(WmtsCapabilities caps, WmtsLayer layer, WmtsMatrixSet set, WmtsMatrix matrix, int row, int column, string format) + { + var values = new Dictionary(layer.Dimensions, StringComparer.OrdinalIgnoreCase) { { "Layer", layer.Id }, { "Style", layer.Style }, { "TileMatrixSet", set.Id }, { "TileMatrix", matrix.Id }, { "TileRow", row.ToString(CultureInfo.InvariantCulture) }, { "TileCol", column.ToString(CultureInfo.InvariantCulture) }, { "Format", format } }; + if (layer.Templates.TryGetValue(format, out var template)) + { + foreach (var v in values) template = template.Replace("{" + v.Key + "}", Uri.EscapeDataString(v.Value), StringComparison.OrdinalIgnoreCase); + if (template.Contains('{')) throw new InvalidDataException("WMTS tile template has an unresolved dimension."); + return template; + } + if (string.IsNullOrWhiteSpace(caps.TileUrl)) throw new InvalidDataException("WMTS advertises neither a usable REST tile template nor KVP GetTile endpoint."); + values["SERVICE"] = "WMTS"; values["REQUEST"] = "GetTile"; values["VERSION"] = "1.0.0"; + return WithQuery(caps.TileUrl, values); + } + public async Task DownloadAsync(WmtsPlan plan, CancellationToken token = default) + { + using var timeout = CancellationTokenSource.CreateLinkedTokenSource(token); timeout.CancelAfter(TimeSpan.FromMinutes(2)); + var key = Convert.ToHexString(SHA256.HashData(Encoding.UTF8.GetBytes(System.Text.Json.JsonSerializer.Serialize(new { plan.Transform, plan.Width, plan.Height, plan.Set.Srs, Urls = plan.Tiles.Select(t => t.Url) })))); + Directory.CreateDirectory(_directory); var target = Path.Combine(_directory, key + ".png"); var gate = Gates.GetOrAdd(target, _ => new(1, 1)); + await gate.WaitAsync(timeout.Token).ConfigureAwait(false); + try + { + bool cached = false; + if (File.Exists(target) && DateTime.UtcNow - File.GetLastWriteTimeUtc(target) < TimeSpan.FromDays(1)) + try { using var cachedStream = File.OpenRead(target); var info = StbImageNext.ImageInfo.FromStream(cachedStream); cached = info.Width == plan.Width && info.Height == plan.Height && File.Exists(target + ".sha256") && File.ReadAllText(target + ".sha256") == Convert.ToHexString(SHA256.HashData(File.ReadAllBytes(target))); } catch { } + if (!cached) + { + var rgba = new byte[checked(plan.Width * plan.Height * 4)]; long total = 0; + foreach (var tile in plan.Tiles) + { + using var response = await _http.GetAsync(tile.Url, HttpCompletionOption.ResponseHeadersRead, timeout.Token).ConfigureAwait(false); response.EnsureSuccessStatusCode(); + await using var input = await response.Content.ReadAsStreamAsync(timeout.Token).ConfigureAwait(false); using var bytes = new MemoryStream(); var buffer = new byte[65536]; int n; + while ((n = await input.ReadAsync(buffer, timeout.Token).ConfigureAwait(false)) > 0) + { total += n; if (bytes.Length + n > 4 * 1024 * 1024 || total > 64L * 1024 * 1024) throw new InvalidDataException("WMTS transfer exceeds tile/batch byte limits."); bytes.Write(buffer, 0, n); } + var pixels = RasterImageCodec.DecodeRgba(bytes.ToArray(), plan.Matrix.TileWidth, plan.Matrix.TileHeight); + for (var row = 0; row < plan.Matrix.TileHeight; row++) Buffer.BlockCopy(pixels, row * plan.Matrix.TileWidth * 4, rgba, ((tile.Row - plan.MinRow) * plan.Matrix.TileHeight * plan.Width + (tile.Column - plan.MinColumn) * plan.Matrix.TileWidth + row * plan.Width) * 4, plan.Matrix.TileWidth * 4); + } + timeout.Token.ThrowIfCancellationRequested(); var temporary = target + "." + Guid.NewGuid().ToString("N") + ".tmp"; + try { RasterImageCodec.WritePng(temporary, plan.Width, plan.Height, rgba); timeout.Token.ThrowIfCancellationRequested(); var checksum = Convert.ToHexString(SHA256.HashData(File.ReadAllBytes(temporary))); File.Move(temporary, target, true); File.WriteAllText(target + ".sha256", checksum); } finally { if (File.Exists(temporary)) File.Delete(temporary); } + } + var raster = new GeoReferencedRasterInfo(target, plan.Set.Srs, plan.Width, plan.Height, plan.Transform.Bounds(plan.Width, plan.Height), new FileInfo(target).Length) { GeoTransform = plan.Transform }; + return new(target, raster, plan, cached, $"{(cached ? "Using cached" : "Loaded")} WMTS '{plan.Layer.Id}', matrix set '{plan.Set.Id}', matrix '{plan.Matrix.Id}', {plan.Set.Srs}, resolution {plan.Matrix.Resolution.ToString("G6", CultureInfo.InvariantCulture)}, {plan.Tiles.Count} tiles, {plan.Width}x{plan.Height}." + (plan.CoverageClipped ? " Declared matrix coverage clips the selected area." : "")); + } + finally { gate.Release(); } + } + private static string Text(XElement e, XName name) => (string?)e.Element(name) ?? throw new InvalidDataException("Missing WMTS " + name.LocalName); + private static double Double(string s) { var d = double.Parse(s, NumberStyles.Float, CultureInfo.InvariantCulture); if (!double.IsFinite(d)) throw new InvalidDataException("Nonfinite WMTS coordinate."); return d; } + private static double[] Numbers(string s) => s.Split((char[]?)null, StringSplitOptions.RemoveEmptyEntries).Select(Double).ToArray(); + private static int Positive(string s) { var n = Nonnegative(s); return n > 0 ? n : throw new InvalidDataException("WMTS size must be positive."); } + private static int Nonnegative(string s) { var n = int.Parse(s, CultureInfo.InvariantCulture); return n >= 0 ? n : throw new InvalidDataException("WMTS limit must be nonnegative."); } + private static string Resolve(Uri basis, string href) { var uri = new Uri(basis, href); if (uri.Scheme is not ("http" or "https") || uri.UserInfo.Length > 0) throw new InvalidDataException("WMTS links require HTTP(S) without embedded credentials."); return uri.AbsoluteUri; } + private static Dictionary Query(string query) => query.TrimStart('?').Split('&', StringSplitOptions.RemoveEmptyEntries).Select(p => p.Split('=', 2)).GroupBy(p => Uri.UnescapeDataString(p[0]), StringComparer.OrdinalIgnoreCase).ToDictionary(g => g.Key, g => g.Last().Length > 1 ? Uri.UnescapeDataString(g.Last()[1]) : "", StringComparer.OrdinalIgnoreCase); + private static string WithQuery(string url, Dictionary values) { var uri = new UriBuilder(url); var query = Query(uri.Query); foreach (var pair in values) query[pair.Key] = pair.Value; uri.Query = string.Join("&", query.Select(p => Uri.EscapeDataString(p.Key) + "=" + Uri.EscapeDataString(p.Value))); return uri.Uri.AbsoluteUri; } +} diff --git a/RhinoSpatial/LoadGeoTiffComponent.cs b/RhinoSpatial/LoadGeoTiffComponent.cs index 81af3c0..67a68bc 100644 --- a/RhinoSpatial/LoadGeoTiffComponent.cs +++ b/RhinoSpatial/LoadGeoTiffComponent.cs @@ -2,6 +2,7 @@ using System.Collections.Generic; using System.Globalization; using System.IO; +using System.Linq; using System.Threading.Tasks; using GH_IO.Serialization; using Grasshopper.Kernel; @@ -55,7 +56,7 @@ public LoadGeoTiffComponent() protected override void RegisterInputParams(GH_InputParamManager pManager) { - pManager.AddTextParameter("GeoTIFF File", "GeoTIFF", "Path to a georeferenced GeoTIFF raster file.", GH_ParamAccess.item); + pManager.AddTextParameter("GeoTIFF File", "GeoTIFF", "Path to a GeoTIFF or TIFF/PNG/JPEG with a world file and resolvable .prj CRS.", GH_ParamAccess.item); pManager.AddTextParameter("Spatial Context", "Spatial Context", "Shared RhinoSpatial spatial context from the Spatial Context component.", GH_ParamAccess.item); } @@ -165,8 +166,8 @@ private SolveResults Compute(RequestData requestData) { return new SolveResults { - Status = $"The GeoTIFF could not be aligned from '{rasterInfo.SrsName}' into the current Spatial Context SRS '{requestData.SpatialContext.ResolvedSrs}'.", - MessageLevel = GH_RuntimeMessageLevel.Error + Status = "The raster footprint does not overlap the selected Spatial Context.", + MessageLevel = GH_RuntimeMessageLevel.Warning }; } @@ -272,112 +273,20 @@ private void ApplySolveResults(IGH_DataAccess dataAccess, SolveResults result) dataAccess.SetData(3, result.Status); } - private static bool TryCreateImageMesh( - GeoReferencedRasterInfo rasterInfo, - SpatialContext2D spatialContext, - out Mesh imageMesh, - out BoundingBox2D transformedBoundingBox, - out bool clippedToSpatialContext, - out double overlapRatio) + internal static bool TryCreateImageMesh( + GeoReferencedRasterInfo rasterInfo, SpatialContext2D spatialContext, + out Mesh imageMesh, out BoundingBox2D transformedBoundingBox, + out bool clippedToSpatialContext, out double overlapRatio) { - imageMesh = new Mesh(); - transformedBoundingBox = new BoundingBox2D(0.0, 0.0, 0.0, 0.0); - clippedToSpatialContext = false; - overlapRatio = 0.0; - - var corners = new[] - { - new Coordinate2D(rasterInfo.BoundingBox.MinX, rasterInfo.BoundingBox.MinY), - new Coordinate2D(rasterInfo.BoundingBox.MaxX, rasterInfo.BoundingBox.MinY), - new Coordinate2D(rasterInfo.BoundingBox.MaxX, rasterInfo.BoundingBox.MaxY), - new Coordinate2D(rasterInfo.BoundingBox.MinX, rasterInfo.BoundingBox.MaxY) - }; - - var transformedCorners = new Point3d[4]; - var offsetX = spatialContext.UseAbsoluteCoordinates ? 0.0 : spatialContext.PlacementOrigin.X; - var offsetY = spatialContext.UseAbsoluteCoordinates ? 0.0 : spatialContext.PlacementOrigin.Y; - - for (var index = 0; index < corners.Length; index++) - { - var sourceCorner = corners[index]; - if (!SpatialReferenceTransform.TryTransformXY( - rasterInfo.SrsName, - spatialContext.ResolvedSrs, - sourceCorner.X, - sourceCorner.Y, - out var x, - out var y)) - { - return false; - } - - transformedCorners[index] = new Point3d(x - offsetX, y - offsetY, 0.0); - } - - transformedBoundingBox = BoundingBox2DFromPoints(transformedCorners); - - var placedSpatialContextBoundingBox = RhinoSpatialContextTools.CreatePlacedBoundingBox( - spatialContext.PlacementBoundingBox, - spatialContext.PlacementOrigin, - spatialContext.UseAbsoluteCoordinates); - - var fullArea = RhinoSpatialContextTools.CalculateBoundingBoxArea(transformedBoundingBox); - if (fullArea > 0.0 && - RhinoSpatialContextTools.TryIntersectBoundingBoxes( - transformedBoundingBox, - placedSpatialContextBoundingBox, - out var intersectionBoundingBox)) - { - overlapRatio = RhinoSpatialContextTools.CalculateBoundingBoxArea(intersectionBoundingBox) / fullArea; - if (overlapRatio > 0.0 && overlapRatio < 0.999) - { - clippedToSpatialContext = true; - - var minU = (float)((intersectionBoundingBox.MinX - transformedBoundingBox.MinX) / (transformedBoundingBox.MaxX - transformedBoundingBox.MinX)); - var maxU = (float)((intersectionBoundingBox.MaxX - transformedBoundingBox.MinX) / (transformedBoundingBox.MaxX - transformedBoundingBox.MinX)); - var minV = (float)((intersectionBoundingBox.MinY - transformedBoundingBox.MinY) / (transformedBoundingBox.MaxY - transformedBoundingBox.MinY)); - var maxV = (float)((intersectionBoundingBox.MaxY - transformedBoundingBox.MinY) / (transformedBoundingBox.MaxY - transformedBoundingBox.MinY)); - - imageMesh = RhinoSpatialContextTools.CreateTexturedBoundingBoxMesh( - intersectionBoundingBox, - new Coordinate2D(0.0, 0.0), - true, - minU, - minV, - maxU, - maxV); - - return true; - } - } - - imageMesh = RhinoSpatialContextTools.CreateTexturedBoundingBoxMesh( - transformedBoundingBox, - new Coordinate2D(0.0, 0.0), - true, - 0.0f, - 0.0f, - 1.0f, - 1.0f); - return true; - } - - private static BoundingBox2D BoundingBox2DFromPoints(IEnumerable points) - { - var minX = double.PositiveInfinity; - var minY = double.PositiveInfinity; - var maxX = double.NegativeInfinity; - var maxY = double.NegativeInfinity; - - foreach (var point in points) - { - minX = Math.Min(minX, point.X); - minY = Math.Min(minY, point.Y); - maxX = Math.Max(maxX, point.X); - maxY = Math.Max(maxY, point.Y); - } - - return new BoundingBox2D(minX, minY, maxX, maxY); + var data=RasterImageMeshBuilder.Build(rasterInfo,spatialContext); + imageMesh=new Mesh(); + foreach(var p in data.Vertices) {imageMesh.Vertices.Add(p.X,p.Y,0);imageMesh.TextureCoordinates.Add((float)p.U,(float)p.V);} + foreach(var f in data.Faces) imageMesh.Faces.AddFace(f.A,f.B,f.C); + imageMesh.Normals.ComputeNormals();imageMesh.Compact(); + transformedBoundingBox=data.Vertices.Count==0 ? new BoundingBox2D(0,0,0,0) : + new BoundingBox2D(data.Vertices.Min(p=>p.X),data.Vertices.Min(p=>p.Y),data.Vertices.Max(p=>p.X),data.Vertices.Max(p=>p.Y)); + clippedToSpatialContext=data.CoverageRatio<0.999; overlapRatio=data.CoverageRatio; + return data.Faces.Count>0; } private static string BuildLargeRasterNote(GeoReferencedRasterInfo rasterInfo) @@ -502,7 +411,7 @@ private static string BuildRequestSignature(string filePath, SpatialContext2D sp return string.Create( CultureInfo.InvariantCulture, - $"{fileInfo.FullName}|{fileInfo.Length}|{fileInfo.LastWriteTimeUtc.Ticks}|" + + $"{fileInfo.FullName}|{fileInfo.Length}|{fileInfo.LastWriteTimeUtc.Ticks}|{RasterSidecars.Signature(filePath)}|" + $"{spatialContext.ResolvedSrs}|{spatialContext.UseAbsoluteCoordinates}|" + $"{FormatBoundingBox(spatialContext.PlacementBoundingBox)}|" + $"{FormatCoordinate(spatialContext.PlacementOrigin)}"); diff --git a/RhinoSpatial/Lod2LoadComponent.cs b/RhinoSpatial/Lod2LoadComponent.cs index c050887..f218e3b 100644 --- a/RhinoSpatial/Lod2LoadComponent.cs +++ b/RhinoSpatial/Lod2LoadComponent.cs @@ -241,7 +241,7 @@ private SolveResults Compute(RequestData requestData, CancellationToken cancella return ComputeLocalCityGmlSource(requestData); } - if (new[] { ".zip", ".gmz", ".cityjson", ".json", ".gml", ".citygml", ".xml" } + if (new[] { ".zip", ".gmz", ".cityjson", ".json", ".jsonl", ".gml", ".citygml", ".xml" } .Contains(Path.GetExtension(new Uri(requestData.Source).AbsolutePath), StringComparer.OrdinalIgnoreCase) && !System.Text.RegularExpressions.Regex.IsMatch(requestData.Source, @"[?&]service=wfs(?:&|$)", System.Text.RegularExpressions.RegexOptions.IgnoreCase)) { @@ -378,7 +378,7 @@ private SolveResults ComputeDownloadedBuildings(RequestData requestData, IReadOn var kind = Path.GetExtension(download.FilePath).Equals(".zip", StringComparison.OrdinalIgnoreCase) ? Lod2SourceKind.Zip : Lod2SourceKind.File; if (EnumerateLocalSourceFiles(download.FilePath, kind).Any(file => { using var stream = file.OpenRead(); return Lod2GmlReader.IsCityGmlDocument(stream); })) Add(false); - if (EnumerateLocalCityJsonSourceFiles(download.FilePath, kind).Any(file => IsCityJsonDocument(file.ReadText()))) Add(true); + if (EnumerateLocalCityJsonSourceFiles(download.FilePath, kind).Any(file => IsCityJsonSourceFile(file))) Add(true); void Add(bool json) { if (!sources.TryGetValue((json, kind), out var files)) sources[(json, kind)] = files = new List(); @@ -405,10 +405,10 @@ void Add(bool json) Status = string.Join(" ", groups.Select(g => g.Status)), MessageLevel = GH_RuntimeMessageLevel.Remark }; } - private static bool IsCityJsonDocument(string text) + private static bool IsCityJsonSourceFile(LocalCityGmlSourceFile file) { - using var document = System.Text.Json.JsonDocument.Parse(text); - return document.RootElement.TryGetProperty("type", out var type) && type.GetString() == "CityJSON"; + using var stream = file.OpenRead(); + return CityJsonReader.IsCityJson(stream); } private SolveResults ComputeLocalCityGmlSource(RequestData requestData) @@ -560,7 +560,7 @@ private SolveResults ComputeLocalCityJsonSource(RequestData requestData) { var sourceFiles = (requestData.LocalSourcePaths ?? new[] { requestData.Source }) .SelectMany(source => EnumerateLocalCityJsonSourceFiles(source, requestData.SourceKind)) - .Where(file => requestData.LocalSourcePaths is null || IsCityJsonDocument(file.ReadText())).ToList(); + .Where(file => requestData.LocalSourcePaths is null || IsCityJsonSourceFile(file)).ToList(); if (sourceFiles.Count == 0) { return new SolveResults @@ -579,8 +579,8 @@ private SolveResults ComputeLocalCityJsonSource(RequestData requestData) CityJsonSourceMetadata metadata; try { - var jsonText = sourceFile.ReadText(); - metadata = CityJsonReader.ReadSourceMetadata(jsonText); + using var sourceStream = sourceFile.OpenRead(); + metadata = CityJsonReader.ReadSourceMetadata(sourceStream); } catch when (requestData.LocalSourcePaths is null) { @@ -660,15 +660,15 @@ private SolveResults ComputeLocalCityJsonSource(RequestData requestData) { try { - var jsonText = candidate.File.ReadText(); + using var cityStream = candidate.File.OpenRead(); var displayName = string.IsNullOrWhiteSpace(requestData.LayerName) ? candidate.File.DisplayName : requestData.LayerName; - returnedBuildings.AddRange(CityJsonReader.ReadBuildings(jsonText, displayName, requestBoundingBox)); + returnedBuildings.AddRange(CityJsonReader.ReadBuildings(cityStream, displayName, requestBoundingBox)); parsedFileCount++; totalByteCount += candidate.File.ByteLength > 0 ? candidate.File.ByteLength - : Encoding.UTF8.GetByteCount(jsonText); + : 0; } catch { @@ -1160,7 +1160,8 @@ private static bool IsCityJsonFileName(string fileName) { var extension = Path.GetExtension(fileName); return extension.Equals(".json", StringComparison.OrdinalIgnoreCase) || - extension.Equals(".cityjson", StringComparison.OrdinalIgnoreCase); + extension.Equals(".cityjson", StringComparison.OrdinalIgnoreCase) || + extension.Equals(".jsonl", StringComparison.OrdinalIgnoreCase); } private static bool LocalSourceContainsCityJson(string source, Lod2SourceKind sourceKind) diff --git a/RhinoSpatial/SpatialTerrainCache.cs b/RhinoSpatial/SpatialTerrainCache.cs index 4398ddd..c451d46 100644 --- a/RhinoSpatial/SpatialTerrainCache.cs +++ b/RhinoSpatial/SpatialTerrainCache.cs @@ -22,7 +22,7 @@ public static void Store( double elevationBase) { var key = RhinoSpatialContextTools.CreateSpatialContextKey(spatialContext); - EntriesByContext[key] = new Entry(terrainSrs, boundingBox, raster, elevationBase); + EntriesByContext[key] = new Entry(raster.GeoTransform is null ? terrainSrs : raster.SrsName, raster.GeoTransform?.Bounds(raster.Width,raster.Height) ?? boundingBox, raster, elevationBase); } public static bool TrySamplePlacedElevation( @@ -82,6 +82,12 @@ private static bool TrySampleRaster(Entry entry, double x, double y, out double var gridX = (x - entry.BoundingBox.MinX) / spanX * (raster.Width - 1); var gridY = (entry.BoundingBox.MaxY - y) / spanY * (raster.Height - 1); + if (raster.GeoTransform is { } geo) + { + var pixel=geo.Pixel(x,y); gridX=pixel.X-0.5; gridY=pixel.Y-0.5; + if(gridX<0 || gridY<0 || gridX>raster.Width-1 || gridY>raster.Height-1) return false; + } + var x0 = Math.Clamp((int)Math.Floor(gridX), 0, raster.Width - 1); var y0 = Math.Clamp((int)Math.Floor(gridY), 0, raster.Height - 1); var x1 = Math.Min(x0 + 1, raster.Width - 1); diff --git a/RhinoSpatial/WmsListLayersComponent.cs b/RhinoSpatial/WmsListLayersComponent.cs index ba2b32c..970e3ca 100644 --- a/RhinoSpatial/WmsListLayersComponent.cs +++ b/RhinoSpatial/WmsListLayersComponent.cs @@ -1,5 +1,6 @@ using System; using System.Collections.Generic; +using System.Linq; using System.Threading.Tasks; using Grasshopper.Kernel; using RhinoSpatial.Core; @@ -26,7 +27,7 @@ private class RequestData public WmsListLayersComponent() : base("List WMS Layers", "WMS Layers", - "List available layers from a WMS GetCapabilities response.", + "List WMS layers or WMTS imagery layers from capabilities.", "RhinoSpatial", "Layers") { } @@ -35,7 +36,7 @@ public WmsListLayersComponent() protected override void RegisterInputParams(GH_InputParamManager pManager) { - pManager.AddTextParameter("WMS Service URL", "WMS URL", "Base URL of the WMS service.", GH_ParamAccess.item); + pManager.AddTextParameter("WMS Service URL", "WMS URL", "WMS service URL or WMTS capabilities URL.", GH_ParamAccess.item); } protected override void RegisterOutputParams(GH_OutputParamManager pManager) @@ -115,7 +116,9 @@ private bool TryGetRequestData(IGH_DataAccess dataAccess, out RequestData reques private SolveResults Compute(RequestData requestData) { - var layers = _wmsClient.LoadLayersAsync(requestData.BaseUrl).GetAwaiter().GetResult(); + var wmts=new WmtsClient().TryLoadCapabilitiesAsync(requestData.BaseUrl,CancelToken).GetAwaiter().GetResult(); + var layers=wmts is null ? _wmsClient.LoadLayersAsync(requestData.BaseUrl).GetAwaiter().GetResult() + : wmts.Layers.Select(l=>new WmsLayerInfo(l.Id,l.Title,l.Links.Where(link=>wmts.Sets.ContainsKey(link.Set)).Select(link=>wmts.Sets[link.Set].Srs).ToList(),l.Wgs84Bounds)).ToList(); return new SolveResults { diff --git a/RhinoSpatial/WmsLoadComponent.cs b/RhinoSpatial/WmsLoadComponent.cs index a0ee1e9..4c70d11 100644 --- a/RhinoSpatial/WmsLoadComponent.cs +++ b/RhinoSpatial/WmsLoadComponent.cs @@ -19,6 +19,7 @@ namespace RhinoSpatial public class WmsLoadComponent : GH_TaskCapableComponent { private readonly WmsClient _wmsClient = new(); + private readonly WmtsClient _wmtsClient = new(); private Mesh? _previewMesh; private DisplayMaterial? _previewMaterial; private string? _previewImageFilePath; @@ -51,8 +52,8 @@ public WmsLoadComponent() protected override void RegisterInputParams(GH_InputParamManager pManager) { - pManager.AddTextParameter("WMS Service URL", "WMS URL", "Base URL of the WMS service. Leave empty to use the built-in fallback map/imagery source sequence.", GH_ParamAccess.item); - pManager.AddTextParameter("Layer", "Layer", "Optional WMS layer name to request. Leave empty to let RhinoSpatial choose a usable requestable layer automatically.", GH_ParamAccess.item, string.Empty); + pManager.AddTextParameter("WMS Service URL", "WMS URL", "WMS service URL or WMTS capabilities URL. Leave empty to use the built-in fallback map/imagery source sequence.", GH_ParamAccess.item); + pManager.AddTextParameter("Layer", "Layer", "Optional WMS or WMTS layer name to request. Leave empty to let RhinoSpatial choose a usable requestable layer automatically.", GH_ParamAccess.item, string.Empty); pManager.AddTextParameter("Spatial Context", "Spatial Context", "Shared RhinoSpatial spatial context from the Spatial Context component. No WFS inputs are required for the WMS workflow.", GH_ParamAccess.item); pManager.AddTextParameter("Format", "Format", "Requested image format.", GH_ParamAccess.item, "image/png"); @@ -329,6 +330,16 @@ private SolveResults Compute(RequestData requestData) private SolveResults ComputeForSource(ResolvedImagerySource source, SpatialContext2D spatialContext, string format) { + var wmts=_wmtsClient.TryLoadCapabilitiesAsync(source.BaseUrl,CancelToken).GetAwaiter().GetResult(); + if(wmts is not null) + { + var plan=WmtsClient.Plan(wmts,source.PreferredLayerName,spatialContext,format); + var image=_wmtsClient.DownloadAsync(plan,CancelToken).GetAwaiter().GetResult(); + if(!LoadGeoTiffComponent.TryCreateImageMesh(image.Raster,spatialContext,out var mesh,out _,out _,out _)) + throw new InvalidOperationException("WMTS imagery does not overlap the selected Spatial Context."); + return new SolveResults { ImageFilePath=image.LocalFilePath, ImageMesh=mesh, + GetMapUrl=string.Join(System.Environment.NewLine,plan.Tiles.Select(t=>t.Url)),Status=image.Status,MessageLevel=GH_RuntimeMessageLevel.Remark }; + } var capabilities = _wmsClient.LoadCapabilitiesAsync(source.BaseUrl).GetAwaiter().GetResult(); var resolvedLayer = ResolveLayerName(capabilities, source.PreferredLayerName); if (string.IsNullOrWhiteSpace(resolvedLayer.LayerName)) diff --git a/THIRD-PARTY-NOTICES.md b/THIRD-PARTY-NOTICES.md index d528557..d7f204e 100644 --- a/THIRD-PARTY-NOTICES.md +++ b/THIRD-PARTY-NOTICES.md @@ -57,6 +57,22 @@ RhinoSpatial uses NetTopologySuite.IO.Esri.Shapefile to read local `.shp` vector RhinoSpatial uses SharpGLTF.Core to decode glTF/GLB tile content for the optional Google 3D Tiles reference viewer. +### StbImageNext + +- Package: `StbImageNext` +- Version: `1.0.0` +- Project URL: [StbImageNext](https://github.com/StbSharp/StbImageNext) +- Package metadata: [NuGet 1.0.0](https://www.nuget.org/packages/StbImageNext/1.0.0) +- Declared license: Unlicense OR MIT +- License texts: [Unlicense](https://licenses.nuget.org/Unlicense) or [MIT](https://licenses.nuget.org/MIT) +- Package authors: StbImageSharpTeam; based on the stb image libraries + +RhinoSpatial uses the managed .NET Standard assembly for bounded PNG/JPEG tile +decoding and local image-header metadata. It adds no native binaries or further +package dependencies. Encoded WMTS mosaics preserve their source pixel colors +and alpha; Google 3D Tiles are unaffected. The dependency is included in package +inspection for this unreleased batch. + ### ProjNET - Package: `ProjNET` diff --git a/docs/COMPATIBILITY_IMPLEMENTATION_PLAN.md b/docs/COMPATIBILITY_IMPLEMENTATION_PLAN.md index a1cfbd2..68cc040 100644 --- a/docs/COMPATIBILITY_IMPLEMENTATION_PLAN.md +++ b/docs/COMPATIBILITY_IMPLEMENTATION_PLAN.md @@ -50,5 +50,5 @@ The [research audit](COMPATIBILITY_RESEARCH.md) records the full backlog. - Live results and exact reproduction inputs are saved in [the example guide](../examples/COMPATIBILITY_WORKFLOWS.md): API-root discovery, five-page capped loading, single- and two-tile DGM1 meshes, direct GeoJSON/Shapefile ZIPs and a declared-CRS CityJSON source. - The Delft-hosted older sample failed TLS on this machine; the cjio maintainer fixture provided a successful alternative. Certificate validation was retained. - New Rhino/Grasshopper visual checks and Windows runtime checks remain on [the manual checklist](../examples/VALIDATION.md#unreleased-source-compatibility), including progress/cancellation, saved definitions and cross-layer XY/Z alignment. -- Priorities 6–10 remain backlog items. No multi-LoD selection, CityJSON sequence/template, affine/world-file, WMTS, COG or GeoPackage implementation is claimed. +- This first batch did not implement priorities 6–10. The subsequent [medium-priority plan](MEDIUM_COMPATIBILITY_PLAN.md) covers 6–8; COG and GeoPackage remain later work. - Public release 0.4.0 and the installed plugin were not changed. diff --git a/docs/COMPATIBILITY_RESEARCH.md b/docs/COMPATIBILITY_RESEARCH.md index e128e8e..16147d5 100644 --- a/docs/COMPATIBILITY_RESEARCH.md +++ b/docs/COMPATIBILITY_RESEARCH.md @@ -2,7 +2,7 @@ Reviewed RhinoSpatial 0.4.0 at `2bd239ee409fdb06f2aa49aa81c2cffbcc1d2be5` against official provider and format documentation. This is a code/documentation audit, not a live reproduction of every provider failure. Priorities are recommendations; no runtime behavior was changed. -Implementation of priorities 1–5 is tracked in the [implementation plan](COMPATIBILITY_IMPLEMENTATION_PLAN.md) and [example guide](../examples/COMPATIBILITY_WORKFLOWS.md). The evidence below describes the audited 0.4.0 baseline, before those changes. +Implementation of priorities 1–5 is tracked in the [implementation plan](COMPATIBILITY_IMPLEMENTATION_PLAN.md) and [example guide](../examples/COMPATIBILITY_WORKFLOWS.md). Priorities 6–8 are implemented in the [medium-priority follow-up](MEDIUM_COMPATIBILITY_PLAN.md), with [saved examples](../examples/MEDIUM_COMPATIBILITY_WORKFLOWS.md). Priorities 9–10 remain later work. The evidence below describes the audited 0.4.0 baseline, before those changes. ## Highest-value improvements diff --git a/docs/COMPONENT_REFERENCE.md b/docs/COMPONENT_REFERENCE.md index 6cc03c2..312fab3 100644 --- a/docs/COMPONENT_REFERENCE.md +++ b/docs/COMPONENT_REFERENCE.md @@ -65,8 +65,8 @@ The component returns one `Spatial Context` output for all aligned source and vi Supported reference metadata includes: - WFS, WMS, and WCS service URLs -- OGC API Features landing/collection metadata and ATOM feed coverage (Unreleased) -- GeoTIFF +- OGC API Features landing/collection metadata, ATOM feed coverage and WMTS capabilities (Unreleased) +- GeoTIFF and supported world-file TIFF/PNG/JPEG imagery - Shapefile - GeoJSON - CityJSON @@ -105,7 +105,7 @@ Public interface: ### List WMS Layers -Connect a WMS service URL to inspect its requestable map layers. Use `List Item` to choose the layer needed by `Load WMS`. +Connect a WMS service URL or WMTS capabilities URL to inspect its imagery layers. Use `List Item` to choose the layer needed by `Load WMS`. Public interface: @@ -173,7 +173,9 @@ Use `Load WMS` for orthophotos, aerial imagery, map layers, and raster overlays 3. Define the area with `Spatial Context`. 4. Connect the WMS URL, layer, format, and context to `Load WMS`. -One component requests one WMS layer. Use several components when multiple image layers should remain explicit. +WMTS is also accepted through these existing components. Use a WMTS capabilities URL with `List WMS Layers` and `Load WMS`; the selected layer uses advertised matrix sets, styles, dimensions and PNG/JPEG formats. `GetMap URL` returns the tile request URLs in WMTS mode. Runtime status reports the selected matrix, resolution, native CRS and tile count. + +One component requests one WMS/WMTS layer. Use several components when multiple image layers should remain explicit. Public inputs: @@ -195,11 +197,13 @@ When a layer does not support the context's primary SRS but advertises another c WMS 1.3 geographic requests use the authority axis order required by the standard. +WMTS supports KVP GetTile and advertised REST templates. It chooses a resolution near a 1024-pixel study view within 64 tiles and 4,194,304 mosaic pixels (8192 per axis). Limits are 4 MiB per tile, 64 MiB per batch and two minutes per mosaic. Matrix limits and geographic axis order are respected; unsupported CRS, ambiguous dimensions and failed required tiles return explicit errors. Dimension defaults are used; an explicit source-URL query value can select a dimension without a new component parameter. Completed PNG mosaics persist under `RhinoSpatial/WmtsCache`, expire after one day, and are checked against SHA-256 before reuse. Delete the cache while idle to reclaim space. No partial mosaic is published after a required tile failure. + ## Load GeoTIFF: local georeferenced rasters Use `Load GeoTIFF` for a local georeferenced image such as an orthophoto, map export, satellite image, or other raster that should align with the selected study area. -Connect a `.tif` or `.tiff` path and the shared `Spatial Context`. +Connect a `.tif`/`.tiff`, `.png`, `.jpg` or `.jpeg` path and the shared `Spatial Context`. PNG/JPEG and ordinary TIFF need a matching world file (`.pgw`, `.jgw`, `.tfw`, long-extension variants or `.wld`) and a resolvable `.prj` unless usable geospatial metadata is embedded. Public inputs: @@ -215,6 +219,8 @@ Outputs include: The file must contain readable geospatial metadata. If RhinoSpatial cannot determine the coordinate system or the raster does not overlap the selected context, it returns a clear warning rather than guessing a placement. +Embedded TIFF georeferencing takes priority. The reader supports finite invertible affine transforms, including rotation, shear and reflection. World-file coefficients refer to pixel centers; GeoTIFF PixelIsArea/PixelIsPoint conventions are honored. Reference extents use all four footprint corners. A bounded mesh grid projects the actual footprint through the shared context, clips it with interpolated UVs, and localizes it without stretching the image over its bounding rectangle. Sidecar changes invalidate cached metadata and component results. A world file without a resolvable CRS fails explicitly. + ## Load Terrain: elevation and ground surfaces Use `Load Terrain` to create ground geometry aligned with the same study area as imagery, vectors, and buildings. @@ -238,6 +244,8 @@ Outputs are the aligned `Terrain` mesh list and a `Status` message. `Coverage` identifies a WCS coverage where required. For local text grids it can also provide an EPSG override such as `EPSG:25832`; otherwise RhinoSpatial uses the current context SRS. +GeoTIFF terrain vertices use the same affine pixel centers as imagery, and cached elevation sampling inverts that transform. Rotated local DEMs are supported; terrain ATOM mosaics still require compatible north-up grids and reject other grids explicitly. + Terrain coordinates are transformed from the source SRS into the shared context before localization, including when the source uses a different projected CRS. Unsupported transformations and non-overlapping WCS coverage return explicit errors. Missing elevation samples leave holes in the surface, including during downsampling; RhinoSpatial does not replace them with zero-height terrain. A raster without connected usable cells returns an error. Esri ASCII Grid samples are placed at cell centers, while reference metadata describes the full cell footprint. Matching `xllcorner`/`yllcorner` and `xllcenter`/`yllcenter` headers are supported. `NODATA_value` is optional and defaults to `-9999`. Cell values may wrap across text lines; their total must match `ncols × nrows`. Invalid headers, truncated data, and extra cell values are rejected. These conventions follow the [Esri ASCII raster format](https://help.arcgis.com/en/arcgisdesktop/10.0/help/009t/009t0000000z000000.htm). @@ -282,7 +290,7 @@ Tile selection supports GeoRSS Simple boxes/polygons and GeoRSS GML envelopes, i Some feeds, including Berlin's, publish a single dataset extent and many download links. These require additional tile information. The supported grid convention is a declared square grid size, an EPSG:25832 or EPSG:25833 category, an enclosing geographic extent, and filenames ending in `_EEE_NNNN` with southwest UTM coordinates in kilometres. Berlin buildings explicitly link a `1x1 Km` tile index; terrain uses a declared `2x2 Km` index. Coarse enclosing coverage allows a one-declared-tile margin for edge/buffer tiles; distant contradictory grids are rejected. This rule depends on metadata and the supported naming convention, not the service hostname. Arbitrary numeric filenames or whole-dataset bounds are never sufficient on their own. A feed with missing, ambiguous or unsupported spatial metadata returns a limitation before downloading building archives. Start from the service feed so its enclosing extent remains available. -ATOM and direct building download URLs support CityGML (`.gml`, `.citygml`, `.xml`) and CityJSON (`.json`, `.cityjson`), including ZIP/GMZ containers. Downloaded CityJSON must declare `metadata.referenceSystem`; sequences and geometry templates are not added by this batch. Local CityJSON/file/folder/ZIP workflows remain supported. Downloaded CityGML must declare a source CRS that can transform into the context; no guessed CRS or approximate bounding-box placement is used. Standard AdV names such as `urn:adv:crs:ETRS89_UTM33*DE_DHHN2016_NH` resolve to UTM XY; published heights are retained without a vertical-datum conversion. +ATOM and direct building download URLs support CityGML (`.gml`, `.citygml`, `.xml`) and CityJSON (`.json`, `.cityjson`), including ZIP/GMZ containers. Downloaded CityJSON/CityJSONSeq must declare the header `metadata.referenceSystem`. Standard CityJSONSeq (`.city.jsonl` / `.jsonl`) and affine row-major GeometryInstance templates are supported. Local CityJSON/file/folder/ZIP workflows remain supported. Downloaded CityGML must declare a source CRS that can transform into the context; no guessed CRS or approximate bounding-box placement is used. Standard AdV names such as `urn:adv:crs:ETRS89_UTM33*DE_DHHN2016_NH` resolve to UTM XY; published heights are retained without a vertical-datum conversion. Use a building dataset, not a terrain feed: Berlin's `/data/a_lod2/atom/` supplies LoD2 buildings, while `/data/dgm1/atom/` supplies CSV terrain grids. Explicit CSV/raster download MIME types are rejected before grid selection with a source-format explanation. Use the DGM1 feed directly with `Load Terrain`; its terrain-specific parser and mosaic path are separate from the building reader. @@ -292,6 +300,12 @@ The component displays tile/download progress while working. `Status` reports se Limits are 64 feeds, 8 subfeed levels, 50,000 download entries and 32 selected tiles per solution; 512 MiB compressed per archive, 256 MiB per expanded file and 1 GiB total expanded per archive. Feed requests time out after 30 seconds, discovery after two minutes, individual downloads after three minutes and the download batch after five minutes. Reduce the study area or use a more specific spatially indexed feed if a limit is reached. ATOM feed coverage and native CRS are supported by `Spatial Context → Reference Source`. Use a service feed containing coverage; a dataset feed lacking coverage reports that limitation. Reference processing reads feed metadata without downloading archives. Saved map selections retain priority. +### CityJSON detail, templates and sequences + +For each CityObject, import all geometry parts at one selected numeric LoD: the highest at or below 2.2, or the lowest higher value if none is available below that threshold. Geometry without any LoD retains legacy behavior; mixing declared and unspecified levels fails because a complete representation cannot be chosen safely. Hierarchy-wide LoD selection and material/texture import are not added. + +CityJSONSeq records are streamed with independent vertex indices and shared header transform, CRS and templates. GeometryInstance vertices use the declared matrix followed by the transformed reference-point anchor; template vertices are not quantized a second time. Invalid indices, duplicate building ids, malformed later records and unsupported selected primitives fail explicitly. Limits are 64 × 1024² characters per record, 512 × 1024² stream characters, 100,000 returned building objects and four million selected source surface points. Local/ZIP/direct downloads use the same reader. Reference metadata can stop at a declared header extent without constructing geometry. See the [medium-priority examples](../examples/MEDIUM_COMPATIBILITY_WORKFLOWS.md). + ### Filtering and performance WFS mode requests a small buffered area to reduce provider BBOX edge misses, then keeps buildings that intersect the actual Spatial Context. diff --git a/docs/MEDIUM_COMPATIBILITY_PLAN.md b/docs/MEDIUM_COMPATIBILITY_PLAN.md new file mode 100644 index 0000000..54f24f9 --- /dev/null +++ b/docs/MEDIUM_COMPATIBILITY_PLAN.md @@ -0,0 +1,36 @@ +# Medium-priority compatibility plan + +Started 2026-11-01 from `8aff1e8` on `codex/medium-source-compatibility`. +This batch builds on draft PR #6; its PR will target that branch until #6 merges. +Scope is research priorities 6–8. COG and GeoPackage remain later work. + +1. **CityJSON:** select one numeric LoD per object (prefer the highest supplied LoD at or below 2.2, otherwise the lowest higher LoD), retain all geometry parts at that LoD, resolve template instances with the specified matrix/anchor, and stream CityJSONSeq records using shared header transform/CRS/templates. Support local, ZIP and validated direct downloads without new component parameters. Fail explicitly on malformed indices/transforms or unsupported primitives instead of producing partial rings. +2. **Raster georeferencing:** share a finite, invertible affine transform for embedded TIFF and world-file imagery. Respect area/point raster conventions, prefer embedded spatial metadata, require a resolvable CRS for sidecars and include sidecars in cache invalidation. Preserve the true raster footprint/UV mapping during projection, clipping and localization. Use the same transform for terrain sample centers; reject unsupported terrain mosaic grids explicitly. +3. **WMTS:** integrate capabilities/layer discovery, advertised matrix sets, bounded tile selection and mosaic loading with existing WMS components and reference metadata. Respect CRS axis order, matrix limits, styles, formats and default dimensions. Support KVP and advertised REST tile templates. Keep unknown CRS or failed required tiles explicit; do not add a provider catalogue or affect Google tiles. + +## Validation and examples + +- Deterministic checks for mixed LoDs, templates, sequences, invalid geometry, affine rotation/shear, pixel-center placement, missing CRS, cache invalidation, WMTS projected matrices, limits, request URLs, cancellation and failures. +- Small live provider/file probes where available; save URLs, layer/matrix choices, CRS/bounds and observed outcomes in an example guide and source manifest. Keep fixtures and live evidence distinct. +- Release build, all existing suites plus the medium-priority suite, JSON/interface/diff checks, and GitHub validation. +- Update reference/catalogue/roadmap/changelog and manual Rhino/Windows checklist. Preserve component GUIDs and parameter layouts, shared context/localization/elevation and metadata-only reference intent. +- Commit, push and open a separate draft PR. No installation, merge, version bump or public publication is part of this batch. + +## Progress + +- [x] Scope, upstream dependency and implementation plan recorded. +- [x] CityJSON LoD selection, templates, sequences and source routing. +- [x] Affine/world-file imagery and terrain placement. +- [x] WMTS discovery, bounded imagery and reference integration. +- [x] Examples, documentation and validation. + +## Recorded results — 2026-11-02 + +- Implemented all three items behind existing components. Component GUIDs and input/output parameter names/order match the previous batch. +- All eight local suites passed, including 27 medium-priority checks. Release solution build passed with zero warnings/errors. Source manifest JSON, unique ids, fixture paths and diff checks passed. +- [Saved recipes](../examples/MEDIUM_COMPATIBILITY_WORKFLOWS.md) include the verified PDOK WMTS URL/layer/native bbox, a public multi-LoD parser fixture, and six bundled declared-CRS synthetic files. Model/sequence/template each parsed one building/six surfaces; the rotated image produced a Core mesh. +- Live PDOK: 19 layers, six supported sets, 36 JPEG tiles at 0.42 m/pixel, 1536 × 1536 mosaic, final clipped Core mesh 512 triangles. Cache reuse and mosaic continuity verified. Public cjio multi-LoD: ten buildings/348 surfaces at LoD 2.2; missing CRS is explicitly labeled parser-only. Existing declared-CRS Delft sample still parsed 160 buildings/5,563 surfaces at LoD1. +- Final raster review added source-window projection for large images, with a global-geographic-to-projected-context regression; UV coverage remains relative to the original image. +- Dependency: managed `StbImageNext` 1.0.0, no native/further package dependencies. Notices updated. An explicitly unreleased temporary package preview contained 17 files / 3,156,137 compressed bytes; the 62,976-byte decoder and dependency manifest were present, stale decoder and AI files absent. Public release staging was preserved. +- New interactive Rhino material/UV/bake/saved-definition and Windows checks remain in [the manual checklist](../examples/VALIDATION.md#unreleased-medium-priority-compatibility). Headless/Core probes are not proof of those native surfaces. +- COG and GeoPackage remain later work. No merge, version bump, public release or installed-plugin replacement occurred. diff --git a/docs/ROADMAP.md b/docs/ROADMAP.md index 22f0862..101d208 100644 --- a/docs/ROADMAP.md +++ b/docs/ROADMAP.md @@ -51,7 +51,7 @@ collection discovery, direct vector/building downloads, terrain ATOM and referen metadata. [The implementation plan](COMPATIBILITY_IMPLEMENTATION_PLAN.md) records validation and remaining work; [example recipes](../examples/COMPATIBILITY_WORKFLOWS.md) save tested URLs and bounds. Interactive Rhino/Windows checks remain before release. -Priorities 6–10 remain follow-on work. +The [medium-priority follow-up](MEDIUM_COMPATIBILITY_PLAN.md) implements priorities 6–8: CityJSON detail/sequence/templates, affine/world-file rasters and WMTS. Its [saved examples](../examples/MEDIUM_COMPATIBILITY_WORKFLOWS.md) include synthetic declared-CRS data and live PDOK imagery. Native Rhino/Windows validation remains; priorities 9–10 (remote COG and GeoPackage) remain later work. ## Now: real-use stabilization diff --git a/docs/TEST_SOURCES.md b/docs/TEST_SOURCES.md index d4bf2eb..34f1ce5 100644 --- a/docs/TEST_SOURCES.md +++ b/docs/TEST_SOURCES.md @@ -66,6 +66,12 @@ Core-placement checks; new interactive Rhino/Grasshopper and Windows validation remains pending. The older Delft-hosted CityJSON URL failed TLS in this batch; historical local-file results below remain historical results. +## Unreleased medium-priority validation (2026-11-02) + +[The plan](MEDIUM_COMPATIBILITY_PLAN.md) and [example guide](../examples/MEDIUM_COMPATIBILITY_WORKFLOWS.md) record priorities 6–8 and exact inputs. `medium-compatibility-check` tests LoD selection, sequence headers/records/cancellation, instance matrices, invalid geometry, world files/CRS/cache invalidation, embedded affine/PixelIsPoint GeoTIFFs, image UV clipping/reprojection, terrain centers, projected/geographic WMTS, matrix limits, tile ordering/alpha, corruption, failures and metadata-only references. The suite uses temporary fixtures and no network. + +Live PDOK capabilities exposed 19 layers/six supported sets; bounded native EPSG:28992 imagery loaded 36 JPEG tiles at 0.42 m/pixel, yielding a 1536 × 1536 PNG and a 512-triangle clipped Core mesh. The mosaic was visually inspected for continuity, not in Rhino. The cjio multi-LoD fixture parsed ten buildings/348 selected surfaces at LoD 2.2; it lacks CRS metadata, so it is parser-only evidence. Bundled declared-CRS synthetic models/sequence/template each parsed one building/six surfaces, and the rotated PNG generated 512 Core triangles. New Rhino/Windows checks remain pending. + ## Deterministic correctness regressions The source-reader checks generate temporary, non-private fixtures for ASCII corner/center origins, reference extents, optional NoData, wrapped/truncated/excess cell values, invalid headers, sparse and irregular XYZ grids, non-finite positions, GeoJSON null attributes, empty geometries, malformed points, and versioned EPSG names. They cover format behavior documented by [Esri](https://help.arcgis.com/en/arcgisdesktop/10.0/help/009t/009t0000000z000000.htm) and [GeoJSON RFC 7946](https://www.rfc-editor.org/rfc/rfc7946.html). diff --git a/examples/MEDIUM_COMPATIBILITY_WORKFLOWS.md b/examples/MEDIUM_COMPATIBILITY_WORKFLOWS.md new file mode 100644 index 0000000..9100335 --- /dev/null +++ b/examples/MEDIUM_COMPATIBILITY_WORKFLOWS.md @@ -0,0 +1,111 @@ +# Medium-priority compatibility examples + +These workflows are **Unreleased** and require the medium-source-compatibility +build, which also contains the previous compatibility batch. Public/installed +0.4.0 is unchanged. Checks below were recorded on 2026-11-02. + +Keep one `Spatial Context` for a study. Existing definitions `02-geotiff-terrain.gh` +and `03-lod2-buildings.gh` can be reused by replacing source inputs. Component +names, GUIDs and parameter layouts are retained; these recipes do not add new +binary Grasshopper definitions. + +## Small local examples with declared coordinates + +The [fixtures/medium](fixtures/medium) folder contains synthetic test data, not +survey data or real aerial imagery: + +| File | Connect to | Expected result | +| --- | --- | --- | +| [building-lods.city.json](fixtures/medium/building-lods.city.json) | `Load LoD2 Buildings → LoD2 Source` | One building, six surfaces, selected LoD 2.2; the alternate LoD1 shell is omitted | +| [building-sequence.city.jsonl](fixtures/medium/building-sequence.city.jsonl) | Same input | Same six surfaces, using the sequence header's transform and CRS | +| [building-template.city.json](fixtures/medium/building-template.city.json) | Same input | One six-surface building from a unit-cube template, scaled and translated from its native anchor | +| [rotated-raster.png](fixtures/medium/rotated-raster.png) | `Load GeoTIFF → GeoTIFF` | A colored raster with its true rotated/sheared footprint and UV mapping | + +Keep `rotated-raster.pgw` and `rotated-raster.prj` beside the PNG. All fixtures use +EPSG:25833 near Berlin; a shared study rectangle +`391296,5820096,391314,5820110` covers them. Use a fixture path as +`Spatial Context → Reference Source` to orient a fresh map, then select the area. +Leave the building `Layer` input empty. Saved map selections still take priority. +The synthetic building base is 32 m; its source Z values remain unchanged before +the normal shared-baseline placement. + +The PNG is 4 × 4 pixels. Its world-file center for pixel (0,0) is +`391300,5820106`; its column vector is `(2,0.5)` and row vector `(1,-2)`. +The image is clipped to the study area after projection, preserving texture +coordinates. A world file alone cannot identify a CRS: removing the `.prj` +must produce an explicit error, rather than assuming the context's CRS. + +Recreate and inspect these files from the repository root: + +```sh +dotnet run --project RhinoSpatial.Sandbox.csproj -c Release -- medium-probe examples examples/fixtures/medium +dotnet run --project RhinoSpatial.Sandbox.csproj -c Release --no-build -- medium-probe cityjson examples/fixtures/medium/building-lods.city.json +dotnet run --project RhinoSpatial.Sandbox.csproj -c Release --no-build -- medium-probe cityjson examples/fixtures/medium/building-sequence.city.jsonl +dotnet run --project RhinoSpatial.Sandbox.csproj -c Release --no-build -- medium-probe cityjson examples/fixtures/medium/building-template.city.json +dotnet run --project RhinoSpatial.Sandbox.csproj -c Release --no-build -- medium-probe raster examples/fixtures/medium/rotated-raster.png +``` + +## Public CityJSON multi-LoD parser example + +```text +https://raw.githubusercontent.com/cityjson/cjio/master/tests/data/multi_lod.json +``` + +The maintainer fixture parsed as 10 buildings and 348 selected surfaces, all at +LoD 2.2. It has **no declared CRS**, so this is a parser example. It does not prove +alignment, and the direct-download validator intentionally rejects it until +coordinate metadata is supplied. Use the bundled declared-CRS fixtures above +for an end-to-end loader example. The public Delft fixture from the previous +guide remains a declared-CRS download example. + +Selection is per CityObject: choose the highest numeric LoD at or below 2.2, or +the lowest higher LoD if none is available below that threshold. Retain all +geometry parts at the selected level. This does not add hierarchy-wide LoD +selection, material/texture import or a public LoD selector. + +The [official CityJSONSeq page](https://www.cityjson.org/cityjsonseq/) links real +sequence samples. Its Delft data host failed TLS from this machine during this +batch; those files are not advertised here as verified downloads. The bundled +sequence exercises shared transform/CRS and independent record vertex indices. + +## Live WMTS imagery from PDOK + +Connect this capabilities URL to `List WMS Layers`, select `2025_ortho25`, and +connect the same URL/layer to `Load WMS`: + +```text +https://service.pdok.nl/hwh/luchtfotorgb/wmts/v1_0?SERVICE=WMTS&REQUEST=GetCapabilities +``` + +The existing WMS component names remain for saved-definition compatibility. +WMTS metadata can also be used as `Reference Source`, with the layer id as +`Reference Layer`; no tiles are requested for this reference step. + +The bounded live check used EPSG:28992 bounds +`84600,447400,85200,447900` near Delft. It discovered 19 layers and six supported +matrix sets, selected native matrix set `EPSG:28992`, matrix `13`, resolution +0.42 m/pixel, and loaded 36 JPEG tiles into a 1536 × 1536 PNG mosaic. The clipped +Core mesh had 512 triangles. The saved mosaic was visually inspected for tile +continuity; this is not a completed Rhino material/alignment test. + +For the map helper, use EPSG:3857, orient near Delft and draw a small area. The +provider advertises compatible matrix sets; grids are read from capabilities. +The precise native check can be reproduced with: + +```sh +dotnet run --project RhinoSpatial.Sandbox.csproj -c Release --no-build -- medium-probe wmts 'https://service.pdok.nl/hwh/luchtfotorgb/wmts/v1_0?SERVICE=WMTS&REQUEST=GetCapabilities' 2025_ortho25 EPSG:28992 84600,447400,85200,447900 +``` + +Read the component's runtime status for matrix/resolution/tile counts. The +existing `GetMap URL` output contains the WMTS tile request URLs in this mode. +Default dimensions are used; a provider with an ambiguous dimension needs an +explicit value in the source URL query. Failed required tiles return an error +and do not publish a partial mosaic. + +## Validation limits + +The `medium-compatibility-check` suite uses deterministic fixtures, including +embedded affine GeoTIFFs, PixelIsPoint, projected/geographic WMTS, matrix limits, +tile order, cache corruption, alpha and cancellation. It makes no external +requests. Rhino preview, texture orientation, baking, saved definitions and +Windows runtime validation remain on [the manual checklist](VALIDATION.md). diff --git a/examples/README.md b/examples/README.md index 71ad0d9..9362139 100644 --- a/examples/README.md +++ b/examples/README.md @@ -61,6 +61,8 @@ collection choices, bounds and observed results for API-root discovery, paging, Berlin DGM1 ATOM terrain, direct GeoJSON/Shapefile ZIPs and CityJSON downloads. These require the development build; the public 0.4.0 package is unchanged. +The [medium-priority examples](MEDIUM_COMPATIBILITY_WORKFLOWS.md) add declared-CRS CityJSON/sequence/template files, a rotated world-file raster, and a verified public WMTS URL with native test bounds. These also require the development build. + ## Source and testing references - [Curated example source list](sources.json) diff --git a/examples/VALIDATION.md b/examples/VALIDATION.md index f39f9d6..e62ba1c 100644 --- a/examples/VALIDATION.md +++ b/examples/VALIDATION.md @@ -161,3 +161,17 @@ version/platform and actual results. These checks are pending for this batch: - [ ] ATOM/API references orient a fresh map without geometry downloads, while saved selections retain priority. - [ ] Direct GeoJSON, Shapefile ZIP and CityJSON URLs load; invalid CRS/companions return clear messages. - [ ] Repeat the relevant checks on Windows before a public release. + + +## Unreleased medium-priority compatibility + +Use [MEDIUM_COMPATIBILITY_WORKFLOWS.md](MEDIUM_COMPATIBILITY_WORKFLOWS.md). These native checks remain pending: + +- [ ] Open existing definitions with unchanged GUIDs and parameter layouts on macOS and Windows. +- [ ] Load each bundled CityJSON/sequence/template example: one six-surface building at the expected coordinates, with no alternate-LoD overlay. +- [ ] Load the rotated PNG with both sidecars, compare its footprint to matching vectors, clip the study area, and verify UV orientation, alpha, preview and baking. +- [ ] Edit a world file while keeping the raster unchanged; recompute and confirm both placement and reference extent update. +- [ ] Verify embedded rotated/PixelIsPoint DEM alignment and cached OSM/building height sampling in localized and absolute modes. +- [ ] List and load PDOK WMTS through existing WMS components, check the selected layer/matrix diagnostics and compare imagery to buildings/vectors in one context. +- [ ] Confirm WMTS Reference Source reads capabilities without tile downloads and saved selections retain priority. +- [ ] Cancel a tile batch and retry; confirm no partial image is presented and cached results remain usable. diff --git a/examples/fixtures/medium/building-lods.city.json b/examples/fixtures/medium/building-lods.city.json new file mode 100644 index 0000000..690b716 --- /dev/null +++ b/examples/fixtures/medium/building-lods.city.json @@ -0,0 +1,228 @@ +{ + "type": "CityJSON", + "version": "2.0", + "metadata": { + "referenceSystem": "https://www.opengis.net/def/crs/EPSG/0/25833", + "geographicalExtent": [ + 391300, + 5820100, + 32, + 391308, + 5820106, + 46 + ] + }, + "transform": { + "scale": [ + 1, + 1, + 1 + ], + "translate": [ + 391300, + 5820100, + 32 + ] + }, + "CityObjects": { + "example-building": { + "type": "Building", + "geometry": [ + { + "type": "Solid", + "lod": "1", + "boundaries": [ + [ + [ + [ + 0, + 3, + 2, + 1 + ] + ], + [ + [ + 0, + 1, + 5, + 4 + ] + ], + [ + [ + 1, + 2, + 6, + 5 + ] + ], + [ + [ + 2, + 3, + 7, + 6 + ] + ], + [ + [ + 3, + 0, + 4, + 7 + ] + ], + [ + [ + 4, + 5, + 6, + 7 + ] + ] + ] + ] + }, + { + "type": "Solid", + "lod": "2.2", + "boundaries": [ + [ + [ + [ + 8, + 11, + 10, + 9 + ] + ], + [ + [ + 8, + 9, + 13, + 12 + ] + ], + [ + [ + 9, + 10, + 14, + 13 + ] + ], + [ + [ + 10, + 11, + 15, + 14 + ] + ], + [ + [ + 11, + 8, + 12, + 15 + ] + ], + [ + [ + 12, + 13, + 14, + 15 + ] + ] + ] + ] + } + ] + } + }, + "vertices": [ + [ + 0, + 0, + 0 + ], + [ + 8, + 0, + 0 + ], + [ + 8, + 6, + 0 + ], + [ + 0, + 6, + 0 + ], + [ + 0, + 0, + 10 + ], + [ + 8, + 0, + 10 + ], + [ + 8, + 6, + 10 + ], + [ + 0, + 6, + 10 + ], + [ + 0, + 0, + 0 + ], + [ + 8, + 0, + 0 + ], + [ + 8, + 6, + 0 + ], + [ + 0, + 6, + 0 + ], + [ + 0, + 0, + 12 + ], + [ + 8, + 0, + 12 + ], + [ + 8, + 6, + 14 + ], + [ + 0, + 6, + 14 + ] + ] +} diff --git a/examples/fixtures/medium/building-sequence.city.jsonl b/examples/fixtures/medium/building-sequence.city.jsonl new file mode 100644 index 0000000..a0e6117 --- /dev/null +++ b/examples/fixtures/medium/building-sequence.city.jsonl @@ -0,0 +1,2 @@ +{"type":"CityJSON","version":"2.0","metadata":{"referenceSystem":"https://www.opengis.net/def/crs/EPSG/0/25833","geographicalExtent":[391300,5820100,32,391308,5820106,46]},"transform":{"scale":[1,1,1],"translate":[391300,5820100,32]},"CityObjects":{},"vertices":[]} +{"type":"CityJSONFeature","id":"example-building","CityObjects":{"example-building":{"type":"Building","geometry":[{"type":"Solid","lod":"1","boundaries":[[[[0,3,2,1]],[[0,1,5,4]],[[1,2,6,5]],[[2,3,7,6]],[[3,0,4,7]],[[4,5,6,7]]]]},{"type":"Solid","lod":"2.2","boundaries":[[[[8,11,10,9]],[[8,9,13,12]],[[9,10,14,13]],[[10,11,15,14]],[[11,8,12,15]],[[12,13,14,15]]]]}]}},"vertices":[[0,0,0],[8,0,0],[8,6,0],[0,6,0],[0,0,10],[8,0,10],[8,6,10],[0,6,10],[0,0,0],[8,0,0],[8,6,0],[0,6,0],[0,0,12],[8,0,12],[8,6,14],[0,6,14]]} diff --git a/examples/fixtures/medium/building-template.city.json b/examples/fixtures/medium/building-template.city.json new file mode 100644 index 0000000..273e3e7 --- /dev/null +++ b/examples/fixtures/medium/building-template.city.json @@ -0,0 +1,156 @@ +{ + "type": "CityJSON", + "version": "2.0", + "metadata": { + "referenceSystem": "https://www.opengis.net/def/crs/EPSG/0/25833", + "geographicalExtent": [ + 391300, + 5820100, + 32, + 391308, + 5820106, + 46 + ] + }, + "CityObjects": { + "template-building": { + "type": "Building", + "geometry": [ + { + "type": "GeometryInstance", + "template": 0, + "boundaries": [ + 0 + ], + "transformationMatrix": [ + 8, + 0, + 0, + 0, + 0, + 6, + 0, + 0, + 0, + 0, + 12, + 0, + 0, + 0, + 0, + 1 + ] + } + ] + } + }, + "vertices": [ + [ + 391300, + 5820100, + 32 + ] + ], + "geometry-templates": { + "templates": [ + { + "type": "Solid", + "lod": "2.2", + "boundaries": [ + [ + [ + [ + 0, + 3, + 2, + 1 + ] + ], + [ + [ + 0, + 1, + 5, + 4 + ] + ], + [ + [ + 1, + 2, + 6, + 5 + ] + ], + [ + [ + 2, + 3, + 7, + 6 + ] + ], + [ + [ + 3, + 0, + 4, + 7 + ] + ], + [ + [ + 4, + 5, + 6, + 7 + ] + ] + ] + ] + } + ], + "vertices-templates": [ + [ + 0, + 0, + 0 + ], + [ + 1, + 0, + 0 + ], + [ + 1, + 1, + 0 + ], + [ + 0, + 1, + 0 + ], + [ + 0, + 0, + 1 + ], + [ + 1, + 0, + 1 + ], + [ + 1, + 1, + 1 + ], + [ + 0, + 1, + 1 + ] + ] + } +} diff --git a/examples/fixtures/medium/rotated-raster.pgw b/examples/fixtures/medium/rotated-raster.pgw new file mode 100644 index 0000000..a25b947 --- /dev/null +++ b/examples/fixtures/medium/rotated-raster.pgw @@ -0,0 +1,6 @@ +2 +0.5 +1 +-2 +391300 +5820106 diff --git a/examples/fixtures/medium/rotated-raster.png b/examples/fixtures/medium/rotated-raster.png new file mode 100644 index 0000000000000000000000000000000000000000..b64d69eca9469b03551cf33df3b13350546aacf1 GIT binary patch literal 99 zcmeAS@N?(olHy`uVBq!ia0vp^EFjFm1|(O0oL2{=v^-rLLn;`VdLAAY>;12=X5-Uy x^#aF~7A*Yuzq##LJ#Vjd!Wo7~4F*084DIRa8!seqHUo7rc)I$ztaD0e0su(fAG`nn literal 0 HcmV?d00001 diff --git a/examples/fixtures/medium/rotated-raster.prj b/examples/fixtures/medium/rotated-raster.prj new file mode 100644 index 0000000..24ff765 --- /dev/null +++ b/examples/fixtures/medium/rotated-raster.prj @@ -0,0 +1 @@ +PROJCS["ETRS89 / UTM zone 33N",GEOGCS["ETRS89",DATUM["European_Terrestrial_Reference_System_1989",SPHEROID["GRS 1980",6378137,298.257222101]],PRIMEM["Greenwich",0],UNIT["degree",0.0174532925199433]],PROJECTION["Transverse_Mercator"],PARAMETER["latitude_of_origin",0],PARAMETER["central_meridian",15],PARAMETER["scale_factor",0.9996],PARAMETER["false_easting",500000],PARAMETER["false_northing",0],UNIT["metre",1],AUTHORITY["EPSG","25833"]] diff --git a/examples/sources.json b/examples/sources.json index 250421b..9ab49e3 100644 --- a/examples/sources.json +++ b/examples/sources.json @@ -975,6 +975,100 @@ 447750.636 ] } + }, + { + "id": "wmts-pdok-ortho25", + "type": "WMTS", + "purpose": "Projected native WMTS selection, JPEG decoding, mosaic and shared image placement", + "url": "https://service.pdok.nl/hwh/luchtfotorgb/wmts/v1_0?SERVICE=WMTS&REQUEST=GetCapabilities", + "layer": "2025_ortho25", + "matrix_set": "EPSG:28992", + "matrix": "13", + "context": { + "srs": "EPSG:28992", + "bbox": [ + 84600, + 447400, + 85200, + 447900 + ] + }, + "notes": [ + "Live: 19 layers and six supported matrix sets; 36 JPEG tiles at 0.42 m/pixel, 1536 x 1536 PNG mosaic, clipped Core mesh 512 triangles.", + "Use List WMS Layers / Load WMS with the same capabilities URL. Map-helper users can select Delft in EPSG:3857.", + "Mosaic continuity inspected; Rhino textures/alignment and Windows runtime pending." + ], + "last_verified": "2026-11-02", + "availability": "Unreleased medium-source-compatibility branch; not the public 0.4.0 package", + "example_guide": "MEDIUM_COMPATIBILITY_WORKFLOWS.md" + }, + { + "id": "cityjson-cjio-multi-lod", + "type": "CityJSON parser fixture", + "purpose": "Public multi-LoD selection reference", + "url": "https://raw.githubusercontent.com/cityjson/cjio/master/tests/data/multi_lod.json", + "notes": [ + "Parsed 10 buildings and 348 selected surfaces at LoD 2.2.", + "No declared CRS: parser-only evidence, not verified placement; direct downloader correctly requires CRS metadata." + ], + "last_verified": "2026-11-02", + "availability": "Unreleased medium-source-compatibility branch; not the public 0.4.0 package", + "example_guide": "MEDIUM_COMPATIBILITY_WORKFLOWS.md" + }, + { + "id": "cityjson-medium-local-examples", + "type": "CityJSON / CityJSONSeq", + "purpose": "Self-contained declared-CRS LoD, template and sequence examples", + "url": null, + "paths": [ + "fixtures/medium/building-lods.city.json", + "fixtures/medium/building-sequence.city.jsonl", + "fixtures/medium/building-template.city.json" + ], + "context": { + "srs": "EPSG:25833", + "bbox": [ + 391296, + 5820096, + 391314, + 5820110 + ] + }, + "notes": [ + "Synthetic Berlin-coordinate examples, not real project buildings. Each parses as one building / six surfaces at LoD 2.2.", + "Connect directly to Load LoD2 Buildings, Layer empty. A local path can also orient Reference Source." + ], + "last_verified": "2026-11-02", + "availability": "Unreleased medium-source-compatibility branch; not the public 0.4.0 package", + "example_guide": "MEDIUM_COMPATIBILITY_WORKFLOWS.md" + }, + { + "id": "raster-medium-affine-local", + "type": "World-file raster", + "purpose": "Rotated/sheared image footprint, pixel-center and UV example", + "url": null, + "path": "fixtures/medium/rotated-raster.png", + "sidecars": [ + "fixtures/medium/rotated-raster.pgw", + "fixtures/medium/rotated-raster.prj" + ], + "context": { + "srs": "EPSG:25833", + "bbox": [ + 391296, + 5820096, + 391314, + 5820110 + ] + }, + "notes": [ + "Synthetic 4 x 4 colored PNG; keep both sidecars alongside it.", + "Pixel (0,0) center 391300,5820106; column vector (2,0.5), row vector (1,-2).", + "Connect to existing Load GeoTIFF. Core mesh produced 512 triangles; native Rhino validation pending." + ], + "last_verified": "2026-11-02", + "availability": "Unreleased medium-source-compatibility branch; not the public 0.4.0 package", + "example_guide": "MEDIUM_COMPATIBILITY_WORKFLOWS.md" } ] } diff --git a/sandbox/MediumCompatibilityChecks.cs b/sandbox/MediumCompatibilityChecks.cs new file mode 100644 index 0000000..eba2e12 --- /dev/null +++ b/sandbox/MediumCompatibilityChecks.cs @@ -0,0 +1,145 @@ +using System.Net; +using System.Text; +using System.Text.Json; +using System.Xml.Linq; +using BitMiracle.LibTiff.Classic; +using RhinoSpatial.Core; + +namespace RhinoSpatial.Sandbox; + +internal static class MediumCompatibilityChecks +{ + private static int _count; + private static void Require(bool value, string message) { if (!value) throw new Exception(message); } + private static void Near(double actual, double expected) => Require(Math.Abs(actual - expected) < 1e-7, $"Expected {expected}, got {actual}."); + private static void Fails(Action action) { try { action(); } catch (Exception e) when (e is InvalidDataException or JsonException or KeyNotFoundException or InvalidOperationException) { return; } throw new Exception("Invalid source accepted."); } + private static async Task Check(string label, Func action) { await action(); _count++; Console.WriteLine("PASS " + label); } + private static Task Sync(Action action) { action(); return Task.CompletedTask; } + internal const string Wkt = "PROJCS[\"ETRS89 / UTM zone 33N\",GEOGCS[\"ETRS89\",DATUM[\"European_Terrestrial_Reference_System_1989\",SPHEROID[\"GRS 1980\",6378137,298.257222101]],PRIMEM[\"Greenwich\",0],UNIT[\"degree\",0.0174532925199433]],PROJECTION[\"Transverse_Mercator\"],PARAMETER[\"latitude_of_origin\",0],PARAMETER[\"central_meridian\",15],PARAMETER[\"scale_factor\",0.9996],PARAMETER[\"false_easting\",500000],PARAMETER[\"false_northing\",0],UNIT[\"metre\",1],AUTHORITY[\"EPSG\",\"25833\"]]"; + internal static string Geometry(string lod, string indices = "[[[0,1,2]]]") => "{\"type\":\"MultiSurface\",\"lod\":" + JsonSerializer.Serialize(lod) + ",\"boundaries\":" + indices + "}"; + internal static string Model(string geometry) => "{\"type\":\"CityJSON\",\"version\":\"2.0\",\"metadata\":{\"referenceSystem\":\"https://www.opengis.net/def/crs/EPSG/0/25833\"},\"CityObjects\":{\"building\":{\"type\":\"Building\",\"geometry\":[" + geometry + "]}},\"vertices\":[[0,0,0],[2,0,0],[0,2,1]]}"; + internal static string Sequence => "{\"type\":\"CityJSON\",\"version\":\"2.0\",\"metadata\":{\"referenceSystem\":\"https://www.opengis.net/def/crs/EPSG/0/25833\"},\"transform\":{\"scale\":[2,3,4],\"translate\":[10,20,30]},\"CityObjects\":{},\"vertices\":[]}\n" + + Model(Geometry("2.2")).Replace("\"CityJSON\"", "\"CityJSONFeature\"").Replace("\"metadata\":{\"referenceSystem\":\"https://www.opengis.net/def/crs/EPSG/0/25833\"},", "") + "\n"; + internal static string Capabilities(string srs = "EPSG:25833", string top = "0 4", string scale = "3571.4285714285716", string extra = "") => $""" + + + orthoOrtho-180 -90180 90image/png{extra}native + native{srs}level{scale}{top}2222 + """; + private sealed class Handler(Func fetch) : HttpMessageHandler + { public List Requests { get; } = new(); protected override Task SendAsync(HttpRequestMessage request, CancellationToken token) { token.ThrowIfCancellationRequested(); Requests.Add(request.RequestUri!); return Task.FromResult(fetch(request)); } } + private static MemoryStream Stream(string text) => new(Encoding.UTF8.GetBytes(text)); + public static async Task RunAsync() + { + var directory = Path.Combine(Path.GetTempPath(), "RhinoSpatial-Medium-" + Guid.NewGuid().ToString("N")); Directory.CreateDirectory(directory); + try + { + await Check("one suitable LoD is selected without overlaying alternate representations", () => Sync(() => + { var b = CityJsonReader.ReadBuildings(Model(Geometry("1") + "," + Geometry("2.2") + "," + Geometry("3")), "test").Single(); Require(b.Surfaces.Count == 1 && b.Attributes["source_lod"] == "2.2", "Mixed LoDs imported."); })); + await Check("same-LoD geometry parts are retained and higher-only LoDs have a deterministic fallback", () => Sync(() => + { var b = CityJsonReader.ReadBuildings(Model(Geometry("3") + "," + Geometry("3") + "," + Geometry("4")), "test").Single(); Require(b.Surfaces.Count == 2 && b.Attributes["source_lod"] == "3", "Geometry part or fallback lost."); })); + await Check("invalid indices fail instead of creating partial rings", () => Sync(() => Fails(() => CityJsonReader.ReadBuildings(Model(Geometry("2", "[[[0,1,99]]]")), "test")))); + await Check("sequence records inherit header transform and source CRS", () => Sync(() => + { using var s = Stream(Sequence); var b = CityJsonReader.ReadBuildings(s, "test").Single(); Near(b.Surfaces[0].OuterPoints[1].X, 14); Near(b.Surfaces[0].OuterPoints[2].Y, 26); Near(b.Surfaces[0].OuterPoints[2].Z, 34); var m = CityJsonReader.ReadSourceMetadata(Sequence); Require(m.SrsName == "EPSG:25833", "Header CRS lost."); Near(m.BoundingBox!.MaxY, 26); })); + await Check("sequence input rejects truncated later records and repeated building ids", () => Sync(() => + { Fails(() => CityJsonReader.ReadBuildings(Sequence + "{", "test")); Fails(() => CityJsonReader.ReadBuildings(Sequence + Sequence.Split('\n')[1] + "\n", "test")); })); + await Check("template vertices use row-major matrix then transformed instance anchor", () => Sync(() => + { + var model = Model("{\"type\":\"GeometryInstance\",\"template\":0,\"boundaries\":[0],\"transformationMatrix\":[0,-2,0,5,3,0,0,7,0,0,4,9,0,0,0,1]}"); + model = model.Replace("\"vertices\":", "\"transform\":{\"scale\":[10,10,10],\"translate\":[100,200,300]},\"geometry-templates\":{\"templates\":[" + Geometry("2.2") + "],\"vertices-templates\":[[0,0,0],[2,0,0],[0,2,1]]},\"vertices\":"); + var b = CityJsonReader.ReadBuildings(model, "test").Single(); Near(b.Surfaces[0].OuterPoints[1].X, 105); Near(b.Surfaces[0].OuterPoints[1].Y, 213); Near(b.Surfaces[0].OuterPoints[2].Z, 313); + var m = CityJsonReader.ReadSourceMetadata(model); Require(m.BoundingBox!.MaxY >= 213, "Template extent omitted."); + })); + await Check("sequence cancellation propagates", () => Sync(() => + { using var token = new CancellationTokenSource(); token.Cancel(); using var s = Stream(Sequence); try { CityJsonReader.ReadBuildings(s, "test", cancellationToken: token.Token); throw new Exception("Cancellation ignored."); } catch (OperationCanceledException) { } })); + await Check("direct sequence downloads validate and cache through shared building path", async () => + { using var http = new HttpClient(new Handler(_ => new(HttpStatusCode.OK) { Content = new StringContent(Sequence) })); var cache = new SourceDownloadCache(AtomDownloadKind.Buildings, http, Path.Combine(directory, "downloads")); var r = await cache.DownloadAsync(new(new Uri("https://fixture.test/model.city.jsonl"), "seq", "", null)); using var s = File.OpenRead(r.FilePath); Require(CityJsonReader.ReadBuildings(s, "test").Count == 1, "Sequence did not survive caching."); }); + var png = Path.Combine(directory, "rotated.png"); var rgba = Enumerable.Range(0, 64).Select(i => (byte)(i * 3)).ToArray(); RasterImageCodec.WritePng(png, 4, 4, rgba); File.WriteAllText(Path.ChangeExtension(png, ".pgw"), "2\n1\n0.5\n-3\n100\n200\n"); File.WriteAllText(Path.ChangeExtension(png, ".prj"), Wkt); + await Check("world-file centers, rotation and shear preserve the true footprint", () => Sync(() => + { var info = GeoTiffReader.ReadImageInfo(png); var g = info.GeoTransform!; Near(g.At(.5, .5).X, 100); Near(g.At(.5, .5).Y, 200); Near(g.At(4, 4).X, 108.75); Near(g.At(4, 4).Y, 193); Near(g.Pixel(100, 200).X, .5); Require(info.SrsName == "EPSG:25833", "Sidecar CRS lost."); })); + await Check("affine image clipping preserves UVs and localization", () => Sync(() => + { var info = GeoTiffReader.ReadImageInfo(png); var context = Lod2AtomChecks.Context("EPSG:25833", new(100, 191, 106, 202)); var mesh = RasterImageMeshBuilder.Build(info, context); Require(mesh.Faces.Count > 0, "Clipped image disappeared."); foreach (var v in mesh.Vertices) { var native = info.GeoTransform!.At(v.U * 4, (1 - v.V) * 4); Near(native.X - context.PlacementOrigin.X, v.X); Near(native.Y - context.PlacementOrigin.Y, v.Y); } Require(mesh.CoverageRatio < 1, "Clip ratio not retained."); })); + await Check("affine imagery reprojects through the shared context without bbox scaling", () => Sync(() => + { + var source = new RasterGeoTransform(2, 0.5, 391300, 0.5, -2, 5820100); var native = source.Bounds(4, 4); + var bounds = Lod2AtomFeedReader.TransformBounds(new("EPSG:25833", native), "EPSG:3857"); var context = Lod2AtomChecks.Context("EPSG:3857", bounds); + var info = new GeoReferencedRasterInfo("test", "EPSG:25833", 4, 4, native, 0) { GeoTransform = source }; var mesh = RasterImageMeshBuilder.Build(info, context); + Require(mesh.Faces.Count > 0, "Reprojected image disappeared."); foreach (var v in mesh.Vertices) { var p = source.At(v.U * 4, (1 - v.V) * 4); Require(SpatialReferenceTransform.TryTransformXY(info.SrsName, context.ResolvedSrs, p.X, p.Y, out var x, out var y), "Projection failed."); Near(v.X, x - context.PlacementOrigin.X); Near(v.Y, y - context.PlacementOrigin.Y); } + })); + await Check("global geographic imagery projects only the selected window with correct UVs",()=>Sync(()=> + { + var g=new RasterGeoTransform(1,0,-180,0,-1,90);var bounds=Lod2AtomFeedReader.TransformBounds(new("EPSG:4326",new(13.3,52.4,13.4,52.5)),"EPSG:3857");var context=Lod2AtomChecks.Context("EPSG:3857",bounds); + var info=new GeoReferencedRasterInfo("global","EPSG:4326",360,180,new(-180,-90,180,90),0){GeoTransform=g};var mesh=RasterImageMeshBuilder.Build(info,context);Require(mesh.Faces.Count>0&&mesh.CoverageRatio<0.001,"Global image was not windowed."); + foreach(var v in mesh.Vertices){var p=g.At(v.U*360,(1-v.V)*180);SpatialReferenceTransform.TryTransformXY("EPSG:4326","EPSG:3857",p.X,p.Y,out var x,out var y);Near(v.X,x-context.PlacementOrigin.X);Near(v.Y,y-context.PlacementOrigin.Y);} + })); + await Check("sidecar edits invalidate cached metadata even with unchanged raster", () => Sync(() => + { var old = GeoTiffInfoCache.GetOrRead(png); File.WriteAllText(Path.ChangeExtension(png, ".pgw"), "2\n1\n0.5\n-3\n110\n200\n"); var updated = GeoTiffInfoCache.GetOrRead(png); Near(updated.GeoTransform!.C - old.GeoTransform!.C, 10); })); + await Check("world file without a CRS and singular affine transforms fail", () => Sync(() => + { var prj = Path.ChangeExtension(png, ".prj"); File.Move(prj, prj + ".off"); Fails(() => GeoTiffReader.ReadImageInfo(png)); File.Move(prj + ".off", prj); Fails(() => new RasterGeoTransform(1, 2, 0, 2, 4, 0).Validate()); })); + await Check("terrain uses affine pixel centers including rotated rows", () => Sync(() => + { var g = new RasterGeoTransform(2, 1, 100, 1, -2, 200); var raster = new TerrainRasterData("test", "EPSG:25833", 2, 2, new(0, 0), new(1, 0), new(0, -1), null, new float[] { 1, 2, 3, 4 }) { GeoTransform = g }; var mesh = TerrainMeshBuilder.Build(raster, g.Bounds(2, 2), Lod2AtomChecks.Context("EPSG:25833", g.Bounds(2, 2)), 2, 0); Near(mesh.Vertices[0].X, g.At(.5, .5).X - 100); Near(mesh.Vertices[2].Y, g.At(.5, 1.5).Y - 196); })); + await Check("embedded affine GeoTIFF and PixelIsPoint retain centers and projected CRS", () => Sync(() => + { + foreach (var point in new[] { false, true }) + { + var path = Path.Combine(directory, "affine-" + point + ".tif"); WriteAffineTiff(path, point); + var info = GeoTiffReader.ReadImageInfo(path); Require(info.SrsName == "EPSG:25833", "Geographic key overrode projected CRS."); + Near(info.GeoTransform!.At(.5, .5).X, point ? 100 : 101.5); Near(info.GeoTransform.At(.5, .5).Y, point ? 200 : 199.5); + var raster = TerrainRasterReader.ReadRaster(path, "test", info.SrsName); Near(raster.Origin.X, point ? 100 : 101.5); Require(raster.Elevations[3] == 4, "TIFF samples changed."); + } + })); + await Check("small geographic affine resolutions remain invertible", () => Sync(() => new RasterGeoTransform(1e-8, 0, 13, 0, -1e-8, 52).Validate())); + var caps = WmtsClient.ReadCapabilities(XDocument.Parse(Capabilities()), new("https://fixture.test/wmts.xml")); var plan = WmtsClient.Plan(caps, "ortho", Lod2AtomChecks.Context("EPSG:25833", new(0, 0, 4, 4))); + await Check("projected WMTS selects exact tile ranges and native mosaic transform", () => Sync(() => + { Require(plan.Tiles.Count == 4 && plan.Width == 4 && plan.Height == 4, "Incorrect tile range."); Near(plan.Transform.At(0, 0).Y, 4); Near(plan.Transform.At(4, 4).X, 4); Require(plan.Tiles[3].Url.Contains("TileRow=1") && plan.Tiles[3].Url.Contains("TileCol=1"), "Wrong KVP tile indices."); })); + await Check("geographic WMTS top-left respects authority axis order", () => Sync(() => + { var c = WmtsClient.ReadCapabilities(XDocument.Parse(Capabilities("urn:ogc:def:crs:EPSG::4326", "90 -180", "279541132.0143589")), new("https://fixture.test/wmts.xml")); Near(c.Sets["native"].Matrices[0].TopLeft.X, -180); Near(c.Sets["native"].Matrices[0].TopLeft.Y, 90); })); + await Check("REST tile templates use default dimensions and URL-escaped identifiers", () => Sync(() => + { var c = WmtsClient.ReadCapabilities(XDocument.Parse(Capabilities(extra: "Time2024")), new("https://fixture.test/wmts.xml")); var p = WmtsClient.Plan(c, "ortho", Lod2AtomChecks.Context("EPSG:25833", new(0, 0, 4, 4))); Require(p.Tiles[0].Url == "https://fixture.test/tiles/2024/level/0/0.png", "REST template not resolved."); })); + await Check("matrix limits exclude unadvertised tiles and report clipped coverage", () => Sync(() => + { var text = Capabilities().Replace("", "level0001"); var p = WmtsClient.Plan(WmtsClient.ReadCapabilities(XDocument.Parse(text), new("https://fixture.test/wmts.xml")), "ortho", Lod2AtomChecks.Context("EPSG:25833", new(0, 0, 4, 4))); Require(p.Tiles.Count == 2 && p.CoverageClipped, "Limits ignored."); })); + await Check("unknown CRS, oversized selections and ambiguous dimensions fail explicitly", () => Sync(() => + { + var unsupported = WmtsClient.ReadCapabilities(XDocument.Parse(Capabilities("EPSG:999999")), new("https://fixture.test/wmts.xml")); Fails(() => WmtsClient.Plan(unsupported, "ortho", Lod2AtomChecks.Context("EPSG:25833", new(0, 0, 4, 4)))); + var wide = WmtsClient.ReadCapabilities(XDocument.Parse(Capabilities().Replace("2", "100").Replace("2", "100")), new("https://fixture.test/wmts.xml")); Fails(() => WmtsClient.Plan(wide, "ortho", Lod2AtomChecks.Context("EPSG:25833", new(0, -196, 200, 4)))); + var dim = WmtsClient.ReadCapabilities(XDocument.Parse(Capabilities(extra: "Time20232024")), new("https://fixture.test/wmts.xml")); Fails(() => WmtsClient.Plan(dim, "ortho", Lod2AtomChecks.Context("EPSG:25833", new(0, 0, 4, 4)))); + })); + var tile = Path.Combine(directory, "tile.png"); RasterImageCodec.WritePng(tile, 2, 2, Enumerable.Repeat(new byte[] { 12, 34, 56, 128 }, 4).SelectMany(x => x).ToArray()); var tileBytes = File.ReadAllBytes(tile); + await Check("WMTS PNG mosaic preserves decoded pixels/alpha and reuses completed cache", async () => + { var handler = new Handler(_ => new(HttpStatusCode.OK) { Content = new ByteArrayContent(tileBytes) }); using var http = new HttpClient(handler); var client = new WmtsClient(http, Path.Combine(directory, "wmts")); var first = await client.DownloadAsync(plan); var pixels = RasterImageCodec.DecodeRgba(File.ReadAllBytes(first.LocalFilePath), 4, 4); Require(pixels[0] == 12 && pixels[^1] == 128, "Mosaic pixel or alpha corrupted."); var second = await client.DownloadAsync(plan); Require(second.UsedCache && handler.Requests.Count == 4, "Completed mosaic was not reused."); }); + await Check("corrupt WMTS cache is reloaded and cancellation publishes no mosaic", async () => + { + var folder = Path.Combine(directory, "corruption"); var handler = new Handler(_ => new(HttpStatusCode.OK) { Content = new ByteArrayContent(tileBytes) }); using var http = new HttpClient(handler); var client = new WmtsClient(http, folder); var first = await client.DownloadAsync(plan); + File.AppendAllText(first.LocalFilePath, "corrupt"); var second = await client.DownloadAsync(plan); Require(!second.UsedCache && handler.Requests.Count == 8, "Corrupt mosaic cache accepted."); + using var token = new CancellationTokenSource(); token.Cancel(); try { await new WmtsClient(http, Path.Combine(directory, "cancelled")).DownloadAsync(plan, token.Token); throw new Exception("Cancellation ignored."); } catch (OperationCanceledException) { } + }); + await Check("tile row/column order is preserved in the mosaic", async () => + { + var images = new Dictionary(); + for (var row = 0; row < 2; row++) for (var col = 0; col < 2; col++) + { var file = Path.Combine(directory, $"tile-{row}-{col}.png"); RasterImageCodec.WritePng(file, 2, 2, Enumerable.Repeat(new byte[] { (byte)(row * 100 + col * 20), 0, 0, 255 }, 4).SelectMany(x => x).ToArray()); images[$"{row},{col}"] = File.ReadAllBytes(file); } + using var http = new HttpClient(new Handler(r => + { var q = r.RequestUri!.Query; var row = q.Contains("TileRow=1") ? 1 : 0; var col = q.Contains("TileCol=1") ? 1 : 0; return new(HttpStatusCode.OK) { Content = new ByteArrayContent(images[$"{row},{col}"]) }; })); + var image = await new WmtsClient(http, Path.Combine(directory, "ordering")).DownloadAsync(plan); var pixels = RasterImageCodec.DecodeRgba(File.ReadAllBytes(image.LocalFilePath), 4, 4); + Require(pixels[0] == 0 && pixels[3 * 4] == 20 && pixels[(3 * 4) * 4] == 100 && pixels[(3 * 4 + 3) * 4] == 120, "Mosaic tiles were flipped or transposed."); + }); + await Check("failed required WMTS tile does not publish a partial mosaic", async () => + { var folder = Path.Combine(directory, "failure"); using var http = new HttpClient(new Handler(r => r.RequestUri!.Query.Contains("TileRow=1") ? new(HttpStatusCode.NotFound) : new(HttpStatusCode.OK) { Content = new ByteArrayContent(tileBytes) })); try { await new WmtsClient(http, folder).DownloadAsync(plan); throw new Exception("Partial mosaic accepted."); } catch (HttpRequestException) { } Require(!Directory.EnumerateFiles(folder).Any(), "Partial output published."); }); + await Check("WMTS reference reads capabilities without requesting tile geometry", async () => + { var handler = new Handler(_ => new(HttpStatusCode.OK) { Content = new StringContent(Capabilities()) }); using var http = new HttpClient(handler); var metadata = await new RemoteReferenceMetadataReader(http).TryReadAsync("https://fixture.test/wmts.xml", "ortho"); Require(metadata?.SourceKind == "WMTS" && handler.Requests.Count == 1, "Reference fetched tiles."); }); + Console.WriteLine($"Medium compatibility checks passed: {_count}"); + } + finally { Directory.Delete(directory, true); } + } + private static void WriteAffineTiff(string path, bool point) + { + using var writer = new BinaryWriter(File.Create(path)); + const uint image = 146, matrix = 150, keys = 278; + writer.Write((ushort)0x4949); writer.Write((ushort)42); writer.Write((uint)8); writer.Write((ushort)11); + void Entry(ushort tag, ushort type, uint count, uint value) { writer.Write(tag); writer.Write(type); writer.Write(count); writer.Write(value); } + Entry(256, 3, 1, 2); Entry(257, 3, 1, 2); Entry(258, 3, 1, 8); Entry(259, 3, 1, 1); Entry(262, 3, 1, 1); Entry(273, 4, 1, image); Entry(277, 3, 1, 1); Entry(278, 4, 1, 2); Entry(279, 4, 1, 4); Entry(34264, 12, 16, matrix); Entry(34735, 3, 16, keys); writer.Write((uint)0); + writer.Write(new byte[] { 1, 2, 3, 4 }); foreach (var v in new double[] { 2, 1, 0, 100, 1, -2, 0, 200, 0, 0, 1, 0, 0, 0, 0, 1 }) writer.Write(v); + foreach (var v in new ushort[] { 1, 1, 0, 3, 1025, 0, 1, (ushort)(point ? 2 : 1), 2048, 0, 1, 4326, 3072, 0, 1, 25833 }) writer.Write(v); + } + +} diff --git a/sandbox/MediumCompatibilityProbe.cs b/sandbox/MediumCompatibilityProbe.cs new file mode 100644 index 0000000..d478ad0 --- /dev/null +++ b/sandbox/MediumCompatibilityProbe.cs @@ -0,0 +1,71 @@ +using System.Globalization; +using RhinoSpatial.Core; + +namespace RhinoSpatial.Sandbox; + +internal static class MediumCompatibilityProbe +{ + public static async Task RunAsync(string[] args) + { + if (args.Length < 3) throw new ArgumentException("medium-probe wmts [layer srs minX,minY,maxX,maxY] | cityjson | raster "); + if (args[1] == "examples") { WriteExamples(args[2]); return; } + if (args[1] == "wmts") + { + var client = new WmtsClient(); var caps = await client.TryLoadCapabilitiesAsync(args[2]) ?? throw new Exception("Not WMTS."); + Console.WriteLine($"Layers: {caps.Layers.Count}; supported matrix sets: {caps.Sets.Count}"); + foreach (var layer in caps.Layers.Take(8)) Console.WriteLine($"{layer.Id} | {layer.Title} | {string.Join(",", layer.Links.Select(l => l.Set))}"); + if (args.Length < 6) return; + var values = args[5].Split(',').Select(s => double.Parse(s, CultureInfo.InvariantCulture)).ToArray(); + var context = Lod2AtomChecks.Context(args[4], new(values[0], values[1], values[2], values[3])); + var plan = WmtsClient.Plan(caps, args[3], context); var result = await client.DownloadAsync(plan); + var mesh = RasterImageMeshBuilder.Build(result.Raster, context); + Console.WriteLine(result.Status); Console.WriteLine($"Image: {result.LocalFilePath}; clipped mesh: {mesh.Vertices.Count} vertices, {mesh.Faces.Count} triangles"); + } + else if (args[1] == "cityjson") + { + using var metadata = File.OpenRead(args[2]); var m = CityJsonReader.ReadSourceMetadata(metadata); + using var input = File.OpenRead(args[2]); var buildings = CityJsonReader.ReadBuildings(input, "probe"); + Console.WriteLine($"{m.SrsName}; buildings: {buildings.Count}; surfaces: {buildings.Sum(b => b.Surfaces.Count)}"); + Console.WriteLine("Selected LoDs: " + string.Join(", ", buildings.Select(b => b.Attributes["source_lod"]).Distinct())); + } + else if (args[1] == "raster") + { + var info = GeoTiffReader.ReadImageInfo(args[2]); var context = Lod2AtomChecks.Context(info.SrsName, info.BoundingBox); + var mesh = RasterImageMeshBuilder.Build(info, context); + Console.WriteLine($"{info.SrsName}; {info.Width}x{info.Height}; affine: {info.GeoTransform}; mesh: {mesh.Faces.Count} triangles"); + } + else throw new ArgumentException("Unknown medium probe mode."); + } + private static void WriteExamples(string directory) + { + Directory.CreateDirectory(directory); + var faces = new[] { new[] { 0, 3, 2, 1 }, new[] { 0, 1, 5, 4 }, new[] { 1, 2, 6, 5 }, new[] { 2, 3, 7, 6 }, new[] { 3, 0, 4, 7 }, new[] { 4, 5, 6, 7 } }; + object Boundaries(int offset) => new[] { faces.Select(face => new[] { face.Select(i => i + offset).ToArray() }).ToArray() }; + object Geometry(string lod, int offset) => new { type = "Solid", lod, boundaries = Boundaries(offset) }; + var vertices = new[] { new[] { 0, 0, 0 }, new[] { 8, 0, 0 }, new[] { 8, 6, 0 }, new[] { 0, 6, 0 }, new[] { 0, 0, 10 }, new[] { 8, 0, 10 }, new[] { 8, 6, 10 }, new[] { 0, 6, 10 }, new[] { 0, 0, 0 }, new[] { 8, 0, 0 }, new[] { 8, 6, 0 }, new[] { 0, 6, 0 }, new[] { 0, 0, 12 }, new[] { 8, 0, 12 }, new[] { 8, 6, 14 }, new[] { 0, 6, 14 } }; + var objects = new Dictionary { { "example-building", new { type = "Building", geometry = new[] { Geometry("1", 0), Geometry("2.2", 8) } } } }; + var metadata = new { referenceSystem = "https://www.opengis.net/def/crs/EPSG/0/25833", geographicalExtent = new[] { 391300d, 5820100d, 32d, 391308d, 5820106d, 46d } }; + var transform = new { scale = new[] { 1, 1, 1 }, translate = new[] { 391300, 5820100, 32 } }; + var model = new { type = "CityJSON", version = "2.0", metadata, transform, CityObjects = objects, vertices }; + var options = new System.Text.Json.JsonSerializerOptions { WriteIndented = true }; + void Json(string name, object value) => File.WriteAllText(Path.Combine(directory, name), System.Text.Json.JsonSerializer.Serialize(value, options) + "\n"); + Json("building-lods.city.json", model); + var header = new { type = "CityJSON", version = "2.0", metadata, transform, CityObjects = new Dictionary(), vertices = Array.Empty() }; + var feature = new { type = "CityJSONFeature", id = "example-building", CityObjects = objects, vertices }; + File.WriteAllText(Path.Combine(directory, "building-sequence.city.jsonl"), System.Text.Json.JsonSerializer.Serialize(header) + "\n" + System.Text.Json.JsonSerializer.Serialize(feature) + "\n"); + var instance = new { type = "GeometryInstance", template = 0, boundaries = new[] { 0 }, transformationMatrix = new[] { 8, 0, 0, 0, 0, 6, 0, 0, 0, 0, 12, 0, 0, 0, 0, 1 } }; + var template = new Dictionary(); + template["type"] = "CityJSON"; template["version"] = "2.0"; template["metadata"] = metadata; + template["CityObjects"] = new Dictionary { { "template-building", new { type = "Building", geometry = new[] { instance } } } }; + template["vertices"] = new[] { new[] { 391300, 5820100, 32 } }; + template["geometry-templates"] = new Dictionary { { "templates", new[] { Geometry("2.2", 0) } }, { "vertices-templates", new[] { new[] { 0, 0, 0 }, new[] { 1, 0, 0 }, new[] { 1, 1, 0 }, new[] { 0, 1, 0 }, new[] { 0, 0, 1 }, new[] { 1, 0, 1 }, new[] { 1, 1, 1 }, new[] { 0, 1, 1 } } } }; + Json("building-template.city.json", template); + var pixels = new byte[4 * 4 * 4]; + for (var row = 0; row < 4; row++) for (var col = 0; col < 4; col++) { var i = (row * 4 + col) * 4; pixels[i] = (byte)(col < 2 ? 240 : 40); pixels[i + 1] = (byte)(row < 2 ? 180 : 40); pixels[i + 2] = (byte)(col < 2 ? 40 : 220); pixels[i + 3] = 255; } + RasterImageCodec.WritePng(Path.Combine(directory, "rotated-raster.png"), 4, 4, pixels); + File.WriteAllText(Path.Combine(directory, "rotated-raster.pgw"), "2\n0.5\n1\n-2\n391300\n5820106\n"); + File.WriteAllText(Path.Combine(directory, "rotated-raster.prj"), MediumCompatibilityChecks.Wkt + "\n"); + Console.WriteLine("Wrote synthetic EPSG:25833 examples to " + Path.GetFullPath(directory)); + } + +} diff --git a/sandbox/Program.cs b/sandbox/Program.cs index cd4def1..b881a43 100644 --- a/sandbox/Program.cs +++ b/sandbox/Program.cs @@ -15,6 +15,10 @@ static async Task Main(string[] args) catch (Exception ex) { Console.Error.WriteLine(ex.Message); Environment.ExitCode = 1; } return; } + if (args.Length > 0 && args[0].Equals("medium-probe", StringComparison.OrdinalIgnoreCase)) + { await MediumCompatibilityProbe.RunAsync(args); return; } + if (args.Length > 0 && args[0].Equals("medium-compatibility-check", StringComparison.OrdinalIgnoreCase)) + { await MediumCompatibilityChecks.RunAsync(); return; } if (args.Length > 0 && args[0].Equals("source-compatibility-check", StringComparison.OrdinalIgnoreCase)) { await SourceCompatibilityChecks.RunAsync();