From 3bf95cd0eb0a28edf7f10c29f6f99e0ddd4ecc8d Mon Sep 17 00:00:00 2001 From: freezy Date: Thu, 3 Sep 2026 00:55:17 +0200 Subject: [PATCH 1/3] packaging: add portable content bundles Add hash-addressed, inert content bundles stored under table/content/, with secure extraction, validation, consent, and LRU cache management. Bundles are deterministic: files are canonically ordered and hashed so identical trees deduplicate regardless of source location, timestamps, or platform. Writers reject absolute/unsafe paths and case collisions; readers verify every file's declared size and SHA-256 while streaming into an atomically published cache directory under persistentDataPath. Split-role IPackagedContentResolver: PackageWriter ingests directories via IPackagedContentSource at write time, while PackageReader and RuntimePackageReader inject a read-only resolver into IPackagedContentConsumer components after a table is restored. Cover round-tripping, malformed manifests, path traversal, and package limits with tests. --- .../Packaging/PackageReader.cs | 8 + .../Packaging/PackageWriter.cs | 30 +- .../Packaging/PackagedContentRefDrawer.cs | 50 +++ .../PackagedContentRefDrawer.cs.meta | 2 + .../Packaging/PackagedContentTests.cs | 324 ++++++++++++++++++ .../Packaging/PackagedContentTests.cs.meta | 2 + .../Packaging/Content.meta | 8 + .../Packaging/Content/PackagedContent.cs | 119 +++++++ .../Packaging/Content/PackagedContent.cs.meta | 2 + .../Content/PackagedContentConsent.cs | 155 +++++++++ .../Content/PackagedContentConsent.cs.meta | 11 + .../Packaging/Content/PackagedContentPath.cs | 71 ++++ .../Content/PackagedContentPath.cs.meta | 2 + .../Content/PackagedContentResolver.cs | 249 ++++++++++++++ .../Content/PackagedContentResolver.cs.meta | 2 + .../Content/PackagedContentValidator.cs | 297 ++++++++++++++++ .../Content/PackagedContentValidator.cs.meta | 2 + .../Content/PackagedContentWriter.cs | 255 ++++++++++++++ .../Content/PackagedContentWriter.cs.meta | 2 + .../VisualPinball.Unity/Packaging/FORMAT.md | 71 ++++ .../Packaging/PackageApi.cs | 3 +- .../VisualPinball.Unity/Packaging/README.md | 10 + .../Packaging/RuntimePackageReader.cs | 7 + 23 files changed, 1680 insertions(+), 2 deletions(-) create mode 100644 VisualPinball.Unity/VisualPinball.Unity.Editor/Packaging/PackagedContentRefDrawer.cs create mode 100644 VisualPinball.Unity/VisualPinball.Unity.Editor/Packaging/PackagedContentRefDrawer.cs.meta create mode 100644 VisualPinball.Unity/VisualPinball.Unity.Test/Packaging/PackagedContentTests.cs create mode 100644 VisualPinball.Unity/VisualPinball.Unity.Test/Packaging/PackagedContentTests.cs.meta create mode 100644 VisualPinball.Unity/VisualPinball.Unity/Packaging/Content.meta create mode 100644 VisualPinball.Unity/VisualPinball.Unity/Packaging/Content/PackagedContent.cs create mode 100644 VisualPinball.Unity/VisualPinball.Unity/Packaging/Content/PackagedContent.cs.meta create mode 100644 VisualPinball.Unity/VisualPinball.Unity/Packaging/Content/PackagedContentConsent.cs create mode 100644 VisualPinball.Unity/VisualPinball.Unity/Packaging/Content/PackagedContentConsent.cs.meta create mode 100644 VisualPinball.Unity/VisualPinball.Unity/Packaging/Content/PackagedContentPath.cs create mode 100644 VisualPinball.Unity/VisualPinball.Unity/Packaging/Content/PackagedContentPath.cs.meta create mode 100644 VisualPinball.Unity/VisualPinball.Unity/Packaging/Content/PackagedContentResolver.cs create mode 100644 VisualPinball.Unity/VisualPinball.Unity/Packaging/Content/PackagedContentResolver.cs.meta create mode 100644 VisualPinball.Unity/VisualPinball.Unity/Packaging/Content/PackagedContentValidator.cs create mode 100644 VisualPinball.Unity/VisualPinball.Unity/Packaging/Content/PackagedContentValidator.cs.meta create mode 100644 VisualPinball.Unity/VisualPinball.Unity/Packaging/Content/PackagedContentWriter.cs create mode 100644 VisualPinball.Unity/VisualPinball.Unity/Packaging/Content/PackagedContentWriter.cs.meta diff --git a/VisualPinball.Unity/VisualPinball.Unity.Editor/Packaging/PackageReader.cs b/VisualPinball.Unity/VisualPinball.Unity.Editor/Packaging/PackageReader.cs index fb6680151..065e0e1ff 100644 --- a/VisualPinball.Unity/VisualPinball.Unity.Editor/Packaging/PackageReader.cs +++ b/VisualPinball.Unity/VisualPinball.Unity.Editor/Packaging/PackageReader.cs @@ -49,8 +49,12 @@ public class PackageReader public PackageReader(string vpePath) { _vpePath = vpePath; + ContentResolver = new PackagedContentResolver(vpePath); } + /// Resolves packaged content through the same cache path used by Player builds. + public IPackagedContentResolver ContentResolver { get; } + public async Task ImportIntoScene(string tableName) { var sw = new Stopwatch(); @@ -91,6 +95,10 @@ public async Task ImportIntoScene(string tableName) comp?.UnpackReferences(stream.GetData(), _tableRoot, _refs, _files); }); + foreach (var consumer in _table.GetComponentsInChildren(true).OfType()) { + consumer.SetPackagedContentResolver(ContentResolver); + } + ReadGlobals(); ReadTableMetadata(); SimulationThreadComponent.EnsureForTable(_tableRoot.gameObject); diff --git a/VisualPinball.Unity/VisualPinball.Unity.Editor/Packaging/PackageWriter.cs b/VisualPinball.Unity/VisualPinball.Unity.Editor/Packaging/PackageWriter.cs index 55a997033..c23d73ac6 100644 --- a/VisualPinball.Unity/VisualPinball.Unity.Editor/Packaging/PackageWriter.cs +++ b/VisualPinball.Unity/VisualPinball.Unity.Editor/Packaging/PackageWriter.cs @@ -36,7 +36,7 @@ namespace VisualPinball.Unity.Editor { - public class PackageWriter + public class PackageWriter : IPackagedContentResolver { private readonly GameObject _table; private readonly PackagedRefs _refs; @@ -53,6 +53,7 @@ public class PackageWriter private IReadOnlyDictionary _originalGltfImageReplacements; private Dictionary _nodeIdByTransform; private readonly SortedSet _usedTypeNames = new(StringComparer.Ordinal); + private readonly List _contentBundles = new(); private const bool ExportActivesOnly = true; @@ -90,6 +91,22 @@ public Task WritePackageAsync(string path, IProgress progress = return WritePackage(path, progress, cancellationToken); } + /// + /// Adds an inert directory to the next package written by this writer. The directory is + /// scanned immediately so its reference can be serialized by table components. + /// + public PackagedContentRef AddDirectory(string kind, string sourceRoot, ContentPackOptions options = null) + { + var prepared = PackagedContentWriter.PrepareDirectory(kind, sourceRoot, options); + _contentBundles.Add(prepared); + return prepared.Reference; + } + + public Task ResolveAsync(PackagedContentRef contentRef, IProgress progress, CancellationToken ct) + { + throw new NotSupportedException("PackageWriter only creates content bundles. Resolve with PackageReader.ContentResolver or RuntimePackageReader.ContentResolver."); + } + private async Task WritePackage(string path, IProgress progress, CancellationToken ct) { var sw = new Stopwatch(); @@ -107,6 +124,7 @@ private async Task WritePackage(string path, IProgress progress, storage = PackageApi.StorageManager.CreateStorage(path); _tableFolder = storage.AddFolder(PackageApi.TableFolder); + var contentWriter = new PackagedContentWriter(_tableFolder); _globalFolder = _tableFolder.AddFolder(PackageApi.GlobalFolder); _metaFolder = _tableFolder.AddFolder(PackageApi.MetaFolder); _files = new PackagedFiles(_tableFolder, _refs); @@ -122,6 +140,16 @@ private async Task WritePackage(string path, IProgress progress, _nodeIdByTransform = VpeNodeIds.AssignIds(_table.transform); _refs.SetNodeIdsForWrite(_nodeIdByTransform); + // Content sources must receive their stable refs before component data is serialized. + foreach (var source in _table.GetComponentsInChildren(true).OfType()) { + ct.ThrowIfCancellationRequested(); + source.PreparePackagedContent(this); + } + foreach (var bundle in _contentBundles.OrderBy(item => item.Reference.Id, StringComparer.Ordinal)) { + ct.ThrowIfCancellationRequested(); + contentWriter.Write(bundle); + } + // prepare scene data ct.ThrowIfCancellationRequested(); Report(progress, 0.03f, "Preparing scene…"); diff --git a/VisualPinball.Unity/VisualPinball.Unity.Editor/Packaging/PackagedContentRefDrawer.cs b/VisualPinball.Unity/VisualPinball.Unity.Editor/Packaging/PackagedContentRefDrawer.cs new file mode 100644 index 000000000..5615f0082 --- /dev/null +++ b/VisualPinball.Unity/VisualPinball.Unity.Editor/Packaging/PackagedContentRefDrawer.cs @@ -0,0 +1,50 @@ +// Visual Pinball Engine +// Copyright (C) 2026 freezy and VPE Team + +using UnityEditor; +using UnityEngine; + +namespace VisualPinball.Unity.Editor +{ + [CustomPropertyDrawer(typeof(PackagedContentRef))] + public sealed class PackagedContentRefDrawer : PropertyDrawer + { + private static readonly string[] Fields = { + "Kind", "EntryPoint", "Id", "ContentHash", "FileCount", "TotalBytes", "SourceDirectory", "ValidationStatus" + }; + + public override float GetPropertyHeight(SerializedProperty property, GUIContent label) + { + if (!property.isExpanded) { + return EditorGUIUtility.singleLineHeight; + } + return (Fields.Length + 1) * (EditorGUIUtility.singleLineHeight + EditorGUIUtility.standardVerticalSpacing); + } + + public override void OnGUI(Rect position, SerializedProperty property, GUIContent label) + { + EditorGUI.BeginProperty(position, label, property); + var line = new Rect(position.x, position.y, position.width, EditorGUIUtility.singleLineHeight); + property.isExpanded = EditorGUI.Foldout(line, property.isExpanded, label, true); + if (property.isExpanded) { + EditorGUI.indentLevel++; + using (new EditorGUI.DisabledScope(true)) { + foreach (var fieldName in Fields) { + line.y += EditorGUIUtility.singleLineHeight + EditorGUIUtility.standardVerticalSpacing; + var child = property.FindPropertyRelative(fieldName); + if (child == null) { + continue; + } + if (fieldName == "TotalBytes") { + EditorGUI.TextField(line, ObjectNames.NicifyVariableName(fieldName), EditorUtility.FormatBytes(child.longValue)); + } else { + EditorGUI.PropertyField(line, child, new GUIContent(ObjectNames.NicifyVariableName(fieldName))); + } + } + } + EditorGUI.indentLevel--; + } + EditorGUI.EndProperty(); + } + } +} diff --git a/VisualPinball.Unity/VisualPinball.Unity.Editor/Packaging/PackagedContentRefDrawer.cs.meta b/VisualPinball.Unity/VisualPinball.Unity.Editor/Packaging/PackagedContentRefDrawer.cs.meta new file mode 100644 index 000000000..252952d32 --- /dev/null +++ b/VisualPinball.Unity/VisualPinball.Unity.Editor/Packaging/PackagedContentRefDrawer.cs.meta @@ -0,0 +1,2 @@ +fileFormatVersion: 2 +guid: eb29e0320799fcd4d8f6c389fd673f1d \ No newline at end of file diff --git a/VisualPinball.Unity/VisualPinball.Unity.Test/Packaging/PackagedContentTests.cs b/VisualPinball.Unity/VisualPinball.Unity.Test/Packaging/PackagedContentTests.cs new file mode 100644 index 000000000..2b85f267f --- /dev/null +++ b/VisualPinball.Unity/VisualPinball.Unity.Test/Packaging/PackagedContentTests.cs @@ -0,0 +1,324 @@ +// Visual Pinball Engine +// Copyright (C) 2026 freezy and VPE Team + +using System; +using System.Diagnostics; +using System.IO; +using System.Linq; +using System.Threading; +using System.Threading.Tasks; +using FluentAssertions; +using NUnit.Framework; + +namespace VisualPinball.Unity.Test.Packaging +{ + [TestFixture] + public class PackagedContentTests + { + private string _root; + + [SetUp] + public void SetUp() + { + _root = Path.Combine(Path.GetTempPath(), "vpe-content-tests", Guid.NewGuid().ToString("N")); + Directory.CreateDirectory(_root); + } + + [TearDown] + public void TearDown() + { + if (Directory.Exists(_root)) { + Directory.Delete(_root, true); + } + } + + [Test] + public async Task RoundTripsNestedTreeInEditorAndRuntimeResolver() + { + var source = CreateFixture(); + var package = Path.Combine(_root, "table.vpe"); + var contentRef = CreatePackage(package, source, new ContentPackOptions { EntryPoint = "machine/config.yaml" }); + var cache = Path.Combine(_root, "cache"); + var resolver = new PackagedContentResolver(package, new PackagedContentCacheOptions { CacheRoot = cache }); + + var progress = 0f; + var resolved = await resolver.ResolveAsync(contentRef, new InlineProgress(value => progress = value), CancellationToken.None); + + File.ReadAllText(Path.Combine(resolved, "machine", "config.yaml")).Should().Be("name: synthetic\n"); + File.ReadAllBytes(Path.Combine(resolved, "media", "large.mp4")).Should().HaveCount(2 * 1024 * 1024); + File.Exists(Path.Combine(resolved, "fonts", "Tést Font.ttf")).Should().BeTrue(); + new FileInfo(Path.Combine(resolved, "empty.py")).Length.Should().Be(0); + File.Exists(Path.Combine(resolved, ".complete")).Should().BeTrue(); + progress.Should().Be(1f); + PackagedContentValidator.ValidatePackage(package).Should().BeEmpty(); + } + + [Test] + public async Task IdenticalTreesAreDeterministicAndShareCacheEntry() + { + var firstSource = CreateFixture("first"); + var secondSource = CreateFixture("second"); + var firstPackage = Path.Combine(_root, "first.vpe"); + var secondPackage = Path.Combine(_root, "second.vpe"); + var firstRef = CreatePackage(firstPackage, firstSource); + var secondRef = CreatePackage(secondPackage, secondSource); + firstRef.Id.Should().Be(secondRef.Id); + firstRef.ContentHash.Should().Be(secondRef.ContentHash); + + var cache = Path.Combine(_root, "shared-cache"); + var firstPath = await new PackagedContentResolver(firstPackage, CacheOptions(cache)).ResolveAsync(firstRef, null, CancellationToken.None); + var secondPath = await new PackagedContentResolver(secondPackage, CacheOptions(cache)).ResolveAsync(secondRef, null, CancellationToken.None); + secondPath.Should().Be(firstPath); + Directory.GetDirectories(cache).Should().ContainSingle(); + } + + [Test] + public void IncludesAndExcludesUseForwardSlashGlobs() + { + var source = CreateFixture(); + var prepared = PackagedContentWriter.PrepareDirectory("test", source, new ContentPackOptions { + IncludeGlobs = new[] { "machine/**", "fonts/*.ttf", "**/*.py" }, + ExcludeGlobs = new[] { "**/*.tmp" }, + }); + prepared.Manifest.Files.Select(file => file.Path).Should().BeEquivalentTo( + "machine/config.yaml", "machine/deep/a/b/c/data.bin", "fonts/Tést Font.ttf", "empty.py"); + } + + [TestCase("../evil")] + [TestCase("C:\\abs")] + [TestCase("C:/abs")] + [TestCase("a\\..\\..\\b")] + [TestCase("a/../../b")] + [TestCase("file:stream")] + [TestCase("/absolute")] + [TestCase("a//b")] + public void RejectsUnsafePathsWithActionableMessages(string path) + { + Action action = () => PackagedContentPath.ValidateRelative(path); + action.Should().Throw().Which.Message.Should().Contain(path); + } + + [Test] + public void RejectsCaseCollisionsInManifest() + { + var reference = new PackagedContentRef(new string('a', 16), "test", null, new string('a', 64)); + var manifest = new PackagedContentManifest { + Format = "vpe-content", + Version = 1, + Kind = "test", + FileCount = 2, + TotalBytes = 0, + ContentHash = reference.ContentHash, + Files = { + new PackagedContentFile { Path = "A.txt", Size = 0, Sha256 = new string('0', 64) }, + new PackagedContentFile { Path = "a.txt", Size = 0, Sha256 = new string('0', 64) }, + }, + }; + Action action = () => PackagedContentValidator.ValidateManifest(manifest, reference); + action.Should().Throw().WithMessage("*differ only by case*"); + } + + [Test] + public async Task CancellationLeavesNoValidLookingPartialAndNextResolveSucceeds() + { + var source = CreateFixture(); + File.WriteAllBytes(Path.Combine(source, "cancel.bin"), new byte[16 * 1024 * 1024]); + var package = Path.Combine(_root, "cancel.vpe"); + var contentRef = CreatePackage(package, source); + var cache = Path.Combine(_root, "cache"); + var cts = new CancellationTokenSource(); + var resolver = new PackagedContentResolver(package, CacheOptions(cache)); + var progress = new InlineProgress(value => { + if (value > 0f) cts.Cancel(); + }); + + Func canceled = () => resolver.ResolveAsync(contentRef, progress, cts.Token); + await canceled.Should().ThrowAsync(); + Directory.EnumerateFiles(cache, ".complete", SearchOption.AllDirectories).Should().BeEmpty(); + + var resolved = await resolver.ResolveAsync(contentRef, null, CancellationToken.None); + File.Exists(Path.Combine(resolved, ".complete")).Should().BeTrue(); + } + + [Test] + public async Task StartupRemovesOrphanTempAndUnmarkedFinalIsReplaced() + { + var source = CreateFixture(); + var package = Path.Combine(_root, "partial.vpe"); + var contentRef = CreatePackage(package, source); + var cache = Path.Combine(_root, "cache"); + var orphan = Path.Combine(cache, contentRef.ContentHash + ".tmp-dead"); + var partial = Path.Combine(cache, contentRef.ContentHash); + Directory.CreateDirectory(orphan); + Directory.CreateDirectory(partial); + File.WriteAllText(Path.Combine(orphan, ".complete"), contentRef.ContentHash); + File.WriteAllText(Path.Combine(partial, "partial"), "bad"); + + var resolver = new PackagedContentResolver(package, CacheOptions(cache)); + Directory.Exists(orphan).Should().BeFalse(); + var resolved = await resolver.ResolveAsync(contentRef, null, CancellationToken.None); + File.Exists(Path.Combine(resolved, "partial")).Should().BeFalse(); + File.Exists(Path.Combine(resolved, ".complete")).Should().BeTrue(); + } + + [Test] + public async Task CacheCapEvictsLeastRecentlyUsedCompletedBundle() + { + var firstSource = CreateFixture("lru-first"); + var secondSource = CreateFixture("lru-second"); + File.WriteAllText(Path.Combine(secondSource, "different.txt"), "different"); + var firstPackage = Path.Combine(_root, "lru-first.vpe"); + var secondPackage = Path.Combine(_root, "lru-second.vpe"); + var firstRef = CreatePackage(firstPackage, firstSource); + var secondRef = CreatePackage(secondPackage, secondSource); + var cache = Path.Combine(_root, "lru-cache"); + var options = new PackagedContentCacheOptions { CacheRoot = cache, CapacityBytes = 3 * 1024 * 1024 }; + var firstPath = await new PackagedContentResolver(firstPackage, options).ResolveAsync(firstRef, null, CancellationToken.None); + Directory.SetLastWriteTimeUtc(firstPath, DateTime.UtcNow.AddHours(-1)); + + var secondPath = await new PackagedContentResolver(secondPackage, options).ResolveAsync(secondRef, null, CancellationToken.None); + + Directory.Exists(firstPath).Should().BeFalse(); + Directory.Exists(secondPath).Should().BeTrue(); + } + + [Test] + public void ValidatorReportsMissingCorruptAndOversizedContent() + { + var source = CreateFixture(); + var prepared = PackagedContentWriter.PrepareDirectory("test", source); + var missingPackage = Path.Combine(_root, "missing.vpe"); + using (var storage = PackageApi.StorageManager.CreateStorage(missingPackage)) { + var table = storage.AddFolder(PackageApi.TableFolder); + var bundle = table.AddFolder(PackageApi.ContentFolder).AddFolder(prepared.Reference.Id); + bundle.AddFile("manifest", PackageApi.Packer.FileExtension).SetData(PackageApi.Packer.Pack(prepared.Manifest)); + bundle.AddFolder("files"); + } + PackagedContentValidator.ValidatePackage(missingPackage).Single().Should().Contain("missing"); + PackagedContentValidator.ValidatePackage(missingPackage, maxBundleBytes: 1).Single().Should().Contain("limit"); + + prepared.Manifest.Files[0].Sha256 = new string('0', 64); + var corruptPackage = Path.Combine(_root, "corrupt.vpe"); + using (var storage = PackageApi.StorageManager.CreateStorage(corruptPackage)) { + var table = storage.AddFolder(PackageApi.TableFolder); + new PackagedContentWriter(table).Write(prepared); + } + PackagedContentValidator.ValidatePackage(corruptPackage).Single().Should().Contain("contentHash"); + } + + [Test] + public void ValidatorRejectsPayloadThatUnderdeclaresItsActualSize() + { + var source = Path.Combine(_root, "underdeclared-source"); + Directory.CreateDirectory(source); + File.WriteAllBytes(Path.Combine(source, "payload.bin"), new byte[] { 1, 2, 3, 4 }); + var prepared = PackagedContentWriter.PrepareDirectory("test", source); + prepared.Manifest.Files[0].Size = 1; + prepared.Manifest.TotalBytes = 1; + var package = Path.Combine(_root, "underdeclared.vpe"); + using (var storage = PackageApi.StorageManager.CreateStorage(package)) { + var table = storage.AddFolder(PackageApi.TableFolder); + new PackagedContentWriter(table).Write(prepared); + } + + PackagedContentValidator.ValidatePackage(package).Single().Should().Contain("manifest declares 1"); + } + + [Test] + public void LinterReportsEntryExecutableDuplicateAndPerFileSizeProblems() + { + var source = Path.Combine(_root, "lint-source"); + Directory.CreateDirectory(source); + File.WriteAllBytes(Path.Combine(source, "first.bin"), new byte[] { 1, 2, 3 }); + File.WriteAllBytes(Path.Combine(source, "second.bin"), new byte[] { 1, 2, 3 }); + File.WriteAllBytes(Path.Combine(source, "helper.exe"), new byte[] { 4, 5, 6 }); + var prepared = PackagedContentWriter.PrepareDirectory("web-show", source); + + var issues = PackagedContentValidator.LintManifest(prepared.Manifest, prepared.Reference, + requireEntryPoint: true, forbidExecutables: true, maxFileBytes: 2); + + issues.Select(issue => issue.Code).Should().Contain(new[] { + "CONTENT_ENTRY_POINT_REQUIRED", + "CONTENT_EXECUTABLE_FORBIDDEN", + "CONTENT_DUPLICATE_PAYLOAD", + "CONTENT_FILE_OVERSIZED", + }); + } + + [Test] + public void SkipsSymbolicLinksInsteadOfFollowingThem() + { + var source = CreateFixture(); + var outside = Path.Combine(_root, "outside.txt"); + File.WriteAllText(outside, "outside"); + var link = Path.Combine(source, "outside-link.txt"); + var createSymbolicLink = typeof(File).GetMethod("CreateSymbolicLink", new[] { typeof(string), typeof(string) }); + if (createSymbolicLink == null) { + Assert.Ignore("Symbolic link creation is unavailable on this runtime."); + } + try { + createSymbolicLink.Invoke(null, new object[] { link, outside }); + } catch (Exception) { + Assert.Ignore("Symbolic link creation is unavailable on this host."); + } + string warning = null; + var prepared = PackagedContentWriter.PrepareDirectory("test", source, new ContentPackOptions { Warning = value => warning = value }); + prepared.Manifest.Files.Select(file => file.Path).Should().NotContain("outside-link.txt"); + warning.Should().Contain("Skipping symbolic link"); + } + + [Test, Category("Performance")] + public async Task Synthetic400MbBundlePerformance() + { + var source = Path.Combine(_root, "perf-source"); + Directory.CreateDirectory(source); + var bigFile = Path.Combine(source, "payload.bin"); + using (var stream = new FileStream(bigFile, FileMode.CreateNew, FileAccess.Write)) { + stream.SetLength(400L * 1024 * 1024); + } + var package = Path.Combine(_root, "perf.vpe"); + var stopwatch = Stopwatch.StartNew(); + var contentRef = CreatePackage(package, source); + var packMs = stopwatch.ElapsedMilliseconds; + var resolver = new PackagedContentResolver(package, CacheOptions(Path.Combine(_root, "perf-cache"))); + stopwatch.Restart(); + await resolver.ResolveAsync(contentRef, null, CancellationToken.None); + var firstLoadMs = stopwatch.ElapsedMilliseconds; + stopwatch.Restart(); + await resolver.ResolveAsync(contentRef, null, CancellationToken.None); + var cachedLoadMs = stopwatch.ElapsedMilliseconds; + var result = $"400 MB: pack={packMs}ms, first-load={firstLoadMs}ms, cached-load={cachedLoadMs}ms"; + Console.WriteLine(result); + } + + private string CreateFixture(string name = "source") + { + var source = Path.Combine(_root, name); + Directory.CreateDirectory(Path.Combine(source, "machine", "deep", "a", "b", "c")); + Directory.CreateDirectory(Path.Combine(source, "media")); + Directory.CreateDirectory(Path.Combine(source, "fonts")); + File.WriteAllText(Path.Combine(source, "machine", "config.yaml"), "name: synthetic\n"); + File.WriteAllBytes(Path.Combine(source, "machine", "deep", "a", "b", "c", "data.bin"), new byte[] { 0, 1, 2, 3 }); + File.WriteAllBytes(Path.Combine(source, "media", "large.mp4"), new byte[2 * 1024 * 1024]); + File.WriteAllText(Path.Combine(source, "fonts", "Tést Font.ttf"), "synthetic font"); + File.WriteAllBytes(Path.Combine(source, "empty.py"), Array.Empty()); + return source; + } + + private static PackagedContentRef CreatePackage(string package, string source, ContentPackOptions options = null) + { + using var storage = PackageApi.StorageManager.CreateStorage(package); + var table = storage.AddFolder(PackageApi.TableFolder); + return new PackagedContentWriter(table).AddDirectory("test", source, options); + } + + private static PackagedContentCacheOptions CacheOptions(string root) => new() { CacheRoot = root }; + + private sealed class InlineProgress : IProgress + { + private readonly Action _report; + public InlineProgress(Action report) => _report = report; + public void Report(float value) => _report(value); + } + } +} diff --git a/VisualPinball.Unity/VisualPinball.Unity.Test/Packaging/PackagedContentTests.cs.meta b/VisualPinball.Unity/VisualPinball.Unity.Test/Packaging/PackagedContentTests.cs.meta new file mode 100644 index 000000000..e98b8b39a --- /dev/null +++ b/VisualPinball.Unity/VisualPinball.Unity.Test/Packaging/PackagedContentTests.cs.meta @@ -0,0 +1,2 @@ +fileFormatVersion: 2 +guid: 3440e6eeb8d86e446b63d54175dc4560 \ No newline at end of file diff --git a/VisualPinball.Unity/VisualPinball.Unity/Packaging/Content.meta b/VisualPinball.Unity/VisualPinball.Unity/Packaging/Content.meta new file mode 100644 index 000000000..96916f9f6 --- /dev/null +++ b/VisualPinball.Unity/VisualPinball.Unity/Packaging/Content.meta @@ -0,0 +1,8 @@ +fileFormatVersion: 2 +guid: 1894b068646189c48bb5b5fb25e30eec +folderAsset: yes +DefaultImporter: + externalObjects: {} + userData: + assetBundleName: + assetBundleVariant: diff --git a/VisualPinball.Unity/VisualPinball.Unity/Packaging/Content/PackagedContent.cs b/VisualPinball.Unity/VisualPinball.Unity/Packaging/Content/PackagedContent.cs new file mode 100644 index 000000000..fdc58d1a3 --- /dev/null +++ b/VisualPinball.Unity/VisualPinball.Unity/Packaging/Content/PackagedContent.cs @@ -0,0 +1,119 @@ +// Visual Pinball Engine +// Copyright (C) 2026 freezy and VPE Team +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +using System; +using System.Collections.Generic; +using System.Threading; +using System.Threading.Tasks; +using Newtonsoft.Json; + +namespace VisualPinball.Unity +{ + /// Stable reference to an inert directory stored in a .vpe package. + [Serializable] + public struct PackagedContentRef + { + public string Id; + public string Kind; + public string EntryPoint; + public string ContentHash; + +#if UNITY_EDITOR + [JsonIgnore] public string SourceDirectory; + [JsonIgnore] public int FileCount; + [JsonIgnore] public long TotalBytes; + [JsonIgnore] public string ValidationStatus; +#endif + + public PackagedContentRef(string id, string kind, string entryPoint, string contentHash) + { + Id = id; + Kind = kind; + EntryPoint = entryPoint; + ContentHash = contentHash; +#if UNITY_EDITOR + SourceDirectory = null; + FileCount = 0; + TotalBytes = 0; + ValidationStatus = null; +#endif + } + } + + public interface IPackagedContentResolver + { + PackagedContentRef AddDirectory(string kind, string sourceRoot, ContentPackOptions options); + Task ResolveAsync(PackagedContentRef contentRef, IProgress progress, CancellationToken ct); + } + + /// + /// Implemented by components that contribute a source directory at package-write time. The + /// component must update its serialized during this callback. + /// + public interface IPackagedContentSource + { + void PreparePackagedContent(IPackagedContentResolver resolver); + } + + /// + /// Implemented by restored components that consume content. Readers inject the resolver before + /// activating a runtime table, so consumers never need access to package or zip internals. + /// + public interface IPackagedContentConsumer + { + void SetPackagedContentResolver(IPackagedContentResolver resolver); + } + + [Serializable] + public sealed class ContentPackOptions + { + public string EntryPoint; + public string[] IncludeGlobs = Array.Empty(); + public string[] ExcludeGlobs = Array.Empty(); + public int MaxFileCount = PackagedContentLimits.DefaultMaxFileCount; + public long MaxTotalBytes = PackagedContentLimits.DefaultMaxBundleBytes; + + [JsonIgnore] + public Action Warning; + } + + public sealed class PackagedContentCacheOptions + { + public string CacheRoot; + public long CapacityBytes = 4L * 1024 * 1024 * 1024; + public int MaxFileCount = PackagedContentLimits.DefaultMaxFileCount; + public long MaxBundleBytes = PackagedContentLimits.DefaultMaxBundleBytes; + } + + public static class PackagedContentLimits + { + public const int DefaultMaxFileCount = 1_000_000; + public const long DefaultMaxBundleBytes = 16L * 1024 * 1024 * 1024; + } + + [Serializable] + public sealed class PackagedContentManifest + { + [JsonProperty("format")] public string Format; + [JsonProperty("version")] public int Version; + [JsonProperty("kind")] public string Kind; + [JsonProperty("entryPoint")] public string EntryPoint; + [JsonProperty("fileCount")] public int FileCount; + [JsonProperty("totalBytes")] public long TotalBytes; + [JsonProperty("contentHash")] public string ContentHash; + [JsonProperty("files")] public List Files = new(); + } + + [Serializable] + public sealed class PackagedContentFile + { + [JsonProperty("path")] public string Path; + [JsonProperty("size")] public long Size; + [JsonProperty("sha256")] public string Sha256; + } +} diff --git a/VisualPinball.Unity/VisualPinball.Unity/Packaging/Content/PackagedContent.cs.meta b/VisualPinball.Unity/VisualPinball.Unity/Packaging/Content/PackagedContent.cs.meta new file mode 100644 index 000000000..83ee4444b --- /dev/null +++ b/VisualPinball.Unity/VisualPinball.Unity/Packaging/Content/PackagedContent.cs.meta @@ -0,0 +1,2 @@ +fileFormatVersion: 2 +guid: ec0f81ce05e24b346a4e7ef6c720abd6 \ No newline at end of file diff --git a/VisualPinball.Unity/VisualPinball.Unity/Packaging/Content/PackagedContentConsent.cs b/VisualPinball.Unity/VisualPinball.Unity/Packaging/Content/PackagedContentConsent.cs new file mode 100644 index 000000000..283993988 --- /dev/null +++ b/VisualPinball.Unity/VisualPinball.Unity/Packaging/Content/PackagedContentConsent.cs @@ -0,0 +1,155 @@ +// Visual Pinball Engine +// Copyright (C) 2026 freezy and VPE Team +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. +// +// You should have received a copy of the GNU General Public License +// along with this program. If not, see . + +using System; +using System.Threading; +using System.Threading.Tasks; +using UnityEngine; + +namespace VisualPinball.Unity +{ + /// + /// Stores a user's explicit permission to execute or render active content extracted from a + /// package. Acknowledgements are keyed by the full content hash, so changed content is always + /// presented to the user again. + /// + public static class PackagedContentConsent + { + private const string KeyPrefix = "vpe.content.consent.v1."; + + /// Test/application hook for supplying a branded consent UI. + public static Func> PromptOverride; + + public static bool IsAcknowledged(PackagedContentRef contentRef) + { + Validate(contentRef); + return PlayerPrefs.GetInt(GetKey(contentRef), 0) == 1; + } + + public static async Task RequireAsync(PackagedContentRef contentRef, string description, CancellationToken ct) + { + Validate(contentRef); + if (IsAcknowledged(contentRef)) { + return; + } + + ct.ThrowIfCancellationRequested(); + var prompt = PromptOverride; + var accepted = prompt != null + ? await prompt(contentRef, description, ct) + : await PromptDefaultAsync(contentRef, description, ct); + ct.ThrowIfCancellationRequested(); + if (!accepted) { + throw new UnauthorizedAccessException( + $"Permission to use packaged {description} was denied. Content hash: {contentRef.ContentHash}." + ); + } + + PlayerPrefs.SetInt(GetKey(contentRef), 1); + PlayerPrefs.Save(); + } + + public static void Forget(PackagedContentRef contentRef) + { + Validate(contentRef); + PlayerPrefs.DeleteKey(GetKey(contentRef)); + } + + private static string GetKey(PackagedContentRef contentRef) => KeyPrefix + contentRef.ContentHash; + + private static void Validate(PackagedContentRef contentRef) + { + if (string.IsNullOrWhiteSpace(contentRef.ContentHash)) { + throw new ArgumentException("Packaged content consent requires a content hash.", nameof(contentRef)); + } + } + + private static Task PromptDefaultAsync(PackagedContentRef contentRef, string description, CancellationToken ct) + { +#if UNITY_EDITOR + ct.ThrowIfCancellationRequested(); + var accepted = UnityEditor.EditorUtility.DisplayDialog( + "Allow packaged table content?", + $"This table contains {description} that will be used outside the package cache.\n\n" + + $"Content hash: {contentRef.ContentHash}\n\nOnly allow tables from sources you trust.", + "Allow", + "Deny" + ); + return Task.FromResult(accepted); +#else + return PackagedContentConsentPrompt.ShowAsync(contentRef, description, ct); +#endif + } + } + +#if !UNITY_EDITOR + internal sealed class PackagedContentConsentPrompt : MonoBehaviour + { + private TaskCompletionSource _completion; + private CancellationTokenRegistration _registration; + private string _message; + private Rect _window = new Rect(0, 0, 560, 260); + + internal static Task ShowAsync(PackagedContentRef contentRef, string description, CancellationToken ct) + { + var gameObject = new GameObject("Packaged Content Consent") { hideFlags = HideFlags.HideAndDontSave }; + DontDestroyOnLoad(gameObject); + var prompt = gameObject.AddComponent(); + prompt._completion = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + prompt._message = $"This table contains {description}. Only allow tables from sources you trust.\n\n" + + $"Content hash: {contentRef.ContentHash}"; + prompt._registration = ct.Register(() => prompt.CompleteCanceled(ct)); + return prompt._completion.Task; + } + + private void OnGUI() + { + _window.x = (Screen.width - _window.width) / 2f; + _window.y = (Screen.height - _window.height) / 2f; + _window = GUI.ModalWindow(GetInstanceID(), _window, DrawWindow, "Allow packaged table content?"); + } + + private void DrawWindow(int id) + { + GUILayout.Space(12); + GUILayout.Label(_message, new GUIStyle(GUI.skin.label) { wordWrap = true }); + GUILayout.FlexibleSpace(); + GUILayout.BeginHorizontal(); + if (GUILayout.Button("Deny", GUILayout.Height(38))) { + Complete(false); + } + if (GUILayout.Button("Allow", GUILayout.Height(38))) { + Complete(true); + } + GUILayout.EndHorizontal(); + GUILayout.Space(8); + } + + private void Complete(bool accepted) + { + _registration.Dispose(); + _completion.TrySetResult(accepted); + Destroy(gameObject); + } + + private void CompleteCanceled(CancellationToken ct) + { + _completion.TrySetCanceled(ct); + Destroy(gameObject); + } + } +#endif +} diff --git a/VisualPinball.Unity/VisualPinball.Unity/Packaging/Content/PackagedContentConsent.cs.meta b/VisualPinball.Unity/VisualPinball.Unity/Packaging/Content/PackagedContentConsent.cs.meta new file mode 100644 index 000000000..2602cf3a0 --- /dev/null +++ b/VisualPinball.Unity/VisualPinball.Unity/Packaging/Content/PackagedContentConsent.cs.meta @@ -0,0 +1,11 @@ +fileFormatVersion: 2 +guid: 58bcb19bb36245ed9ad5da3b13dcfda1 +MonoImporter: + externalObjects: {} + serializedVersion: 2 + defaultReferences: [] + executionOrder: 0 + icon: {instanceID: 0} + userData: + assetBundleName: + assetBundleVariant: diff --git a/VisualPinball.Unity/VisualPinball.Unity/Packaging/Content/PackagedContentPath.cs b/VisualPinball.Unity/VisualPinball.Unity/Packaging/Content/PackagedContentPath.cs new file mode 100644 index 000000000..edcc39b0c --- /dev/null +++ b/VisualPinball.Unity/VisualPinball.Unity/Packaging/Content/PackagedContentPath.cs @@ -0,0 +1,71 @@ +// Visual Pinball Engine +// Copyright (C) 2026 freezy and VPE Team +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. +// +// You should have received a copy of the GNU General Public License +// along with this program. If not, see . + +using System; +using System.Collections.Generic; +using System.IO; +using System.Text.RegularExpressions; + +namespace VisualPinball.Unity +{ + public static class PackagedContentPath + { + private static readonly Regex DrivePrefix = new("^[A-Za-z]:", RegexOptions.CultureInvariant); + + public static string ValidateRelative(string path, string description = "content path") + { + if (string.IsNullOrWhiteSpace(path)) { + throw new InvalidDataException($"The {description} is empty."); + } + if (path.IndexOf('\\') >= 0) { + throw new InvalidDataException($"The {description} '{path}' contains a backslash; package paths must use '/'."); + } + if (path.IndexOf(':') >= 0) { + throw new InvalidDataException($"The {description} '{path}' contains ':' (drive prefixes and alternate data streams are forbidden)."); + } + if (Path.IsPathRooted(path) || path.StartsWith("/", StringComparison.Ordinal) || DrivePrefix.IsMatch(path)) { + throw new InvalidDataException($"The {description} '{path}' is absolute."); + } + + var parts = path.Split('/'); + foreach (var part in parts) { + if (part.Length == 0 || part == "." || part == "..") { + throw new InvalidDataException($"The {description} '{path}' contains an empty, '.' or '..' segment."); + } + } + return string.Join("/", parts); + } + + public static string GetContainedPath(string root, string relativePath) + { + var safeRelativePath = ValidateRelative(relativePath); + var fullRoot = Path.GetFullPath(root).TrimEnd(Path.DirectorySeparatorChar, Path.AltDirectorySeparatorChar); + var candidate = Path.GetFullPath(Path.Combine(fullRoot, safeRelativePath.Replace('/', Path.DirectorySeparatorChar))); + var prefix = fullRoot + Path.DirectorySeparatorChar; + var comparison = OperatingSystemPathComparison; + if (!candidate.StartsWith(prefix, comparison)) { + throw new InvalidDataException($"Content path '{relativePath}' escapes cache root '{fullRoot}'."); + } + return candidate; + } + + public static StringComparer FileSystemNameComparer => + Path.DirectorySeparatorChar == '\\' ? StringComparer.OrdinalIgnoreCase : StringComparer.Ordinal; + + private static StringComparison OperatingSystemPathComparison => + Path.DirectorySeparatorChar == '\\' ? StringComparison.OrdinalIgnoreCase : StringComparison.Ordinal; + } +} diff --git a/VisualPinball.Unity/VisualPinball.Unity/Packaging/Content/PackagedContentPath.cs.meta b/VisualPinball.Unity/VisualPinball.Unity/Packaging/Content/PackagedContentPath.cs.meta new file mode 100644 index 000000000..4cd277250 --- /dev/null +++ b/VisualPinball.Unity/VisualPinball.Unity/Packaging/Content/PackagedContentPath.cs.meta @@ -0,0 +1,2 @@ +fileFormatVersion: 2 +guid: 7f21dc97308daa84fa8fc0ab3034dc5b \ No newline at end of file diff --git a/VisualPinball.Unity/VisualPinball.Unity/Packaging/Content/PackagedContentResolver.cs b/VisualPinball.Unity/VisualPinball.Unity/Packaging/Content/PackagedContentResolver.cs new file mode 100644 index 000000000..9b530ba38 --- /dev/null +++ b/VisualPinball.Unity/VisualPinball.Unity/Packaging/Content/PackagedContentResolver.cs @@ -0,0 +1,249 @@ +// Visual Pinball Engine +// Copyright (C) 2026 freezy and VPE Team +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. +// +// You should have received a copy of the GNU General Public License +// along with this program. If not, see . + +using System; +using System.Collections.Generic; +using System.IO; +using System.Linq; +using System.Security.Cryptography; +using System.Text; +using System.Threading; +using System.Threading.Tasks; +using UnityEngine; + +namespace VisualPinball.Unity +{ + /// Editor- and Player-safe resolver for content stored in a .vpe package. + public sealed class PackagedContentResolver : IPackagedContentResolver + { + private const string CompleteMarker = ".complete"; + private readonly string _packagePath; + private readonly PackagedContentCacheOptions _options; + private readonly string _cacheRoot; + + public PackagedContentResolver(string packagePath, PackagedContentCacheOptions options = null) + { + _packagePath = Path.GetFullPath(packagePath ?? throw new ArgumentNullException(nameof(packagePath))); + _options = options ?? new PackagedContentCacheOptions(); + _cacheRoot = Path.GetFullPath(string.IsNullOrWhiteSpace(_options.CacheRoot) + ? Path.Combine(Application.persistentDataPath, "ContentCache") + : _options.CacheRoot); + Directory.CreateDirectory(_cacheRoot); + CleanupTemporaryDirectories(); + } + + public PackagedContentRef AddDirectory(string kind, string sourceRoot, ContentPackOptions options) + { + throw new NotSupportedException("This resolver is read-only. Use PackagedContentWriter while creating a package."); + } + + public async Task ResolveAsync(PackagedContentRef contentRef, IProgress progress, CancellationToken ct) + { + PackagedContentValidator.ValidateReference(contentRef); + if (!File.Exists(_packagePath)) { + throw new FileNotFoundException("The .vpe package containing this content does not exist.", _packagePath); + } + + var destination = Path.Combine(_cacheRoot, contentRef.ContentHash); + if (IsComplete(destination, contentRef.ContentHash)) { + Directory.SetLastWriteTimeUtc(destination, DateTime.UtcNow); + progress?.Report(1f); + return destination; + } + + var temporary = Path.Combine(_cacheRoot, contentRef.ContentHash + ".tmp-" + Guid.NewGuid().ToString("N")); + Directory.CreateDirectory(temporary); + try { + using var storage = PackageApi.StorageManager.OpenStorage(_packagePath); + var bundleFolder = GetBundleFolder(storage, contentRef.Id); + var manifest = ReadManifest(bundleFolder, contentRef.Id); + PackagedContentValidator.ValidateManifest(manifest, contentRef, _options.MaxFileCount, _options.MaxBundleBytes); + var filesFolder = bundleFolder.GetFolder("files"); + long writtenBytes = 0; + foreach (var entry in manifest.Files.OrderBy(file => file.Path, StringComparer.Ordinal)) { + ct.ThrowIfCancellationRequested(); + var targetPath = PackagedContentPath.GetContainedPath(temporary, entry.Path); + Directory.CreateDirectory(Path.GetDirectoryName(targetPath)!); + var packageFile = GetFile(filesFolder, entry.Path); + using var source = packageFile.AsStream(); + using var target = new FileStream(targetPath, FileMode.CreateNew, FileAccess.Write, FileShare.None, 128 * 1024, true); + using var sha = SHA256.Create(); + var buffer = new byte[128 * 1024]; + long fileBytes = 0; + while (true) { + var read = await source.ReadAsync(buffer, 0, buffer.Length, ct); + if (read == 0) { + break; + } + await target.WriteAsync(buffer, 0, read, ct); + sha.TransformBlock(buffer, 0, read, null, 0); + fileBytes += read; + writtenBytes += read; + if (fileBytes > entry.Size || writtenBytes > manifest.TotalBytes) { + throw new InvalidDataException($"Content file '{entry.Path}' is larger than declared in its manifest."); + } + progress?.Report(manifest.TotalBytes == 0 ? 1f : System.Math.Min(1f, writtenBytes / (float)manifest.TotalBytes)); + } + sha.TransformFinalBlock(Array.Empty(), 0, 0); + if (fileBytes != entry.Size) { + throw new InvalidDataException($"Content file '{entry.Path}' has {fileBytes} bytes; manifest declares {entry.Size}."); + } + if (!string.Equals(PackagedContentWriter.ToHex(sha.Hash), entry.Sha256, StringComparison.Ordinal)) { + throw new InvalidDataException($"Content file '{entry.Path}' failed SHA-256 verification."); + } + } + + ct.ThrowIfCancellationRequested(); + File.WriteAllText(Path.Combine(temporary, CompleteMarker), contentRef.ContentHash, new UTF8Encoding(false)); + Publish(temporary, destination, contentRef.ContentHash); + progress?.Report(1f); + TrimCache(contentRef.ContentHash); + return destination; + } catch { + TryDeleteDirectory(temporary); + throw; + } + } + + private static IPackageFolder GetBundleFolder(IPackageStorage storage, string id) + { + var table = storage.GetFolder(PackageApi.TableFolder); + if (!table.TryGetFolder(PackageApi.ContentFolder, out var content) || !content.TryGetFolder(id, out var bundle)) { + throw new InvalidDataException($"Package does not contain referenced content bundle '{id}'."); + } + return bundle; + } + + private static PackagedContentManifest ReadManifest(IPackageFolder bundle, string id) + { + if (!bundle.TryGetFile("manifest", out var file, PackageApi.Packer.FileExtension)) { + throw new InvalidDataException($"Content bundle '{id}' is missing manifest.json."); + } + try { + return PackageApi.Packer.Unpack(file.GetData()); + } catch (Exception ex) { + throw new InvalidDataException($"Content bundle '{id}' has a corrupt manifest.json.", ex); + } + } + + private static IPackageFile GetFile(IPackageFolder root, string path) + { + var parts = PackagedContentPath.ValidateRelative(path).Split('/'); + var folder = root; + for (var i = 0; i < parts.Length - 1; i++) { + if (!folder.TryGetFolder(parts[i], out folder)) { + throw new InvalidDataException($"Content file '{path}' is missing from the package."); + } + } + if (!folder.TryGetFile(parts[^1], out var file)) { + throw new InvalidDataException($"Content file '{path}' is missing from the package."); + } + return file; + } + + private static void Publish(string temporary, string destination, string hash) + { + if (Directory.Exists(destination)) { + if (IsComplete(destination, hash)) { + TryDeleteDirectory(temporary); + return; + } + TryDeleteDirectory(destination); + } + try { + Directory.Move(temporary, destination); + } catch (IOException) when (IsComplete(destination, hash)) { + TryDeleteDirectory(temporary); + } + } + + private static bool IsComplete(string directory, string hash) + { + try { + var marker = Path.Combine(directory, CompleteMarker); + return Directory.Exists(directory) && File.Exists(marker) && File.ReadAllText(marker).Trim() == hash; + } catch { + return false; + } + } + + private void CleanupTemporaryDirectories() + { + foreach (var directory in Directory.EnumerateDirectories(_cacheRoot, "*.tmp-*", SearchOption.TopDirectoryOnly)) { + TryDeleteDirectory(directory); + } + } + + private void TrimCache(string protectedHash) + { + if (_options.CapacityBytes < 0) { + return; + } + var entries = Directory.EnumerateDirectories(_cacheRoot) + .Where(path => Path.GetFileName(path).IndexOf(".tmp-", StringComparison.Ordinal) < 0) + .Select(path => new CacheEntry(path, GetDirectorySize(path), Directory.GetLastWriteTimeUtc(path))) + .OrderBy(entry => entry.LastAccessUtc) + .ToList(); + var total = entries.Sum(entry => entry.Size); + foreach (var entry in entries) { + if (total <= _options.CapacityBytes) { + break; + } + if (Path.GetFileName(entry.Path) == protectedHash) { + continue; + } + TryDeleteDirectory(entry.Path); + total -= entry.Size; + } + } + + private static long GetDirectorySize(string directory) + { + try { + return Directory.EnumerateFiles(directory, "*", SearchOption.AllDirectories).Sum(path => new FileInfo(path).Length); + } catch { + return 0; + } + } + + private static void TryDeleteDirectory(string directory) + { + try { + if (Directory.Exists(directory)) { + Directory.Delete(directory, true); + } + } catch (IOException) { + // A second resolver/process may still be publishing or reading this directory. + } catch (UnauthorizedAccessException) { + // Best-effort cache maintenance must not prevent content from loading. + } + } + + private readonly struct CacheEntry + { + public readonly string Path; + public readonly long Size; + public readonly DateTime LastAccessUtc; + + public CacheEntry(string path, long size, DateTime lastAccessUtc) + { + Path = path; + Size = size; + LastAccessUtc = lastAccessUtc; + } + } + } +} diff --git a/VisualPinball.Unity/VisualPinball.Unity/Packaging/Content/PackagedContentResolver.cs.meta b/VisualPinball.Unity/VisualPinball.Unity/Packaging/Content/PackagedContentResolver.cs.meta new file mode 100644 index 000000000..1fc88089b --- /dev/null +++ b/VisualPinball.Unity/VisualPinball.Unity/Packaging/Content/PackagedContentResolver.cs.meta @@ -0,0 +1,2 @@ +fileFormatVersion: 2 +guid: 14487df31df6a36429c3a3bc45056077 \ No newline at end of file diff --git a/VisualPinball.Unity/VisualPinball.Unity/Packaging/Content/PackagedContentValidator.cs b/VisualPinball.Unity/VisualPinball.Unity/Packaging/Content/PackagedContentValidator.cs new file mode 100644 index 000000000..6e9a39312 --- /dev/null +++ b/VisualPinball.Unity/VisualPinball.Unity/Packaging/Content/PackagedContentValidator.cs @@ -0,0 +1,297 @@ +// Visual Pinball Engine +// Copyright (C) 2026 freezy and VPE Team +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. +// +// You should have received a copy of the GNU General Public License +// along with this program. If not, see . + +using System; +using System.Collections.Generic; +using System.IO; +using System.Linq; +using System.Text.RegularExpressions; + +namespace VisualPinball.Unity +{ + public enum PackagedContentLintSeverity + { + Warning, + Error, + } + + public sealed class PackagedContentLintIssue + { + public string Code { get; } + public PackagedContentLintSeverity Severity { get; } + public string Message { get; } + public string Path { get; } + + public PackagedContentLintIssue(string code, PackagedContentLintSeverity severity, string message, string path = null) + { + Code = code; + Severity = severity; + Message = message; + Path = path; + } + + public override string ToString() => string.IsNullOrEmpty(Path) + ? $"{Severity} {Code}: {Message}" + : $"{Severity} {Code} ({Path}): {Message}"; + } + + public static class PackagedContentValidator + { + private static readonly Regex Sha256 = new("^[0-9a-f]{64}$", RegexOptions.CultureInvariant); + private static readonly HashSet ExecutableExtensions = new(StringComparer.OrdinalIgnoreCase) { + ".app", ".bat", ".cmd", ".com", ".dll", ".dylib", ".exe", ".msi", ".ps1", ".scr", ".so", + }; + + /// + /// Returns actionable authoring diagnostics in addition to the normative manifest validation. + /// Consumers choose whether their bundle kind requires an entry point or forbids executable payloads. + /// + public static IReadOnlyList LintManifest(PackagedContentManifest manifest, + PackagedContentRef contentRef, bool requireEntryPoint = false, bool forbidExecutables = false, + long maxFileBytes = PackagedContentLimits.DefaultMaxBundleBytes, + int maxFileCount = PackagedContentLimits.DefaultMaxFileCount, + long maxBundleBytes = PackagedContentLimits.DefaultMaxBundleBytes) + { + var issues = new List(); + try { + ValidateReference(contentRef); + ValidateManifest(manifest, contentRef, maxFileCount, maxBundleBytes); + } catch (Exception ex) when (ex is InvalidDataException or OverflowException) { + issues.Add(new PackagedContentLintIssue("CONTENT_MANIFEST_INVALID", PackagedContentLintSeverity.Error, ex.Message)); + } + + if (manifest == null) { + return issues; + } + if (requireEntryPoint && string.IsNullOrWhiteSpace(manifest.EntryPoint)) { + issues.Add(new PackagedContentLintIssue("CONTENT_ENTRY_POINT_REQUIRED", PackagedContentLintSeverity.Error, + $"Content kind '{manifest.Kind ?? ""}' requires an entryPoint.")); + } + + var hashes = new Dictionary(StringComparer.Ordinal); + foreach (var file in manifest.Files ?? new List()) { + if (file == null) { + issues.Add(new PackagedContentLintIssue("CONTENT_FILE_NULL", PackagedContentLintSeverity.Error, + "The content manifest contains a null file entry.")); + continue; + } + if (file.Size > maxFileBytes) { + issues.Add(new PackagedContentLintIssue("CONTENT_FILE_OVERSIZED", PackagedContentLintSeverity.Error, + $"File is {file.Size:N0} bytes; the configured per-file limit is {maxFileBytes:N0} bytes.", file.Path)); + } + if (forbidExecutables && ExecutableExtensions.Contains(System.IO.Path.GetExtension(file.Path ?? string.Empty))) { + issues.Add(new PackagedContentLintIssue("CONTENT_EXECUTABLE_FORBIDDEN", PackagedContentLintSeverity.Error, + "Executable payloads are forbidden for this content kind. Use player-owned runtimes and dependencies.", file.Path)); + } + if (!string.IsNullOrEmpty(file.Sha256) && hashes.TryGetValue(file.Sha256, out var firstPath)) { + issues.Add(new PackagedContentLintIssue("CONTENT_DUPLICATE_PAYLOAD", PackagedContentLintSeverity.Warning, + $"File duplicates '{firstPath}' byte-for-byte; remove one copy or reference the shared asset.", file.Path)); + } else if (!string.IsNullOrEmpty(file.Sha256)) { + hashes[file.Sha256] = file.Path; + } + } + return issues; + } + + /// Validates every content manifest and payload in a package without extracting it. + public static IReadOnlyList ValidatePackage(string packagePath, + int maxFileCount = PackagedContentLimits.DefaultMaxFileCount, + long maxBundleBytes = PackagedContentLimits.DefaultMaxBundleBytes) + { + var errors = new List(); + try { + using var storage = PackageApi.StorageManager.OpenStorage(packagePath); + var table = storage.GetFolder(PackageApi.TableFolder); + if (!table.TryGetFolder(PackageApi.ContentFolder, out var content)) { + return errors; + } + content.VisitFolders(bundle => { + try { + if (!Regex.IsMatch(bundle.Name, "^[0-9a-f]{16}$")) { + throw new InvalidDataException($"Content folder id '{bundle.Name}' is not 16 lowercase hexadecimal characters."); + } + if (!bundle.TryGetFile("manifest", out var manifestFile, PackageApi.Packer.FileExtension)) { + throw new InvalidDataException($"Content bundle '{bundle.Name}' is missing manifest.json."); + } + PackagedContentManifest manifest; + try { + manifest = PackageApi.Packer.Unpack(manifestFile.GetData()); + } catch (Exception ex) { + throw new InvalidDataException($"Content bundle '{bundle.Name}' has a corrupt manifest.json.", ex); + } + var contentRef = new PackagedContentRef(bundle.Name, manifest.Kind, manifest.EntryPoint, manifest.ContentHash); + ValidateReference(contentRef); + ValidateManifest(manifest, contentRef, maxFileCount, maxBundleBytes); + if (!bundle.TryGetFolder("files", out var filesFolder)) { + throw new InvalidDataException($"Content bundle '{bundle.Name}' is missing its files directory."); + } + foreach (var entry in manifest.Files) { + var file = FindFile(filesFolder, entry.Path); + using var stream = file.AsStream(); + if (stream.CanSeek && stream.Length != entry.Size) { + throw new InvalidDataException($"Content file '{entry.Path}' in '{bundle.Name}' has {stream.Length:N0} bytes; manifest declares {entry.Size:N0}."); + } + using var sha = System.Security.Cryptography.SHA256.Create(); + long actualBytes = 0; + var buffer = new byte[128 * 1024]; + int read; + while ((read = stream.Read(buffer, 0, buffer.Length)) > 0) { + actualBytes += read; + sha.TransformBlock(buffer, 0, read, null, 0); + } + sha.TransformFinalBlock(Array.Empty(), 0, 0); + if (actualBytes != entry.Size) { + throw new InvalidDataException($"Content file '{entry.Path}' in '{bundle.Name}' has {actualBytes:N0} bytes; manifest declares {entry.Size:N0}."); + } + var hash = sha.Hash; + if (!string.Equals(PackagedContentWriter.ToHex(hash), entry.Sha256, StringComparison.Ordinal)) { + throw new InvalidDataException($"Content file '{entry.Path}' in '{bundle.Name}' failed SHA-256 verification."); + } + } + } catch (Exception ex) { + var detail = ex.InnerException == null ? ex.Message : $"{ex.Message} {ex.InnerException.Message}"; + errors.Add(detail); + } + }); + } catch (Exception ex) { + errors.Add($"Cannot validate package '{packagePath}': {ex.Message}"); + } + return errors; + } + + public static void ValidateReference(PackagedContentRef contentRef) + { + if (string.IsNullOrWhiteSpace(contentRef.Id) || !Regex.IsMatch(contentRef.Id, "^[0-9a-f]{16}$")) { + throw new InvalidDataException($"Content reference id '{contentRef.Id}' must be 16 lowercase hexadecimal characters."); + } + if (string.IsNullOrWhiteSpace(contentRef.Kind)) { + throw new InvalidDataException("Content reference has no kind."); + } + if (string.IsNullOrWhiteSpace(contentRef.ContentHash) || !Sha256.IsMatch(contentRef.ContentHash)) { + throw new InvalidDataException("Content reference hash must be a 64-character lowercase SHA-256 value."); + } + if (!contentRef.ContentHash.StartsWith(contentRef.Id, StringComparison.Ordinal)) { + throw new InvalidDataException("Content reference id does not match its content hash."); + } + if (!string.IsNullOrEmpty(contentRef.EntryPoint)) { + PackagedContentPath.ValidateRelative(contentRef.EntryPoint, "content entry point"); + } + } + + public static void ValidateManifest(PackagedContentManifest manifest, PackagedContentRef contentRef, + int maxFileCount = PackagedContentLimits.DefaultMaxFileCount, + long maxBundleBytes = PackagedContentLimits.DefaultMaxBundleBytes) + { + if (manifest == null) { + throw new InvalidDataException("Content manifest is empty."); + } + if (manifest.Format != "vpe-content" || manifest.Version != 1) { + throw new InvalidDataException($"Unsupported content manifest '{manifest.Format}' version {manifest.Version}; expected vpe-content version 1."); + } + if (manifest.ContentHash != contentRef.ContentHash || manifest.Kind != contentRef.Kind || manifest.EntryPoint != contentRef.EntryPoint) { + throw new InvalidDataException($"Content manifest for '{contentRef.Id}' does not match its PackagedContentRef."); + } + if (!Sha256.IsMatch(manifest.ContentHash ?? string.Empty)) { + throw new InvalidDataException("Content manifest has an invalid contentHash."); + } + if (manifest.FileCount < 0 || manifest.FileCount > maxFileCount) { + throw new InvalidDataException($"Content manifest declares {manifest.FileCount:N0} files; limit is {maxFileCount:N0}."); + } + if (manifest.TotalBytes < 0 || manifest.TotalBytes > maxBundleBytes) { + throw new InvalidDataException($"Content manifest declares {manifest.TotalBytes:N0} bytes; limit is {maxBundleBytes:N0} bytes."); + } + if (manifest.Files == null || manifest.Files.Count != manifest.FileCount) { + throw new InvalidDataException("Content manifest fileCount does not match its files list."); + } + + long totalBytes = 0; + var exactPaths = new HashSet(StringComparer.Ordinal); + var casePaths = new HashSet(StringComparer.OrdinalIgnoreCase); + foreach (var file in manifest.Files) { + PackagedContentPath.ValidateRelative(file.Path); + if (!exactPaths.Add(file.Path)) { + throw new InvalidDataException($"Content manifest contains duplicate path '{file.Path}'."); + } + if (!casePaths.Add(file.Path)) { + throw new InvalidDataException($"Content manifest contains paths that differ only by case at '{file.Path}'."); + } + if (file.Size < 0 || !Sha256.IsMatch(file.Sha256 ?? string.Empty)) { + throw new InvalidDataException($"Content manifest entry '{file.Path}' has an invalid size or SHA-256."); + } + checked { totalBytes += file.Size; } + } + if (totalBytes != manifest.TotalBytes) { + throw new InvalidDataException($"Content manifest totalBytes is {manifest.TotalBytes}; file list totals {totalBytes}."); + } + if (!string.IsNullOrEmpty(manifest.EntryPoint) && !exactPaths.Contains(manifest.EntryPoint)) { + throw new InvalidDataException($"Content entry point '{manifest.EntryPoint}' is not in the files list."); + } + if (!IsCanonicalOrder(manifest.Files)) { + throw new InvalidDataException("Content manifest files are not in deterministic ordinal path order."); + } + var canonicalHash = ComputeCanonicalHash(manifest.Files); + if (canonicalHash != manifest.ContentHash) { + throw new InvalidDataException("Content manifest contentHash does not match its canonical file list."); + } + } + + private static bool IsCanonicalOrder(IReadOnlyList files) + { + for (var i = 1; i < files.Count; i++) { + if (string.CompareOrdinal(files[i - 1].Path, files[i].Path) >= 0) { + return false; + } + } + return true; + } + + private static string ComputeCanonicalHash(IEnumerable files) + { + using var hash = System.Security.Cryptography.IncrementalHash.CreateHash(System.Security.Cryptography.HashAlgorithmName.SHA256); + foreach (var file in files) { + hash.AppendData(System.Text.Encoding.UTF8.GetBytes(file.Path)); + hash.AppendData(new byte[] { 0 }); + hash.AppendData(HexToBytes(file.Sha256)); + hash.AppendData(new byte[] { (byte)'\n' }); + } + return PackagedContentWriter.ToHex(hash.GetHashAndReset()); + } + + private static byte[] HexToBytes(string value) + { + var bytes = new byte[value.Length / 2]; + for (var i = 0; i < bytes.Length; i++) { + bytes[i] = Convert.ToByte(value.Substring(i * 2, 2), 16); + } + return bytes; + } + + private static IPackageFile FindFile(IPackageFolder root, string path) + { + var parts = PackagedContentPath.ValidateRelative(path).Split('/'); + var folder = root; + for (var i = 0; i < parts.Length - 1; i++) { + if (!folder.TryGetFolder(parts[i], out folder)) { + throw new InvalidDataException($"Content file '{path}' is missing from the package."); + } + } + if (!folder.TryGetFile(parts[^1], out var file)) { + throw new InvalidDataException($"Content file '{path}' is missing from the package."); + } + return file; + } + } +} diff --git a/VisualPinball.Unity/VisualPinball.Unity/Packaging/Content/PackagedContentValidator.cs.meta b/VisualPinball.Unity/VisualPinball.Unity/Packaging/Content/PackagedContentValidator.cs.meta new file mode 100644 index 000000000..1eeb0e31a --- /dev/null +++ b/VisualPinball.Unity/VisualPinball.Unity/Packaging/Content/PackagedContentValidator.cs.meta @@ -0,0 +1,2 @@ +fileFormatVersion: 2 +guid: c111b80465107694c8b990ab422fd0ff \ No newline at end of file diff --git a/VisualPinball.Unity/VisualPinball.Unity/Packaging/Content/PackagedContentWriter.cs b/VisualPinball.Unity/VisualPinball.Unity/Packaging/Content/PackagedContentWriter.cs new file mode 100644 index 000000000..8845ca05b --- /dev/null +++ b/VisualPinball.Unity/VisualPinball.Unity/Packaging/Content/PackagedContentWriter.cs @@ -0,0 +1,255 @@ +// Visual Pinball Engine +// Copyright (C) 2026 freezy and VPE Team +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. +// +// You should have received a copy of the GNU General Public License +// along with this program. If not, see . + +using System; +using System.Collections.Generic; +using System.IO; +using System.Linq; +using System.Security.Cryptography; +using System.Text; +using System.Text.RegularExpressions; + +namespace VisualPinball.Unity +{ + /// Deterministically ingests a directory into table/content. + public sealed class PackagedContentWriter + { + private readonly IPackageFolder _tableFolder; + + public PackagedContentWriter(IPackageFolder tableFolder) + { + _tableFolder = tableFolder ?? throw new ArgumentNullException(nameof(tableFolder)); + } + + public PackagedContentRef AddDirectory(string kind, string sourceRoot, ContentPackOptions options = null) + { + var prepared = PrepareDirectory(kind, sourceRoot, options); + Write(prepared); + return prepared.Reference; + } + + public static PreparedContent PrepareDirectory(string kind, string sourceRoot, ContentPackOptions options = null) + { + if (string.IsNullOrWhiteSpace(kind)) { + throw new ArgumentException("A content kind is required.", nameof(kind)); + } + if (string.IsNullOrWhiteSpace(sourceRoot)) { + throw new ArgumentException("A source directory is required.", nameof(sourceRoot)); + } + + options ??= new ContentPackOptions(); + var root = Path.GetFullPath(sourceRoot); + if (!Directory.Exists(root)) { + throw new DirectoryNotFoundException($"Content source directory does not exist: {root}"); + } + + var files = EnumerateFiles(root, options) + .OrderBy(file => file.RelativePath, StringComparer.Ordinal) + .ToList(); + if (files.Count > options.MaxFileCount) { + throw new InvalidDataException($"Content directory has {files.Count:N0} files; the configured limit is {options.MaxFileCount:N0}."); + } + + long totalBytes = 0; + var caseNames = new HashSet(StringComparer.OrdinalIgnoreCase); + using var canonicalHash = IncrementalHash.CreateHash(HashAlgorithmName.SHA256); + foreach (var file in files) { + if (!caseNames.Add(file.RelativePath)) { + throw new InvalidDataException($"Content contains duplicate paths that differ only by case: '{file.RelativePath}'. Rename one file for cross-platform packages."); + } + + file.Size = new FileInfo(file.FullPath).Length; + checked { totalBytes += file.Size; } + if (totalBytes > options.MaxTotalBytes) { + throw new InvalidDataException($"Content directory is {totalBytes:N0} bytes; the configured limit is {options.MaxTotalBytes:N0} bytes."); + } + file.Hash = ComputeFileHash(file.FullPath); + var pathBytes = Encoding.UTF8.GetBytes(file.RelativePath); + canonicalHash.AppendData(pathBytes); + canonicalHash.AppendData(new byte[] { 0 }); + canonicalHash.AppendData(file.Hash); + canonicalHash.AppendData(new byte[] { (byte)'\n' }); + } + + var contentHash = ToHex(canonicalHash.GetHashAndReset()); + var entryPoint = string.IsNullOrWhiteSpace(options.EntryPoint) + ? null + : PackagedContentPath.ValidateRelative(options.EntryPoint, "content entry point"); + if (entryPoint != null && files.All(file => file.RelativePath != entryPoint)) { + throw new InvalidDataException($"Content entry point '{entryPoint}' is not an included file."); + } + + var id = contentHash.Substring(0, 16); + var contentRef = new PackagedContentRef(id, kind, entryPoint, contentHash); +#if UNITY_EDITOR + contentRef.SourceDirectory = root; + contentRef.FileCount = files.Count; + contentRef.TotalBytes = totalBytes; + contentRef.ValidationStatus = "Valid"; +#endif + var manifest = new PackagedContentManifest { + Format = "vpe-content", + Version = 1, + Kind = kind, + EntryPoint = entryPoint, + FileCount = files.Count, + TotalBytes = totalBytes, + ContentHash = contentHash, + Files = files.Select(file => new PackagedContentFile { + Path = file.RelativePath, + Size = file.Size, + Sha256 = ToHex(file.Hash), + }).ToList(), + }; + return new PreparedContent(root, files, manifest, contentRef); + } + + public void Write(PreparedContent prepared) + { + var contentFolder = GetOrAddFolder(_tableFolder, PackageApi.ContentFolder); + if (contentFolder.TryGetFolder(prepared.Reference.Id, out var existing)) { + if (!existing.TryGetFile("manifest", out var manifestFile, PackageApi.Packer.FileExtension)) { + throw new InvalidDataException($"Content id collision at '{prepared.Reference.Id}': existing bundle has no manifest."); + } + var existingManifest = PackageApi.Packer.Unpack(manifestFile.GetData()); + if (existingManifest.ContentHash != prepared.Reference.ContentHash) { + throw new InvalidDataException($"Content id collision at '{prepared.Reference.Id}'."); + } + return; + } + + var bundleFolder = contentFolder.AddFolder(prepared.Reference.Id); + bundleFolder.AddFile("manifest", PackageApi.Packer.FileExtension).SetData(PackageApi.Packer.Pack(prepared.Manifest)); + var filesFolder = bundleFolder.AddFolder("files"); + foreach (var file in prepared.Files) { + var targetFolder = filesFolder; + var parts = file.RelativePath.Split('/'); + for (var i = 0; i < parts.Length - 1; i++) { + targetFolder = GetOrAddFolder(targetFolder, parts[i]); + } + + using var source = new FileStream(file.FullPath, FileMode.Open, FileAccess.Read, FileShare.Read); + using var target = targetFolder.AddFile(parts[^1]).AsStream(); + using var sha = SHA256.Create(); + var buffer = new byte[128 * 1024]; + int read; + while ((read = source.Read(buffer, 0, buffer.Length)) > 0) { + target.Write(buffer, 0, read); + sha.TransformBlock(buffer, 0, read, null, 0); + } + sha.TransformFinalBlock(Array.Empty(), 0, 0); + if (!sha.Hash.SequenceEqual(file.Hash)) { + throw new IOException($"Content file changed while the package was being written: {file.RelativePath}"); + } + } + } + + private static IEnumerable EnumerateFiles(string root, ContentPackOptions options) + { + var pending = new Stack(); + pending.Push(root); + while (pending.Count > 0) { + var directory = pending.Pop(); + foreach (var childDirectory in Directory.EnumerateDirectories(directory).OrderBy(path => path, StringComparer.Ordinal)) { + if (IsLink(childDirectory)) { + options.Warning?.Invoke($"Skipping symbolic link or junction '{childDirectory}'."); + continue; + } + pending.Push(childDirectory); + } + foreach (var fullPath in Directory.EnumerateFiles(directory).OrderBy(path => path, StringComparer.Ordinal)) { + if (IsLink(fullPath)) { + options.Warning?.Invoke($"Skipping symbolic link '{fullPath}'."); + continue; + } + var relative = Path.GetRelativePath(root, fullPath); + if (Path.DirectorySeparatorChar == '\\') { + relative = relative.Replace('\\', '/'); + } + relative = PackagedContentPath.ValidateRelative(relative, "source-relative content path"); + if (MatchesOptions(relative, options)) { + yield return new PreparedFile(fullPath, relative); + } + } + } + } + + private static bool IsLink(string path) => (File.GetAttributes(path) & FileAttributes.ReparsePoint) != 0; + + private static bool MatchesOptions(string path, ContentPackOptions options) + { + var included = options.IncludeGlobs == null || options.IncludeGlobs.Length == 0 || + options.IncludeGlobs.Any(glob => GlobMatches(path, glob)); + return included && (options.ExcludeGlobs == null || !options.ExcludeGlobs.Any(glob => GlobMatches(path, glob))); + } + + internal static bool GlobMatches(string path, string glob) + { + if (string.IsNullOrWhiteSpace(glob)) { + return false; + } + glob = glob.Replace('\\', '/'); + var pattern = "^" + Regex.Escape(glob) + .Replace(@"\*\*/", "(?:.*/)?") + .Replace(@"\*\*", ".*") + .Replace(@"\*", "[^/]*") + .Replace(@"\?", "[^/]") + "$"; + return Regex.IsMatch(path, pattern, RegexOptions.CultureInvariant); + } + + private static byte[] ComputeFileHash(string path) + { + using var stream = new FileStream(path, FileMode.Open, FileAccess.Read, FileShare.Read); + using var sha = SHA256.Create(); + return sha.ComputeHash(stream); + } + + internal static string ToHex(byte[] bytes) => string.Concat(bytes.Select(value => value.ToString("x2"))); + + private static IPackageFolder GetOrAddFolder(IPackageFolder parent, string name) => + parent.TryGetFolder(name, out var folder) ? folder : parent.AddFolder(name); + + public sealed class PreparedContent + { + public readonly string Root; + public readonly List Files; + public readonly PackagedContentManifest Manifest; + public readonly PackagedContentRef Reference; + + public PreparedContent(string root, List files, PackagedContentManifest manifest, PackagedContentRef reference) + { + Root = root; + Files = files; + Manifest = manifest; + Reference = reference; + } + } + + public sealed class PreparedFile + { + public readonly string FullPath; + public readonly string RelativePath; + public long Size; + public byte[] Hash; + + public PreparedFile(string fullPath, string relativePath) + { + FullPath = fullPath; + RelativePath = relativePath; + } + } + } +} diff --git a/VisualPinball.Unity/VisualPinball.Unity/Packaging/Content/PackagedContentWriter.cs.meta b/VisualPinball.Unity/VisualPinball.Unity/Packaging/Content/PackagedContentWriter.cs.meta new file mode 100644 index 000000000..006798b29 --- /dev/null +++ b/VisualPinball.Unity/VisualPinball.Unity/Packaging/Content/PackagedContentWriter.cs.meta @@ -0,0 +1,2 @@ +fileFormatVersion: 2 +guid: 9b43e4d3203a925419878f990ca88bf9 \ No newline at end of file diff --git a/VisualPinball.Unity/VisualPinball.Unity/Packaging/FORMAT.md b/VisualPinball.Unity/VisualPinball.Unity/Packaging/FORMAT.md index 288ccb656..111109d35 100644 --- a/VisualPinball.Unity/VisualPinball.Unity/Packaging/FORMAT.md +++ b/VisualPinball.Unity/VisualPinball.Unity/Packaging/FORMAT.md @@ -49,6 +49,10 @@ Already-compressed payloads (the two GLBs, everything under `textures/`) are wri zip entries; JSON entries are deflated. Entry timestamps are fixed (DOS epoch) so identical content produces byte-identical packages. +Optional inert consumer content is stored below `table/content//` with a +`manifest.json` and a `files/` directory. The normative layout and security rules are specified +in [content bundles](#content----inert-directory-bundles). + ## manifest.json The root-level manifest identifies the file and declares versions. It is the first thing a reader @@ -242,6 +246,45 @@ prefab linkage. Shared `ScriptableObject`-style assets, grouped by registered type name, one JSON per asset plus a `.meta.json` carrying the instance id used by component references. +## content/ — inert directory bundles + +Consumer-defined directory trees (for example a web application or an MPF machine) are stored as +bytes under `table/content//files/**`. VPE never executes a bundle. `` is +the first 16 lowercase hexadecimal characters of the bundle's canonical SHA-256. The canonical +input is each file, sorted by ordinal forward-slash path, encoded as UTF-8 path + NUL + the raw +32-byte file SHA-256 + LF. Identical trees therefore deduplicate independent of source location, +timestamps, platform, or enumeration order. Empty directories are not represented. + +`table/content//manifest.json` is normative: + +```json +{ + "format": "vpe-content", + "version": 1, + "kind": "consumer-defined", + "entryPoint": "relative/start.file", + "fileCount": 2, + "totalBytes": 1234, + "contentHash": "<64 lowercase hex characters>", + "files": [ + { "path": "relative/start.file", "size": 12, "sha256": "<64 lowercase hex characters>" } + ] +} +``` + +`entryPoint` may be null. `files` is in ordinal path order and supplies the per-file size and hash +needed for streaming extraction verification. Writers reject absolute paths, drive prefixes, +empty/`.`/`..` segments, backslashes, colons (including NTFS alternate data streams), and paths +that differ only by case. Symbolic links and junctions are skipped with a warning. + +Readers extract bundles to `/ContentCache//`. Every output path is +re-normalized and checked to remain under the temporary cache root, every byte count and SHA-256 +is verified, and cancellation removes the temporary tree. Extraction happens in a sibling +`.tmp-` directory; `.complete` is written last and the directory is then +renamed atomically. Only a final directory with a matching `.complete` marker may be reused. +Orphan temporary directories are removed on resolver startup. Completed entries are retained by +directory modification time under a configurable LRU cap (4 GiB by default). + ## screenshots/ Generated preview renders (`*.jpg`), the user-supplied `backglass.jpg`, and `bounds.json` (table @@ -257,3 +300,31 @@ crop bounds). Informational; not needed to load the table. 4. Ignore unknown JSON fields everywhere. 5. Resolve all node references through the `vpeId` extras of `table.glb`. 6. Sanitize zip entry names before writing anything to disk. +7. Validate every content manifest, reject missing/oversized/duplicate/unsafe entries, and verify + every content file's declared size and SHA-256 before publishing it to the content cache. + +## Content-bundle linting and migration + +`PackagedContentValidator.ValidatePackage()` is the normative integrity gate. It verifies the v1 +manifest, canonical hash, declared and actual file sizes, per-file hashes, entry point, ordering, +path safety, file count, and total size without publishing an extraction. The resolver repeats the +size/hash/path checks while streaming into its temporary cache directory. + +Authors can call `PackagedContentValidator.LintManifest()` before writing. In addition to manifest +validity it reports stable issue codes for a required-but-missing entry point, forbidden executable +extensions, duplicate byte payloads, and a configurable per-file size ceiling. Bundle consumers +must select policy explicitly: a browser or machine bundle normally requires an entry point and +forbids packaged executables because runtimes and native dependencies are player-owned. Python, +HTML, JavaScript, fonts, media, and WASM are data under this policy; executable native formats are +not. Warnings about duplicate bytes should be resolved or documented before release. + +Readers accept exactly `vpe-content` version 1. Unknown content-manifest versions are rejected before +extraction. Optional manifest fields may be added within v1 only when old readers can ignore them +without changing integrity or path semantics. Any change to canonical hashing, path normalization, +content identity, required fields, or extraction rules requires a new version. + +There is no older public content-bundle version to migrate. Future migration creates a new canonical +tree and manifest, which necessarily produces a new `contentHash` and cache location; readers never +rewrite a package or reuse the old identity. Migration tooling must validate source bytes, write the +new version into a separate package, run package validation and consumer-specific linting, then +resolve it into a clean cache as an end-to-end check. diff --git a/VisualPinball.Unity/VisualPinball.Unity/Packaging/PackageApi.cs b/VisualPinball.Unity/VisualPinball.Unity/Packaging/PackageApi.cs index eaf383642..930f379d7 100644 --- a/VisualPinball.Unity/VisualPinball.Unity/Packaging/PackageApi.cs +++ b/VisualPinball.Unity/VisualPinball.Unity/Packaging/PackageApi.cs @@ -50,7 +50,8 @@ public static class PackageApi public const string LampsFile = "lamps"; public const string AssetFolder = "assets"; public const string SoundFolder = "sounds"; - public const string ScreenshotsFolder = "screenshots"; + public const string ScreenshotsFolder = "screenshots"; + public const string ContentFolder = "content"; // Source textures are plain image files under table/textures/, one zip entry per // texture, referenced by file name from the materials payload. The zip central directory diff --git a/VisualPinball.Unity/VisualPinball.Unity/Packaging/README.md b/VisualPinball.Unity/VisualPinball.Unity/Packaging/README.md index 20478cbc7..b9bf9aa62 100644 --- a/VisualPinball.Unity/VisualPinball.Unity/Packaging/README.md +++ b/VisualPinball.Unity/VisualPinball.Unity/Packaging/README.md @@ -35,6 +35,16 @@ shaped the design. > the metadata to which it would apply is minuscule compared to the rest, the performance > advantage was quickly outweighed by its unreadability and the hassle to set up. +## Content author checklist + +Use `PackagedContentWriter.PrepareDirectory()` to produce deterministic `vpe-content` version 1 +input. Set an explicit kind and entry point, restrict includes/excludes, and choose file/count/total +limits appropriate for the consumer. Review skipped-link warnings. Run +`PackagedContentValidator.LintManifest()` with entry-point and executable policy enabled for web and +machine bundles, write the package, then run `ValidatePackage()` before distribution. Finally, +resolve it into an empty cache and launch the consumer on a clean player without source paths or +author-installed runtimes. The normative version and migration rules are in `FORMAT.md`. + ## Design rationale Decisions worth knowing the "why" of: diff --git a/VisualPinball.Unity/VisualPinball.Unity/Packaging/RuntimePackageReader.cs b/VisualPinball.Unity/VisualPinball.Unity/Packaging/RuntimePackageReader.cs index cb4a26d84..95f59453c 100644 --- a/VisualPinball.Unity/VisualPinball.Unity/Packaging/RuntimePackageReader.cs +++ b/VisualPinball.Unity/VisualPinball.Unity/Packaging/RuntimePackageReader.cs @@ -57,8 +57,12 @@ public class RuntimePackageReader public RuntimePackageReader(string vpePath) { _vpePath = vpePath; + ContentResolver = new PackagedContentResolver(vpePath); } + /// Resolves inert content bundles for runtime consumers without exposing zip internals. + public IPackagedContentResolver ContentResolver { get; } + public async Task ImportIntoScene(Transform parent = null, CancellationToken cancellationToken = default) { return await ImportIntoScene(parent, null, cancellationToken); @@ -259,6 +263,9 @@ await ReadPackablesAsync( ReportProgress(progress, RuntimePackageLoadStage.RestoringMaterials, 1f, "Material profiles applied."); materialsStopwatch.Stop(); Logger.Info($"RuntimePackageReader: Restored material profiles in {materialsStopwatch.ElapsedMilliseconds}ms."); + foreach (var consumer in _table.GetComponentsInChildren(true).OfType()) { + consumer.SetPackagedContentResolver(ContentResolver); + } loadSucceeded = true; } finally { From c8d14d4ddccaba4c6e745589d78f6662e4162824 Mon Sep 17 00:00:00 2001 From: freezy Date: Thu, 3 Sep 2026 01:03:53 +0200 Subject: [PATCH 2/3] packaging: harden content temp cleanup and link re-check Address review feedback on the content bundle subsystem. Startup temp-directory cleanup only reaps orphans older than one hour. Resolvers share the default cache root, so a blanket wipe of *.tmp-* could delete the extraction another concurrently constructed resolver is still writing. An in-progress extraction keeps its directory young, so an age guard reaps crashed-run orphans without touching live work. Re-check for a reparse point before measuring and hashing each source file. Enumeration skips links, but a source tree mutated mid-export could swap a regular file for a symlink pointing outside sourceRoot before it is hashed; the captured hash then also guards the later write. --- .../Content/PackagedContentResolver.cs | 21 +++++++++++++++++++ .../Content/PackagedContentWriter.cs | 9 ++++++++ .../VisualPinball.Unity/Packaging/FORMAT.md | 3 ++- 3 files changed, 32 insertions(+), 1 deletion(-) diff --git a/VisualPinball.Unity/VisualPinball.Unity/Packaging/Content/PackagedContentResolver.cs b/VisualPinball.Unity/VisualPinball.Unity/Packaging/Content/PackagedContentResolver.cs index 9b530ba38..fba6bdeaa 100644 --- a/VisualPinball.Unity/VisualPinball.Unity/Packaging/Content/PackagedContentResolver.cs +++ b/VisualPinball.Unity/VisualPinball.Unity/Packaging/Content/PackagedContentResolver.cs @@ -30,6 +30,11 @@ namespace VisualPinball.Unity public sealed class PackagedContentResolver : IPackagedContentResolver { private const string CompleteMarker = ".complete"; + + // Temp directories younger than this are assumed to belong to a concurrent extraction and are + // left alone; older ones are orphans from a crashed run and are safe to reap on startup. + private static readonly TimeSpan TemporaryDirectoryStaleAfter = TimeSpan.FromHours(1); + private readonly string _packagePath; private readonly PackagedContentCacheOptions _options; private readonly string _cacheRoot; @@ -182,7 +187,23 @@ private static bool IsComplete(string directory, string hash) private void CleanupTemporaryDirectories() { + // Only reap orphans left by a previous run. Resolvers share the default cache root, so a + // second resolver may be constructed while another is still extracting into its own + // GUID-suffixed temp directory. An in-progress extraction keeps its directory young + // (creating entries touches the directory's write time), so an age guard avoids deleting + // a tree another resolver is still publishing, on file systems where an open handle does + // not block the delete. + var now = DateTime.UtcNow; foreach (var directory in Directory.EnumerateDirectories(_cacheRoot, "*.tmp-*", SearchOption.TopDirectoryOnly)) { + DateTime lastWriteUtc; + try { + lastWriteUtc = Directory.GetLastWriteTimeUtc(directory); + } catch { + continue; + } + if (now - lastWriteUtc < TemporaryDirectoryStaleAfter) { + continue; + } TryDeleteDirectory(directory); } } diff --git a/VisualPinball.Unity/VisualPinball.Unity/Packaging/Content/PackagedContentWriter.cs b/VisualPinball.Unity/VisualPinball.Unity/Packaging/Content/PackagedContentWriter.cs index 8845ca05b..595a0d727 100644 --- a/VisualPinball.Unity/VisualPinball.Unity/Packaging/Content/PackagedContentWriter.cs +++ b/VisualPinball.Unity/VisualPinball.Unity/Packaging/Content/PackagedContentWriter.cs @@ -71,6 +71,15 @@ public static PreparedContent PrepareDirectory(string kind, string sourceRoot, C throw new InvalidDataException($"Content contains duplicate paths that differ only by case: '{file.RelativePath}'. Rename one file for cross-platform packages."); } + // Re-check for a reparse point swapped in after enumeration. Enumeration skips links, + // but a source tree mutated mid-export could redirect this entry outside sourceRoot + // before it is measured and hashed, which would otherwise package external bytes. The + // hash captured here also guards the later write: bytes that change afterwards fail the + // verification in Write(). + if (IsLink(file.FullPath)) { + throw new InvalidDataException($"Content file '{file.RelativePath}' became a symbolic link after enumeration; aborting to avoid packaging content from outside the source directory."); + } + file.Size = new FileInfo(file.FullPath).Length; checked { totalBytes += file.Size; } if (totalBytes > options.MaxTotalBytes) { diff --git a/VisualPinball.Unity/VisualPinball.Unity/Packaging/FORMAT.md b/VisualPinball.Unity/VisualPinball.Unity/Packaging/FORMAT.md index 111109d35..02b97e37b 100644 --- a/VisualPinball.Unity/VisualPinball.Unity/Packaging/FORMAT.md +++ b/VisualPinball.Unity/VisualPinball.Unity/Packaging/FORMAT.md @@ -282,7 +282,8 @@ re-normalized and checked to remain under the temporary cache root, every byte c is verified, and cancellation removes the temporary tree. Extraction happens in a sibling `.tmp-` directory; `.complete` is written last and the directory is then renamed atomically. Only a final directory with a matching `.complete` marker may be reused. -Orphan temporary directories are removed on resolver startup. Completed entries are retained by +Orphan temporary directories older than one hour are removed on resolver startup; younger ones are +left untouched so a concurrent resolver's in-progress extraction is never deleted. Completed entries are retained by directory modification time under a configurable LRU cap (4 GiB by default). ## screenshots/ From 0cc109235098de005e427f4c6a1d562550b93ea8 Mon Sep 17 00:00:00 2001 From: freezy Date: Thu, 3 Sep 2026 01:11:45 +0200 Subject: [PATCH 3/3] packaging: heartbeat live extractions, bracket content link check Follow-up to review on the content bundle subsystem. An extraction that spends a long time writing a single large file never creates another directory entry, so its temp directory's write time would go stale and a concurrently constructed resolver could reap it mid-write. The extraction now refreshes its temp directory's write time every few minutes, keeping it well under the one-hour stale cutoff. Bracket the source-file measure-and-hash with reparse-point checks on both sides instead of one, and read size and hash from a single handle so the path is resolved once. A link present at either boundary is rejected before its bytes are packaged. The residual is the sub-syscall window inside the read, which cannot be closed without no-follow open semantics that netstandard2.1 does not expose; the writer operates on the author's own local tree at export time. --- .../Content/PackagedContentResolver.cs | 16 ++++++++++++ .../Content/PackagedContentWriter.cs | 25 +++++++++++-------- 2 files changed, 31 insertions(+), 10 deletions(-) diff --git a/VisualPinball.Unity/VisualPinball.Unity/Packaging/Content/PackagedContentResolver.cs b/VisualPinball.Unity/VisualPinball.Unity/Packaging/Content/PackagedContentResolver.cs index fba6bdeaa..b24fe4277 100644 --- a/VisualPinball.Unity/VisualPinball.Unity/Packaging/Content/PackagedContentResolver.cs +++ b/VisualPinball.Unity/VisualPinball.Unity/Packaging/Content/PackagedContentResolver.cs @@ -35,6 +35,11 @@ public sealed class PackagedContentResolver : IPackagedContentResolver // left alone; older ones are orphans from a crashed run and are safe to reap on startup. private static readonly TimeSpan TemporaryDirectoryStaleAfter = TimeSpan.FromHours(1); + // While extracting, the temp directory's write time is refreshed at least this often so that a + // long single-file copy (which never creates another directory entry) still reads as live and + // is not reaped by a concurrently constructed resolver. Must stay well below the stale cutoff. + private static readonly TimeSpan TemporaryDirectoryHeartbeat = TimeSpan.FromMinutes(5); + private readonly string _packagePath; private readonly PackagedContentCacheOptions _options; private readonly string _cacheRoot; @@ -78,6 +83,7 @@ public async Task ResolveAsync(PackagedContentRef contentRef, IProgress< PackagedContentValidator.ValidateManifest(manifest, contentRef, _options.MaxFileCount, _options.MaxBundleBytes); var filesFolder = bundleFolder.GetFolder("files"); long writtenBytes = 0; + var lastHeartbeatUtc = DateTime.UtcNow; foreach (var entry in manifest.Files.OrderBy(file => file.Path, StringComparer.Ordinal)) { ct.ThrowIfCancellationRequested(); var targetPath = PackagedContentPath.GetContainedPath(temporary, entry.Path); @@ -100,6 +106,16 @@ public async Task ResolveAsync(PackagedContentRef contentRef, IProgress< if (fileBytes > entry.Size || writtenBytes > manifest.TotalBytes) { throw new InvalidDataException($"Content file '{entry.Path}' is larger than declared in its manifest."); } + var utcNow = DateTime.UtcNow; + if (utcNow - lastHeartbeatUtc >= TemporaryDirectoryHeartbeat) { + lastHeartbeatUtc = utcNow; + try { + Directory.SetLastWriteTimeUtc(temporary, utcNow); + } catch { + // Best-effort liveness signal; failing to touch the directory only risks a + // benign reap by another resolver, which the caller retries. + } + } progress?.Report(manifest.TotalBytes == 0 ? 1f : System.Math.Min(1f, writtenBytes / (float)manifest.TotalBytes)); } sha.TransformFinalBlock(Array.Empty(), 0, 0); diff --git a/VisualPinball.Unity/VisualPinball.Unity/Packaging/Content/PackagedContentWriter.cs b/VisualPinball.Unity/VisualPinball.Unity/Packaging/Content/PackagedContentWriter.cs index 595a0d727..d54c46d2e 100644 --- a/VisualPinball.Unity/VisualPinball.Unity/Packaging/Content/PackagedContentWriter.cs +++ b/VisualPinball.Unity/VisualPinball.Unity/Packaging/Content/PackagedContentWriter.cs @@ -71,21 +71,25 @@ public static PreparedContent PrepareDirectory(string kind, string sourceRoot, C throw new InvalidDataException($"Content contains duplicate paths that differ only by case: '{file.RelativePath}'. Rename one file for cross-platform packages."); } - // Re-check for a reparse point swapped in after enumeration. Enumeration skips links, - // but a source tree mutated mid-export could redirect this entry outside sourceRoot - // before it is measured and hashed, which would otherwise package external bytes. The - // hash captured here also guards the later write: bytes that change afterwards fail the - // verification in Write(). + // Bracket the measure-and-hash read with reparse-point checks on both sides. Enumeration + // skips links, but a source tree mutated mid-export could swap this entry for a symlink + // pointing outside sourceRoot; a link seen at either boundary is rejected before its + // bytes are packaged. Size and hash are taken from a single handle so the path is + // resolved once, and the captured hash later guards the write via the verification in + // Write(). The residual is the sub-syscall window inside the read, which cannot be + // closed without no-follow open semantics that netstandard2.1 does not expose. if (IsLink(file.FullPath)) { - throw new InvalidDataException($"Content file '{file.RelativePath}' became a symbolic link after enumeration; aborting to avoid packaging content from outside the source directory."); + throw new InvalidDataException($"Content file '{file.RelativePath}' is a symbolic link; links are not packaged."); + } + (file.Size, file.Hash) = MeasureAndHash(file.FullPath); + if (IsLink(file.FullPath)) { + throw new InvalidDataException($"Content file '{file.RelativePath}' was replaced by a symbolic link while being hashed; aborting to avoid packaging content from outside the source directory."); } - file.Size = new FileInfo(file.FullPath).Length; checked { totalBytes += file.Size; } if (totalBytes > options.MaxTotalBytes) { throw new InvalidDataException($"Content directory is {totalBytes:N0} bytes; the configured limit is {options.MaxTotalBytes:N0} bytes."); } - file.Hash = ComputeFileHash(file.FullPath); var pathBytes = Encoding.UTF8.GetBytes(file.RelativePath); canonicalHash.AppendData(pathBytes); canonicalHash.AppendData(new byte[] { 0 }); @@ -219,11 +223,12 @@ internal static bool GlobMatches(string path, string glob) return Regex.IsMatch(path, pattern, RegexOptions.CultureInvariant); } - private static byte[] ComputeFileHash(string path) + private static (long Size, byte[] Hash) MeasureAndHash(string path) { using var stream = new FileStream(path, FileMode.Open, FileAccess.Read, FileShare.Read); using var sha = SHA256.Create(); - return sha.ComputeHash(stream); + var hash = sha.ComputeHash(stream); + return (stream.Length, hash); } internal static string ToHex(byte[] bytes) => string.Concat(bytes.Select(value => value.ToString("x2")));