From 68aa81779f4e3fb6156c3e9735aea68b4ed4422e Mon Sep 17 00:00:00 2001 From: adnan-bhaldar Date: Mon, 26 Oct 2026 12:28:21 +0530 Subject: [PATCH 1/6] Add .sql as a plain text input format (#607) SQL scripts (e.g. saved from SSMS) can now be converted to TXT, and from there to PDF, DOCX and the other text routes. The extension is added to the existing plain text format; no new converter is needed. --- src/Filee.Core/Formats/FormatRegistry.cs | 4 +- src/Filee.Engines/EngineRegistry.cs | 1 + src/Filee.Engines/Text/SqlTextConverter.cs | 63 +++++++++++++++++++ .../SqlTextConverterTests.cs | 48 ++++++++++++++ 4 files changed, 114 insertions(+), 2 deletions(-) create mode 100644 src/Filee.Engines/Text/SqlTextConverter.cs create mode 100644 tests/Filee.Engines.Tests/SqlTextConverterTests.cs diff --git a/src/Filee.Core/Formats/FormatRegistry.cs b/src/Filee.Core/Formats/FormatRegistry.cs index 41ea7c2db..e84bb2813 100644 --- a/src/Filee.Core/Formats/FormatRegistry.cs +++ b/src/Filee.Core/Formats/FormatRegistry.cs @@ -141,8 +141,8 @@ public static class FormatRegistry // Plain text / markup // Text logs, source/proxy scripts and style sheets are read as plain text, never executed (issues #273, #284, #254, - // #388, #368). - new("txt", "TXT", FormatCategory.Text, ["txt", "log", "py", "pac", "bat", "command", "lua", "css", "ps1"]), + // #388, #368, #607). + new("txt", "TXT", FormatCategory.Text, ["txt", "log", "py", "pac", "bat", "command", "lua", "css", "ps1", "sql"]), new("md", "Markdown", FormatCategory.Text, ["md", "markdown"]), new("html", "HTML", FormatCategory.Text, ["html", "htm", "xhtml"]), // A web page saved with its pictures in one MIME file (Chrome, Edge, Word); read as the page it holds. diff --git a/src/Filee.Engines/EngineRegistry.cs b/src/Filee.Engines/EngineRegistry.cs index 5212f3c27..c4905f74f 100644 --- a/src/Filee.Engines/EngineRegistry.cs +++ b/src/Filee.Engines/EngineRegistry.cs @@ -46,6 +46,7 @@ public static IReadOnlyList CreateAll(EngineEnvironment env, ILogger new RhwpConverter(), new UnhwpConverter(), new MarkdownConverter(), + new SqlTextConverter(), new SpreadsheetConverter(), new OoxmlConverter(), new HwpxConverter(), diff --git a/src/Filee.Engines/Text/SqlTextConverter.cs b/src/Filee.Engines/Text/SqlTextConverter.cs new file mode 100644 index 000000000..895088f9c --- /dev/null +++ b/src/Filee.Engines/Text/SqlTextConverter.cs @@ -0,0 +1,63 @@ +// SQL script → TXT. Pure .NET, no external engine. +// SSMS saves .sql files as UTF-16 (with BOM), UTF-8 (with or without BOM) or the Windows ANSI code page, +// so the text is decoded with the right encoding and always written back as UTF-8. + +using System.Text; +using Filee.Core.Conversion; + +namespace Filee.Engines.Text; + +public sealed class SqlTextConverter : IConverter +{ + public string Id => "sqltext"; + public string DisplayName => "SQL script"; + public int MaxParallelism => 0; + + public IReadOnlyList Edges { get; } = + [ + new("sql", "txt"), + ]; + + public EngineStatus GetStatus() => EngineStatus.Available(); + + public Task> ConvertAsync(ConversionStep step, IProgress? progress, CancellationToken ct) => + Task.Run>(async () => + { + var output = step.Output.Allocate("txt"); + if (output is null) + return []; + + var bytes = await File.ReadAllBytesAsync(step.InputPath, ct); + ct.ThrowIfCancellationRequested(); + + var text = Decode(bytes); + await File.WriteAllTextAsync(output, text, new UTF8Encoding(encoderShouldEmitUTF8Identifier: false), ct); + + progress?.Report(1); + return [output]; + }, ct); + + internal static string Decode(byte[] bytes) + { + // 1. A byte order mark tells us exactly what the file is. + if (bytes.Length >= 3 && bytes[0] == 0xEF && bytes[1] == 0xBB && bytes[2] == 0xBF) + return Encoding.UTF8.GetString(bytes, 3, bytes.Length - 3); + + if (bytes.Length >= 2 && bytes[0] == 0xFF && bytes[1] == 0xFE) // UTF-16 LE (SSMS default "Unicode") + return Encoding.Unicode.GetString(bytes, 2, bytes.Length - 2); + + if (bytes.Length >= 2 && bytes[0] == 0xFE && bytes[1] == 0xFF) // UTF-16 BE + return Encoding.BigEndianUnicode.GetString(bytes, 2, bytes.Length - 2); + + // 2. No BOM: valid UTF-8 (this includes plain ASCII). + try + { + return new UTF8Encoding(encoderShouldEmitUTF8Identifier: false, throwOnInvalidBytes: true).GetString(bytes); + } + catch (DecoderFallbackException) + { + // 3. Not UTF-8: SSMS "ANSI" files use the Windows code page, Windows-1252 in most of the world. + return CodePagesEncodingProvider.Instance.GetEncoding(1252)!.GetString(bytes); + } + } +} \ No newline at end of file diff --git a/tests/Filee.Engines.Tests/SqlTextConverterTests.cs b/tests/Filee.Engines.Tests/SqlTextConverterTests.cs new file mode 100644 index 000000000..e9fe77562 --- /dev/null +++ b/tests/Filee.Engines.Tests/SqlTextConverterTests.cs @@ -0,0 +1,48 @@ +using System.Text; +using Filee.Engines.Text; + +namespace Filee.Engines.Tests; + +public class SqlTextConverterTests +{ + private const string Script = "SELECT 'café' AS Name;\r\nGO\r\n"; + + [Fact] + public void Decodes_utf16_with_bom_like_ssms_unicode() + { + var bytes = Encoding.Unicode.GetPreamble().Concat(Encoding.Unicode.GetBytes(Script)).ToArray(); + Assert.Equal(Script, SqlTextConverter.Decode(bytes)); + } + + [Fact] + public void Decodes_utf8_with_bom() + { + var bytes = Encoding.UTF8.GetPreamble().Concat(Encoding.UTF8.GetBytes(Script)).ToArray(); + Assert.Equal(Script, SqlTextConverter.Decode(bytes)); + } + + [Fact] + public void Decodes_utf8_without_bom() + { + Assert.Equal(Script, SqlTextConverter.Decode(Encoding.UTF8.GetBytes(Script))); + } + + [Fact] + public void Decodes_windows_1252_ansi() + { + // 0xE9 alone is "é" in Windows-1252 and is not valid UTF-8. + var bytes = new byte[] { (byte)'c', (byte)'a', (byte)'f', 0xE9 }; + Assert.Equal("café", SqlTextConverter.Decode(bytes)); + } +} + +// End-to-end check, same style as the other engine tests. Add it inside a class that has the `fx` EngineFixture: +// +// [Fact] +// public async Task Sql_to_txt() +// { +// var path = Path.Combine(Path.GetTempPath(), $"{Guid.NewGuid():N}.sql"); +// await File.WriteAllTextAsync(path, "SELECT 1;", Encoding.Unicode); +// var job = await fx.ConvertAsync([path], new Preset { TargetFormat = "txt" }); +// Assert.Equal(JobState.Completed, job.State); +// } \ No newline at end of file From 63e34bfc631b1829594885b310f8ea99b718ae49 Mon Sep 17 00:00:00 2001 From: adnan-bhaldar Date: Mon, 26 Oct 2026 12:32:49 +0530 Subject: [PATCH 2/6] Add SQL support to Text & Markdown engine for UTF-8 TXT conversion --- docs/ENGINES.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/ENGINES.md b/docs/ENGINES.md index 3b233a9f3..94e820750 100644 --- a/docs/ENGINES.md +++ b/docs/ENGINES.md @@ -12,7 +12,7 @@ Filee chooses engines automatically. Settings → *Engines* shows their status, | PDFsharp | built in (library) | images → PDF, merge, split, page ranges, page organising, lossless rotate and crop-box edits, spreadsheet print-studio assembly, **compression** (smaller PDFs, see below), PDFs without their metadata | | PDFium | built in (library) | PDF → images (multi-page TIFF/GIF supported) | | Unhwp | built in (library) | HWP/HWPX → TXT, Markdown, HTML (pictures in a `_files` folder next to the Markdown, embedded in the HTML) | -| Text & Markdown (Markdig) | built in (library) | **Markdown → HTML, TXT**; **TXT, LOG, PY, PAC, BAT, COMMAND, LUA, CSS, PS1 → UTF-8 TXT**, preserving source text without running scripts (and → PDF / DOCX through the HWPX writer, rhwp or LibreOffice) | +| Text & Markdown (Markdig) | built in (library) | **Markdown → HTML, TXT**; **TXT, LOG, PY, PAC, BAT, COMMAND, LUA, CSS, PS1, SQL → UTF-8 TXT**, preserving source text without running scripts (and → PDF / DOCX through the HWPX writer, rhwp or LibreOffice) | | Spreadsheets (ExcelDataReader for XLS) | built in (library) | **XLSX, XLS, ODS, CSV, TSV → XLSX, ODS, CSV, TSV** (one CSV / TSV per sheet) | | Office Open XML | built in | **DOCM / DOTX / DOTM ↔ DOCX, XLSM / XLTX ↔ XLSX, PPTM / POTX / POTM / PPSX ↔ PPTX** (macros removed for macro-free types) | | Fonts | built in | **TTF, OTF, WOFF, WOFF2, EOT ↔ each other**; CFF (PostScript) outlines become TrueType for TTF and EOT | From 5eacbd51a5e1fc60a8b866b9194d38ccc8ea94d3 Mon Sep 17 00:00:00 2001 From: adnan-bhaldar Date: Mon, 26 Oct 2026 13:07:45 +0530 Subject: [PATCH 3/6] Remove commented-out end-to-end SQL to TXT test from SqlTextConverterTests --- tests/Filee.Engines.Tests/SqlTextConverterTests.cs | 13 +------------ 1 file changed, 1 insertion(+), 12 deletions(-) diff --git a/tests/Filee.Engines.Tests/SqlTextConverterTests.cs b/tests/Filee.Engines.Tests/SqlTextConverterTests.cs index e9fe77562..8cd6869eb 100644 --- a/tests/Filee.Engines.Tests/SqlTextConverterTests.cs +++ b/tests/Filee.Engines.Tests/SqlTextConverterTests.cs @@ -34,15 +34,4 @@ public void Decodes_windows_1252_ansi() var bytes = new byte[] { (byte)'c', (byte)'a', (byte)'f', 0xE9 }; Assert.Equal("café", SqlTextConverter.Decode(bytes)); } -} - -// End-to-end check, same style as the other engine tests. Add it inside a class that has the `fx` EngineFixture: -// -// [Fact] -// public async Task Sql_to_txt() -// { -// var path = Path.Combine(Path.GetTempPath(), $"{Guid.NewGuid():N}.sql"); -// await File.WriteAllTextAsync(path, "SELECT 1;", Encoding.Unicode); -// var job = await fx.ConvertAsync([path], new Preset { TargetFormat = "txt" }); -// Assert.Equal(JobState.Completed, job.State); -// } \ No newline at end of file +} \ No newline at end of file From fdf37ba323a8d901df6594781be7991bb48d1a3a Mon Sep 17 00:00:00 2001 From: adnan-bhaldar Date: Mon, 26 Oct 2026 13:16:12 +0530 Subject: [PATCH 4/6] style: fix final newlines and line endings --- src/Filee.Engines/Text/SqlTextConverter.cs | 2 +- tests/Filee.Engines.Tests/SqlTextConverterTests.cs | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/Filee.Engines/Text/SqlTextConverter.cs b/src/Filee.Engines/Text/SqlTextConverter.cs index 895088f9c..ee9f18643 100644 --- a/src/Filee.Engines/Text/SqlTextConverter.cs +++ b/src/Filee.Engines/Text/SqlTextConverter.cs @@ -60,4 +60,4 @@ internal static string Decode(byte[] bytes) return CodePagesEncodingProvider.Instance.GetEncoding(1252)!.GetString(bytes); } } -} \ No newline at end of file +} diff --git a/tests/Filee.Engines.Tests/SqlTextConverterTests.cs b/tests/Filee.Engines.Tests/SqlTextConverterTests.cs index 8cd6869eb..0f5dd54d4 100644 --- a/tests/Filee.Engines.Tests/SqlTextConverterTests.cs +++ b/tests/Filee.Engines.Tests/SqlTextConverterTests.cs @@ -34,4 +34,4 @@ public void Decodes_windows_1252_ansi() var bytes = new byte[] { (byte)'c', (byte)'a', (byte)'f', 0xE9 }; Assert.Equal("café", SqlTextConverter.Decode(bytes)); } -} \ No newline at end of file +} From 86943ad5bd90ab8657c8aa6a51b21c29fea7ba3c Mon Sep 17 00:00:00 2001 From: adnan-bhaldar Date: Mon, 26 Oct 2026 15:55:12 +0530 Subject: [PATCH 5/6] fix(text): drop the separate SQL converter, use the txt extension (#607) The extra converter had no description in the i18n files, so the Engines page test failed. .sql is now handled by adding it to the existing txt format, like .py and .bat. --- src/Filee.Engines/EngineRegistry.cs | 1 - src/Filee.Engines/Text/SqlTextConverter.cs | 63 ------------------- tests/Filee.App.Tests/RenderTests.cs | 2 +- .../SqlTextConverterTests.cs | 37 ----------- 4 files changed, 1 insertion(+), 102 deletions(-) delete mode 100644 src/Filee.Engines/Text/SqlTextConverter.cs delete mode 100644 tests/Filee.Engines.Tests/SqlTextConverterTests.cs diff --git a/src/Filee.Engines/EngineRegistry.cs b/src/Filee.Engines/EngineRegistry.cs index c4905f74f..5212f3c27 100644 --- a/src/Filee.Engines/EngineRegistry.cs +++ b/src/Filee.Engines/EngineRegistry.cs @@ -46,7 +46,6 @@ public static IReadOnlyList CreateAll(EngineEnvironment env, ILogger new RhwpConverter(), new UnhwpConverter(), new MarkdownConverter(), - new SqlTextConverter(), new SpreadsheetConverter(), new OoxmlConverter(), new HwpxConverter(), diff --git a/src/Filee.Engines/Text/SqlTextConverter.cs b/src/Filee.Engines/Text/SqlTextConverter.cs deleted file mode 100644 index ee9f18643..000000000 --- a/src/Filee.Engines/Text/SqlTextConverter.cs +++ /dev/null @@ -1,63 +0,0 @@ -// SQL script → TXT. Pure .NET, no external engine. -// SSMS saves .sql files as UTF-16 (with BOM), UTF-8 (with or without BOM) or the Windows ANSI code page, -// so the text is decoded with the right encoding and always written back as UTF-8. - -using System.Text; -using Filee.Core.Conversion; - -namespace Filee.Engines.Text; - -public sealed class SqlTextConverter : IConverter -{ - public string Id => "sqltext"; - public string DisplayName => "SQL script"; - public int MaxParallelism => 0; - - public IReadOnlyList Edges { get; } = - [ - new("sql", "txt"), - ]; - - public EngineStatus GetStatus() => EngineStatus.Available(); - - public Task> ConvertAsync(ConversionStep step, IProgress? progress, CancellationToken ct) => - Task.Run>(async () => - { - var output = step.Output.Allocate("txt"); - if (output is null) - return []; - - var bytes = await File.ReadAllBytesAsync(step.InputPath, ct); - ct.ThrowIfCancellationRequested(); - - var text = Decode(bytes); - await File.WriteAllTextAsync(output, text, new UTF8Encoding(encoderShouldEmitUTF8Identifier: false), ct); - - progress?.Report(1); - return [output]; - }, ct); - - internal static string Decode(byte[] bytes) - { - // 1. A byte order mark tells us exactly what the file is. - if (bytes.Length >= 3 && bytes[0] == 0xEF && bytes[1] == 0xBB && bytes[2] == 0xBF) - return Encoding.UTF8.GetString(bytes, 3, bytes.Length - 3); - - if (bytes.Length >= 2 && bytes[0] == 0xFF && bytes[1] == 0xFE) // UTF-16 LE (SSMS default "Unicode") - return Encoding.Unicode.GetString(bytes, 2, bytes.Length - 2); - - if (bytes.Length >= 2 && bytes[0] == 0xFE && bytes[1] == 0xFF) // UTF-16 BE - return Encoding.BigEndianUnicode.GetString(bytes, 2, bytes.Length - 2); - - // 2. No BOM: valid UTF-8 (this includes plain ASCII). - try - { - return new UTF8Encoding(encoderShouldEmitUTF8Identifier: false, throwOnInvalidBytes: true).GetString(bytes); - } - catch (DecoderFallbackException) - { - // 3. Not UTF-8: SSMS "ANSI" files use the Windows code page, Windows-1252 in most of the world. - return CodePagesEncodingProvider.Instance.GetEncoding(1252)!.GetString(bytes); - } - } -} diff --git a/tests/Filee.App.Tests/RenderTests.cs b/tests/Filee.App.Tests/RenderTests.cs index 5412324cc..671a254aa 100644 --- a/tests/Filee.App.Tests/RenderTests.cs +++ b/tests/Filee.App.Tests/RenderTests.cs @@ -2530,4 +2530,4 @@ internal static void Save(TopLevel window, string name) using var stream = File.Create(Path.Combine(TestServices.ScreenshotDirectory, name)); frame.Save(stream, new Avalonia.Media.Imaging.PngBitmapEncoderOptions()); } -} +} \ No newline at end of file diff --git a/tests/Filee.Engines.Tests/SqlTextConverterTests.cs b/tests/Filee.Engines.Tests/SqlTextConverterTests.cs deleted file mode 100644 index 0f5dd54d4..000000000 --- a/tests/Filee.Engines.Tests/SqlTextConverterTests.cs +++ /dev/null @@ -1,37 +0,0 @@ -using System.Text; -using Filee.Engines.Text; - -namespace Filee.Engines.Tests; - -public class SqlTextConverterTests -{ - private const string Script = "SELECT 'café' AS Name;\r\nGO\r\n"; - - [Fact] - public void Decodes_utf16_with_bom_like_ssms_unicode() - { - var bytes = Encoding.Unicode.GetPreamble().Concat(Encoding.Unicode.GetBytes(Script)).ToArray(); - Assert.Equal(Script, SqlTextConverter.Decode(bytes)); - } - - [Fact] - public void Decodes_utf8_with_bom() - { - var bytes = Encoding.UTF8.GetPreamble().Concat(Encoding.UTF8.GetBytes(Script)).ToArray(); - Assert.Equal(Script, SqlTextConverter.Decode(bytes)); - } - - [Fact] - public void Decodes_utf8_without_bom() - { - Assert.Equal(Script, SqlTextConverter.Decode(Encoding.UTF8.GetBytes(Script))); - } - - [Fact] - public void Decodes_windows_1252_ansi() - { - // 0xE9 alone is "é" in Windows-1252 and is not valid UTF-8. - var bytes = new byte[] { (byte)'c', (byte)'a', (byte)'f', 0xE9 }; - Assert.Equal("café", SqlTextConverter.Decode(bytes)); - } -} From 936b523a8e510567088ff12954422dad8d087dde Mon Sep 17 00:00:00 2001 From: adnan-bhaldar Date: Mon, 26 Oct 2026 15:59:27 +0530 Subject: [PATCH 6/6] chore: restore RenderTests.cs --- tests/Filee.App.Tests/RenderTests.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/Filee.App.Tests/RenderTests.cs b/tests/Filee.App.Tests/RenderTests.cs index 671a254aa..5412324cc 100644 --- a/tests/Filee.App.Tests/RenderTests.cs +++ b/tests/Filee.App.Tests/RenderTests.cs @@ -2530,4 +2530,4 @@ internal static void Save(TopLevel window, string name) using var stream = File.Create(Path.Combine(TestServices.ScreenshotDirectory, name)); frame.Save(stream, new Avalonia.Media.Imaging.PngBitmapEncoderOptions()); } -} \ No newline at end of file +}