From d2bfa56c6b66230548de59c88ab502d097041cb8 Mon Sep 17 00:00:00 2001 From: chsword Date: Tue, 15 Sep 2026 15:32:58 +0000 Subject: [PATCH 1/4] =?UTF-8?q?=E6=B5=81=E5=BC=8F=E5=AF=BC=E5=87=BA?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ObjectToExcelStream 边取数据边写入调用方给出的流，内存中只保留 StreamingRowWindow 指定的若干行（默认 100），走 NPOI 的 SXSSF。实测二十万行五列：内存导出峰值 710 MB / 6.4 秒，流式导出 89 MB / 3.7 秒，且内存不随行数增长。 为此把要回看数据的两处改为写入时算出：自动列宽随写随量，按值合并的各段随写随记，数据因而只遍历一遍（内存导出同样受益，AutoColumnWidth 不再走两遍）。多张长下拉列表改为一次写完隐藏表，否则第二列会落在已刷出内存的行上而丢失；跨表公式改为向底下的 XSSF 取表头，SXSSF 那一层读不到随源工作簿带进来的行。 流式写入要 SkiaSharp（SXSSF 刷行时用它量字符宽度），NPOI 把该依赖标为不随包传递，故由使用方自行引用，缺失时在写出任何内容之前报错说明。库本身不引入，以免只用内存导出的使用方为此背上数十兆的本机库。 版本 2.10.0。 Generated with [Claude Code](https://claude.ai/code) via [Happy](https://happy.engineering) Co-Authored-By: Claude Opus 5 Co-Authored-By: Happy --- CLAUDE.md | 4 + .../Chsword.Excel2Object.Tests.csproj | 5 + .../StreamingExportTest.cs | 388 ++++++++++++++++++ .../TestTempDirectory.cs | 39 ++ .../Chsword.Excel2Object.csproj | 2 +- Chsword.Excel2Object/ExcelExporter.cs | 327 +++++++++------ Chsword.Excel2Object/ExcelHelper.cs | 14 + .../Internal/DropdownValidation.cs | 69 +++- .../Internal/MergedRegions.cs | 177 +++++--- .../Internal/StreamingSupport.cs | 43 ++ Chsword.Excel2Object/Internal/TypeConvert.cs | 88 +++- Chsword.Excel2Object/Models/SheetModel.cs | 6 +- .../Options/ExcelExporterOptions.cs | 7 + README.md | 7 + README_EN.md | 7 + docs/README.md | 1 + docs/versions/v2.10.0.md | 120 ++++++ 17 files changed, 1078 insertions(+), 226 deletions(-) create mode 100644 Chsword.Excel2Object.Tests/StreamingExportTest.cs create mode 100644 Chsword.Excel2Object.Tests/TestTempDirectory.cs create mode 100644 Chsword.Excel2Object/Internal/StreamingSupport.cs create mode 100644 docs/versions/v2.10.0.md diff --git a/CLAUDE.md b/CLAUDE.md index c22fb91..e02aa78 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -47,6 +47,10 @@ dotnet test Chsword.Excel2Object.Tests # 不指定框架，逐个运行 应让其类型如实反映（例如把内部模型的属性定为非空），并在真正可能为空的入口处校验。注意 `string.IsNullOrEmpty` 在较早的目标框架上没有 `NotNullWhen` 标注，需显式判空才能各框架一致。 +测试项目引用了 `SkiaSharp` 与 `SkiaSharp.NativeAssets.Linux.NoDependencies`：流式导出走 NPOI 的 SXSSF，后者 +把行刷出内存时要用它测量字符宽度，而 NPOI 把该依赖标为不随包传递。库本身不引用它，使用方亦须自行引用， +故不可将其加入 `Chsword.Excel2Object.csproj`。 + 各目标框架须使用同一语言版本（`LangVersion`）：否则低版本框架会退回较早的 C#，同一份测试代码可能 在一处编译通过、在另一处失败。 diff --git a/Chsword.Excel2Object.Tests/Chsword.Excel2Object.Tests.csproj b/Chsword.Excel2Object.Tests/Chsword.Excel2Object.Tests.csproj index 0244b1a..af9f411 100644 --- a/Chsword.Excel2Object.Tests/Chsword.Excel2Object.Tests.csproj +++ b/Chsword.Excel2Object.Tests/Chsword.Excel2Object.Tests.csproj @@ -26,6 +26,11 @@ runtime; build; native; contentfiles; analyzers; buildtransitive + + + diff --git a/Chsword.Excel2Object.Tests/StreamingExportTest.cs b/Chsword.Excel2Object.Tests/StreamingExportTest.cs new file mode 100644 index 0000000..1f8db90 --- /dev/null +++ b/Chsword.Excel2Object.Tests/StreamingExportTest.cs @@ -0,0 +1,388 @@ +using System; +using System.Collections; +using System.Collections.Generic; +using System.IO; +using System.Linq; +using Chsword.Excel2Object.Options; +using Chsword.Excel2Object.Styles; +using Microsoft.VisualStudio.TestTools.UnitTesting; +using NPOI.SS.UserModel; + +namespace Chsword.Excel2Object.Tests; + +/// +/// 流式导出：边取数据边写，内存中只留若干行。已刷出内存的行无从读回，故凡是要回看数据的特性 +/// （自动列宽、按值合并）都改在写入过程中算出，这里逐一验证其结果与内存导出一致。 +/// +[TestClass] +public class StreamingExportTest +{ + /// 刻意远小于行数，好让写入过程中确有大批行被刷出内存。 + private const int Window = 5; + + public class Model + { + [ExcelColumn("省份")] public string Province { get; set; } = ""; + [ExcelColumn("城市")] public string City { get; set; } = ""; + [ExcelColumn("金额", Format = "#,##0.00")] public decimal Amount { get; set; } + [ExcelColumn("下单时间", Format = "yyyy-MM-dd")] public DateTime PlacedAt { get; set; } + [ExcelColumn("状态")] public string Status { get; set; } = ""; + } + + private static List Rows(int count) + { + return Enumerable.Range(0, count).Select(i => new Model + { + // 每 10 行换一个省份、每 5 行换一个城市，于是合并区域必然跨越被刷出内存的行 + Province = "省" + i / 10, + City = "市" + i / 5, + Amount = 100 + i, + PlacedAt = new DateTime(2026, 1, 1).AddDays(i), + Status = i % 2 == 0 ? "已发货" : "待付款" + }).ToList(); + } + + private static ISheet Streamed(IEnumerable rows, Action? configure = null, + int window = Window) + { + using var output = new MemoryStream(); + new ExcelExporter().ObjectToExcelStream(rows, output, options => + { + options.ExcelType = ExcelType.Xlsx; + options.StreamingRowWindow = window; + configure?.Invoke(options); + }); + + return WorkbookFactory.Create(new MemoryStream(output.ToArray())).GetSheetAt(0); + } + + private static ISheet InMemory(IEnumerable rows, Action? configure = null) + { + var bytes = new ExcelExporter().ObjectToExcelBytes(rows, options => + { + options.ExcelType = ExcelType.Xlsx; + configure?.Invoke(options); + }); + Assert.IsNotNull(bytes); + return WorkbookFactory.Create(new MemoryStream(bytes!)).GetSheetAt(0); + } + + /// 合并区域，按列再按行排序，与写入顺序一致。 + private static string[] Regions(ISheet sheet) + { + return sheet.MergedRegions.OrderBy(r => r.FirstColumn).ThenBy(r => r.FirstRow) + .Select(r => r.FormatAsString()).ToArray(); + } + + /// 逐格比对两张表：值、数字格式、字体与底色都取自文件，而非只看值。 + private static void AssertSameSheet(ISheet expected, ISheet actual) + { + Assert.AreEqual(expected.LastRowNum, actual.LastRowNum, "行数"); + for (var r = 0; r <= expected.LastRowNum; r++) + { + var expectedRow = expected.GetRow(r); + var actualRow = actual.GetRow(r); + Assert.AreEqual(expectedRow.LastCellNum, actualRow.LastCellNum, $"第 {r} 行的列数"); + for (var c = 0; c < expectedRow.LastCellNum; c++) + { + var a = expectedRow.GetCell(c); + var b = actualRow.GetCell(c); + Assert.AreEqual(a.CellType, b.CellType, $"{a.Address} 的类型"); + Assert.AreEqual(a.ToString(), b.ToString(), $"{a.Address} 的值"); + Assert.AreEqual(a.CellStyle.GetDataFormatString(), b.CellStyle.GetDataFormatString(), + $"{a.Address} 的数字格式"); + Assert.AreEqual(a.CellStyle.FillForegroundColor, b.CellStyle.FillForegroundColor, + $"{a.Address} 的底色"); + Assert.AreEqual(a.CellStyle.GetFont(expected.Workbook).IsBold, + b.CellStyle.GetFont(actual.Workbook).IsBold, $"{a.Address} 的字体"); + } + } + } + + [TestMethod] + public void StreamedFileMatchesTheInMemoryOne() + { + var rows = Rows(120); + Action configure = options => options.Styles + .Header(s => s.Bold().Background("#4472C4").Color("#FFFFFF")) + .EvenRows(s => s.Background("#F2F2F2")) + .Column("金额", s => s.Right()); + + AssertSameSheet(InMemory(rows, configure), Streamed(rows, configure)); + } + + [TestMethod] + public void MergesSpanRowsAlreadyFlushedFromMemory() + { + var rows = Rows(30); + Action configure = options => + { + options.MergeRepeatedColumns.Add("省份"); + options.MergeRepeatedColumns.Add("城市"); + }; + + // 每段 10 行、窗口 5 行：任何一段都跨越已刷出内存的行 + var streamed = Regions(Streamed(rows, configure)); + var provinces = streamed.Where(r => r.StartsWith("A")).ToArray(); + CollectionAssert.AreEqual(new[] {"A2:A11", "A12:A21", "A22:A31"}, provinces, string.Join(" ", streamed)); + CollectionAssert.AreEqual(Regions(InMemory(rows, configure)), streamed, string.Join(" ", streamed)); + } + + [TestMethod] + public void DeclaredRegionsAreResolvedAgainstTheRowsWritten() + { + var sheet = Streamed(Rows(30), options => + options.MergedRegions.Add(new MergedRegion("状态") {FirstRow = 1, LastRow = 30})); + + CollectionAssert.AreEqual(new[] {"E2:E31"}, Regions(sheet)); + } + + [TestMethod] + public void HeaderViewDropdownAndConditionalFormatSurvive() + { + var sheet = Streamed(Rows(40), options => + { + options.FreezeHeader = true; + options.AutoFilter = true; + options.Dropdowns["状态"] = new[] {"已发货", "待付款", "已取消"}; + options.ConditionalFormats.Add(new ConditionalFormat("金额") + { + Operator = ConditionalOperator.GreaterThan, + Value = 120, + FontColor = ExcelStyleColor.Red + }); + }); + + Assert.AreEqual(1, sheet.PaneInformation.HorizontalSplitPosition, "冻结位置"); + Assert.AreEqual(1, sheet.GetDataValidations().Count, "数据验证"); + Assert.AreEqual(1, sheet.SheetConditionalFormatting.NumConditionalFormattings, "条件格式"); + Assert.IsNotNull(sheet.Workbook.GetName("_xlnm._FilterDatabase"), "筛选区域"); + } + + [TestMethod] + public void TwoLongDropdownsBothLandOnTheHiddenSheet() + { + // 超过 255 字符的列表放不进验证本身，要写到隐藏表上。两张列表即两列，流式导出的工作表只能 + // 自上而下写一遍，故须一次写完，否则后写的那一列会丢。 + var first = Enumerable.Range(0, 60).Select(i => "甲选项" + i).ToArray(); + var second = Enumerable.Range(0, 80).Select(i => "乙选项" + i).ToArray(); + + using var output = new MemoryStream(); + new ExcelExporter().ObjectToExcelStream(Rows(20), output, options => + { + options.ExcelType = ExcelType.Xlsx; + options.StreamingRowWindow = Window; + options.Dropdowns["省份"] = first; + options.Dropdowns["城市"] = second; + }); + + var workbook = WorkbookFactory.Create(new MemoryStream(output.ToArray())); + var lists = workbook.GetSheet("_excel2object_lists"); + Assert.IsNotNull(lists, "隐藏的列表工作表"); + for (var i = 0; i < second.Length; i++) + { + var row = lists.GetRow(i); + Assert.AreEqual(i < first.Length ? first[i] : null, row.GetCell(0)?.StringCellValue, $"第 {i} 行第一张列表"); + Assert.AreEqual(second[i], row.GetCell(1)?.StringCellValue, $"第 {i} 行第二张列表"); + } + } + + [TestMethod] + public void AutoColumnWidthMatchesTheInMemoryExport() + { + var rows = Rows(60); + Action configure = options => options.AutoColumnWidth = true; + + var streamed = Streamed(rows, configure); + var inMemory = InMemory(rows, configure); + for (var i = 0; i < 5; i++) + Assert.AreEqual(inMemory.GetColumnWidth(i), streamed.GetColumnWidth(i), $"第 {i} 列的宽度"); + } + + [TestMethod] + public void TheDataIsEnumeratedOnlyOnce() + { + var source = new CountingSequence(Rows(40)); + using var output = new MemoryStream(); + new ExcelExporter().ObjectToExcelStream(source, output, options => + { + options.ExcelType = ExcelType.Xlsx; + options.StreamingRowWindow = Window; + // 自动列宽此前要先把数据过一遍量宽度，如今与写入同一遍完成 + options.AutoColumnWidth = true; + }); + + Assert.AreEqual(1, source.Enumerations); + } + + [TestMethod] + public void TheOutputStreamIsLeftOpen() + { + using var output = new MemoryStream(); + new ExcelExporter().ObjectToExcelStream(Rows(20), output, options => options.StreamingRowWindow = Window); + + // 关闭了的 MemoryStream 连长度都读不到 + Assert.IsTrue(output.Length > 0); + output.WriteByte(0); + } + + [TestMethod] + public void NothingIsWrittenWhenAMergeColumnIsUnknown() + { + using var output = new MemoryStream(); + var e = Assert.ThrowsException(() => + new ExcelExporter().ObjectToExcelStream(Rows(40), output, options => + { + options.StreamingRowWindow = Window; + options.MergeRepeatedColumns.Add("不存在的列"); + })); + + StringAssert.Contains(e.Message, "不存在的列"); + // 列名在写入开始前即解析，故调用方的流上不会留下半个文件 + Assert.AreEqual(0, output.Length); + } + + [TestMethod] + public void TheRowWindowMustBePositive() + { + using var output = new MemoryStream(); + var e = Assert.ThrowsException(() => + new ExcelExporter().ObjectToExcelStream(Rows(10), output, options => options.StreamingRowWindow = 0)); + + StringAssert.Contains(e.Message, "StreamingRowWindow"); + } + + [TestMethod] + public void XlsIsWrittenToTheStreamToo() + { + // .xls 无从流式写入，仍应写出一份可读的文件 + using var output = new MemoryStream(); + new ExcelExporter().ObjectToExcelStream(Rows(20), output, options => options.ExcelType = ExcelType.Xls); + + var sheet = WorkbookFactory.Create(new MemoryStream(output.ToArray())).GetSheetAt(0); + Assert.AreEqual(20, sheet.LastRowNum); + Assert.AreEqual("省0", sheet.GetRow(1).GetCell(0).StringCellValue); + } + + [TestMethod] + public void TheStreamedFileImportsBack() + { + // 流式写入的文本不进共享字符串表，而是直接写在单元格里（SXSSF 的做法，否则各不相同的字符串 + // 会全数留在内存中）。读的一侧不受影响，此处以导回对象确认。 + var rows = Rows(30); + using var output = new MemoryStream(); + new ExcelExporter().ObjectToExcelStream(rows, output, options => + { + options.ExcelType = ExcelType.Xlsx; + options.StreamingRowWindow = Window; + }); + + var imported = new ExcelImporter().ExcelToObject(output.ToArray()).ToList(); + Assert.AreEqual(rows.Count, imported.Count); + for (var i = 0; i < rows.Count; i++) + { + Assert.AreEqual(rows[i].Province, imported[i].Province, $"第 {i} 行的省份"); + Assert.AreEqual(rows[i].Amount, imported[i].Amount, $"第 {i} 行的金额"); + Assert.AreEqual(rows[i].PlacedAt, imported[i].PlacedAt, $"第 {i} 行的下单时间"); + } + } + + [TestMethod] + public void TempFilesAreWrittenWhileExportingAndRemovedAtTheEnd() + { + // 临时目录已由 TestTempDirectory 改指到本进程自己的目录，故这里数到的只会是自己写下的文件 + var duringExport = -1; + var rows = Rows(100).Select((row, i) => + { + if (i == 90) duringExport = PoiFiles(); + return row; + }); + + using var output = new MemoryStream(); + new ExcelExporter().ObjectToExcelStream(rows, output, options => + { + options.ExcelType = ExcelType.Xlsx; + options.StreamingRowWindow = Window; + }); + + Assert.AreEqual(1, duringExport, "写入过程中应有一个临时文件装着已刷出内存的行"); + Assert.AreEqual(0, PoiFiles(), "导出结束后临时文件应已删除"); + } + + /// NPOI 把刷出内存的行写在临时目录下的这个位置。 + private static int PoiFiles() + { + var dir = Path.Combine(Path.GetTempPath(), "poifiles"); + return Directory.Exists(dir) ? Directory.GetFiles(dir, "poi-sxssf-sheet*").Length : 0; + } + + [TestMethod] + public void FormulaColumnsAreWrittenPerRow() + { + var sheet = Streamed(Rows(30), options => options.FormulaColumns.Add(new FormulaColumn + { + Title = "两倍金额", + Formula = c => c["金额"] * 2 + })); + + // 公式按各自所在的行写出，行号不因先前的行离开内存而错位 + Assert.AreEqual("C2*2", sheet.GetRow(1).GetCell(5).CellFormula); + Assert.AreEqual("C21*2", sheet.GetRow(20).GetCell(5).CellFormula); + Assert.AreEqual("C31*2", sheet.GetRow(30).GetCell(5).CellFormula); + } + + [TestMethod] + public void FormulasCanReferToASheetAlreadyInTheWorkbook() + { + // 先写出一张工作簿，再以流式导出向其追加一张表，其中的公式引用前一张表的列 + var first = new ExcelExporter().ObjectToExcelBytes(Rows(10), options => + { + options.ExcelType = ExcelType.Xlsx; + options.SheetTitle = "上月"; + }); + + using var output = new MemoryStream(); + new ExcelExporter().ObjectToExcelStream(Rows(30), output, options => + { + options.ExcelType = ExcelType.Xlsx; + options.StreamingRowWindow = Window; + options.SheetTitle = "本月"; + options.SourceExcelBytes = first; + options.FormulaColumns.Add(new FormulaColumn + { + Title = "环比", + Formula = c => c["金额"] - c.Sheet("上月")["金额"] + }); + }); + + var workbook = WorkbookFactory.Create(new MemoryStream(output.ToArray())); + var sheet = workbook.GetSheet("本月"); + // 表名含非 ASCII 字符，读回时 NPOI 会给它加上引号 + Assert.AreEqual("C2-'上月'!C2", sheet.GetRow(1).GetCell(5).CellFormula); + Assert.AreEqual(10, workbook.GetSheet("上月").LastRowNum, "原有的表应原样保留"); + } + + /// 记下被遍历的次数，遍历第二遍即失败——流式导出只应过一遍数据。 + private sealed class CountingSequence : IEnumerable + { + private readonly List _rows; + + public CountingSequence(List rows) + { + _rows = rows; + } + + public int Enumerations { get; private set; } + + public IEnumerator GetEnumerator() + { + Enumerations++; + return _rows.GetEnumerator(); + } + + IEnumerator IEnumerable.GetEnumerator() + { + return GetEnumerator(); + } + } +} diff --git a/Chsword.Excel2Object.Tests/TestTempDirectory.cs b/Chsword.Excel2Object.Tests/TestTempDirectory.cs new file mode 100644 index 0000000..fba31f4 --- /dev/null +++ b/Chsword.Excel2Object.Tests/TestTempDirectory.cs @@ -0,0 +1,39 @@ +using System; +using System.IO; +using Microsoft.VisualStudio.TestTools.UnitTesting; + +namespace Chsword.Excel2Object.Tests; + +/// +/// 把临时目录改指到本进程自己的目录。流式导出会把刷出内存的行写进临时目录下的 poifiles， +/// 而 NPOI 只在首次使用时解析一次该目录，各目标框架的测试进程又是并行跑的：共用系统临时目录， +/// 「导出结束后临时文件已删除」这类断言就会数到别的进程头上。 +/// +[TestClass] +public static class TestTempDirectory +{ + private static string? _dir; + + [AssemblyInitialize] + public static void Redirect(TestContext context) + { + _dir = Path.Combine(Path.GetTempPath(), "e2o-tests-" + Guid.NewGuid().ToString("N")); + Directory.CreateDirectory(_dir); + foreach (var name in new[] {"TMPDIR", "TMP", "TEMP"}) Environment.SetEnvironmentVariable(name, _dir); + } + + [AssemblyCleanup] + public static void Remove() + { + if (_dir == null) return; + + try + { + Directory.Delete(_dir, true); + } + catch (IOException) + { + // 清理失败不必连累测试结论 + } + } +} diff --git a/Chsword.Excel2Object/Chsword.Excel2Object.csproj b/Chsword.Excel2Object/Chsword.Excel2Object.csproj index 358367c..e8f7af6 100644 --- a/Chsword.Excel2Object/Chsword.Excel2Object.csproj +++ b/Chsword.Excel2Object/Chsword.Excel2Object.csproj @@ -15,7 +15,7 @@ true Chsword.Excel2Object Library Zou Jian - 2.9.1 + 2.10.0 Copyright ? 2014-2025 https://github.com/chsword/Excel2Object/ README.md diff --git a/Chsword.Excel2Object/ExcelExporter.cs b/Chsword.Excel2Object/ExcelExporter.cs index 025219b..4bb3e63 100644 --- a/Chsword.Excel2Object/ExcelExporter.cs +++ b/Chsword.Excel2Object/ExcelExporter.cs @@ -8,6 +8,7 @@ using NPOI.SS.Formula; using NPOI.SS.UserModel; using NPOI.SS.Util; +using NPOI.XSSF.Streaming; using NPOI.XSSF.UserModel; using HorizontalAlignment = Chsword.Excel2Object.Styles.HorizontalAlignment; @@ -79,15 +80,110 @@ public class ExcelExporter return ObjectToExcelBytes(excel, options); } + /// + /// 边取数据边写入 ，内存中只保留 + /// 指定的若干行，适用于行数多到不宜先在 + /// 内存中建好整个工作簿的导出。 + /// + /// + /// + /// 仅 能够流式写入； 的格式决定了 + /// 必须先在内存中建好再写出，此时本方法只是把结果写进 ，并不省内存 + /// （该格式至多 65,536 行，本也不适合大文件）。 + /// + /// + /// 流式写入依赖 NPOI 的 SXSSF，后者在把行刷出内存时要测量默认字符宽度，为此需要 SkiaSharp， + /// 而 NPOI 把该依赖标为不随包传递。缺少时本方法会在写入开始前抛出 + /// 并说明需要引用哪些包。 + /// + /// 由调用方负责关闭，本方法不会关闭它。 + /// + /// + /// + /// using var file = File.Create("orders.xlsx"); + /// new ExcelExporter().ObjectToExcelStream(ReadOrders(), file, options => + /// { + /// options.ExcelType = ExcelType.Xlsx; + /// options.FreezeHeader = true; + /// }); + /// + /// + public void ObjectToExcelStream(IEnumerable data, Stream output, + Action? optionsAction = null) + { + var options = new ExcelExporterOptions(); + optionsAction?.Invoke(options); + var excel = data is IEnumerable> models + ? TypeConvert.ConvertDictionaryToExcelModel(models, options) + : TypeConvert.ConvertObjectToExcelModel(data, options); + + ObjectToExcelStream(excel, options, output); + } + + /// + public void ObjectToExcelStream(DataTable dt, Stream output, + Action? optionsAction = null) + { + var options = new ExcelExporterOptions(); + optionsAction?.Invoke(options); + ObjectToExcelStream(TypeConvert.ConvertDataSetToExcelModel(dt, options), options, output); + } + + private void ObjectToExcelStream(ExcelModel excel, ExcelExporterOptions options, Stream output) + { + if (output == null) throw new ArgumentNullException(nameof(output)); + + // 字节数组那两个入口在源工作簿读不出来时返回 null，此处没有 null 可返回，故如实报错 + if (!Write(excel, options, output, true)) + throw new Excel2ObjectException("SourceExcelBytes 不是能够打开的工作簿。"); + } + private byte[]? ObjectToExcelBytes(ExcelModel excel, ExcelExporterOptions options) { - var excelType = options.ExcelType; + using var output = new MemoryStream(); + return Write(excel, options, output, false) ? output.ToArray() : null; + } + + /// 写出整个工作簿；源工作簿读不出来时返回 false。 + private bool Write(ExcelModel excel, ExcelExporterOptions options, Stream output, bool streaming) + { + var workbook = OpenWorkbook(options, streaming); + if (workbook == null) return false; + + try + { + if (options.MappingColumnAction == null) options.MappingColumnAction = (s, _) => s; + var styleFactory = new CellStyleFactory(workbook); + + if (excel.Sheets != null) + foreach (var excelSheet in excel.Sheets) + WriteSheet(workbook, excelSheet, options, styleFactory); + + // 调用方给的流由调用方关闭 + workbook.Write(output, true); + return true; + } + finally + { + // SXSSF 把刷出内存的行写在临时文件里，不释放则留在磁盘上 + (workbook as SXSSFWorkbook)?.Dispose(); + } + } + /// + /// 建立或读入工作簿。流式导出用 SXSSF：它只在内存中保留最近的若干行，其余写入临时文件。 + /// + private static IWorkbook? OpenWorkbook(ExcelExporterOptions options, bool streaming) + { IWorkbook workbook; if (options.SourceExcelBytes == null) - workbook = Workbook(excelType); + { + workbook = streaming && options.ExcelType == ExcelType.Xlsx + ? new SXSSFWorkbook(Window(options)) + : Workbook(options.ExcelType); + } else - // read work book + { try { using var memoryStream = new MemoryStream(options.SourceExcelBytes); @@ -98,84 +194,102 @@ public class ExcelExporter return null; } - if (options.MappingColumnAction == null) options.MappingColumnAction = (s, _) => s; - var styleFactory = new CellStyleFactory(workbook); - - if (excel.Sheets != null) - foreach (var excelSheet in excel.Sheets) - { - var sheet = string.IsNullOrWhiteSpace(excelSheet.Title) - ? workbook.CreateSheet() - : workbook.CreateSheet(excelSheet.Title); - sheet.ForceFormulaRecalculation = true; - var columns = excelSheet.Columns.OrderBy(c => c.Order).ToArray(); - - // Calculate column widths - if (options.AutoColumnWidth) - { - var columnWidths = CalculateColumnWidths(columns, excelSheet.Rows, options, options.MappingColumnAction); - for (var i = 0; i < columns.Length; i++) - { - sheet.SetColumnWidth(i, columnWidths[i] * ExcelConstants.DefaultColumnWidthMultiplier); - } - } - else - { - for (var i = 0; i < columns.Length; i++) - { - sheet.SetColumnWidth(i, options.DefaultColumnWidth * ExcelConstants.DefaultColumnWidthMultiplier); - } - } - - var headerRow = sheet.CreateRow(ExcelConstants.DefaultHeaderRowIndex); - for (var i = 0; i < columns.Length; i++) - { - var cell = headerRow.CreateCell(i); - cell.SetCellType(CellType.String); - cell.SetCellValue(options.MappingColumnAction(columns[i].Title, columns[i].Type)); - var headerStyle = styleFactory.Get(StyleResolver.Header(columns[i], options.Styles)); - if (headerStyle != null) cell.CellStyle = headerStyle; - } + // .xls 无从流式写入，续写时就按原样在内存中完成 + if (streaming && workbook is XSSFWorkbook xssf) workbook = new SXSSFWorkbook(xssf, Window(options)); + } + + if (workbook is SXSSFWorkbook streamed) StreamingSupport.Ensure(streamed); + return workbook; + } + + private static int Window(ExcelExporterOptions options) + { + if (options.StreamingRowWindow > 0) return options.StreamingRowWindow; + + throw new Excel2ObjectException( + $"StreamingRowWindow 为 {options.StreamingRowWindow}：内存中保留的行数须为正数。"); + } - // one look per column per kind of row, rather than one worked out per cell - var oddStyles = columns - .Select(c => StyleResolver.Resolve(c, options.Styles, ExcelConstants.DefaultDataStartRowIndex)) - .ToArray(); - var evenStyles = columns - .Select(c => StyleResolver.Resolve(c, options.Styles, ExcelConstants.DefaultDataStartRowIndex + 1)) - .ToArray(); - - var columnTitles = columns.Select(c => c.Title).ToArray(); - var sheetColumnsResolver = BuildSheetColumnsResolver(workbook, sheet, columnTitles!); - var rowNumber = ExcelConstants.DefaultDataStartRowIndex; - var data = excelSheet.Rows; - foreach (var item in data) + private void WriteSheet(IWorkbook workbook, SheetModel excelSheet, ExcelExporterOptions options, + CellStyleFactory styleFactory) + { + var sheet = string.IsNullOrWhiteSpace(excelSheet.Title) + ? workbook.CreateSheet() + : workbook.CreateSheet(excelSheet.Title); + sheet.ForceFormulaRecalculation = true; + var columns = excelSheet.Columns.OrderBy(c => c.Order).ToArray(); + + // 要合并哪些列在此处即解析：写入尚未开始，列名写错也就不会留下写了一半的工作表 + var merges = new MergedRegions(columns, options); + var tracking = merges.TracksAnyColumn; + + // 自动列宽按内容而定，故边写边量，量完再设；数据只经过一遍，流式导出也就无需回看 + var widths = options.AutoColumnWidth ? new int[columns.Length] : null; + if (widths == null) + for (var i = 0; i < columns.Length; i++) + sheet.SetColumnWidth(i, options.DefaultColumnWidth * ExcelConstants.DefaultColumnWidthMultiplier); + + var headerRow = sheet.CreateRow(ExcelConstants.DefaultHeaderRowIndex); + for (var i = 0; i < columns.Length; i++) + { + var title = options.MappingColumnAction!(columns[i].Title, columns[i].Type); + var cell = headerRow.CreateCell(i); + cell.SetCellType(CellType.String); + cell.SetCellValue(title); + var headerStyle = styleFactory.Get(StyleResolver.Header(columns[i], options.Styles)); + if (headerStyle != null) cell.CellStyle = headerStyle; + if (widths != null) widths[i] = CalculateTextWidth(title); + } + + // one look per column per kind of row, rather than one worked out per cell + var oddStyles = columns + .Select(c => StyleResolver.Resolve(c, options.Styles, ExcelConstants.DefaultDataStartRowIndex)) + .ToArray(); + var evenStyles = columns + .Select(c => StyleResolver.Resolve(c, options.Styles, ExcelConstants.DefaultDataStartRowIndex + 1)) + .ToArray(); + + var columnTitles = columns.Select(c => c.Title).ToArray(); + var sheetColumnsResolver = BuildSheetColumnsResolver(workbook, sheet, columnTitles); + var rowNumber = ExcelConstants.DefaultDataStartRowIndex; + foreach (var item in excelSheet.Rows) + { + var row = sheet.CreateRow(rowNumber++); + var rowStyles = (row.RowNum - ExcelConstants.DefaultDataStartRowIndex) % 2 == 0 + ? oddStyles + : evenStyles; + for (var i = 0; i < columns.Length; i++) + { + var column = columns[i]; + var cell = row.CreateCell(i); + var raw = item.TryGetValue(column.Title, out var value) ? value : null; + if (raw is DBNull) raw = null; + var val = raw?.ToString() ?? ""; + SetCellValue(options, column, rowStyles[i], cell, raw, val, columnTitles, + sheetColumnsResolver, styleFactory); + + // 一格量一次宽：单元格显示成什么样由所在列的样式决定，隔行底色于此无关， + // 故一律按奇数行那一套来量，与逐列解析一次的做法一致 + if (widths != null) { - var row = sheet.CreateRow(rowNumber++); - var rowStyles = (row.RowNum - ExcelConstants.DefaultDataStartRowIndex) % 2 == 0 - ? oddStyles - : evenStyles; - for (var i = 0; i < columns.Length; i++) - { - var column = columns[i]; - var cell = row.CreateCell(i); - var raw = column.Title != null && item.TryGetValue(column.Title, out var value) - ? value - : null; - if (raw is DBNull) raw = null; - var val = raw?.ToString() ?? ""; - SetCellValue(options, column, rowStyles[i], cell, raw, val, columnTitles, - sheetColumnsResolver, styleFactory); - } + var width = CalculateTextWidth(CellText(raw, oddStyles[i])); + if (width > widths[i]) widths[i] = width; } - ApplyHeaderView(sheet, options, columns.Length, rowNumber - 1); - DropdownValidation.Apply(sheet, columns, rowNumber - 1); - ConditionalFormatting.Apply(sheet, columns, rowNumber - 1, options.ConditionalFormats); - MergedRegions.Apply(sheet, columns, rowNumber - 1, options); + if (tracking && merges.Tracks(i)) merges.Observe(i, row.RowNum, cell); } + } - return ToBytes(workbook); + if (widths != null) + for (var i = 0; i < columns.Length; i++) + sheet.SetColumnWidth(i, + Math.Max(options.MinColumnWidth, Math.Min(options.MaxColumnWidth, widths[i])) * + ExcelConstants.DefaultColumnWidthMultiplier); + + ApplyHeaderView(sheet, options, columns.Length, rowNumber - 1); + DropdownValidation.Apply(sheet, columns, rowNumber - 1); + ConditionalFormatting.Apply(sheet, columns, rowNumber - 1, options.ConditionalFormats); + merges.Apply(sheet, rowNumber - 1); } /// @@ -257,7 +371,7 @@ private static IWorkbook Workbook(ExcelType excelType) var other = workbook.GetSheet(sheetTitle) ?? throw new Excel2ObjectException( $"refers to sheet [{sheetTitle}], which is not in the workbook. " + "Write that sheet first, e.g. via AppendObjectToExcelBytes."); - var header = other.GetRow(ExcelConstants.DefaultHeaderRowIndex); + var header = HeaderRow(workbook, sheetTitle, other); if (header != null) // GetCell(i) rather than Cells: the latter skips blank cells and would shift indexes. titles = Enumerable.Range(0, header.LastCellNum) @@ -270,6 +384,18 @@ private static IWorkbook Workbook(ExcelType excelType) }; } + /// + /// 另一张表的表头行。流式导出时 SXSSF 只是在工作簿外面套了一层：随源工作簿带进来的表，其行 + /// 并不在这一层里，须向底下的 XSSF 去取。 + /// + private static IRow? HeaderRow(IWorkbook workbook, string sheetTitle, ISheet sheet) + { + var header = sheet.GetRow(ExcelConstants.DefaultHeaderRowIndex); + if (header != null || workbook is not SXSSFWorkbook streamed) return header; + + return streamed.XssfWorkbook.GetSheet(sheetTitle)?.GetRow(ExcelConstants.DefaultHeaderRowIndex); + } + private void SetCellValue(ExcelExporterOptions options, ExcelColumn column, ResolvedColumnStyle resolved, ICell cell, object? raw, string val, string[] columnTitles, Func sheetColumnsResolver, CellStyleFactory styleFactory) @@ -414,55 +540,6 @@ private static void SetStyle(ICell cell, ICellStyle? style) if (style != null) cell.CellStyle = style; } - /// - /// Calculate optimal column widths based on content - /// - private int[] CalculateColumnWidths(ExcelColumn[] columns, IEnumerable> data, - ExcelExporterOptions options, Func mappingColumnAction) - { - var columnWidths = new int[columns.Length]; - - // Initialize with header widths - for (var i = 0; i < columns.Length; i++) - { - var headerText = mappingColumnAction(columns[i].Title ?? "", columns[i].Type); - columnWidths[i] = CalculateTextWidth(headerText); - } - - // a cell is measured as it is shown, so the styles of its column decide the text; the stripes - // make no difference to that, and resolving once per column keeps it out of the row loop - var styles = columns - .Select(c => StyleResolver.Resolve(c, options.Styles, ExcelConstants.DefaultDataStartRowIndex)) - .ToArray(); - - // Calculate widths based on data content - foreach (var item in data) - { - for (var i = 0; i < columns.Length; i++) - { - var column = columns[i]; - if (column.Title != null && item.TryGetValue(column.Title, out var value)) - { - var cellText = CellText(value, styles[i]); - var textWidth = CalculateTextWidth(cellText); - if (textWidth > columnWidths[i]) - { - columnWidths[i] = textWidth; - } - } - } - } - - // Apply min/max constraints - for (var i = 0; i < columnWidths.Length; i++) - { - columnWidths[i] = Math.Max(options.MinColumnWidth, - Math.Min(options.MaxColumnWidth, columnWidths[i])); - } - - return columnWidths; - } - /// /// The text a cell shows, which is what the width of an auto-sized column has to fit. A date /// column given a number format - which only a format written for that very column can do - shows diff --git a/Chsword.Excel2Object/ExcelHelper.cs b/Chsword.Excel2Object/ExcelHelper.cs index 870f805..d1f090f 100644 --- a/Chsword.Excel2Object/ExcelHelper.cs +++ b/Chsword.Excel2Object/ExcelHelper.cs @@ -111,6 +111,20 @@ public static void ObjectToExcel(IEnumerable data, string path, return excelExporter.ObjectToExcelBytes(data, optionsAction); } + /// + public static void ObjectToExcelStream(IEnumerable data, Stream output, + Action? optionsAction = null) + { + new ExcelExporter().ObjectToExcelStream(data, output, optionsAction); + } + + /// + public static void ObjectToExcelStream(DataTable dt, Stream output, + Action? optionsAction = null) + { + new ExcelExporter().ObjectToExcelStream(dt, output, optionsAction); + } + private static void WriteExcelBytesToFile(byte[]? bytes, string path) { if (bytes != null) diff --git a/Chsword.Excel2Object/Internal/DropdownValidation.cs b/Chsword.Excel2Object/Internal/DropdownValidation.cs index a39fb71..966a4cc 100644 --- a/Chsword.Excel2Object/Internal/DropdownValidation.cs +++ b/Chsword.Excel2Object/Internal/DropdownValidation.cs @@ -31,15 +31,31 @@ public static void Apply(ISheet sheet, ExcelColumn[] columns, int lastDataRowInd var lastRow = Math.Max(lastDataRowIndex, ExcelConstants.DefaultDataStartRowIndex); var helper = sheet.GetDataValidationHelper(); + // 放不进验证本身的列先收齐，好把隐藏表一次写完：流式导出的工作表只能自上而下写一遍， + // 回头往写过的行上补一列会丢掉那些值 + var longLists = new List(); + var listIndexes = new Dictionary(); for (var i = 0; i < columns.Length; i++) { var values = columns[i].Dropdown; if (values == null || values.Length == 0) continue; Validate(values, columns[i].Title); + if (FitsInline(values)) continue; - var constraint = FitsInline(values) - ? helper.CreateExplicitListConstraint(values) - : helper.CreateFormulaListConstraint(WriteToListSheet(sheet.Workbook, values)); + listIndexes[i] = longLists.Count; + longLists.Add(values); + } + + var names = WriteToListSheet(sheet.Workbook, longLists); + + for (var i = 0; i < columns.Length; i++) + { + var values = columns[i].Dropdown; + if (values == null || values.Length == 0) continue; + + var constraint = listIndexes.TryGetValue(i, out var list) + ? helper.CreateFormulaListConstraint(names[list]) + : helper.CreateExplicitListConstraint(values); var validation = helper.CreateValidation(constraint, new CellRangeAddressList(ExcelConstants.DefaultDataStartRowIndex, lastRow, i, i)); @@ -72,34 +88,47 @@ private static bool FitsInline(string[] values) } /// - /// Puts the values in their own column on the hidden sheet and returns the defined name covering - /// them. A validation reads the name rather than the range itself, because the .xls format has no - /// way to point a validation at another sheet directly. + /// Puts each list in its own column on the hidden sheet and returns the defined name covering it. + /// A validation reads the name rather than the range itself, because the .xls format has no way to + /// point a validation at another sheet directly. /// - private static string WriteToListSheet(IWorkbook workbook, string[] values) + /// + /// 所有列表一并写入，逐行填满各列，而非一列写完再写下一列：流式导出的工作表写过的行会被刷出 + /// 内存，回头再写即告丢失。 + /// + private static string[] WriteToListSheet(IWorkbook workbook, List lists) { + if (lists.Count == 0) return []; + var sheet = ListSheet(workbook); - // the sheet grows to the right, one column per list, so lists written earlier keep their range - var column = 0; + // 该表向右增长，一列一张列表，先前写下的列表因而保有其区域 + var first = 0; var header = sheet.GetRow(0); - if (header != null) column = header.LastCellNum < 0 ? 0 : header.LastCellNum; + if (header != null) first = header.LastCellNum < 0 ? 0 : header.LastCellNum; - for (var i = 0; i < values.Length; i++) + var rows = lists.Max(values => values.Length); + for (var i = 0; i < rows; i++) { var row = sheet.GetRow(i) ?? sheet.CreateRow(i); - row.CreateCell(column).SetCellValue(values[i]); + for (var list = 0; list < lists.Count; list++) + if (i < lists[list].Length) + row.CreateCell(first + list).SetCellValue(lists[list][i]); } - var letter = CellReference.ConvertNumToColString(column); - // the sheet name needs no quoting, which matters: xlsx keeps the formula as written while xls - // reparses it - var range = $"{sheet.SheetName}!${letter}$1:${letter}${values.Length}"; + var names = new string[lists.Count]; + for (var list = 0; list < lists.Count; list++) + { + var letter = CellReference.ConvertNumToColString(first + list); + // the sheet name needs no quoting, which matters: xlsx keeps the formula as written while xls + // reparses it + var name = workbook.CreateName(); + name.NameName = FreeName(workbook); + name.RefersToFormula = $"{sheet.SheetName}!${letter}$1:${letter}${lists[list].Length}"; + names[list] = name.NameName; + } - var name = workbook.CreateName(); - name.NameName = FreeName(workbook); - name.RefersToFormula = range; - return name.NameName; + return names; } private static string FreeName(IWorkbook workbook) diff --git a/Chsword.Excel2Object/Internal/MergedRegions.cs b/Chsword.Excel2Object/Internal/MergedRegions.cs index 246aa03..8e81a0b 100644 --- a/Chsword.Excel2Object/Internal/MergedRegions.cs +++ b/Chsword.Excel2Object/Internal/MergedRegions.cs @@ -9,25 +9,63 @@ namespace Chsword.Excel2Object.Internal; /// /// 合并单元格：把某列中连续相同的值并成一格，以及合并调用方另行指定的区域。 /// -internal static class MergedRegions +/// +/// 各段是在写入过程中逐格记下的，而非事后回看工作表：流式导出会把先前的行刷出内存，届时已无从读回。 +/// +internal sealed class MergedRegions { - public static void Apply(ISheet sheet, ExcelColumn[] columns, int lastDataRowIndex, - ExcelExporterOptions options) + private readonly ExcelColumn[] _columns; + private readonly ExcelExporterOptions _options; + + /// 各列当前那一段的起始行与取值，按列序号存放；未跟踪的列为空。 + private readonly Dictionary _open = new(); + + private readonly Dictionary> _runs = new(); + + public MergedRegions(ExcelColumn[] columns, ExcelExporterOptions options) { - // 同一列内的若干段天然互不相交，故按列分别算出后整体写入，无需两两比对 - var byColumn = new Dictionary>(); + _columns = columns; + _options = options; + + // 列名与公式列在此处即校验：写入尚未开始，报错也就不会留下写了一半的工作表 foreach (var title in options.MergeRepeatedColumns.Distinct(StringComparer.Ordinal)) { var column = IndexOf(columns, title); RejectFormula(columns[column]); - byColumn[column] = Runs(sheet, column, lastDataRowIndex); + _open[column] = new Run(ExcelConstants.DefaultDataStartRowIndex, null); + _runs[column] = new List(); } + } + + /// 是否有按值合并的列，没有则连每格的记录都不必做。 + public bool TracksAnyColumn => _open.Count > 0; + + public bool Tracks(int column) + { + return _open.ContainsKey(column); + } + + /// 记下刚写好的一格。行号须递增，即按写入顺序调用。 + public void Observe(int column, int rowIndex, ICell cell) + { + if (!_open.TryGetValue(column, out var run)) return; + + var value = CellValue.Of(cell); + if (value != null && run.Value != null && value.Equals(run.Value)) return; + + CloseRun(column, run, rowIndex - 1); + _open[column] = new Run(rowIndex, value); + } + + public void Apply(ISheet sheet, int lastDataRowIndex) + { + foreach (var pair in _open) CloseRun(pair.Key, pair.Value, lastDataRowIndex); var declared = new List(); - foreach (var region in options.MergedRegions) + foreach (var region in _options.MergedRegions) { - var range = Resolve(region, columns, lastDataRowIndex); - var clash = Overlapping(range, byColumn, declared); + var range = Resolve(region, _columns, lastDataRowIndex); + var clash = Overlapping(range, _runs, declared); if (clash != null) throw new Excel2ObjectException( $"区域 [{region}] 与已合并的 [{clash.FormatAsString()}] 重叠。"); @@ -35,12 +73,19 @@ public static void Apply(ISheet sheet, ExcelColumn[] columns, int lastDataRowInd declared.Add(range); } - foreach (var range in byColumn.Values.SelectMany(list => list).Concat(declared)) + foreach (var range in _runs.Values.SelectMany(list => list).Concat(declared)) // 重叠已在此处校验过，故绕开 NPOI 自带的两两比对：它对每个新区域都要扫描已有全部区域， // 十万行的分组表会因此退化为平方级 AddUnsafe(sheet, range); } + /// 一段就此收尾：只有跨越多行时才算一处合并区域。 + private void CloseRun(int column, Run run, int lastRow) + { + if (run.Value != null && lastRow > run.FirstRow) + _runs[column].Add(new CellRangeAddress(run.FirstRow, lastRow, column, column)); + } + private static void AddUnsafe(ISheet sheet, CellRangeAddress range) { switch (sheet) @@ -94,53 +139,6 @@ private static int IndexOf(ExcelColumn[] columns, string title) throw new Excel2ObjectException($"要合并的列 [{title}] 不在该工作表中。"); } - /// 该列中连续相同的值各占一段，仅相邻且相等的行参与。 - private static List Runs(ISheet sheet, int column, int lastDataRowIndex) - { - var runs = new List(); - var start = ExcelConstants.DefaultDataStartRowIndex; - if (lastDataRowIndex <= start) return runs; - - for (var row = start + 1; row <= lastDataRowIndex + 1; row++) - { - // 多走一行，好让最后一段也能收尾 - if (row <= lastDataRowIndex && Same(Cell(sheet, start, column), Cell(sheet, row, column))) continue; - - if (row - start > 1) runs.Add(new CellRangeAddress(start, row - 1, column, column)); - start = row; - } - - return runs; - } - - private static ICell? Cell(ISheet sheet, int rowIndex, int column) - { - return sheet.GetRow(rowIndex)?.GetCell(column); - } - - /// - /// 两个单元格是否装着同一个值。数值直接比较其值，不经由文本：.NET Framework 上 "R" 与 G17 - /// 两种格式都可能把不同的数渲染成同一串字符，比较文本会把它们并成一格。空单元格一律不相同， - /// 因而不参与合并。 - /// - private static bool Same(ICell? a, ICell? b) - { - if (a == null || b == null || a.CellType != b.CellType) return false; - - switch (a.CellType) - { - case CellType.Numeric: - return a.NumericCellValue.Equals(b.NumericCellValue); - case CellType.String: - var text = a.StringCellValue; - return !string.IsNullOrEmpty(text) && text == b.StringCellValue; - case CellType.Boolean: - return a.BooleanCellValue == b.BooleanCellValue; - default: - return false; - } - } - /// 把按标题与数据行序号写下的区域，换算成工作表中的坐标。 private static CellRangeAddress Resolve(MergedRegion region, ExcelColumn[] columns, int lastDataRowIndex) { @@ -211,4 +209,69 @@ private static bool Intersects(CellRangeAddress a, CellRangeAddress b) return a.FirstRow <= b.LastRow && b.FirstRow <= a.LastRow && a.FirstColumn <= b.LastColumn && b.FirstColumn <= a.LastColumn; } + + /// 某一列中正在延续的一段。 + private readonly struct Run + { + public Run(int firstRow, CellValue? value) + { + FirstRow = firstRow; + Value = value; + } + + public int FirstRow { get; } + + /// 该段的取值；为空表示这一行不参与合并（空单元格、公式等）。 + public CellValue? Value { get; } + } + + /// + /// 一格的值，于写入时即记下。数值记其值而不记文本：.NET Framework 上 "R" 与 G17 两种格式都可能 + /// 把不同的数渲染成同一串字符，比较文本会把它们并成一格。 + /// + private sealed class CellValue + { + private readonly bool _flag; + private readonly double _number; + private readonly string? _text; + private readonly CellType _type; + + private CellValue(CellType type, double number, string? text, bool flag) + { + _type = type; + _number = number; + _text = text; + _flag = flag; + } + + /// 该格参与合并时的取值；空单元格、公式等一律返回 null，即不参与。 + public static CellValue? Of(ICell cell) + { + switch (cell.CellType) + { + case CellType.Numeric: + return new CellValue(CellType.Numeric, cell.NumericCellValue, null, false); + case CellType.String: + var text = cell.StringCellValue; + return string.IsNullOrEmpty(text) ? null : new CellValue(CellType.String, 0, text, false); + case CellType.Boolean: + return new CellValue(CellType.Boolean, 0, null, cell.BooleanCellValue); + default: + return null; + } + } + + public bool Equals(CellValue other) + { + if (_type != other._type) return false; + + return _type switch + { + CellType.Numeric => _number.Equals(other._number), + CellType.String => _text == other._text, + CellType.Boolean => _flag == other._flag, + _ => false + }; + } + } } diff --git a/Chsword.Excel2Object/Internal/StreamingSupport.cs b/Chsword.Excel2Object/Internal/StreamingSupport.cs new file mode 100644 index 0000000..382403a --- /dev/null +++ b/Chsword.Excel2Object/Internal/StreamingSupport.cs @@ -0,0 +1,43 @@ +using NPOI.SS.UserModel; +using NPOI.SS.Util; + +namespace Chsword.Excel2Object.Internal; + +/// +/// 流式写入开始前的一道检查。 +/// +/// +/// NPOI 的 SXSSF 每把一行刷出内存，都要先量一次默认字符宽度（为自动列宽备着的），而这一步用的是 +/// SkiaSharp。NPOI 把 SkiaSharp 列为不随包传递的依赖，应用若未自行引用，缺失要到第一次刷行时才 +/// 暴露，届时已写下若干行，抛出的又是 FileNotFoundException 之类与 Excel 无关的异常。 +/// 故在此处先量一次：代价是一次字体度量，换来的是在任何内容写出之前给出一句说得清的错。 +/// +internal static class StreamingSupport +{ + private const string SkiaSharp = "SkiaSharp"; + + public static void Ensure(IWorkbook workbook) + { + try + { + SheetUtil.GetDefaultCharWidth(workbook); + } + catch (Exception e) when (NeedsSkiaSharp(e)) + { + throw new Excel2ObjectException( + "流式导出需要 SkiaSharp：NPOI 把行刷出内存时要测量字符宽度，而它把 SkiaSharp 列为不随包" + + "传递的依赖。请在应用中引用 SkiaSharp（Linux 上还需 SkiaSharp.NativeAssets.Linux." + + "NoDependencies），或改用 ObjectToExcelBytes 在内存中完成导出。", e); + } + } + + private static bool NeedsSkiaSharp(Exception e) + { + for (var current = e; current != null; current = current.InnerException) + if (current is TypeLoadException or FileNotFoundException or BadImageFormatException && + current.Message.IndexOf(SkiaSharp, StringComparison.OrdinalIgnoreCase) >= 0) + return true; + + return false; + } +} diff --git a/Chsword.Excel2Object/Internal/TypeConvert.cs b/Chsword.Excel2Object/Internal/TypeConvert.cs index 1fb058f..4548f82 100644 --- a/Chsword.Excel2Object/Internal/TypeConvert.cs +++ b/Chsword.Excel2Object/Internal/TypeConvert.cs @@ -1,5 +1,6 @@ using System.Data; using System.Linq.Expressions; +using System.Reflection; using Chsword.Excel2Object.Options; namespace Chsword.Excel2Object.Internal; @@ -13,13 +14,30 @@ public static ExcelModel ConvertDictionaryToExcelModel(IEnumerable()}; var sheet = SheetModel.Create(sheetTitle); excel.Sheets.Add(sheet); - var list = data.ToList(); - var title = list.FirstOrDefault(); - if (title == null) return excel; - var columns = title.Keys.Select((c, i) => new ExcelColumn(c, typeof(string)) {Order = i}).ToList(); + // 列由首行的键决定，故首行须先取出；其余各行留待写入时逐行取用，以免整份数据都进内存 + var rows = data.GetEnumerator(); + bool any; + try + { + any = rows.MoveNext(); + } + catch + { + rows.Dispose(); + throw; + } + + if (!any) + { + rows.Dispose(); + return excel; + } + + var first = rows.Current; + var columns = first.Keys.Select((c, i) => new ExcelColumn(c, typeof(string)) {Order = i}).ToList(); sheet.Columns = AttachColumns(columns, options); - sheet.Rows = list; + sheet.Rows = FirstThenRest(first, rows); return excel; } @@ -57,18 +75,7 @@ public static ExcelModel ConvertObjectToExcelModel(IEnumerable d } sheet.Columns = AttachColumns(columns, options); - foreach (var item in data.Where(c => c != null)) - { - var row = new Dictionary(); - foreach (var column in objKeysArray) - { - // 值为 null 的属性不入字典：导出时取不到值与取到 null 同样写成空白单元格 - var value = column.Key.GetValue(item, null); - if (value != null) row[column.Value.Title] = value; - } - - sheet.Rows.Add(row); - } + sheet.Rows = ToRows(data, objKeysArray); return excel; } @@ -85,16 +92,53 @@ internal static ExcelModel ConvertDataSetToExcelModel(DataTable dt, ExcelExporte .Select((item, i) => new ExcelColumn(item.ColumnName, item.DataType) {Order = i}).ToList(); sheet.Columns = AttachColumns(columns, options); - var data = dt.Rows.Cast().ToArray(); - foreach (var item in data.Where(c => c != null)) + sheet.Rows = ToRows(dt, dataSetColumnArray); + + return excel; + } + + /// 逐行取值，写一行取一行，而非先把整份数据摊成字典列表。 + private static IEnumerable> ToRows(IEnumerable data, + KeyValuePair[] columns) + { + foreach (var item in data) { + if (item == null) continue; + var row = new Dictionary(); - foreach (var column in dataSetColumnArray) row[column.ColumnName] = item[column.ColumnName]; + foreach (var column in columns) + { + // 值为 null 的属性不入字典：导出时取不到值与取到 null 同样写成空白单元格 + var value = column.Key.GetValue(item, null); + if (value != null) row[column.Value.Title] = value; + } - sheet.Rows.Add(row); + yield return row; } + } - return excel; + private static IEnumerable> ToRows(DataTable dt, DataColumn[] columns) + { + foreach (DataRow item in dt.Rows) + { + if (item == null) continue; + + var row = new Dictionary(); + foreach (var column in columns) row[column.ColumnName] = item[column.ColumnName]; + + yield return row; + } + } + + /// 把已取出的首行与余下各行接回一条序列，并在遍历结束时释放枚举器。 + private static IEnumerable> FirstThenRest(Dictionary first, + IEnumerator> rest) + { + using (rest) + { + yield return first; + while (rest.MoveNext()) yield return rest.Current; + } } private static List AttachColumns(List columns, ExcelExporterOptions options) diff --git a/Chsword.Excel2Object/Models/SheetModel.cs b/Chsword.Excel2Object/Models/SheetModel.cs index 35d1315..bdef751 100644 --- a/Chsword.Excel2Object/Models/SheetModel.cs +++ b/Chsword.Excel2Object/Models/SheetModel.cs @@ -4,7 +4,11 @@ internal class SheetModel { public List Columns { get; set; } = new(); public int Index { get; set; } - public List> Rows { get; set; } = new(); + /// + /// 数据行。声明为惰性序列而非列表：流式导出要边取边写，一次只在内存里留一行。 + /// 该序列只会被遍历一次。 + /// + public IEnumerable> Rows { get; set; } = Enumerable.Empty>(); public string Title { get; private set; } = null!; public static SheetModel Create(string? title) diff --git a/Chsword.Excel2Object/Options/ExcelExporterOptions.cs b/Chsword.Excel2Object/Options/ExcelExporterOptions.cs index 53e399b..88e1bb2 100644 --- a/Chsword.Excel2Object/Options/ExcelExporterOptions.cs +++ b/Chsword.Excel2Object/Options/ExcelExporterOptions.cs @@ -97,6 +97,13 @@ public class ExcelExporterOptions /// public IDictionary Dropdowns { get; set; } = new Dictionary(); + /// + /// 流式导出时内存中保留的数据行数（默认 100），仅 + /// 用得到。行一旦被刷出内存即写入临时 + /// 文件，此后不再能读回，因此该值并非越小越好：它同时决定了写入过程中可回看的范围。 + /// + public int StreamingRowWindow { get; set; } = 100; + /// /// Put Excel's filter dropdowns on the header row, over the data written in this export /// (default: false). diff --git a/README.md b/README.md index 15d4439..9a2add2 100644 --- a/README.md +++ b/README.md @@ -59,12 +59,19 @@ excel2obj generate-model orders.xlsx --class Order # 由表头生成 - [x] 数据验证（下拉列表）✅ **v2.5.0 新增** - [x] 条件格式 ✅ **v2.6.0 新增** - [x] 样式表（CSS 风格）：背景色、边框、隔行底色、表头样式 ✅ **v2.7.0 新增** +- [x] 合并单元格：按列合并连续相同值、指定区域 ✅ **v2.9.1 新增** +- [x] 大文件流式导出：边取边写，内存不随行数增长 ✅ **v2.10.0 新增** - 查看 [docs/versions/v2.10.0.md](docs/versions/v2.10.0.md) - [x] 支持 Excel 日期/日期时间/时间格式 ✅ **v2.0.4 新增**，导出为真正的日期单元格 ✅ **v2.4.0 新增** - 查看 [DateTimeFormats.md](DateTimeFormats.md) - [x] 公式列引用同一工作簿的其他 sheet ✅ **v2.1.0 新增** - 查看 [ExcelFunctions.md](ExcelFunctions.md) - [x] 公式内置函数库 ✅ **v2.3.0 新增** - 334 个 Excel 函数，10 个类别 - 查看 [ExcelFunctions.md](ExcelFunctions.md) ### 发布说明 +* **2026.09.15** - v2.10.0 +- [x] ✨ **新增:** 流式导出：`ExcelHelper.ObjectToExcelStream(data, stream, options => ...)` 边取数据边写入调用方给出的流，内存中只保留 `StreamingRowWindow` 指定的若干行（默认 100），占用不再随行数增长。实测二十万行五列：内存导出峰值 710 MB / 6.4 秒，流式导出 89 MB / 3.7 秒。冻结、筛选、下拉、条件格式、样式表、公式列与合并单元格在流式导出下同样生效；仅 `.xlsx` 能够流式写入，`.xls` 的格式决定了必须先在内存中建好。流式写入基于 NPOI 的 SXSSF，后者刷行时要用 SkiaSharp 测量字符宽度，而 NPOI 把该依赖标为不随包传递，故应用需自行引用 `SkiaSharp`（Linux 上另需 `SkiaSharp.NativeAssets.Linux.NoDependencies`），缺失时会在写出任何内容之前报错说明 - 查看 [docs/versions/v2.10.0.md](docs/versions/v2.10.0.md) +- [x] 🔧 自动列宽与按值合并改在写入过程中算出，不再回看数据：流式导出得以成立，内存导出亦少走一遍数据。开启 `AutoColumnWidth` 时不再把数据源遍历两遍，只能遍历一次的序列因而也可用 + + * **2026.09.15** - v2.9.1 - [x] ✨ **新增:** 合并单元格：`options.MergeRepeatedColumns.Add("省份")` 将该列中连续相同的值并成一格（仅相邻且相等的行参与，空单元格不参与，各列彼此独立判断），`options.MergedRegions.Add(new MergedRegion("省份", "城市"))` 另行指定区域（列写标题、行写数据行序号，已知布局时也可直接写 `"A1:C1"`）。被并入的单元格仍保留各自的值，导回对象时每一行的数据依然完整；区域重叠会在导出时报错并指出与哪一个重叠 - 查看 [docs/versions/v2.9.1.md](docs/versions/v2.9.1.md) - [x] 🔧 空引用检查由 `annotations` 改为 `enable`，并将 nullable 警告视为错误。改动过程中修正数处：公式列缺少标题时不再静默导出无名列，而是在加入 `options.FormulaColumns` 时抛出异常；内部列模型的 `Title` 与 `Type` 定为非空；取值为 `null` 的属性不再写入行字典（行为不变） diff --git a/README_EN.md b/README_EN.md index 872a9ce..e49a88a 100644 --- a/README_EN.md +++ b/README_EN.md @@ -57,12 +57,19 @@ See [Chsword.Excel2Object.Cli/README.md](Chsword.Excel2Object.Cli/README.md). - [x] Data validation (dropdown lists) ✅ **New in v2.5.0** - [x] Conditional formatting ✅ **New in v2.6.0** - [x] A CSS-flavoured stylesheet: backgrounds, borders, striped rows, header styles ✅ **New in v2.7.0** +- [x] Merged cells: consecutive equal values in a column, or a range of your own ✅ **New in v2.9.1** +- [x] Streaming export for large files: written as the data is read, with a footprint that does not grow with the row count ✅ **New in v2.10.0** - See [docs/versions/v2.10.0.md](docs/versions/v2.10.0.md) - [x] Support date/datetime/time formats in Excel ✅ **New in v2.0.4**, exported as real date cells ✅ **New in v2.4.0** - See [DateTimeFormats.md](DateTimeFormats.md) - [x] Formula columns referencing other sheets of the same workbook ✅ **New in v2.1.0** - See [ExcelFunctions.md](ExcelFunctions.md) - [x] Built-in formula function library ✅ **New in v2.3.0** - 334 Excel functions in 10 categories - See [ExcelFunctions.md](ExcelFunctions.md) ### Release Notes +* **2026.09.15** - v2.10.0 +- [x] ✨ **NEW:** Streaming export: `ExcelHelper.ObjectToExcelStream(data, stream, options => ...)` writes to the stream as it reads the data, keeping only `StreamingRowWindow` rows in memory (100 by default), so the footprint no longer grows with the row count. Measured on 200,000 rows of five columns: 710 MB / 6.4 s in memory against 89 MB / 3.7 s streamed. Frozen headers, filters, dropdowns, conditional formats, the stylesheet, formula columns and merged cells all work the same way; only `.xlsx` can be streamed, as the `.xls` format has to be assembled in memory. Streaming goes through NPOI's SXSSF, which measures character widths with SkiaSharp when it flushes rows, and NPOI marks that dependency as not flowing to consumers - so the application needs its own `SkiaSharp` reference (plus `SkiaSharp.NativeAssets.Linux.NoDependencies` on Linux); when it is missing the export says so before writing anything - See [docs/versions/v2.10.0.md](docs/versions/v2.10.0.md) +- [x] 🔧 Auto column widths and merge-by-value are now worked out as the rows are written rather than by reading the data back, which is what makes streaming possible - and costs the in-memory export one pass less. `AutoColumnWidth` no longer enumerates the source twice, so a sequence that can only be enumerated once now works + + * **2026.09.15** - v2.9.1 - [x] ✨ **NEW:** Merged cells: `options.MergeRepeatedColumns.Add("Province")` merges consecutive equal values in that column into one cell (only adjacent equal rows, never blanks, each column judged on its own), and `options.MergedRegions.Add(new MergedRegion("Province", "City"))` merges a range named by column titles and data-row ordinals (an `"A1:C1"` address still works where the layout is known). The cells merged away keep their own values, so every row still imports in full; overlapping regions are refused at export time, naming the region they overlap - [x] 🔧 Null checking moved from `annotations` to `enable`, with nullable warnings treated as errors. Several things surfaced and were fixed: a formula column without a title is now refused when it is added to `options.FormulaColumns` rather than silently exported as a nameless column; the internal column model's `Title` and `Type` became non-nullable; a property whose value is `null` no longer enters the row dictionary (no change in behaviour) diff --git a/docs/README.md b/docs/README.md index d5d4f34..d92e0fc 100644 --- a/docs/README.md +++ b/docs/README.md @@ -6,6 +6,7 @@ | 版本 | 发布日期 | 主要内容 | | --- | --- | --- | +| [v2.10.0](versions/v2.10.0.md) | 2026-09-15 | 流式导出：边取边写，内存不随行数增长 | | [v2.9.1](versions/v2.9.1.md) | 2026-09-15 | 合并单元格：按列合并连续相同值、指定区域；空引用检查改为强制 | | [v2.8.1](versions/v2.8.1.md) | 2026-09-13 | 修复 .NET Framework 上样式互相覆盖、导入数值丢精度 | | [v2.8.0](versions/v2.8.0.md) | 2026-09-13 | 导入错误改由回调上报；公式求值器按工作簿创建 | diff --git a/docs/versions/v2.10.0.md b/docs/versions/v2.10.0.md new file mode 100644 index 0000000..b59fdc0 --- /dev/null +++ b/docs/versions/v2.10.0.md @@ -0,0 +1,120 @@ +# v2.10.0 + +发布日期：2026-09-15 + +## 概述 + +本版本新增流式导出：边取数据边写入调用方给出的流，内存中只保留若干行，导出所占内存不再随行数增长。为此，自动列宽与按值合并的计算方式一并改为在写入过程中完成，数据因而只被遍历一遍。 + +## 变更明细 + +### 1. 新增：流式导出 + +此前的导出一律先在内存中建好整个工作簿，再整体转成 `byte[]`。行数达到十万量级时，工作簿与字节数组同时驻留内存，占用相当可观。 + +新增的 `ObjectToExcelStream` 直接写入调用方给出的流： + +```csharp +using var file = File.Create("orders.xlsx"); +new ExcelExporter().ObjectToExcelStream(ReadOrders(), file, options => +{ + options.ExcelType = ExcelType.Xlsx; + options.FreezeHeader = true; + options.MergeRepeatedColumns.Add("省份"); +}); +``` + +`ExcelHelper` 上有同名的静态方法，`DataTable` 亦有对应重载： + +```csharp +ExcelHelper.ObjectToExcelStream(orders, stream, options => options.AutoFilter = true); +ExcelHelper.ObjectToExcelStream(dataTable, stream); +``` + +在 ASP.NET Core 中可直接写入响应体，无需先在内存中备齐整个文件： + +```csharp +[HttpGet("orders.xlsx")] +public IActionResult Download() +{ + // Kestrel 默认不允许同步写入响应体，而 NPOI 的写入是同步的，故按请求开启 + HttpContext.Features.Get()!.AllowSynchronousIO = true; + + Response.ContentType = "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet"; + Response.Headers.ContentDisposition = "attachment; filename=orders.xlsx"; + ExcelHelper.ObjectToExcelStream(ReadOrders(), Response.Body, o => o.ExcelType = ExcelType.Xlsx); + return new EmptyResult(); +} +``` + +数据源逐行取用，因此可以是数据库游标、迭代器等不宜全数载入内存的序列。传入的流由调用方负责关闭，本方法不会关闭它。 + +本机实测（五列，含文本、金额与日期，net8.0）： + +| 行数 | 内存导出 | 流式导出 | +| --- | --- | --- | +| 5 万 | 245 MB / 2.6 秒 | 89 MB / 2.2 秒 | +| 20 万 | 710 MB / 6.4 秒 | 89 MB / 3.7 秒 | + +流式导出的内存占用不随行数增长：被刷出内存的行写入临时文件，导出结束时删除。 + +### 2. 前置条件：应用需自行引用 SkiaSharp + +流式写入基于 NPOI 的 SXSSF，后者每把一行刷出内存，都要先测量一次默认字符宽度，而这一步用的是 SkiaSharp。NPOI 把 SkiaSharp 列为**不随包传递**的依赖，应用若未自行引用，运行时将无从加载。 + +```xml + + + +``` + +本库不替使用方引入该依赖：它会为每个运行时标识带来数十兆的本机库，而只用内存导出的使用方本无须为此付出代价。缺失时，`ObjectToExcelStream` 会在任何内容写出之前抛出 `Excel2ObjectException`，说明需要引用哪些包，而不是在写下若干行之后抛出一个与 Excel 无关的 `FileNotFoundException`。 + +### 3. 流式导出与内存导出的差别 + +各项特性在流式导出下同样生效：冻结表头、自动筛选、下拉、条件格式、样式表、公式列、按值合并与指定区域合并，均有用例覆盖。两处差别需要说明： + +- **仅 `.xlsx` 能够流式写入。** `.xls` 的格式决定了必须先在内存中建好再写出，此时 `ObjectToExcelStream` 只是把结果写进流，并不省内存。该格式至多 65,536 行，本也不适合大文件。 +- **文本不进共享字符串表，而是直接写在单元格里。** 这是 SXSSF 的做法：共享字符串表要把所有不重复的字符串留在内存中，与流式写入的初衷相悖。读的一侧不受影响，Excel 与本库的导入都照常；文件体积则取决于数据——重复值多时会略大，各不相同时反而略小。 + +内存中保留的行数由 `StreamingRowWindow` 指定，默认 100： + +```csharp +options.StreamingRowWindow = 500; +``` + +该值并非越小越好：行一旦刷出内存即写入临时文件，此后不可再读回，它同时决定了写入过程中可回看的范围。取值须为正数，否则抛出 `Excel2ObjectException`。 + +### 4. 改进：自动列宽与按值合并改在写入过程中算出 + +这两项此前都要回看数据：自动列宽先把数据遍历一遍量出宽度，再从头写入；按值合并在全部写完之后逐格读回工作表，找出连续相同的值。流式导出下两者都行不通——先前的行已不在内存中。 + +现改为写入时逐格记下：宽度随写随量，合并的各段随写随记，写完即可落定。除了使流式导出得以成立，内存导出亦因此少走一遍数据。 + +### 5. 改进：多张长下拉列表一次写入隐藏工作表 + +超过 255 字符的下拉列表放不进数据验证本身，须写到隐藏工作表上，一张列表占一列。此前逐张写入，第二张要回头从首行开始再写一遍；在流式写入的工作表上，这些行已被刷出内存，那一列会悄然丢失。现改为收齐全部列表后逐行填满各列，一次写完。 + +### 6. 流式导出下的跨表公式 + +公式列可以引用同一工作簿中已有的表（见 v2.1.0），本库为此要读出那张表的表头，以便把列标题换算成列号。流式导出时 SXSSF 只是套在工作簿外面的一层，随 `SourceExcelBytes` 带进来的表，其行并不在这一层里，读到的表头因而为空，公式会被判为「引用了不存在的列」。现改为向底下的工作簿去取表头： + +```csharp +using var output = new MemoryStream(); +new ExcelExporter().ObjectToExcelStream(thisMonth, output, options => +{ + options.ExcelType = ExcelType.Xlsx; + options.SheetTitle = "本月"; + options.SourceExcelBytes = lastMonthBytes; + options.FormulaColumns.Add(new FormulaColumn + { + Title = "环比", + Formula = c => c["金额"] - c.Sheet("上月")["金额"] + }); +}); +``` + +## 行为变更 + +- **数据源只被遍历一次。** 此前开启 `AutoColumnWidth` 时，`ObjectToExcelBytes` 会把数据遍历两遍（一遍量宽度、一遍写入），只能遍历一次的序列因而不可用。现在两个入口都只遍历一遍。 +- **数据源抛出的异常改在写入过程中浮现。** 行的取值由「建模型时全部取完」改为「写一行取一行」，`GetValue` 等处抛出的异常因而发生在导出过程中，而非导出开始之前。所有情形下调用方收到的仍是同一个异常。 From 669cb77e9abde3744bcf3e1f2c3a66fe0e90fa12 Mon Sep 17 00:00:00 2001 From: chsword Date: Tue, 15 Sep 2026 15:37:24 +0000 Subject: [PATCH 2/4] =?UTF-8?q?=E6=8C=89=20CodeFactor=20=E7=9A=84=E5=9B=9B?= =?UTF-8?q?=E6=9D=A1=E6=84=8F=E8=A7=81=E6=94=B6=E6=8B=BE?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit WriteSheet 拆成表头、行、列宽三段，写一张表时一路带着的列与样式收进 SheetLayout；SetCellValue 与 SetDateTimeCellValue 本就不用实例状态，改为静态。去掉 ObjectToExcelStream 里的显式判空：本仓库其余公开入口都不做这件事，且该参数在可空检查下本就是非空的。测试数据里的除法补上括号。 Generated with [Claude Code](https://claude.ai/code) via [Happy](https://happy.engineering) Co-Authored-By: Claude Opus 5 Co-Authored-By: Happy --- .../StreamingExportTest.cs | 4 +- Chsword.Excel2Object/ExcelExporter.cs | 143 +++++++++++------- 2 files changed, 94 insertions(+), 53 deletions(-) diff --git a/Chsword.Excel2Object.Tests/StreamingExportTest.cs b/Chsword.Excel2Object.Tests/StreamingExportTest.cs index 1f8db90..7092640 100644 --- a/Chsword.Excel2Object.Tests/StreamingExportTest.cs +++ b/Chsword.Excel2Object.Tests/StreamingExportTest.cs @@ -34,8 +34,8 @@ private static List Rows(int count) return Enumerable.Range(0, count).Select(i => new Model { // 每 10 行换一个省份、每 5 行换一个城市，于是合并区域必然跨越被刷出内存的行 - Province = "省" + i / 10, - City = "市" + i / 5, + Province = "省" + (i / 10), + City = "市" + (i / 5), Amount = 100 + i, PlacedAt = new DateTime(2026, 1, 1).AddDays(i), Status = i % 2 == 0 ? "已发货" : "待付款" diff --git a/Chsword.Excel2Object/ExcelExporter.cs b/Chsword.Excel2Object/ExcelExporter.cs index 4bb3e63..930db1f 100644 --- a/Chsword.Excel2Object/ExcelExporter.cs +++ b/Chsword.Excel2Object/ExcelExporter.cs @@ -131,8 +131,6 @@ public void ObjectToExcelStream(DataTable dt, Stream output, private void ObjectToExcelStream(ExcelModel excel, ExcelExporterOptions options, Stream output) { - if (output == null) throw new ArgumentNullException(nameof(output)); - // 字节数组那两个入口在源工作簿读不出来时返回 null，此处没有 null 可返回，故如实报错 if (!Write(excel, options, output, true)) throw new Excel2ObjectException("SourceExcelBytes 不是能够打开的工作簿。"); @@ -210,7 +208,7 @@ private static int Window(ExcelExporterOptions options) $"StreamingRowWindow 为 {options.StreamingRowWindow}：内存中保留的行数须为正数。"); } - private void WriteSheet(IWorkbook workbook, SheetModel excelSheet, ExcelExporterOptions options, + private static void WriteSheet(IWorkbook workbook, SheetModel excelSheet, ExcelExporterOptions options, CellStyleFactory styleFactory) { var sheet = string.IsNullOrWhiteSpace(excelSheet.Title) @@ -221,75 +219,78 @@ private void WriteSheet(IWorkbook workbook, SheetModel excelSheet, ExcelExporter // 要合并哪些列在此处即解析：写入尚未开始，列名写错也就不会留下写了一半的工作表 var merges = new MergedRegions(columns, options); - var tracking = merges.TracksAnyColumn; + var layout = new SheetLayout(columns, options); - // 自动列宽按内容而定，故边写边量，量完再设；数据只经过一遍，流式导出也就无需回看 - var widths = options.AutoColumnWidth ? new int[columns.Length] : null; - if (widths == null) - for (var i = 0; i < columns.Length; i++) - sheet.SetColumnWidth(i, options.DefaultColumnWidth * ExcelConstants.DefaultColumnWidthMultiplier); + WriteHeader(sheet, layout, options, styleFactory); + var lastRowIndex = WriteRows(sheet, excelSheet.Rows, layout, options, styleFactory, + BuildSheetColumnsResolver(workbook, sheet, layout.Titles), merges); + ApplyColumnWidths(sheet, layout, options); + + ApplyHeaderView(sheet, options, columns.Length, lastRowIndex); + DropdownValidation.Apply(sheet, columns, lastRowIndex); + ConditionalFormatting.Apply(sheet, columns, lastRowIndex, options.ConditionalFormats); + merges.Apply(sheet, lastRowIndex); + } - var headerRow = sheet.CreateRow(ExcelConstants.DefaultHeaderRowIndex); - for (var i = 0; i < columns.Length; i++) + private static void WriteHeader(ISheet sheet, SheetLayout layout, ExcelExporterOptions options, + CellStyleFactory styleFactory) + { + var row = sheet.CreateRow(ExcelConstants.DefaultHeaderRowIndex); + for (var i = 0; i < layout.Columns.Length; i++) { - var title = options.MappingColumnAction!(columns[i].Title, columns[i].Type); - var cell = headerRow.CreateCell(i); + var title = options.MappingColumnAction!(layout.Columns[i].Title, layout.Columns[i].Type); + var cell = row.CreateCell(i); cell.SetCellType(CellType.String); cell.SetCellValue(title); - var headerStyle = styleFactory.Get(StyleResolver.Header(columns[i], options.Styles)); - if (headerStyle != null) cell.CellStyle = headerStyle; - if (widths != null) widths[i] = CalculateTextWidth(title); + var style = styleFactory.Get(StyleResolver.Header(layout.Columns[i], options.Styles)); + if (style != null) cell.CellStyle = style; + layout.Measure(i, title); } + } - // one look per column per kind of row, rather than one worked out per cell - var oddStyles = columns - .Select(c => StyleResolver.Resolve(c, options.Styles, ExcelConstants.DefaultDataStartRowIndex)) - .ToArray(); - var evenStyles = columns - .Select(c => StyleResolver.Resolve(c, options.Styles, ExcelConstants.DefaultDataStartRowIndex + 1)) - .ToArray(); - - var columnTitles = columns.Select(c => c.Title).ToArray(); - var sheetColumnsResolver = BuildSheetColumnsResolver(workbook, sheet, columnTitles); + /// 逐行写入，返回最后一行的行号；行只经过一遍，写过便可离开内存。 + private static int WriteRows(ISheet sheet, IEnumerable> rows, SheetLayout layout, + ExcelExporterOptions options, CellStyleFactory styleFactory, Func sheetColumnsResolver, + MergedRegions merges) + { + var tracking = merges.TracksAnyColumn; var rowNumber = ExcelConstants.DefaultDataStartRowIndex; - foreach (var item in excelSheet.Rows) + foreach (var item in rows) { var row = sheet.CreateRow(rowNumber++); var rowStyles = (row.RowNum - ExcelConstants.DefaultDataStartRowIndex) % 2 == 0 - ? oddStyles - : evenStyles; - for (var i = 0; i < columns.Length; i++) + ? layout.OddStyles + : layout.EvenStyles; + for (var i = 0; i < layout.Columns.Length; i++) { - var column = columns[i]; + var column = layout.Columns[i]; var cell = row.CreateCell(i); var raw = item.TryGetValue(column.Title, out var value) ? value : null; if (raw is DBNull) raw = null; var val = raw?.ToString() ?? ""; - SetCellValue(options, column, rowStyles[i], cell, raw, val, columnTitles, + SetCellValue(options, column, rowStyles[i], cell, raw, val, layout.Titles, sheetColumnsResolver, styleFactory); - // 一格量一次宽：单元格显示成什么样由所在列的样式决定，隔行底色于此无关， - // 故一律按奇数行那一套来量，与逐列解析一次的做法一致 - if (widths != null) - { - var width = CalculateTextWidth(CellText(raw, oddStyles[i])); - if (width > widths[i]) widths[i] = width; - } + // 一格量一次宽：单元格显示成什么样由所在列的样式决定，隔行底色于此无关，故一律按 + // 奇数行那一套来量，与逐列解析一次的做法一致 + layout.Measure(i, CellText(raw, layout.OddStyles[i])); if (tracking && merges.Tracks(i)) merges.Observe(i, row.RowNum, cell); } } - if (widths != null) - for (var i = 0; i < columns.Length; i++) - sheet.SetColumnWidth(i, - Math.Max(options.MinColumnWidth, Math.Min(options.MaxColumnWidth, widths[i])) * - ExcelConstants.DefaultColumnWidthMultiplier); + return rowNumber - 1; + } - ApplyHeaderView(sheet, options, columns.Length, rowNumber - 1); - DropdownValidation.Apply(sheet, columns, rowNumber - 1); - ConditionalFormatting.Apply(sheet, columns, rowNumber - 1, options.ConditionalFormats); - merges.Apply(sheet, rowNumber - 1); + private static void ApplyColumnWidths(ISheet sheet, SheetLayout layout, ExcelExporterOptions options) + { + for (var i = 0; i < layout.Columns.Length; i++) + { + var width = layout.Widths == null + ? options.DefaultColumnWidth + : Math.Max(options.MinColumnWidth, Math.Min(options.MaxColumnWidth, layout.Widths[i])); + sheet.SetColumnWidth(i, width * ExcelConstants.DefaultColumnWidthMultiplier); + } } /// @@ -396,7 +397,7 @@ private static IWorkbook Workbook(ExcelType excelType) return streamed.XssfWorkbook.GetSheet(sheetTitle)?.GetRow(ExcelConstants.DefaultHeaderRowIndex); } - private void SetCellValue(ExcelExporterOptions options, ExcelColumn column, ResolvedColumnStyle resolved, + private static void SetCellValue(ExcelExporterOptions options, ExcelColumn column, ResolvedColumnStyle resolved, ICell cell, object? raw, string val, string[] columnTitles, Func sheetColumnsResolver, CellStyleFactory styleFactory) { @@ -504,7 +505,7 @@ private void SetCellValue(ExcelExporterOptions options, ExcelColumn column, Reso /// format that shows the same thing; restores /// the text export of earlier versions. /// - private void SetDateTimeCellValue(ExcelExporterOptions options, ResolvedColumnStyle resolved, ICell cell, + private static void SetDateTimeCellValue(ExcelExporterOptions options, ResolvedColumnStyle resolved, ICell cell, DateTime date, string val, CellStyleFactory styleFactory) { var asText = options.DateTimeAsText; @@ -630,4 +631,44 @@ private static int CalculateTextWidth(string text) return (int)Math.Ceiling(width) + 2; // Add padding } -} \ No newline at end of file + + /// + /// 写一张表时一路带着的东西：各列、各列在奇偶行上的样式，以及正在量的列宽。 + /// + private sealed class SheetLayout + { + public SheetLayout(ExcelColumn[] columns, ExcelExporterOptions options) + { + Columns = columns; + Titles = columns.Select(c => c.Title).ToArray(); + // one look per column per kind of row, rather than one worked out per cell + OddStyles = columns + .Select(c => StyleResolver.Resolve(c, options.Styles, ExcelConstants.DefaultDataStartRowIndex)) + .ToArray(); + EvenStyles = columns + .Select(c => StyleResolver.Resolve(c, options.Styles, ExcelConstants.DefaultDataStartRowIndex + 1)) + .ToArray(); + // 自动列宽按内容而定，故边写边量，量完再设；数据只经过一遍，流式导出也就无需回看 + Widths = options.AutoColumnWidth ? new int[columns.Length] : null; + } + + public ExcelColumn[] Columns { get; } + + public string[] Titles { get; } + + public ResolvedColumnStyle[] OddStyles { get; } + + public ResolvedColumnStyle[] EvenStyles { get; } + + /// 各列已量到的宽度；未开启自动列宽时为 null。 + public int[]? Widths { get; } + + public void Measure(int column, string text) + { + if (Widths == null) return; + + var width = CalculateTextWidth(text); + if (width > Widths[column]) Widths[column] = width; + } + } +} From 2f21bda0f21f08cdfde7b8621f6354521113170f Mon Sep 17 00:00:00 2001 From: chsword Date: Tue, 15 Sep 2026 15:44:22 +0000 Subject: [PATCH 3/4] =?UTF-8?q?=E6=8C=89=20Copilot=20=E7=9A=84=E5=A4=8D?= =?UTF-8?q?=E6=9F=A5=E6=84=8F=E8=A7=81=E4=BF=AE=E6=AD=A3=E4=B8=89=E5=A4=84?= =?UTF-8?q?=EF=BC=8C=E5=B9=B6=E6=94=B9=E6=AD=A3=E6=8E=AA=E8=BE=9E?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 合并区域在流式工作表上落回了 NPOI 自带的 AddMergedRegion：SXSSFSheet 既非 XSSFSheet 也非 HSSFSheet，两两比对因而照旧，绕开它的用意落空。补上该分支，并加一个 2000 行、400 个区域的用例。 下拉的隐藏列表表，流式追加时会撞上 NPOI 的「该行已写入磁盘」：随源工作簿带进来的表，其行不在流式这一层里，既读不回也无从从首行重写。改为此时另起一张列表表，原有的列表与指向它的定义名称原样保留。 字典入口为取列名要先读一行，枚举器因而在写入开始之前即已打开；写入若未能开始（选项不合法、缺少 SkiaSharp 等），它不会被释放。改由模型带着，导出结束时一并释放。 另外，列名映射不再以 ! 压制可空警告，也不再改写调用方的选项对象；措辞改正：SXSSF 省下的是内存而非等待时间，最终的包要等数据取完才一次写入调用方的流，文档此前说成了「边取边写入流」。 Generated with [Claude Code](https://claude.ai/code) via [Happy](https://happy.engineering) Co-Authored-By: Claude Opus 5 Co-Authored-By: Happy --- .../StreamingExportTest.cs | 53 +++++++++++++++++++ Chsword.Excel2Object/ExcelExporter.cs | 24 +++++---- .../Internal/DropdownValidation.cs | 17 +++++- .../Internal/MergedRegions.cs | 4 ++ Chsword.Excel2Object/Internal/TypeConvert.cs | 1 + Chsword.Excel2Object/Models/ExcelModel.cs | 7 +++ README.md | 4 +- README_EN.md | 4 +- docs/versions/v2.10.0.md | 8 +-- 9 files changed, 105 insertions(+), 17 deletions(-) diff --git a/Chsword.Excel2Object.Tests/StreamingExportTest.cs b/Chsword.Excel2Object.Tests/StreamingExportTest.cs index 7092640..bfaead1 100644 --- a/Chsword.Excel2Object.Tests/StreamingExportTest.cs +++ b/Chsword.Excel2Object.Tests/StreamingExportTest.cs @@ -187,6 +187,59 @@ public void TwoLongDropdownsBothLandOnTheHiddenSheet() } } + [TestMethod] + public void ALongDropdownGoesToItsOwnListSheetWhenAppending() + { + // 源工作簿里已有一张写着长列表的隐藏表。流式写入无从往那张表里再添一列——它的行不在流式 + // 这一层里，既读不回也无从从首行重写——故应另起一张，原有的列表原样保留。 + var existing = Enumerable.Range(0, 60).Select(i => "旧选项" + i).ToArray(); + var added = Enumerable.Range(0, 70).Select(i => "新选项" + i).ToArray(); + + var first = new ExcelExporter().ObjectToExcelBytes(Rows(5), options => + { + options.ExcelType = ExcelType.Xlsx; + options.SheetTitle = "上月"; + options.Dropdowns["省份"] = existing; + }); + + using var output = new MemoryStream(); + new ExcelExporter().ObjectToExcelStream(Rows(20), output, options => + { + options.ExcelType = ExcelType.Xlsx; + options.StreamingRowWindow = Window; + options.SheetTitle = "本月"; + options.SourceExcelBytes = first; + options.Dropdowns["城市"] = added; + }); + + var workbook = WorkbookFactory.Create(new MemoryStream(output.ToArray())); + var old = workbook.GetSheet("_excel2object_lists"); + var fresh = workbook.GetSheet("_excel2object_lists2"); + Assert.IsNotNull(fresh, "应另起一张列表工作表"); + Assert.AreEqual(existing[0], old.GetRow(0).GetCell(0).StringCellValue, "原有的列表应原样保留"); + Assert.AreEqual(existing[59], old.GetRow(59).GetCell(0).StringCellValue, "原有的列表应原样保留"); + Assert.AreEqual(added[0], fresh.GetRow(0).GetCell(0).StringCellValue); + Assert.AreEqual(added[69], fresh.GetRow(69).GetCell(0).StringCellValue); + Assert.AreEqual(SheetVisibility.Hidden, workbook.GetSheetVisibility(workbook.GetSheetIndex(fresh))); + + // 两张列表各有自己的定义名称，先前那个仍指向原处 + Assert.AreEqual("_excel2object_lists!$A$1:$A$60", workbook.GetName("_excel2object_list1").RefersToFormula); + Assert.AreEqual("_excel2object_lists2!$A$1:$A$70", workbook.GetName("_excel2object_list2").RefersToFormula); + } + + [TestMethod] + public void ManyMergedRegionsAreWrittenOnAStreamedSheet() + { + // 分组表的合并区域数量与行数同阶：2000 行、每 5 行一组即 400 个区域。流式工作表同样绕开 + // NPOI 自带的两两比对，否则这一步会退化为平方级。 + var sheet = Streamed(Rows(2000), options => options.MergeRepeatedColumns.Add("城市"), 100); + + var regions = Regions(sheet); + Assert.AreEqual(400, regions.Length); + Assert.AreEqual("B2:B6", regions[0]); + Assert.AreEqual("B1997:B2001", regions[399]); + } + [TestMethod] public void AutoColumnWidthMatchesTheInMemoryExport() { diff --git a/Chsword.Excel2Object/ExcelExporter.cs b/Chsword.Excel2Object/ExcelExporter.cs index 930db1f..155dcb8 100644 --- a/Chsword.Excel2Object/ExcelExporter.cs +++ b/Chsword.Excel2Object/ExcelExporter.cs @@ -81,12 +81,16 @@ public class ExcelExporter } /// - /// 边取数据边写入 ，内存中只保留 - /// 指定的若干行，适用于行数多到不宜先在 - /// 内存中建好整个工作簿的导出。 + /// 逐行取数据并写出，内存中只保留 指定的 + /// 若干行，适用于行数多到不宜先在内存中建好整个工作簿的导出。 /// /// /// + /// 省下的是内存，不是等待时间：写过的行随即离开内存，落到临时文件，而最终的 .xlsx + /// 包要等数据取完才一次写入 。因此调用方的流在导出过程中并不会 + /// 陆续收到内容。 + /// + /// /// 仅 能够流式写入； 的格式决定了 /// 必须先在内存中建好再写出，此时本方法只是把结果写进 ，并不省内存 /// （该格式至多 65,536 行，本也不适合大文件）。 @@ -150,12 +154,12 @@ private bool Write(ExcelModel excel, ExcelExporterOptions options, Stream output try { - if (options.MappingColumnAction == null) options.MappingColumnAction = (s, _) => s; + var mapColumn = options.MappingColumnAction ?? ((title, _) => title); var styleFactory = new CellStyleFactory(workbook); if (excel.Sheets != null) foreach (var excelSheet in excel.Sheets) - WriteSheet(workbook, excelSheet, options, styleFactory); + WriteSheet(workbook, excelSheet, options, styleFactory, mapColumn); // 调用方给的流由调用方关闭 workbook.Write(output, true); @@ -165,6 +169,8 @@ private bool Write(ExcelModel excel, ExcelExporterOptions options, Stream output { // SXSSF 把刷出内存的行写在临时文件里，不释放则留在磁盘上 (workbook as SXSSFWorkbook)?.Dispose(); + // 数据尚未取完即告失败时，取数据的枚举器也要释放 + excel.RowSource?.Dispose(); } } @@ -209,7 +215,7 @@ private static int Window(ExcelExporterOptions options) } private static void WriteSheet(IWorkbook workbook, SheetModel excelSheet, ExcelExporterOptions options, - CellStyleFactory styleFactory) + CellStyleFactory styleFactory, Func mapColumn) { var sheet = string.IsNullOrWhiteSpace(excelSheet.Title) ? workbook.CreateSheet() @@ -221,7 +227,7 @@ private static void WriteSheet(IWorkbook workbook, SheetModel excelSheet, ExcelE var merges = new MergedRegions(columns, options); var layout = new SheetLayout(columns, options); - WriteHeader(sheet, layout, options, styleFactory); + WriteHeader(sheet, layout, options, styleFactory, mapColumn); var lastRowIndex = WriteRows(sheet, excelSheet.Rows, layout, options, styleFactory, BuildSheetColumnsResolver(workbook, sheet, layout.Titles), merges); ApplyColumnWidths(sheet, layout, options); @@ -233,12 +239,12 @@ private static void WriteSheet(IWorkbook workbook, SheetModel excelSheet, ExcelE } private static void WriteHeader(ISheet sheet, SheetLayout layout, ExcelExporterOptions options, - CellStyleFactory styleFactory) + CellStyleFactory styleFactory, Func mapColumn) { var row = sheet.CreateRow(ExcelConstants.DefaultHeaderRowIndex); for (var i = 0; i < layout.Columns.Length; i++) { - var title = options.MappingColumnAction!(layout.Columns[i].Title, layout.Columns[i].Type); + var title = mapColumn(layout.Columns[i].Title, layout.Columns[i].Type); var cell = row.CreateCell(i); cell.SetCellType(CellType.String); cell.SetCellValue(title); diff --git a/Chsword.Excel2Object/Internal/DropdownValidation.cs b/Chsword.Excel2Object/Internal/DropdownValidation.cs index 966a4cc..c52a7e7 100644 --- a/Chsword.Excel2Object/Internal/DropdownValidation.cs +++ b/Chsword.Excel2Object/Internal/DropdownValidation.cs @@ -1,5 +1,6 @@ using NPOI.SS.UserModel; using NPOI.SS.Util; +using NPOI.XSSF.Streaming; namespace Chsword.Excel2Object.Internal; @@ -131,6 +132,19 @@ private static string[] WriteToListSheet(IWorkbook workbook, List list return names; } + /// + /// 该表还能不能再添一列。流式写入的表只能自上而下写一遍，而随源工作簿带进来的表，其行并不在 + /// 流式这一层里：既读不回，也无从从首行重写。此时另起一张列表表，原有的列表与指向它的定义 + /// 名称原样保留。 + /// + private static bool Extendable(IWorkbook workbook, ISheet sheet) + { + if (workbook is not SXSSFWorkbook streamed) return true; + + var underlying = streamed.XssfWorkbook.GetSheet(sheet.SheetName); + return underlying == null || underlying.PhysicalNumberOfRows == 0; + } + private static string FreeName(IWorkbook workbook) { for (var i = 1;; i++) @@ -158,7 +172,8 @@ private static ISheet ListSheet(IWorkbook workbook) return sheet; } - if (workbook.GetSheetVisibility(workbook.GetSheetIndex(existing)) != SheetVisibility.Visible) + if (workbook.GetSheetVisibility(workbook.GetSheetIndex(existing)) != SheetVisibility.Visible && + Extendable(workbook, existing)) return existing; name = ListSheetName + suffix; diff --git a/Chsword.Excel2Object/Internal/MergedRegions.cs b/Chsword.Excel2Object/Internal/MergedRegions.cs index 8e81a0b..506db2f 100644 --- a/Chsword.Excel2Object/Internal/MergedRegions.cs +++ b/Chsword.Excel2Object/Internal/MergedRegions.cs @@ -2,6 +2,7 @@ using NPOI.HSSF.UserModel; using NPOI.SS.UserModel; using NPOI.SS.Util; +using NPOI.XSSF.Streaming; using NPOI.XSSF.UserModel; namespace Chsword.Excel2Object.Internal; @@ -90,6 +91,9 @@ private static void AddUnsafe(ISheet sheet, CellRangeAddress range) { switch (sheet) { + case SXSSFSheet streamed: + streamed.AddMergedRegionUnsafe(range); + break; case XSSFSheet xssf: xssf.AddMergedRegionUnsafe(range); break; diff --git a/Chsword.Excel2Object/Internal/TypeConvert.cs b/Chsword.Excel2Object/Internal/TypeConvert.cs index 4548f82..76155d3 100644 --- a/Chsword.Excel2Object/Internal/TypeConvert.cs +++ b/Chsword.Excel2Object/Internal/TypeConvert.cs @@ -38,6 +38,7 @@ public static ExcelModel ConvertDictionaryToExcelModel(IEnumerable? Sheets { get; set; } + + /// + /// 取数据的枚举器，仅字典入口用得到：列由首行的键决定，故建模型时就得先取一行，枚举器因而 + /// 在写入开始之前即已打开。写入若未能开始（例如选项不合法），它也须被释放，故由模型带着， + /// 导出结束时一并释放。 + /// + public IDisposable? RowSource { get; set; } } \ No newline at end of file diff --git a/README.md b/README.md index 9a2add2..8f6e5b2 100644 --- a/README.md +++ b/README.md @@ -60,7 +60,7 @@ excel2obj generate-model orders.xlsx --class Order # 由表头生成 - [x] 条件格式 ✅ **v2.6.0 新增** - [x] 样式表（CSS 风格）：背景色、边框、隔行底色、表头样式 ✅ **v2.7.0 新增** - [x] 合并单元格：按列合并连续相同值、指定区域 ✅ **v2.9.1 新增** -- [x] 大文件流式导出：边取边写，内存不随行数增长 ✅ **v2.10.0 新增** - 查看 [docs/versions/v2.10.0.md](docs/versions/v2.10.0.md) +- [x] 大文件流式导出：内存不随行数增长 ✅ **v2.10.0 新增** - 查看 [docs/versions/v2.10.0.md](docs/versions/v2.10.0.md) - [x] 支持 Excel 日期/日期时间/时间格式 ✅ **v2.0.4 新增**，导出为真正的日期单元格 ✅ **v2.4.0 新增** - 查看 [DateTimeFormats.md](DateTimeFormats.md) - [x] 公式列引用同一工作簿的其他 sheet ✅ **v2.1.0 新增** - 查看 [ExcelFunctions.md](ExcelFunctions.md) - [x] 公式内置函数库 ✅ **v2.3.0 新增** - 334 个 Excel 函数，10 个类别 - 查看 [ExcelFunctions.md](ExcelFunctions.md) @@ -68,7 +68,7 @@ excel2obj generate-model orders.xlsx --class Order # 由表头生成 ### 发布说明 * **2026.09.15** - v2.10.0 -- [x] ✨ **新增:** 流式导出：`ExcelHelper.ObjectToExcelStream(data, stream, options => ...)` 边取数据边写入调用方给出的流，内存中只保留 `StreamingRowWindow` 指定的若干行（默认 100），占用不再随行数增长。实测二十万行五列：内存导出峰值 710 MB / 6.4 秒，流式导出 89 MB / 3.7 秒。冻结、筛选、下拉、条件格式、样式表、公式列与合并单元格在流式导出下同样生效；仅 `.xlsx` 能够流式写入，`.xls` 的格式决定了必须先在内存中建好。流式写入基于 NPOI 的 SXSSF，后者刷行时要用 SkiaSharp 测量字符宽度，而 NPOI 把该依赖标为不随包传递，故应用需自行引用 `SkiaSharp`（Linux 上另需 `SkiaSharp.NativeAssets.Linux.NoDependencies`），缺失时会在写出任何内容之前报错说明 - 查看 [docs/versions/v2.10.0.md](docs/versions/v2.10.0.md) +- [x] ✨ **新增:** 流式导出：`ExcelHelper.ObjectToExcelStream(data, stream, options => ...)` 逐行取数据并写出，内存中只保留 `StreamingRowWindow` 指定的若干行（默认 100），占用不再随行数增长（省下的是内存而非等待时间：写过的行落到临时文件，最终的包在数据取完后一次写入调用方的流）。实测二十万行五列：内存导出峰值 710 MB / 6.4 秒，流式导出 89 MB / 3.7 秒。冻结、筛选、下拉、条件格式、样式表、公式列与合并单元格在流式导出下同样生效；仅 `.xlsx` 能够流式写入，`.xls` 的格式决定了必须先在内存中建好。流式写入基于 NPOI 的 SXSSF，后者刷行时要用 SkiaSharp 测量字符宽度，而 NPOI 把该依赖标为不随包传递，故应用需自行引用 `SkiaSharp`（Linux 上另需 `SkiaSharp.NativeAssets.Linux.NoDependencies`），缺失时会在写出任何内容之前报错说明 - 查看 [docs/versions/v2.10.0.md](docs/versions/v2.10.0.md) - [x] 🔧 自动列宽与按值合并改在写入过程中算出，不再回看数据：流式导出得以成立，内存导出亦少走一遍数据。开启 `AutoColumnWidth` 时不再把数据源遍历两遍，只能遍历一次的序列因而也可用 diff --git a/README_EN.md b/README_EN.md index e49a88a..429a297 100644 --- a/README_EN.md +++ b/README_EN.md @@ -58,7 +58,7 @@ See [Chsword.Excel2Object.Cli/README.md](Chsword.Excel2Object.Cli/README.md). - [x] Conditional formatting ✅ **New in v2.6.0** - [x] A CSS-flavoured stylesheet: backgrounds, borders, striped rows, header styles ✅ **New in v2.7.0** - [x] Merged cells: consecutive equal values in a column, or a range of your own ✅ **New in v2.9.1** -- [x] Streaming export for large files: written as the data is read, with a footprint that does not grow with the row count ✅ **New in v2.10.0** - See [docs/versions/v2.10.0.md](docs/versions/v2.10.0.md) +- [x] Streaming export for large files: a memory footprint that does not grow with the row count ✅ **New in v2.10.0** - See [docs/versions/v2.10.0.md](docs/versions/v2.10.0.md) - [x] Support date/datetime/time formats in Excel ✅ **New in v2.0.4**, exported as real date cells ✅ **New in v2.4.0** - See [DateTimeFormats.md](DateTimeFormats.md) - [x] Formula columns referencing other sheets of the same workbook ✅ **New in v2.1.0** - See [ExcelFunctions.md](ExcelFunctions.md) - [x] Built-in formula function library ✅ **New in v2.3.0** - 334 Excel functions in 10 categories - See [ExcelFunctions.md](ExcelFunctions.md) @@ -66,7 +66,7 @@ See [Chsword.Excel2Object.Cli/README.md](Chsword.Excel2Object.Cli/README.md). ### Release Notes * **2026.09.15** - v2.10.0 -- [x] ✨ **NEW:** Streaming export: `ExcelHelper.ObjectToExcelStream(data, stream, options => ...)` writes to the stream as it reads the data, keeping only `StreamingRowWindow` rows in memory (100 by default), so the footprint no longer grows with the row count. Measured on 200,000 rows of five columns: 710 MB / 6.4 s in memory against 89 MB / 3.7 s streamed. Frozen headers, filters, dropdowns, conditional formats, the stylesheet, formula columns and merged cells all work the same way; only `.xlsx` can be streamed, as the `.xls` format has to be assembled in memory. Streaming goes through NPOI's SXSSF, which measures character widths with SkiaSharp when it flushes rows, and NPOI marks that dependency as not flowing to consumers - so the application needs its own `SkiaSharp` reference (plus `SkiaSharp.NativeAssets.Linux.NoDependencies` on Linux); when it is missing the export says so before writing anything - See [docs/versions/v2.10.0.md](docs/versions/v2.10.0.md) +- [x] ✨ **NEW:** Streaming export: `ExcelHelper.ObjectToExcelStream(data, stream, options => ...)` reads the data row by row and writes each row out, keeping only `StreamingRowWindow` rows in memory (100 by default), so the footprint no longer grows with the row count. What it saves is memory rather than latency: rows written go to a temp file, and the final package reaches the caller's stream in one go once the data has been read. Measured on 200,000 rows of five columns: 710 MB / 6.4 s in memory against 89 MB / 3.7 s streamed. Frozen headers, filters, dropdowns, conditional formats, the stylesheet, formula columns and merged cells all work the same way; only `.xlsx` can be streamed, as the `.xls` format has to be assembled in memory. Streaming goes through NPOI's SXSSF, which measures character widths with SkiaSharp when it flushes rows, and NPOI marks that dependency as not flowing to consumers - so the application needs its own `SkiaSharp` reference (plus `SkiaSharp.NativeAssets.Linux.NoDependencies` on Linux); when it is missing the export says so before writing anything - See [docs/versions/v2.10.0.md](docs/versions/v2.10.0.md) - [x] 🔧 Auto column widths and merge-by-value are now worked out as the rows are written rather than by reading the data back, which is what makes streaming possible - and costs the in-memory export one pass less. `AutoColumnWidth` no longer enumerates the source twice, so a sequence that can only be enumerated once now works diff --git a/docs/versions/v2.10.0.md b/docs/versions/v2.10.0.md index b59fdc0..4edcde3 100644 --- a/docs/versions/v2.10.0.md +++ b/docs/versions/v2.10.0.md @@ -4,7 +4,7 @@ ## 概述 -本版本新增流式导出：边取数据边写入调用方给出的流，内存中只保留若干行，导出所占内存不再随行数增长。为此，自动列宽与按值合并的计算方式一并改为在写入过程中完成，数据因而只被遍历一遍。 +本版本新增流式导出：逐行取数据并写出，内存中只保留若干行，导出所占内存不再随行数增长。为此，自动列宽与按值合并的计算方式一并改为在写入过程中完成，数据因而只被遍历一遍。 ## 变更明细 @@ -12,7 +12,7 @@ 此前的导出一律先在内存中建好整个工作簿，再整体转成 `byte[]`。行数达到十万量级时，工作簿与字节数组同时驻留内存，占用相当可观。 -新增的 `ObjectToExcelStream` 直接写入调用方给出的流： +新增的 `ObjectToExcelStream` 逐行取数据并写出，写过的行随即离开内存： ```csharp using var file = File.Create("orders.xlsx"); @@ -31,7 +31,7 @@ ExcelHelper.ObjectToExcelStream(orders, stream, options => options.AutoFilter = ExcelHelper.ObjectToExcelStream(dataTable, stream); ``` -在 ASP.NET Core 中可直接写入响应体，无需先在内存中备齐整个文件： +在 ASP.NET Core 中可直接写入响应体，不必先在内存中备好整个文件的字节数组： ```csharp [HttpGet("orders.xlsx")] @@ -49,6 +49,8 @@ public IActionResult Download() 数据源逐行取用，因此可以是数据库游标、迭代器等不宜全数载入内存的序列。传入的流由调用方负责关闭，本方法不会关闭它。 +需要说明的是，**省下的是内存，不是等待时间**：写过的行落到临时文件，而最终的 `.xlsx` 包（其本身是个 zip）要等数据取完才一次写入调用方的流。上面那个接口不会一边生成一边向浏览器送出内容，它的意义在于生成十万行的文件时不必把整个工作簿连同字节数组一并留在内存里。 + 本机实测（五列，含文本、金额与日期，net8.0）： | 行数 | 内存导出 | 流式导出 | From 1a5076a936e9cc36345af044551a42618d8e5976 Mon Sep 17 00:00:00 2001 From: chsword Date: Tue, 15 Sep 2026 16:35:40 +0000 Subject: [PATCH 4/4] =?UTF-8?q?=E6=8C=89=E5=A4=8D=E6=9F=A5=E6=84=8F?= =?UTF-8?q?=E8=A7=81=E6=94=B6=E6=8B=BE=E4=B8=89=E5=A4=84?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 建立工作簿这一步本身也会失败——选项不合法、源工作簿读不出来、缺少 SkiaSharp——而它原先在 try 之外，字典入口预先取走一行所打开的枚举器因而不会被释放，数据库游标之类的东西就这么悬着。挪进 try 之内。 行窗口原先只在 SXSSF 那条路上校验，.xls 上的非法取值无声通过，与选项文档所说不符。改为流式导出一律校验。 删掉 ToBytes：ObjectToExcelBytes 改走 Write 之后它已无人调用，留着还少了 SXSSF 的释放。 Generated with [Claude Code](https://claude.ai/code) via [Happy](https://happy.engineering) Co-Authored-By: Claude Opus 5 Co-Authored-By: Happy --- .../StreamingExportTest.cs | 88 ++++++++++++++++++- Chsword.Excel2Object/ExcelExporter.cs | 31 +++---- 2 files changed, 100 insertions(+), 19 deletions(-) diff --git a/Chsword.Excel2Object.Tests/StreamingExportTest.cs b/Chsword.Excel2Object.Tests/StreamingExportTest.cs index bfaead1..3ba093b 100644 --- a/Chsword.Excel2Object.Tests/StreamingExportTest.cs +++ b/Chsword.Excel2Object.Tests/StreamingExportTest.cs @@ -298,11 +298,41 @@ public void NothingIsWrittenWhenAMergeColumnIsUnknown() [TestMethod] public void TheRowWindowMustBePositive() { - using var output = new MemoryStream(); - var e = Assert.ThrowsException(() => - new ExcelExporter().ObjectToExcelStream(Rows(10), output, options => options.StreamingRowWindow = 0)); + // .xls 那条路上虽用不着这个值，取值不合法同样说明调用方想要的与得到的并不一致 + foreach (var excelType in new[] {ExcelType.Xlsx, ExcelType.Xls}) + { + using var output = new MemoryStream(); + var e = Assert.ThrowsException(() => + new ExcelExporter().ObjectToExcelStream(Rows(10), output, options => + { + options.ExcelType = excelType; + options.StreamingRowWindow = 0; + }), excelType.ToString()); + + StringAssert.Contains(e.Message, "StreamingRowWindow"); + Assert.AreEqual(0, output.Length, excelType.ToString()); + } + } - StringAssert.Contains(e.Message, "StreamingRowWindow"); + [TestMethod] + public void TheSourceIsReleasedWhenTheExportNeverStarts() + { + // 字典入口为取列名要先读一行，枚举器因而在写入开始之前即已打开。写入若根本没能开始， + // 它也须被释放——数据库游标之类的东西正是这样悬着的。 + var badWindow = new TrackedDictionaries(); + Assert.ThrowsException(() => + new ExcelExporter().ObjectToExcelStream(badWindow, new MemoryStream(), + options => options.StreamingRowWindow = 0)); + Assert.IsTrue(badWindow.Disposed, "行窗口不合法时"); + + var badSource = new TrackedDictionaries(); + Assert.ThrowsException(() => + new ExcelExporter().ObjectToExcelStream(badSource, new MemoryStream(), options => + { + options.StreamingRowWindow = Window; + options.SourceExcelBytes = new byte[] {1, 2, 3}; + })); + Assert.IsTrue(badSource.Disposed, "源工作簿读不出来时"); } [TestMethod] @@ -415,6 +445,56 @@ public void FormulasCanReferToASheetAlreadyInTheWorkbook() Assert.AreEqual(10, workbook.GetSheet("上月").LastRowNum, "原有的表应原样保留"); } + /// 记下自己有没有被释放：字典入口会预先取走一行，那个枚举器的归属正是要验的东西。 + private sealed class TrackedDictionaries : IEnumerable> + { + public bool Disposed { get; private set; } + + public IEnumerator> GetEnumerator() + { + return new Enumerator(this); + } + + IEnumerator IEnumerable.GetEnumerator() + { + return GetEnumerator(); + } + + private sealed class Enumerator : IEnumerator> + { + private readonly TrackedDictionaries _owner; + private int _index; + + public Enumerator(TrackedDictionaries owner) + { + _owner = owner; + } + + public Dictionary Current { get; private set; } = new(); + + object IEnumerator.Current => Current; + + public bool MoveNext() + { + if (_index >= 3) return false; + + Current = new Dictionary {{"省份", "省" + _index}, {"金额", 100 + _index}}; + _index++; + return true; + } + + public void Reset() + { + _index = 0; + } + + public void Dispose() + { + _owner.Disposed = true; + } + } + } + /// 记下被遍历的次数，遍历第二遍即失败——流式导出只应过一遍数据。 private sealed class CountingSequence : IEnumerable { diff --git a/Chsword.Excel2Object/ExcelExporter.cs b/Chsword.Excel2Object/ExcelExporter.cs index 155dcb8..026d6f5 100644 --- a/Chsword.Excel2Object/ExcelExporter.cs +++ b/Chsword.Excel2Object/ExcelExporter.cs @@ -149,11 +149,15 @@ private void ObjectToExcelStream(ExcelModel excel, ExcelExporterOptions options, /// 写出整个工作簿；源工作簿读不出来时返回 false。 private bool Write(ExcelModel excel, ExcelExporterOptions options, Stream output, bool streaming) { - var workbook = OpenWorkbook(options, streaming); - if (workbook == null) return false; - + IWorkbook? workbook = null; try { + // 建立工作簿这一步本身也会失败（选项不合法、源工作簿读不出来、缺少 SkiaSharp）， + // 故一并放在 try 之内：字典入口为取列名已先读了一行，那个枚举器无论如何都要释放 + var window = streaming ? Window(options) : 0; + workbook = OpenWorkbook(options, streaming, window); + if (workbook == null) return false; + var mapColumn = options.MappingColumnAction ?? ((title, _) => title); var styleFactory = new CellStyleFactory(workbook); @@ -175,15 +179,16 @@ private bool Write(ExcelModel excel, ExcelExporterOptions options, Stream output } /// - /// 建立或读入工作簿。流式导出用 SXSSF：它只在内存中保留最近的若干行，其余写入临时文件。 + /// 建立或读入工作簿。流式导出用 SXSSF：它只在内存中保留最近的 行， + /// 其余写入临时文件。 /// - private static IWorkbook? OpenWorkbook(ExcelExporterOptions options, bool streaming) + private static IWorkbook? OpenWorkbook(ExcelExporterOptions options, bool streaming, int window) { IWorkbook workbook; if (options.SourceExcelBytes == null) { workbook = streaming && options.ExcelType == ExcelType.Xlsx - ? new SXSSFWorkbook(Window(options)) + ? new SXSSFWorkbook(window) : Workbook(options.ExcelType); } else @@ -199,13 +204,17 @@ private bool Write(ExcelModel excel, ExcelExporterOptions options, Stream output } // .xls 无从流式写入，续写时就按原样在内存中完成 - if (streaming && workbook is XSSFWorkbook xssf) workbook = new SXSSFWorkbook(xssf, Window(options)); + if (streaming && workbook is XSSFWorkbook xssf) workbook = new SXSSFWorkbook(xssf, window); } if (workbook is SXSSFWorkbook streamed) StreamingSupport.Ensure(streamed); return workbook; } + /// + /// 内存中保留的行数。无论哪种格式都要校验：`.xls` 那条路上虽用不着这个值，取值不合法却同样 + /// 说明调用方想要的与得到的并不一致。 + /// private static int Window(ExcelExporterOptions options) { if (options.StreamingRowWindow > 0) return options.StreamingRowWindow; @@ -333,14 +342,6 @@ private static T Switch(ExcelType excelType, Func funcXlsHssf, Func fun return obj; } - private static byte[] ToBytes(IWorkbook workbook) - { - using var output = new MemoryStream(); - workbook.Write(output, true); - var bytes = output.ToArray(); - return bytes; - } - private static IWorkbook Workbook(ExcelType excelType) { IWorkbook workbook = excelType switch