diff --git a/Chsword.Excel2Object.Tests/Chsword.Excel2Object.Tests.csproj b/Chsword.Excel2Object.Tests/Chsword.Excel2Object.Tests.csproj index af9f411..03a0f54 100644 --- a/Chsword.Excel2Object.Tests/Chsword.Excel2Object.Tests.csproj +++ b/Chsword.Excel2Object.Tests/Chsword.Excel2Object.Tests.csproj @@ -37,6 +37,12 @@ + + + + + + diff --git a/Chsword.Excel2Object.Tests/StreamingImportTest.cs b/Chsword.Excel2Object.Tests/StreamingImportTest.cs new file mode 100644 index 0000000..b5b486c --- /dev/null +++ b/Chsword.Excel2Object.Tests/StreamingImportTest.cs @@ -0,0 +1,379 @@ +using System; +using System.Collections.Generic; +using System.IO; +using System.Linq; +using Chsword.Excel2Object.Options; +using Microsoft.VisualStudio.TestTools.UnitTesting; +using NPOI.SS.UserModel; +using NPOI.XSSF.UserModel; + +namespace Chsword.Excel2Object.Tests; + +/// +/// 流式导入：逐行读出 .xlsx 的一张工作表，不把整个工作簿建进内存。转换那一套与整份读入共用， +/// 故这里验的是「读出来的东西一样」，以及流式独有的几处：公式取缓存值、表名定位、惰性取用。 +/// +[TestClass] +public class StreamingImportTest +{ + public enum Status + { + 待付款 = 1, + 已发货 = 2 + } + + public class Model + { + [ExcelColumn("文本")] public string Text { get; set; } = ""; + [ExcelColumn("金额", Format = "#,##0.00")] public decimal Amount { get; set; } + [ExcelColumn("数量")] public int Count { get; set; } + [ExcelColumn("下单时间", Format = "yyyy-MM-dd HH:mm")] public DateTime PlacedAt { get; set; } + [ExcelColumn("完成时间")] public DateTime? FinishedAt { get; set; } + [ExcelColumn("已付")] public bool Paid { get; set; } + [ExcelColumn("状态")] public Status Status { get; set; } + [ExcelColumn("链接")] public Uri? Link { get; set; } + } + + private static List Rows(int count) + { + return Enumerable.Range(0, count).Select(i => new Model + { + Text = i % 3 == 0 ? "值" + i : "重复的值", + Amount = 100.5m + i, + Count = i, + PlacedAt = new DateTime(2026, 1, 1, 9, 30, 0).AddMinutes(i), + FinishedAt = i % 2 == 0 ? new DateTime(2026, 2, 1).AddDays(i) : null, + Paid = i % 2 == 0, + Status = i % 2 == 0 ? Status.已发货 : Status.待付款, + Link = new Uri("https://example.com/" + i) + }).ToList(); + } + + private static byte[] Export(IEnumerable rows, ExcelType excelType = ExcelType.Xlsx, + string? sheetTitle = null) + { + var bytes = new ExcelExporter().ObjectToExcelBytes(rows, options => + { + options.ExcelType = excelType; + options.SheetTitle = sheetTitle; + }); + Assert.IsNotNull(bytes); + return bytes!; + } + + private static List Streamed(byte[] bytes, Action? configure = null) + { + using var input = new MemoryStream(bytes); + return new ExcelImporter().ExcelStreamToObject(input, configure).ToList(); + } + + private static void AssertSame(IReadOnlyList expected, IReadOnlyList actual) + { + Assert.AreEqual(expected.Count, actual.Count, "行数"); + for (var i = 0; i < expected.Count; i++) + { + Assert.AreEqual(expected[i].Text, actual[i].Text, $"第 {i} 行的文本"); + Assert.AreEqual(expected[i].Amount, actual[i].Amount, $"第 {i} 行的金额"); + Assert.AreEqual(expected[i].Count, actual[i].Count, $"第 {i} 行的数量"); + Assert.AreEqual(expected[i].PlacedAt, actual[i].PlacedAt, $"第 {i} 行的下单时间"); + Assert.AreEqual(expected[i].FinishedAt, actual[i].FinishedAt, $"第 {i} 行的完成时间"); + Assert.AreEqual(expected[i].Paid, actual[i].Paid, $"第 {i} 行的已付"); + Assert.AreEqual(expected[i].Status, actual[i].Status, $"第 {i} 行的状态"); + Assert.AreEqual(expected[i].Link, actual[i].Link, $"第 {i} 行的链接"); + } + } + + [TestMethod] + public void StreamedImportMatchesTheInMemoryOne() + { + // 共享字符串表版本：内存导出写出的文件把文本收进 sharedStrings.xml + var bytes = Export(Rows(50)); + var inMemory = new ExcelImporter().ExcelToObject(bytes).ToList(); + + AssertSame(inMemory, Streamed(bytes)); + AssertSame(Rows(50), Streamed(bytes)); + } + + [TestMethod] + public void InlineStringsAreReadToo() + { + // 流式导出写出的文件把文本直接写在单元格里，不进共享字符串表 + using var output = new MemoryStream(); + new ExcelExporter().ObjectToExcelStream(Rows(30), output, options => + { + options.ExcelType = ExcelType.Xlsx; + options.StreamingRowWindow = 5; + }); + + var bytes = output.ToArray(); + Assert.IsFalse(Parts(bytes).Contains("xl/sharedStrings.xml"), "前提：该文件确实没有共享字符串表"); + AssertSame(Rows(30), Streamed(bytes)); + } + + [TestMethod] + public void BlanksAndGapsKeepTheirColumns() + { + var rows = Rows(4); + rows[1].Text = ""; + rows[2].Link = null; + rows[3].FinishedAt = null; + + var streamed = Streamed(Export(rows)); + Assert.AreEqual("", streamed[1].Text); + Assert.IsNull(streamed[2].Link); + Assert.IsNull(streamed[3].FinishedAt); + // 空白格不影响同一行上其他列的对位 + Assert.AreEqual(rows[2].Amount, streamed[2].Amount); + Assert.AreEqual(rows[3].Status, streamed[3].Status); + } + + [TestMethod] + public void TheSheetIsFoundByTitle() + { + var first = Export(Rows(3), sheetTitle: "上月"); + var both = new ExcelExporter().AppendObjectToExcelBytes(first, Rows(6), options => + { + options.SheetTitle = "本月"; + options.ExcelType = ExcelType.Xlsx; + }); + Assert.IsNotNull(both); + + Assert.AreEqual(3, Streamed(both!, o => o.SheetTitle = "上月").Count); + Assert.AreEqual(6, Streamed(both!, o => o.SheetTitle = "本月").Count); + // 不指定表名时取第一张 + Assert.AreEqual(3, Streamed(both!).Count); + } + + [TestMethod] + public void AnUnknownSheetIsRefusedBeforeAnyRowIsRead() + { + using var input = new MemoryStream(Export(Rows(3))); + + // 表名写错当场即报，不必等到取第一行 + var e = Assert.ThrowsException(() => + new ExcelImporter().ExcelStreamToObject(input, o => o.SheetTitle = "没有这张表")); + StringAssert.Contains(e.Message, "没有这张表"); + } + + [TestMethod] + public void TitleSkipLineIsHonoured() + { + // 表头之上多出两行说明文字 + var bytes = Export(Rows(5)); + var workbook = new XSSFWorkbook(new MemoryStream(bytes)); + var sheet = workbook.GetSheetAt(0); + sheet.ShiftRows(0, sheet.LastRowNum, 2); + sheet.CreateRow(0).CreateCell(0).SetCellValue("导出说明"); + sheet.CreateRow(1).CreateCell(0).SetCellValue("第二行说明"); + using var shifted = new MemoryStream(); + workbook.Write(shifted, true); + + var rows = Streamed(shifted.ToArray(), o => o.TitleSkipLine = 2); + Assert.AreEqual(5, rows.Count); + Assert.AreEqual("值0", rows[0].Text); + } + + [TestMethod] + public void AFormulaCellReadsItsStoredResult() + { + // Excel 存盘时会写下公式的计算结果，流式读取用的正是这个值 + var workbook = new XSSFWorkbook(); + var sheet = workbook.CreateSheet("数据"); + var header = sheet.CreateRow(0); + header.CreateCell(0).SetCellValue("文本"); + header.CreateCell(1).SetCellValue("数量"); + var row = sheet.CreateRow(1); + row.CreateCell(0).SetCellValue("甲"); + var formula = row.CreateCell(1); + formula.SetCellFormula("2*21"); + formula.SetCellValue(42); + using var bytes = new MemoryStream(); + workbook.Write(bytes, true); + + var streamed = Streamed(bytes.ToArray()); + Assert.AreEqual(42, streamed[0].Count); + } + + /// 公式列对应到一个可空属性，好看清「读作空白」究竟读成了什么。 + public class FormulaModel + { + [ExcelColumn("文本")] public string Text { get; set; } = ""; + [ExcelColumn("数量")] public int? Count { get; set; } + } + + [TestMethod] + public void AFormulaWithoutAStoredResultReadsAsBlank() + { + // 本库导出的文件里公式没有算出的结果，要等 Excel 打开时才算 + var bytes = new ExcelExporter().ObjectToExcelBytes( + new[] {new FormulaModel {Text = "甲"}, new FormulaModel {Text = "乙"}}, options => + { + options.ExcelType = ExcelType.Xlsx; + options.FormulaColumns.Add(new FormulaColumn {Title = "数量", Formula = c => c["文本"]}); + }); + Assert.IsNotNull(bytes); + + var errors = new List(); + using var input = new MemoryStream(bytes!); + var streamed = new ExcelImporter() + .ExcelStreamToObject(input, options => options.OnCellError = errors.Add).ToList(); + + Assert.AreEqual(2, streamed.Count); + Assert.IsNull(streamed[0].Count, "没有存下结果的公式读作空白"); + // 空白不是读取失败：这样的格常写成一个空的 ，不应当作读不出来上报 + Assert.AreEqual(0, errors.Count, string.Join("；", errors.Select(e => e.Exception.Message))); + // 其余各列照常 + Assert.AreEqual("甲", streamed[0].Text); + } + + [TestMethod] + public void ABlankCellBehavesTheSameWayAsWhenReadWhole() + { + // 空白格落在非可空的数值列上，两条路都以转换失败中止——此处只验二者一致 + var workbook = new XSSFWorkbook(); + var sheet = workbook.CreateSheet("数据"); + var header = sheet.CreateRow(0); + header.CreateCell(0).SetCellValue("文本"); + header.CreateCell(1).SetCellValue("数量"); + var row = sheet.CreateRow(1); + row.CreateCell(0).SetCellValue("甲"); + row.CreateCell(1).SetBlank(); + using var bytes = new MemoryStream(); + workbook.Write(bytes, true); + + Assert.ThrowsException(() => + new ExcelImporter().ExcelToObject(bytes.ToArray()).ToList()); + Assert.ThrowsException(() => Streamed(bytes.ToArray())); + + // 同一格落在可空列上则读作 null + using var input = new MemoryStream(bytes.ToArray()); + var nullable = new ExcelImporter().ExcelStreamToObject(input).ToList(); + Assert.IsNull(nullable[0].Count); + } + + [TestMethod] + public void PrefixedElementsMissingReferencesAndEmptyRowsAreRead() + { + // 带前缀的写法（）同样合法，行与格的 r 属性也可以不写，空行可以自闭合。 + // 这类文件由别的工具写出，此处手工造一份。 + var bytes = HandWritten(); + + var rows = Streamed(bytes, options => options.TitleSkipLine = 1); + Assert.AreEqual(2, rows.Count); + Assert.AreEqual("甲", rows[0].Text); + Assert.AreEqual(12, rows[0].Count); + Assert.AreEqual("乙", rows[1].Text); + Assert.AreEqual(34, rows[1].Count); + } + + /// 一份手写的 .xlsx：元素带前缀、行与格不写 r、开头有一个自闭合的空行。 + private static byte[] HandWritten() + { + const string main = "http://schemas.openxmlformats.org/spreadsheetml/2006/main"; + const string rels = "http://schemas.openxmlformats.org/officeDocument/2006/relationships"; + var parts = new Dictionary + { + ["[Content_Types].xml"] = + "" + + "" + + "" + + "" + + "" + + "", + ["_rels/.rels"] = + "" + + "" + + "" + + "", + ["xl/_rels/workbook.xml.rels"] = + "" + + "" + + "" + + "", + ["xl/workbook.xml"] = + "" + + "" + + "", + ["xl/worksheets/sheet1.xml"] = + "" + + "" + + "" + + "文本" + + "数量" + + "甲12" + + "乙34" + + "" + }; + + using var buffer = new MemoryStream(); + using (var zip = new System.IO.Compression.ZipArchive(buffer, System.IO.Compression.ZipArchiveMode.Create, true)) + foreach (var part in parts) + { + using var entry = new StreamWriter(zip.CreateEntry(part.Key).Open()); + entry.Write(part.Value); + } + + return buffer.ToArray(); + } + + [TestMethod] + public void ReadFailuresAreReportedWithTheirPosition() + { + var bytes = Export(Rows(3)); + var workbook = new XSSFWorkbook(new MemoryStream(bytes)); + // 第 3 行的「下单时间」改写成一段不是日期的文本 + var cell = workbook.GetSheetAt(0).GetRow(3).GetCell(3); + cell.SetCellType(CellType.String); + cell.SetCellValue("待定"); + using var broken = new MemoryStream(); + workbook.Write(broken, true); + + var errors = new List(); + using var input = new MemoryStream(broken.ToArray()); + var rows = new ExcelImporter() + .ExcelStreamToObject(input, options => options.OnCellError = errors.Add).ToList(); + + Assert.AreEqual(3, rows.Count); + Assert.AreEqual(1, errors.Count); + Assert.AreEqual("D4", errors[0].CellReference); + StringAssert.Contains(errors[0].Exception.Message, "待定"); + Assert.AreEqual(default, rows[2].PlacedAt); + } + + [TestMethod] + public void RowsAreTakenOneAtATime() + { + // 惰性取用：取够了就可以不再往下读 + using var input = new MemoryStream(Export(Rows(500))); + var first = new ExcelImporter().ExcelStreamToObject(input).Take(3).ToList(); + + Assert.AreEqual(3, first.Count); + Assert.AreEqual("值0", first[0].Text); + } + + [TestMethod] + public void XlsGoesThroughTheSameEntry() + { + // .xls 无从逐行读出，整份读入后结果一致 + AssertSame(Rows(10), Streamed(Export(Rows(10), ExcelType.Xls))); + } + + [TestMethod] + public void TheDictionaryFormIsReadToo() + { + using var input = new MemoryStream(Export(Rows(3))); + var rows = new ExcelImporter().ExcelStreamToObject>(input).ToList(); + + Assert.AreEqual(3, rows.Count); + Assert.AreEqual("值0", rows[0]["文本"]); + Assert.AreEqual("100.5", rows[0]["金额"]); + // 日期列读作它显示的样子，而非 Excel 存的序列号 + Assert.AreEqual("2026-01-01 09:30:00", rows[0]["下单时间"]); + } + + private static string[] Parts(byte[] bytes) + { + using var archive = new System.IO.Compression.ZipArchive(new MemoryStream(bytes)); + return archive.Entries.Select(e => e.FullName).ToArray(); + } +} diff --git a/Chsword.Excel2Object/Chsword.Excel2Object.csproj b/Chsword.Excel2Object/Chsword.Excel2Object.csproj index e8f7af6..12ac32e 100644 --- a/Chsword.Excel2Object/Chsword.Excel2Object.csproj +++ b/Chsword.Excel2Object/Chsword.Excel2Object.csproj @@ -15,7 +15,7 @@ true Chsword.Excel2Object Library Zou Jian - 2.10.0 + 2.11.0 Copyright ? 2014-2025 https://github.com/chsword/Excel2Object/ README.md diff --git a/Chsword.Excel2Object/ExcelHelper.cs b/Chsword.Excel2Object/ExcelHelper.cs index d1f090f..074bbdb 100644 --- a/Chsword.Excel2Object/ExcelHelper.cs +++ b/Chsword.Excel2Object/ExcelHelper.cs @@ -111,6 +111,14 @@ public static void ObjectToExcel(IEnumerable data, string path, return excelExporter.ObjectToExcelBytes(data, optionsAction); } + /// + public static IEnumerable ExcelStreamToObject(Stream input, + Action? optionAction = null) + where TModel : class, new() + { + return new ExcelImporter().ExcelStreamToObject(input, optionAction); + } + /// public static void ObjectToExcelStream(IEnumerable data, Stream output, Action? optionsAction = null) diff --git a/Chsword.Excel2Object/ExcelImporter.cs b/Chsword.Excel2Object/ExcelImporter.cs index 12d8603..e78bc9b 100644 --- a/Chsword.Excel2Object/ExcelImporter.cs +++ b/Chsword.Excel2Object/ExcelImporter.cs @@ -9,7 +9,7 @@ namespace Chsword.Excel2Object; public class ExcelImporter { - private static readonly Dictionary> SpecialConvertDict = + private static readonly Dictionary> SpecialConvertDict = new() { [typeof(DateTime)] = GetCellDateTime, @@ -17,6 +17,13 @@ public class ExcelImporter [typeof(Uri)] = GetCellUri }; + /// + /// What each number format seen so far displays, so NPOI's format parsing runs once per format + /// rather than once per cell. marks a format that is no + /// date at all. + /// + private static readonly ConcurrentDictionary FormatParts = new(); + public IEnumerable? ExcelToObject(string path, string? sheetTitle) where TModel : class, new() { @@ -40,12 +47,8 @@ public IEnumerable ExcelToObject(byte[] bytes, var options = new ExcelImporterOptions(); optionAction?.Invoke(options); var context = new ImportContext(options); - var result = GetDataRows(bytes, options); - if (typeof(TModel) == typeof(Dictionary)) - return (InternalExcelToDictionary(result, context) as IEnumerable)!; - - var list = InternalExcelToObject(result, context); - return list; + var rows = GetDataRows(bytes, options, context); + return ToModels(rows, context); } public IEnumerable ExcelToObject(byte[] bytes, string? sheetTitle) @@ -54,76 +57,164 @@ public IEnumerable ExcelToObject(byte[] bytes, string? sheetTitl return ExcelToObject(bytes, options => { options.SheetTitle = sheetTitle; }); } - private static IEnumerable> InternalExcelToDictionary(IEnumerator? result, - ImportContext context) + /// + /// 逐行读出 中的一张工作表，不把整个工作簿建进内存，适用于行数多到 + /// 不宜整份读入的导入。 + /// + /// + /// + /// 取到的序列是惰性的：取一行、转一行，读过的行随即可以回收。留在内存里的只有共享字符串表 + /// 与样式表——前者是文件中所有不重复的字符串，后者通常只有几十条。 + /// + /// + /// 与整份读入相比有一处不同：公式格读的是文件中存着的上一次计算结果，而非 + /// 当场求值。Excel 存盘时会写下这个结果，本库导出的文件则没有（要等 Excel 打开时才算出）， + /// 此时该格读作空白，与其他空白格一样。 + /// + /// + /// 只有 .xlsx 能够逐行读出；传入 .xls 时会照旧整份读入内存，结果一致。 + /// 由本方法读取，遍历结束（或中途放弃）时关闭其上的工作簿；与 + /// 一样，取到的序列须被遍历。 + /// + /// + /// + /// + /// using var file = File.OpenRead("orders.xlsx"); + /// foreach (var order in new ExcelImporter().ExcelStreamToObject<Order>(file)) + /// Handle(order); + /// + /// + public IEnumerable ExcelStreamToObject(Stream input, + Action? optionAction = null) + where TModel : class, new() { - var list = new List>(); + // .xls 的格式无从逐行读出，整份读入后结果与既有方式一致。此处先于建立选项判断， + // optionAction 才不会被调用两次——它可能是有状态的 + if (!LooksLikeXlsx(input)) return ExcelToObject(ReadAll(input), optionAction); - if (result == null) - return list; - var rows = result; - var titleRow = rows.Current; - if (titleRow == null) return list; - var columns = titleRow.Cells.ToDictionary(c => c.StringCellValue, c => c.ColumnIndex); + var options = new ExcelImporterOptions(); + optionAction?.Invoke(options); + var context = new ImportContext(options); + var rows = XlsxRowReader.Rows(input, options, context).GetEnumerator(); + rows.MoveNext(); + for (var i = 0; i < options.TitleSkipLine; i++) rows.MoveNext(); + return ToModels(rows, context); + } - while (rows.MoveNext()) + /// + /// .xlsx 是个 zip，头两个字节为 PK。流不可定位时无从先看一眼，此时按 .xlsx 处理， + /// 真不是的话打开那一步会说清楚。 + /// + private static bool LooksLikeXlsx(Stream input) + { + if (!input.CanSeek) return true; + + var position = input.Position; + var head = new byte[2]; + var read = input.Read(head, 0, 2); + input.Position = position; + return read == 2 && head[0] == (byte) 'P' && head[1] == (byte) 'K'; + } + + private static byte[] ReadAll(Stream input) + { + using var buffer = new MemoryStream(); + input.CopyTo(buffer); + return buffer.ToArray(); + } + + /// 行从哪里来并不影响其后的转换：字典与模型两条路都只认 。 + private static IEnumerable ToModels(IEnumerator? rows, ImportContext context) + where TModel : class, new() + { + if (typeof(TModel) == typeof(Dictionary)) + return (InternalExcelToDictionary(rows, context) as IEnumerable)!; + + return InternalExcelToObject(rows, context); + } + + private static IEnumerable> InternalExcelToDictionary(IEnumerator? result, + ImportContext context) + { + if (result == null) yield break; + + // 取行的枚举器在此释放：流式导入由它持有着打开的工作簿，中途放弃（Take、break）时也须关上 + using (result) { - var row = rows.Current; - if (row == null || row.Cells?.Count == 0) - continue; + var titleRow = result.Current; + if (titleRow == null) yield break; + + // 同名的标题以最左边那一列为准，与模型列的对应方式一致 + var columns = new Dictionary(); + foreach (var cell in titleRow.Cells) + { + var title = TextOf(cell.Value); + if (title != null && !columns.ContainsKey(title)) columns[title] = cell.Key; + } - var model = new Dictionary(); + while (result.MoveNext()) + { + var row = result.Current; + if (row == null || row.CellCount == 0) + continue; - foreach (var column in columns) - model[column.Key] = GetCellValue(row.GetCell(column.Value), context, datesAsText: true) ?? ""; + var model = new Dictionary(); - list.Add(model); - } + foreach (var column in columns) + model[column.Key] = TextOf(row.Cell(column.Value), true) ?? ""; - return list; + yield return model; + } + } } - private static IEnumerable InternalExcelToObject(IEnumerator? result, + private static IEnumerable InternalExcelToObject(IEnumerator? result, ImportContext context) where TModel : class, new() { if (result == null) yield break; - - var dictColumns = BuildColumnMappings(result); - while (result.MoveNext()) + // 取行的枚举器在此释放：流式导入由它持有着打开的工作簿，中途放弃（Take、break）时也须关上 + using (result) { - var row = result.Current; + var dictColumns = BuildColumnMappings(result); + + while (result.MoveNext()) + { + var row = result.Current; - if (row == null || row.Cells?.Count == 0) - continue; + if (row == null || row.CellCount == 0) + continue; - var model = new TModel(); - PopulateModelFromRow(model, row, dictColumns, context); - yield return model; + var model = new TModel(); + PopulateModelFromRow(model, row, dictColumns, context); + yield return model; + } } } - private static Dictionary> BuildColumnMappings(IEnumerator result) + private static Dictionary> BuildColumnMappings( + IEnumerator result) where TModel : class, new() { var dict = ExcelUtil.GetPropertiesAttributesDict(); var dictColumns = new Dictionary>(); var titleRow = result.Current; - + if (titleRow != null) foreach (var cell in titleRow.Cells) { - var prop = dict.FirstOrDefault(c => cell.StringCellValue == c.Value.Title); - if (prop.Key != null && !dictColumns.ContainsKey(cell.ColumnIndex)) - dictColumns.Add(cell.ColumnIndex, prop); + var title = TextOf(cell.Value); + var prop = dict.FirstOrDefault(c => title == c.Value.Title); + if (prop.Key != null && !dictColumns.ContainsKey(cell.Key)) + dictColumns.Add(cell.Key, prop); } - + return dictColumns; } - private static void PopulateModelFromRow(TModel model, IRow row, + private static void PopulateModelFromRow(TModel model, IImportRow row, Dictionary> dictColumns, ImportContext context) where TModel : class, new() { @@ -131,26 +222,23 @@ private static void PopulateModelFromRow(TModel model, IRow row, { var propType = pair.Value.Key.PropertyType; var type = TypeUtil.GetUnNullableType(propType); - - object? value = type.IsEnum + + var value = type.IsEnum ? GetEnum(row, pair.Key, type, context) : GetCellValueByType(row, pair.Key, propType, type, context); - + pair.Value.Key.SetValue(model, value, null); } } - private static object? GetCellValueByType(IRow row, int columnIndex, Type propType, Type type, + private static object? GetCellValueByType(IImportRow row, int columnIndex, Type propType, Type type, ImportContext context) { - if (SpecialConvertDict.TryGetValue(type, out var special)) - { - return special(row, columnIndex, context); - } + if (SpecialConvertDict.TryGetValue(type, out var special)) return special(row, columnIndex, context); // a date cell reads as the date it shows into a string, and as the serial number Excel stores // into anything numeric - var cellValue = GetCellValue(row.GetCell(columnIndex), context, type == typeof(string)); + var cellValue = TextOf(row.Cell(columnIndex), type == typeof(string)); // 读取失败（已上报）与读到空值不同：前者取该类型的默认值，不再尝试转换，否则会以 // Convert.ChangeType("") 抛出的 FormatException 掩盖真正的原因，并把同一格上报两次 @@ -164,12 +252,12 @@ private static void PopulateModelFromRow(TModel model, IRow row, try { - return Convert.ChangeType(cellValue, type); + return System.Convert.ChangeType(cellValue, type); } catch (Exception e) { // 转换失败照旧向外抛出并中止导入，上报只是让调用方知道是哪一个单元格 - context.Report(row, columnIndex, e); + context.Report(row.SheetTitle, row.RowIndex, columnIndex, e); throw; } } @@ -182,67 +270,64 @@ private static void PopulateModelFromRow(TModel model, IRow row, return Activator.CreateInstance(type); } - private static object? GetCellBoolean(IRow row, int key, ImportContext context) + private static object? GetCellBoolean(IImportRow row, int key, ImportContext context) { - var cellValue = GetCellValue(row.GetCell(key), context); + var cellValue = TextOf(row.Cell(key)); // 显式判空而非 string.IsNullOrEmpty：后者在较早的目标框架上没有 NotNullWhen 标注 if (cellValue == null || cellValue.Length == 0) return null; if (bool.TryParse(cellValue, out var value)) return value; - + var lowerValue = cellValue.ToLower(); if (ExcelConstants.BooleanValues.TrueValues.Any(v => v.Equals(lowerValue, StringComparison.OrdinalIgnoreCase))) return true; if (ExcelConstants.BooleanValues.FalseValues.Any(v => v.Equals(lowerValue, StringComparison.OrdinalIgnoreCase))) return false; - + try { - return Convert.ToBoolean(cellValue); + return System.Convert.ToBoolean(cellValue); } catch (Exception e) { - context.Report(row, key, e); + context.Report(row.SheetTitle, row.RowIndex, key, e); throw; } } - private static object? GetCellDateTime(IRow row, int index, ImportContext context) + private static object? GetCellDateTime(IImportRow row, int index, ImportContext context) { - var cell = row.GetCell(index); - if (cell == null) return null; + var cell = row.Cell(index); + if (!cell.Exists) return null; - // 取文本的这一步自行上报失败，放在 try 之外，同一次失败才不会被上报两次 - var cellText = GetCellValue(cell, context); + // 取文本的这一步自行上报失败，放在其外，同一次失败才不会被上报两次 + var cellText = TextOf(cell); if (cellText == null || cellText.Length == 0) return null; - try - { - switch (cell.CellType) - { - case CellType.Numeric: - // 序列号超出 Excel 日历时 DateCellValue 会抛出，此时该单元格取 null - return cell.DateCellValue; - case CellType.String: - var text = cell.StringCellValue; - var parsed = GetDateTimeFromString(text); - if (parsed == null && !string.IsNullOrWhiteSpace(text)) - // 文本不是日期是最常见的导入失败，同样要让调用方知道 - context.Report(cell, new FormatException($"[{text}] 不是可识别的日期。")); - return parsed; - default: - return null; - } - } - catch (Exception e) + switch (cell.Kind) { - context.Report(cell, e); - return null; + case CellType.Numeric: + var date = cell.Date(); + if (date == null) + // 序列号超出 Excel 的日历，此时该单元格取 null + context.Report(row.SheetTitle, row.RowIndex, index, + new ArgumentException($"[{cell.NumberText()}] 不在 Excel 的日历范围内，无从读作日期。")); + return date; + case CellType.String: + var text = cell.Text; + var parsed = GetDateTimeFromString(text); + if (parsed == null && !string.IsNullOrWhiteSpace(text)) + // 文本不是日期是最常见的导入失败，同样要让调用方知道 + context.Report(row.SheetTitle, row.RowIndex, index, + new FormatException($"[{text}] 不是可识别的日期。")); + return parsed; + default: + return null; } } - private static object? GetCellUri(IRow row, int key, ImportContext context) + private static object? GetCellUri(IImportRow row, int key, ImportContext context) { - var cellValue = GetCellValue(row.GetCell(key), context); + var cellValue = TextOf(row.Cell(key)); if (cellValue == null || cellValue.Length == 0) return null; try @@ -251,7 +336,7 @@ private static void PopulateModelFromRow(TModel model, IRow row, } catch (Exception e) { - context.Report(row, key, e); + context.Report(row.SheetTitle, row.RowIndex, key, e); throw; } } @@ -265,91 +350,44 @@ private static void PopulateModelFromRow(TModel model, IRow row, /// 单元格的文本；null 表示读取失败——失败已经上报，调用方据此取默认值即可，不应再 /// 按空字符串继续转换。 /// - private static string? GetCellValue(ICell? cell, ImportContext context, bool datesAsText = false) + private static string? TextOf(CellData cell, bool datesAsText = false) { - var result = string.Empty; - if (cell == null) return result; - try - { - switch (cell.CellType) - { - case CellType.Numeric: - result = (datesAsText ? DateCellText(cell) : null) ?? NumberText(cell.NumericCellValue); - break; - case CellType.String: - result = cell.StringCellValue; - break; - case CellType.Blank: - result = string.Empty; - break; - case CellType.Formula: - // 求值器按工作簿创建一次，不再逐单元格创建 - result = GetCellValue(context.Evaluator(cell.Sheet.Workbook).EvaluateInCell(cell), context, - datesAsText); - break; - default: - // Boolean ("TRUE"/"FALSE"), Error and _None all render acceptably through ToString. - result = cell.ToString(); - break; - } - } - catch (Exception e) + if (cell.Failed) return null; + if (!cell.Exists) return string.Empty; + + var result = cell.Kind switch { - context.Report(cell, e); - return null; - } + CellType.Numeric => (datesAsText ? DateCellText(cell) : null) ?? cell.NumberText(), + CellType.String => cell.Text, + CellType.Blank => string.Empty, + // Boolean ("TRUE"/"FALSE") and Error both read as the text they carry + _ => cell.Text + }; return (result ?? "").Trim(); } - /// - /// 数值的文本表示，必须能原样解析回同一个 double：数值列正是经由这段文本转换的。 - /// - /// - /// .NET Framework 的默认格式为 G15，会丢掉低位；其 "R" 对少数值给出的表示同样无法往返 - /// （已知缺陷）。故先按 "R" 渲染并验证一次，确实不能往返时才退回 17 位有效数字——这样 - /// 常见数值不会平白变成 0.10000000000000001 这样的写法。 - /// - private static string NumberText(double value) - { - var text = value.ToString("R", CultureInfo.InvariantCulture); - if (double.TryParse(text, NumberStyles.Float, CultureInfo.InvariantCulture, out var back) && - back.Equals(value)) - return text; - - return value.ToString("G17", CultureInfo.InvariantCulture); - } - - /// - /// What each number format seen so far displays, so NPOI's format parsing runs once per format - /// rather than once per cell. marks a format that is no - /// date at all. - /// - private static readonly ConcurrentDictionary FormatParts = new(); - /// /// A date cell is a number with a date format. Read as text - into a string property or a /// dictionary - it comes out as the date it shows, not as the serial number Excel stores: an ISO /// date, a time of day or both, whichever its format displays. Null when the cell is no date; /// elapsed time ([h]:mm) counts as none, the number being the best that can be offered then. /// - private static string? DateCellText(ICell cell) + private static string? DateCellText(CellData cell) { - var style = cell.CellStyle; - var format = style?.GetDataFormatString(); + var format = cell.Format; if (format == null) return null; if (!FormatParts.TryGetValue(format, out var parts)) { - parts = DateUtil.IsADateFormat(style!.DataFormat, format) + parts = DateUtil.IsADateFormat(cell.FormatIndex, format) ? ExcelDateFormat.PartsShown(format) : ExcelDateFormat.Parts.None; FormatParts[format] = parts; } - if (parts == ExcelDateFormat.Parts.None || !DateUtil.IsValidExcelDate(cell.NumericCellValue)) - return null; - var date = cell.DateCellValue; + if (parts == ExcelDateFormat.Parts.None) return null; + var date = cell.Date(); if (date == null) return null; var pattern = parts == ExcelDateFormat.Parts.Date ? ExcelDateFormat.IsoDate : @@ -357,7 +395,8 @@ private static string NumberText(double value) return date.Value.ToString(pattern, CultureInfo.InvariantCulture); } - private static IEnumerator? GetDataRows(byte[]? bytes, ExcelImporterOptions options) + private static IEnumerator? GetDataRows(byte[]? bytes, ExcelImporterOptions options, + ImportContext context) { if (bytes == null || bytes.Length == 0) return null; @@ -384,16 +423,26 @@ private static string NumberText(double value) throw new Excel2ObjectException($"The specified sheet:[{options.SheetTitle}] does not exist"); } - var rows = sheet.GetEnumerator(); + var rows = NpoiRows(sheet, context).GetEnumerator(); rows.MoveNext(); for (var i = 0; i < options.TitleSkipLine; i++) rows.MoveNext(); return rows; } + /// 把整份读入内存的工作表的各行，包成与来源无关的行。 + private static IEnumerable NpoiRows(ISheet sheet, ImportContext context) + { + // 1904 日期系统是整个工作簿的属性，逐格去问一遍并无意义 + var date1904 = sheet.Workbook.IsDate1904(); + foreach (IRow row in sheet) + if (row != null) + yield return new NpoiRow(row, context, date1904); + } + private static DateTime? GetDateTimeFromString(string str) { DateTime dt; - + // Handle Chinese date formats (年月日) if (str.EndsWith(ExcelConstants.DateFormats.YearSuffix)) { @@ -410,19 +459,19 @@ private static string NumberText(double value) // Try standard parsing first if (DateTime.TryParse(str, out dt)) return dt; - + // Try parsing with specific formats - if (DateTime.TryParseExact(str, ExcelConstants.DateFormats.CommonDateTimeFormats, + if (DateTime.TryParseExact(str, ExcelConstants.DateFormats.CommonDateTimeFormats, CultureInfo.InvariantCulture, DateTimeStyles.None, out dt)) return dt; - + // Try parsing with current culture - if (DateTime.TryParseExact(str, ExcelConstants.DateFormats.CommonDateTimeFormats, + if (DateTime.TryParseExact(str, ExcelConstants.DateFormats.CommonDateTimeFormats, CultureInfo.CurrentCulture, DateTimeStyles.None, out dt)) return dt; - + // Handle time-only formats - combine with today's date - if (DateTime.TryParseExact(str, ExcelConstants.DateFormats.CommonDateTimeFormats, + if (DateTime.TryParseExact(str, ExcelConstants.DateFormats.CommonDateTimeFormats, CultureInfo.InvariantCulture, DateTimeStyles.NoCurrentDateDefault, out dt)) { // If only time is provided, combine with today's date @@ -432,7 +481,7 @@ private static string NumberText(double value) } return dt; } - + // Fallback for partial dates if (DateTime.TryParse((str + ExcelConstants.DateFormats.DefaultYearMonthSuffix).Replace(ExcelConstants.DateFormats.YearSuffix, "").Replace(ExcelConstants.DateFormats.MonthSuffix, ""), out dt)) return dt; @@ -446,15 +495,15 @@ private static string NumberText(double value) return null; } - private static object? GetEnum(IRow row, int key, Type enumType, ImportContext context) + private static object? GetEnum(IImportRow row, int key, Type enumType, ImportContext context) { - var cellValue = GetCellValue(row.GetCell(key), context); + var cellValue = TextOf(row.Cell(key)); if (cellValue == null || cellValue.Length == 0) return null; if (Enum.GetNames(enumType).Contains(cellValue)) return Enum.Parse(enumType, cellValue); // 取值不在枚举中：沿用既有行为取 0，但不再悄无声息 - context.Report(row, key, + context.Report(row.SheetTitle, row.RowIndex, key, new FormatException($"[{cellValue}] 不是 {enumType.Name} 的取值，按 0 处理。")); return Enum.ToObject(enumType, 0); } -} \ No newline at end of file +} diff --git a/Chsword.Excel2Object/Internal/CellData.cs b/Chsword.Excel2Object/Internal/CellData.cs new file mode 100644 index 0000000..1a27cde --- /dev/null +++ b/Chsword.Excel2Object/Internal/CellData.cs @@ -0,0 +1,123 @@ +using System.Globalization; +using NPOI.SS.UserModel; + +namespace Chsword.Excel2Object.Internal; + +/// +/// 一格读到的东西，与它从哪里来无关：可以是 NPOI 的单元格，也可以是流式读取解出的一段 XML。 +/// +/// +/// 导入的类型转换原本直接写在 之上，而流式读取拿到的是原始的 XML，并没有 +/// 单元格对象。把两者都归到这个结构上，转换那一套便只有一份。 +/// +internal readonly struct CellData +{ + /// 该格不存在——空单元格并不占位。 + public static readonly CellData None = default; + + /// + /// 这一格读取失败，且已上报。与「不存在」不同：不存在读作空白，读取失败则应取该类型的默认值， + /// 不再按空字符串继续转换，否则真正的原因会被转换时抛出的异常掩盖。 + /// + public static readonly CellData Failure = new(true); + + /// + /// 文本以可空字段存放而非自动属性：结构的默认值（不存在的那一格）本就没有文本，让它读作空串， + /// 各处便无须为此判空。 + /// + private readonly string? _text; + + private CellData(bool failed) + { + Failed = failed; + Exists = false; + Kind = CellType.Blank; + _text = null; + Number = 0; + Flag = false; + FormatIndex = 0; + Format = null; + Date1904 = false; + } + + private CellData(CellType kind, string? text, double number, bool flag, short formatIndex, string? format, + bool date1904) + { + Failed = false; + Exists = true; + Kind = kind; + _text = text; + Number = number; + Flag = flag; + FormatIndex = formatIndex; + Format = format; + Date1904 = date1904; + } + + public bool Exists { get; } + + public bool Failed { get; } + + public CellType Kind { get; } + + /// 文本格即其文本；数值、布尔与错误格则是其渲染出的样子。 + public string Text => _text ?? string.Empty; + + public double Number { get; } + + public bool Flag { get; } + + /// 数字格式的序号与格式串，用于判断这一格是不是日期。 + public short FormatIndex { get; } + + public string? Format { get; } + + /// 该工作簿是否采用 1904 日期系统；序列号换算成日期时须据此而定。 + public bool Date1904 { get; } + + public static CellData Blank(short formatIndex = 0, string? format = null) + { + return new CellData(CellType.Blank, null, 0, false, formatIndex, format, false); + } + + public static CellData OfNumber(double value, short formatIndex, string? format, bool date1904) + { + return new CellData(CellType.Numeric, null, value, false, formatIndex, format, date1904); + } + + public static CellData OfText(string? text) + { + return new CellData(CellType.String, text, 0, false, 0, null, false); + } + + public static CellData OfBoolean(bool value) + { + // 与 NPOI 的单元格渲染一致 + return new CellData(CellType.Boolean, value ? "TRUE" : "FALSE", 0, value, 0, null, false); + } + + public static CellData OfError(string? text) + { + return new CellData(CellType.Error, text, 0, false, 0, null, false); + } + + /// 数值格的文本表示；必须能原样解析回同一个 double，数值列正是经由这段文本转换的。 + public string NumberText() + { + var text = Number.ToString("R", CultureInfo.InvariantCulture); + if (double.TryParse(text, NumberStyles.Float, CultureInfo.InvariantCulture, out var back) && + back.Equals(Number)) + return text; + + // .NET Framework 的 "R" 对少数值无法往返（已知缺陷），此时退回 17 位有效数字 + return Number.ToString("G17", CultureInfo.InvariantCulture); + } + + /// 该格作为日期的取值；序列号超出 Excel 日历时为 null。 + public DateTime? Date() + { + if (Kind != CellType.Numeric || !DateUtil.IsValidExcelDate(Number)) return null; + + return DateUtil.GetJavaDate(Number, Date1904); + } +} diff --git a/Chsword.Excel2Object/Internal/ImportContext.cs b/Chsword.Excel2Object/Internal/ImportContext.cs index accb9f9..602c9a8 100644 --- a/Chsword.Excel2Object/Internal/ImportContext.cs +++ b/Chsword.Excel2Object/Internal/ImportContext.cs @@ -61,7 +61,8 @@ public void Report(IRow? row, int columnIndex, Exception exception) Report(row?.Sheet?.SheetName, row?.RowNum ?? -1, columnIndex, exception); } - private void Report(string? sheetTitle, int rowIndex, int columnIndex, Exception exception) + /// 位置由调用方给出：流式读取没有单元格对象，只有行列号。 + public void Report(string? sheetTitle, int rowIndex, int columnIndex, Exception exception) { if (_options.OnCellError == null) return; diff --git a/Chsword.Excel2Object/Internal/ImportRow.cs b/Chsword.Excel2Object/Internal/ImportRow.cs new file mode 100644 index 0000000..2690b47 --- /dev/null +++ b/Chsword.Excel2Object/Internal/ImportRow.cs @@ -0,0 +1,91 @@ +using NPOI.SS.UserModel; + +namespace Chsword.Excel2Object.Internal; + +/// +/// 一行数据，与它从哪里来无关：整份读入内存的工作簿，或流式读取解出的一行。 +/// +internal interface IImportRow +{ + string? SheetTitle { get; } + + int RowIndex { get; } + + /// 该行存在的各格，按列序号给出。 + IEnumerable> Cells { get; } + + /// 该行有几格。整行为空的行不参与导入，判断这一点不必把每格都取出来。 + int CellCount { get; } + + /// 某一列上的那一格；不存在时为 。 + CellData Cell(int columnIndex); +} + +/// 整份读入内存时的一行，取值经由 NPOI 的单元格。 +internal sealed class NpoiRow : IImportRow +{ + private readonly ImportContext _context; + private readonly bool _date1904; + private readonly IRow _row; + + public NpoiRow(IRow row, ImportContext context, bool date1904) + { + _row = row; + _context = context; + _date1904 = date1904; + } + + public string? SheetTitle => _row.Sheet?.SheetName; + + public int RowIndex => _row.RowNum; + + public int CellCount => _row.Cells.Count; + + public IEnumerable> Cells + { + get + { + foreach (var cell in _row.Cells) + yield return new KeyValuePair(cell.ColumnIndex, Of(cell, _context, _date1904)); + } + } + + public CellData Cell(int columnIndex) + { + var cell = _row.GetCell(columnIndex); + return cell == null ? CellData.None : Of(cell, _context, _date1904); + } + + /// + /// 把 NPOI 的单元格取成与来源无关的一格。公式在此求值：求值器按工作簿创建一次，其结果的类型 + /// 与取值即为这一格的类型与取值。 + /// + private static CellData Of(ICell cell, ImportContext context, bool date1904) + { + try + { + switch (cell.CellType) + { + case CellType.Numeric: + var style = cell.CellStyle; + return CellData.OfNumber(cell.NumericCellValue, style?.DataFormat ?? 0, + style?.GetDataFormatString(), date1904); + case CellType.String: + return CellData.OfText(cell.StringCellValue); + case CellType.Boolean: + return CellData.OfBoolean(cell.BooleanCellValue); + case CellType.Blank: + return CellData.Blank(); + case CellType.Formula: + return Of(context.Evaluator(cell.Sheet.Workbook).EvaluateInCell(cell), context, date1904); + default: + return CellData.OfError(cell.ToString() ?? string.Empty); + } + } + catch (Exception e) + { + context.Report(cell.Sheet?.SheetName, cell.RowIndex, cell.ColumnIndex, e); + return CellData.Failure; + } + } +} diff --git a/Chsword.Excel2Object/Internal/XlsxRowReader.cs b/Chsword.Excel2Object/Internal/XlsxRowReader.cs new file mode 100644 index 0000000..7f0a677 --- /dev/null +++ b/Chsword.Excel2Object/Internal/XlsxRowReader.cs @@ -0,0 +1,394 @@ +using System.Globalization; +using System.Xml; +using Chsword.Excel2Object.Options; +using NPOI.OpenXml4Net.OPC; +using NPOI.SS.UserModel; +using NPOI.XSSF.EventUserModel; +using NPOI.XSSF.Model; + +namespace Chsword.Excel2Object.Internal; + +/// +/// 逐行读出 .xlsx 的一张工作表，不把整个工作簿建进内存。 +/// +/// +/// +/// 读的是工作表的那份 XML：一行解出来即交给调用方，随后便可回收。留在内存里的只有共享字符串表 +/// 与样式表——前者是文件中所有不重复的字符串，后者通常只有几十条，二者都远小于整份工作簿。 +/// +/// +/// 公式格读的是文件中存着的上一次计算结果。本库导出的文件里公式没有这个结果（要等 Excel 打开时 +/// 才算出），此时该格读作空白。 +/// +/// +internal static class XlsxRowReader +{ + private const string RelationshipNamespace = + "http://schemas.openxmlformats.org/officeDocument/2006/relationships"; + + public static IEnumerable Rows(Stream input, ExcelImporterOptions options, ImportContext context) + { + OPCPackage package; + try + { + package = OPCPackage.Open(input); + } + catch (Exception e) + { + throw new Excel2ObjectException("这不是一份能够打开的 .xlsx 文件。", e); + } + + // 打开与定位在此处即完成：表名写错应当当场报错，而不是等到第一次取行 + try + { + var reader = new XSSFReader(package); + var workbook = ReadWorkbook(reader); + var sheet = Locate(workbook, options.SheetTitle); + return Iterate(package, reader, sheet, workbook.Date1904, context); + } + catch + { + package.Close(); + throw; + } + } + + /// + /// 自此按行取用。工作簿由这条序列持有：遍历完毕或中途放弃（foreach 均会释放枚举器） + /// 时关闭；与 一样，取到的序列须被遍历。 + /// + private static IEnumerable Iterate(OPCPackage package, XSSFReader reader, SheetInfo sheet, + bool date1904, ImportContext context) + { + try + { + var styles = new NumberFormats(reader.StylesTable); + var strings = new ReadOnlySharedStringsTable(package); + using var sheetData = reader.GetSheet(sheet.RelationshipId); + + foreach (var row in ReadRows(sheetData, sheet.Title, date1904, styles, strings, context)) + yield return row; + } + finally + { + package.Close(); + } + } + + /// 工作簿本身那份 XML：各表的名字与其部件编号，以及所用的日期系统。 + private static WorkbookInfo ReadWorkbook(XSSFReader reader) + { + var sheets = new List(); + var date1904 = false; + + using var data = reader.WorkbookData; + using var xml = XmlReader.Create(data, new XmlReaderSettings {IgnoreWhitespace = true, IgnoreComments = true}); + while (xml.Read()) + { + if (xml.NodeType != XmlNodeType.Element) continue; + + switch (xml.LocalName) + { + case "workbookPr": + date1904 = IsTrue(xml.GetAttribute("date1904")) || IsTrue(xml.GetAttribute("date1904Compat")); + break; + case "sheet": + var id = xml.GetAttribute("id", RelationshipNamespace) ?? xml.GetAttribute("r:id"); + if (id != null) sheets.Add(new SheetInfo(xml.GetAttribute("name") ?? string.Empty, id)); + break; + } + } + + return new WorkbookInfo(sheets, date1904); + } + + private static SheetInfo Locate(WorkbookInfo workbook, string? title) + { + if (workbook.Sheets.Count == 0) throw new Excel2ObjectException("该工作簿中没有工作表。"); + + if (string.IsNullOrEmpty(title)) return workbook.Sheets[0]; + + foreach (var sheet in workbook.Sheets) + if (sheet.Title == title) + return sheet; + + throw new Excel2ObjectException($"The specified sheet:[{title}] does not exist"); + } + + private static IEnumerable ReadRows(Stream sheetData, string sheetTitle, bool date1904, + NumberFormats styles, ReadOnlySharedStringsTable strings, ImportContext context) + { + using var xml = XmlReader.Create(sheetData, + new XmlReaderSettings {IgnoreWhitespace = true, IgnoreComments = true}); + + var cells = new Dictionary(); + var rowIndex = -1; + var column = -1; + + while (xml.Read()) + { + if (xml.NodeType == XmlNodeType.Element && xml.LocalName == "row") + { + // 行号自 1 计起，该属性可以不写，此时按上一行顺延 + var declared = xml.GetAttribute("r"); + rowIndex = declared == null ? rowIndex + 1 : Math.Max(Number(declared) - 1, 0); + cells = new Dictionary(); + column = -1; + if (!xml.IsEmptyElement) continue; + + // 自闭合的空行没有结束标记，就地给出，否则 TitleSkipLine 会把它数漏 + yield return new StreamedRow(sheetTitle, rowIndex, cells); + continue; + } + + if (xml.NodeType == XmlNodeType.Element && xml.LocalName == "c") + { + var reference = xml.GetAttribute("r"); + column = reference == null ? column + 1 : ColumnOf(reference); + var type = xml.GetAttribute("t"); + var style = xml.GetAttribute("s"); + var cell = ReadCell(xml, type, style, date1904, styles, strings, sheetTitle, rowIndex, column, + context); + if (cell.Exists || cell.Failed) cells[column] = cell; + continue; + } + + if (xml.NodeType == XmlNodeType.EndElement && xml.LocalName == "row") + yield return new StreamedRow(sheetTitle, rowIndex, cells); + } + } + + /// + /// 一格的取值。t 说明它是什么：s 指向共享字符串表，inlineStr 与 str + /// 自带文本，b 是布尔，e 是错误值，其余为数值。 + /// + private static CellData ReadCell(XmlReader xml, string? type, string? style, bool date1904, + NumberFormats styles, ReadOnlySharedStringsTable strings, string sheetTitle, int rowIndex, int column, + ImportContext context) + { + var text = CellText(xml); + + // 没有算出结果的公式常写成一个空的 ，这样的内容一律读作空白，不去解析 + if (text != null && text.Length == 0 && type != "inlineStr" && type != "str") text = null; + + try + { + switch (type) + { + case "s": + // 共享字符串表：值是其中的序号 + return text == null ? CellData.Blank() : CellData.OfText(strings.GetEntryAt(int.Parse(text, CultureInfo.InvariantCulture))); + case "inlineStr": + case "str": + return text == null ? CellData.Blank() : CellData.OfText(text); + case "b": + return text == null ? CellData.Blank() : CellData.OfBoolean(text == "1" || text == "TRUE"); + case "e": + return CellData.OfError(text); + case "d": + // ISO 8601 写法的日期，少见 + return text == null + ? CellData.Blank() + : CellData.OfText(text); + default: + if (text == null) return CellData.Blank(); + + var format = styles.Of(style); + return CellData.OfNumber(double.Parse(text, NumberStyles.Float, CultureInfo.InvariantCulture), + format.Index, format.Text, date1904); + } + } + catch (Exception e) + { + context.Report(sheetTitle, rowIndex, column, e); + return CellData.Failure; + } + } + + /// 一格里写着的内容：<v> 中的值，或 <is> 中的行内字符串。 + private static string? CellText(XmlReader xml) + { + if (xml.IsEmptyElement) return null; + + string? value = null; + var depth = xml.Depth; + // ReadElementContentAsString 会一并越过该元素，故读到内容之后不再 Read，否则将跳过下一个节点 + while (!(xml.NodeType == XmlNodeType.EndElement && xml.Depth == depth)) + { + if (xml.NodeType == XmlNodeType.Element) + { + if (xml.LocalName == "v") + { + value = xml.ReadElementContentAsString(); + continue; + } + + if (xml.LocalName == "is") + { + // 行内字符串优先：它与 不会同时出现 + return InlineText(xml); + } + } + + if (!xml.Read()) break; + } + + return value; + } + + /// 行内字符串：<is> 之下可有若干段 <t>，依次相接。 + private static string InlineText(XmlReader xml) + { + var depth = xml.Depth; + var text = string.Empty; + while (!(xml.NodeType == XmlNodeType.EndElement && xml.Depth == depth)) + { + if (xml.NodeType == XmlNodeType.Element && xml.LocalName == "t") + { + text += xml.ReadElementContentAsString(); + continue; + } + + if (!xml.Read()) break; + } + + return text; + } + + private static bool IsTrue(string? value) + { + return value == "1" || string.Equals(value, "true", StringComparison.OrdinalIgnoreCase); + } + + private static int Number(string? value) + { + return value != null && int.TryParse(value, NumberStyles.Integer, CultureInfo.InvariantCulture, out var n) + ? n + : 0; + } + + /// 由 B12 这样的地址取出列序号，自 0 计起。 + private static int ColumnOf(string reference) + { + var column = 0; + foreach (var c in reference) + { + if (c is < 'A' or > 'Z') + { + if (c is >= 'a' and <= 'z') + { + column = (column * 26) + (c - 'a' + 1); + continue; + } + + break; + } + + column = (column * 26) + (c - 'A' + 1); + } + + return column - 1; + } + + /// 流式读出的一行：各格已解好，按列序号存放。 + private sealed class StreamedRow : IImportRow + { + private readonly Dictionary _cells; + + public StreamedRow(string? sheetTitle, int rowIndex, Dictionary cells) + { + SheetTitle = sheetTitle; + RowIndex = rowIndex; + _cells = cells; + } + + public string? SheetTitle { get; } + + public int RowIndex { get; } + + public IEnumerable> Cells => _cells.OrderBy(c => c.Key); + + public int CellCount => _cells.Count; + + public CellData Cell(int columnIndex) + { + return _cells.TryGetValue(columnIndex, out var cell) ? cell : CellData.None; + } + } + + /// + /// 各样式所用的数字格式。判断一格是不是日期要看它的格式，而样式通常只有几十条，逐格去问 + /// 样式表并无必要。 + /// + private sealed class NumberFormats + { + private readonly Dictionary _cache = new(); + private readonly StylesTable _styles; + + public NumberFormats(StylesTable styles) + { + _styles = styles; + } + + public Format Of(string? styleIndex) + { + if (styleIndex == null) return default; + + var index = Number(styleIndex); + if (_cache.TryGetValue(index, out var format)) return format; + + try + { + var style = _styles.GetStyleAt(index); + format = style == null ? default : new Format(style.DataFormat, style.GetDataFormatString()); + } + catch (Exception) + { + // 样式表里没有这一条：按没有格式处理，读到的仍是那个数 + format = default; + } + + _cache[index] = format; + return format; + } + } + + private readonly struct Format + { + public Format(short index, string? text) + { + Index = index; + Text = text; + } + + public short Index { get; } + + public string? Text { get; } + } + + private readonly struct SheetInfo + { + public SheetInfo(string title, string relationshipId) + { + Title = title; + RelationshipId = relationshipId; + } + + public string Title { get; } + + public string RelationshipId { get; } + } + + private readonly struct WorkbookInfo + { + public WorkbookInfo(List sheets, bool date1904) + { + Sheets = sheets; + Date1904 = date1904; + } + + public List Sheets { get; } + + public bool Date1904 { get; } + } +} diff --git a/README.md b/README.md index b9d6ed4..8782876 100644 --- a/README.md +++ b/README.md @@ -61,12 +61,18 @@ excel2obj generate-model orders.xlsx --class Order # 由表头生成 - [x] 样式表（CSS 风格）：背景色、边框、隔行底色、表头样式 ✅ **v2.7.0 新增** - [x] 合并单元格：按列合并连续相同值、指定区域 ✅ **v2.9.1 新增** - [x] 大文件流式导出：内存不随行数增长 ✅ **v2.10.0 新增** - 查看 [docs/versions/v2.10.0.md](docs/versions/v2.10.0.md) +- [x] 大文件流式导入：逐行读出，不把整个工作簿建进内存 ✅ **v2.11.0 新增** - 查看 [docs/versions/v2.11.0.md](docs/versions/v2.11.0.md) - [x] 支持 Excel 日期/日期时间/时间格式 ✅ **v2.0.4 新增**，导出为真正的日期单元格 ✅ **v2.4.0 新增** - 查看 [DateTimeFormats.md](DateTimeFormats.md) - [x] 公式列引用同一工作簿的其他 sheet ✅ **v2.1.0 新增** - 查看 [ExcelFunctions.md](ExcelFunctions.md) - [x] 公式内置函数库 ✅ **v2.3.0 新增** - 334 个 Excel 函数，10 个类别 - 查看 [ExcelFunctions.md](ExcelFunctions.md) ### 发布说明 +* **2026.09.16** - v2.11.0 +- [x] ✨ **新增:** 流式导入：`ExcelHelper.ExcelStreamToObject(stream)` 逐行读出 `.xlsx` 的一张工作表，不把整个工作簿建进内存。实测二十万行五列：整份读入峰值 1265 MB / 8.4 秒，流式导入 194 MB / 2.9 秒，两者读出的数据逐字段一致。取到的序列是惰性的，`Take` 一类的操作真的能少读。三处不同：公式格读的是文件中存着的上一次计算结果而非当场求值（本库导出的文件里公式没有这个结果，读作空白，与其他空白格一样）、只有 `.xlsx` 能逐行读出（`.xls` 照旧整份读入）、工作表在一开始就定位 - 查看 [docs/versions/v2.11.0.md](docs/versions/v2.11.0.md) +- [x] 🔧 导入的类型转换不再依赖 NPOI 的单元格：整份读入与逐行读出共用同一套转换，两条路的行为不会各自漂移。字典形式的导入随之改为惰性给出；标题行中同名的标题以最左一列为准，非文本的标题不再中断导入 + + * **2026.09.16** - v2.10.0 - [x] ✨ **新增:** 流式导出：`ExcelHelper.ObjectToExcelStream(data, stream, options => ...)` 逐行取数据并写出，内存中只保留 `StreamingRowWindow` 指定的若干行（默认 100），占用不再随行数增长（省下的是内存而非等待时间：写过的行落到临时文件，最终的包在数据取完后一次写入调用方的流）。实测二十万行五列：内存导出峰值 710 MB / 6.4 秒，流式导出 89 MB / 3.7 秒。冻结、筛选、下拉、条件格式、样式表、公式列与合并单元格在流式导出下同样生效；仅 `.xlsx` 能够流式写入，`.xls` 的格式决定了必须先在内存中建好。流式写入基于 NPOI 的 SXSSF，后者刷行时要用 SkiaSharp 测量字符宽度，而 NPOI 把该依赖标为不随包传递，故应用需自行引用 `SkiaSharp`（Linux 上另需 `SkiaSharp.NativeAssets.Linux.NoDependencies`），缺失时会在写出任何内容之前报错说明 - 查看 [docs/versions/v2.10.0.md](docs/versions/v2.10.0.md) - [x] 🔧 自动列宽与按值合并改在写入过程中算出，不再回看数据：流式导出得以成立，内存导出亦少走一遍数据。开启 `AutoColumnWidth` 时不再把数据源遍历两遍，只能遍历一次的序列因而也可用 diff --git a/README_EN.md b/README_EN.md index ebfc8ff..dfc65af 100644 --- a/README_EN.md +++ b/README_EN.md @@ -59,12 +59,18 @@ See [Chsword.Excel2Object.Cli/README.md](Chsword.Excel2Object.Cli/README.md). - [x] A CSS-flavoured stylesheet: backgrounds, borders, striped rows, header styles ✅ **New in v2.7.0** - [x] Merged cells: consecutive equal values in a column, or a range of your own ✅ **New in v2.9.1** - [x] Streaming export for large files: a memory footprint that does not grow with the row count ✅ **New in v2.10.0** - See [docs/versions/v2.10.0.md](docs/versions/v2.10.0.md) +- [x] Streaming import for large files: read row by row instead of building the whole workbook ✅ **New in v2.11.0** - See [docs/versions/v2.11.0.md](docs/versions/v2.11.0.md) - [x] Support date/datetime/time formats in Excel ✅ **New in v2.0.4**, exported as real date cells ✅ **New in v2.4.0** - See [DateTimeFormats.md](DateTimeFormats.md) - [x] Formula columns referencing other sheets of the same workbook ✅ **New in v2.1.0** - See [ExcelFunctions.md](ExcelFunctions.md) - [x] Built-in formula function library ✅ **New in v2.3.0** - 334 Excel functions in 10 categories - See [ExcelFunctions.md](ExcelFunctions.md) ### Release Notes +* **2026.09.16** - v2.11.0 +- [x] ✨ **NEW:** Streaming import: `ExcelHelper.ExcelStreamToObject(stream)` reads one sheet of an `.xlsx` row by row instead of building the whole workbook in memory. Measured on 200,000 rows of five columns: 1265 MB / 8.4 s read whole against 194 MB / 2.9 s streamed, field for field the same data. The sequence is lazy, so `Take` and friends really do read less. Three differences: a formula cell reads the result stored in the file rather than being evaluated (files this library writes carry no such result, so those cells read as blank, like any other blank cell), only `.xlsx` can be read row by row (`.xls` is still read whole), and the sheet is located up front - See [docs/versions/v2.11.0.md](docs/versions/v2.11.0.md) +- [x] 🔧 Import conversion no longer depends on NPOI's cell objects: reading whole and reading row by row share one conversion, so the two cannot drift apart. The dictionary form of the import became lazy as a result; duplicate header titles now resolve to the leftmost column, and a non-text header no longer aborts the import + + * **2026.09.16** - v2.10.0 - [x] ✨ **NEW:** Streaming export: `ExcelHelper.ObjectToExcelStream(data, stream, options => ...)` reads the data row by row and writes each row out, keeping only `StreamingRowWindow` rows in memory (100 by default), so the footprint no longer grows with the row count. What it saves is memory rather than latency: rows written go to a temp file, and the final package reaches the caller's stream in one go once the data has been read. Measured on 200,000 rows of five columns: 710 MB / 6.4 s in memory against 89 MB / 3.7 s streamed. Frozen headers, filters, dropdowns, conditional formats, the stylesheet, formula columns and merged cells all work the same way; only `.xlsx` can be streamed, as the `.xls` format has to be assembled in memory. Streaming goes through NPOI's SXSSF, which measures character widths with SkiaSharp when it flushes rows, and NPOI marks that dependency as not flowing to consumers - so the application needs its own `SkiaSharp` reference (plus `SkiaSharp.NativeAssets.Linux.NoDependencies` on Linux); when it is missing the export says so before writing anything - See [docs/versions/v2.10.0.md](docs/versions/v2.10.0.md) - [x] 🔧 Auto column widths and merge-by-value are now worked out as the rows are written rather than by reading the data back, which is what makes streaming possible - and costs the in-memory export one pass less. `AutoColumnWidth` no longer enumerates the source twice, so a sequence that can only be enumerated once now works diff --git a/docs/README.md b/docs/README.md index 7d82de3..0ed0cf1 100644 --- a/docs/README.md +++ b/docs/README.md @@ -6,6 +6,7 @@ | 版本 | 发布日期 | 主要内容 | | --- | --- | --- | +| [v2.11.0](versions/v2.11.0.md) | 2026-09-16 | 流式导入：逐行读出，不把整个工作簿建进内存 | | [v2.10.0](versions/v2.10.0.md) | 2026-09-16 | 流式导出：边取边写，内存不随行数增长 | | [v2.9.1](versions/v2.9.1.md) | 2026-09-15 | 合并单元格：按列合并连续相同值、指定区域；空引用检查改为强制 | | [v2.8.1](versions/v2.8.1.md) | 2026-09-13 | 修复 .NET Framework 上样式互相覆盖、导入数值丢精度 | diff --git a/docs/versions/v2.11.0.md b/docs/versions/v2.11.0.md new file mode 100644 index 0000000..00cc33c --- /dev/null +++ b/docs/versions/v2.11.0.md @@ -0,0 +1,69 @@ +# v2.11.0 + +发布日期：2026-09-16 + +## 概述 + +本版本新增流式导入：逐行读出 `.xlsx` 的一张工作表，不把整个工作簿建进内存。上一版让导出不再受行数牵制，这一版补上读的一侧。为此，导入的类型转换与 NPOI 的单元格解耦，整份读入与逐行读出共用同一套转换。 + +## 变更明细 + +### 1. 新增：流式导入 + +此前的导入一律先把整个工作簿读进内存：`WorkbookFactory.Create` 会为每一格建一个对象，二十万行的文件因此要占用一吉字节以上。 + +新增的 `ExcelStreamToObject` 逐行读出： + +```csharp +using var file = File.OpenRead("orders.xlsx"); +foreach (var order in ExcelHelper.ExcelStreamToObject(file)) + Handle(order); +``` + +`ExcelImporter` 上有同名的实例方法，选项与既有入口一致（`SheetTitle`、`TitleSkipLine`、`OnCellError`）： + +```csharp +var errors = new List(); +using var file = File.OpenRead("orders.xlsx"); +var orders = new ExcelImporter().ExcelStreamToObject(file, options => +{ + options.SheetTitle = "本月"; + options.OnCellError = errors.Add; +}); + +foreach (var order in orders) Handle(order); // 取一行、转一行 +``` + +取到的序列是惰性的：取一行、转一行，读过的行随即可以回收。`Take`、`Where` 一类的操作因此真的能少读——取够了便不再往下读。 + +本机实测（五列，含文本、金额与日期，net8.0）： + +| 行数 | 整份读入 | 流式导入 | +| --- | --- | --- | +| 5 万 | 393 MB / 3.1 秒 | 103 MB / 1.1 秒 | +| 20 万 | 1265 MB / 8.4 秒 | 194 MB / 2.9 秒 | + +两种方式读出的数据逐字段一致（实测中以求和校验）。流式导入快出一截，是因为不必先为每一格建一个对象。 + +需要说明的是，流式导入的内存并非常数：文件中的共享字符串表会整个留在内存里（Excel 把重复的文本收在那里），调用方自己攒下的对象也照样占内存。省掉的是「每一格一个对象」这一份。 + +### 2. 与整份读入的三处不同 + +- **公式格读的是文件中存着的上一次计算结果**，而非当场求值。Excel 存盘时会写下这个结果，故 Excel 保存过的文件照常读出；本库导出的文件里公式没有这个结果（要等 Excel 打开时才算出），此时该格读作空白——与其他空白格一样：写进可空列得到 `null`，写进非可空的数值列则照旧以转换失败中止，这一点两条路相同。整份读入的那条路仍会当场求值，行为不变。 +- **只有 `.xlsx` 能够逐行读出。** 传入 `.xls` 时会照旧整份读入内存，结果一致——该格式的数据并非顺序存放，无从逐行取出。 +- **工作表要在一开始就定位。** 表名写错时当场抛出 `Excel2ObjectException`，而不是等到取第一行。 + +传入的流由本方法读取，遍历结束或中途放弃时关闭其上的工作簿。与 `File.ReadLines` 一样，取到的序列须被遍历——只取不用则文件不会被关上。 + +### 3. 内部：导入的类型转换不再依赖 NPOI 的单元格 + +导入的类型转换原本直接写在 `ICell` 之上，而逐行读出拿到的是原始的 XML，并没有单元格对象。现把两者都归到内部的 `IImportRow` 与 `CellData` 上：一格读到的东西与它从哪里来无关，转换那一套因而只有一份，两条路的行为不会各自漂移。 + +## 行为变更 + +以下几处对既有入口同样生效，均为读取行为的收敛，不影响正常的导入结果： + +- **字典形式的导入改为惰性给出。** `ExcelToObject>` 此前先把全部行收成一个 `List` 再返回，现与模型形式一致，逐行给出。 +- **标题行中同名的标题以最左边那一列为准。** 此前字典形式遇到重复标题会抛出 `ArgumentException`（`ToDictionary` 的行为），现与模型形式的做法一致。 +- **标题行中的非文本单元格不再中断导入。** 此前取标题时直接取字符串值，遇到数值标题会抛出异常；现读作它显示的文本，对不上的列照旧忽略。 +- **日期序列号超出 Excel 日历时上报的异常改为本库抛出的 `ArgumentException`**，并写明该序列号；此前上报的是 NPOI 抛出的异常。该单元格仍取 `null`，导入继续。