diff --git a/listenarr.application/Search/Indexers/MyAnonamouse/MyAnonamousePublishDateParser.cs b/listenarr.application/Search/Indexers/MyAnonamouse/MyAnonamousePublishDateParser.cs index 3b6365844..69efaa0bc 100644 --- a/listenarr.application/Search/Indexers/MyAnonamouse/MyAnonamousePublishDateParser.cs +++ b/listenarr.application/Search/Indexers/MyAnonamouse/MyAnonamousePublishDateParser.cs @@ -8,6 +8,7 @@ * (at your option) any later version. */ +using System.Globalization; using System.Text.Json; using Microsoft.Extensions.Logging; @@ -65,16 +66,20 @@ internal static class MyAnonamousePublishDateParser double? hours = null; double? minutes = null; + // These three ages are machine format like every other number in this + // indexer's JSON, so they are pinned for uniformity. No damage was + // measured here: the values seen in practice are whole numbers, which + // read the same under every culture. // Prefer explicit ageHours/ageMinutes if present if (item.TryGetProperty("ageHours", out var ah) && (ah.ValueKind == JsonValueKind.Number || ah.ValueKind == JsonValueKind.String)) { if (ah.ValueKind == JsonValueKind.Number) hours = ah.GetDouble(); - else if (double.TryParse(ah.GetString(), out var htmp)) hours = htmp; + else if (double.TryParse(ah.GetString(), NumberStyles.Float, CultureInfo.InvariantCulture, out var htmp)) hours = htmp; } if (item.TryGetProperty("ageMinutes", out var am) && (am.ValueKind == JsonValueKind.Number || am.ValueKind == JsonValueKind.String)) { if (am.ValueKind == JsonValueKind.Number) minutes = am.GetDouble(); - else if (double.TryParse(am.GetString(), out var mtmp)) minutes = mtmp; + else if (double.TryParse(am.GetString(), NumberStyles.Float, CultureInfo.InvariantCulture, out var mtmp)) minutes = mtmp; } // Fallback to 'age' if present. Heuristic: small values (<=48) likely hours; otherwise treat as days. @@ -86,7 +91,8 @@ internal static class MyAnonamousePublishDateParser if (a <= 48) hours = a; else days = (int)Math.Floor(a); } - else if (ageElem.ValueKind == JsonValueKind.String && double.TryParse(ageElem.GetString(), out var adtmp)) + else if (ageElem.ValueKind == JsonValueKind.String + && double.TryParse(ageElem.GetString(), NumberStyles.Float, CultureInfo.InvariantCulture, out var adtmp)) { var a = adtmp; if (a <= 48) hours = a; diff --git a/listenarr.application/Search/Indexers/MyAnonamouse/MyAnonamouseSizeParser.cs b/listenarr.application/Search/Indexers/MyAnonamouse/MyAnonamouseSizeParser.cs index bc85fb0fb..fe7a20f68 100644 --- a/listenarr.application/Search/Indexers/MyAnonamouse/MyAnonamouseSizeParser.cs +++ b/listenarr.application/Search/Indexers/MyAnonamouse/MyAnonamouseSizeParser.cs @@ -33,7 +33,7 @@ public static long ExtractFromDescription(string? description, ILogger logger) var sizeValue = match.Groups[1].Value.Replace(",", ""); var unit = match.Groups[2].Value.ToUpper(); - if (double.TryParse(sizeValue, out var value)) + if (double.TryParse(sizeValue, NumberStyles.Float, CultureInfo.InvariantCulture, out var value)) { var result = ParseDecimalUnit(value, unit, binary: true); logger.LogDebug("Extracted size from MyAnonamouse description formatted: {Value} {Unit} = {Result} bytes", value, unit, result); @@ -46,7 +46,7 @@ public static long ExtractFromDescription(string? description, ILogger logger) { var sizeValue = match.Groups[1].Value.Replace(",", ""); var unit = match.Groups[2].Value.ToUpper(); - if (double.TryParse(sizeValue, out var value)) + if (double.TryParse(sizeValue, NumberStyles.Float, CultureInfo.InvariantCulture, out var value)) { var result = ParseDecimalUnit(value, unit, binary: true); logger.LogDebug("Extracted size from MyAnonamouse description (no bytes): {Value} {Unit} = {Result} bytes", value, unit, result); diff --git a/listenarr.application/Search/Indexers/Torznab/TorznabResponseParser.SizeParsing.cs b/listenarr.application/Search/Indexers/Torznab/TorznabResponseParser.SizeParsing.cs index 669f512af..18bc98d29 100644 --- a/listenarr.application/Search/Indexers/Torznab/TorznabResponseParser.SizeParsing.cs +++ b/listenarr.application/Search/Indexers/Torznab/TorznabResponseParser.SizeParsing.cs @@ -8,7 +8,15 @@ namespace Listenarr.Application.Search.Indexers.Torznab { internal sealed partial class TorznabResponseParser { - private long ParseSizeString(string sizeStr) + /// + /// Reads a size out of a Torznab attribute value, either as plain bytes or as a + /// formatted string such as "1.5 GiB". + /// + /// + /// Internal rather than private so the culture behaviour of the unit match can be + /// asserted against the real method rather than a copy of it. + /// + internal long ParseSizeString(string sizeStr) { if (string.IsNullOrEmpty(sizeStr)) return 0; @@ -26,7 +34,10 @@ private long ParseSizeString(string sizeStr) if (match.Success && double.TryParse(match.Groups[1].Value, System.Globalization.NumberStyles.Float, System.Globalization.CultureInfo.InvariantCulture, out var value)) { - var unit = match.Groups[2].Value.ToUpper(); + // ToUpperInvariant, not ToUpper: under tr-TR the 'i' of "GiB" uppercases to + // 'I' with a dot (U+0130), no binary arm matches, and the size falls through + // to the (long)value default, which is the raw mantissa in bytes. + var unit = match.Groups[2].Value.ToUpperInvariant(); return unit switch { "B" => (long)value, diff --git a/listenarr.infrastructure/DownloadClients/Sabnzbd/SabnzbdResponseMapper.cs b/listenarr.infrastructure/DownloadClients/Sabnzbd/SabnzbdResponseMapper.cs index bb5f289c2..5d26822c8 100644 --- a/listenarr.infrastructure/DownloadClients/Sabnzbd/SabnzbdResponseMapper.cs +++ b/listenarr.infrastructure/DownloadClients/Sabnzbd/SabnzbdResponseMapper.cs @@ -198,7 +198,10 @@ public static double ParseSpeed(string speedStr) var parts = speedStr.Split(' ', StringSplitOptions.RemoveEmptyEntries); if (parts.Length == 0) return 0; - if (!double.TryParse(parts[0], out var value)) return 0; + // NumberStyles.Float rather than Any: Any carries AllowThousands, AllowCurrencySymbol + // and AllowParentheses, so under the invariant culture "1,5" reads as 15 and "(1.5)" + // as -1.5. SABnzbd emits a plain decimal, and a value that is not one should fail. + if (!double.TryParse(parts[0], NumberStyles.Float, CultureInfo.InvariantCulture, out var value)) return 0; if (parts.Length > 1) { @@ -317,7 +320,9 @@ private static double GetDouble(JsonElement element, string propertyName) if (property.ValueKind == JsonValueKind.Number) return property.GetDouble(); - if (property.ValueKind == JsonValueKind.String && double.TryParse(property.GetString() ?? "0", out var value)) + // Float rather than Any, for the reason given on ParseSpeed above. + if (property.ValueKind == JsonValueKind.String + && double.TryParse(property.GetString() ?? "0", NumberStyles.Float, CultureInfo.InvariantCulture, out var value)) return value; return 0; diff --git a/listenarr.infrastructure/Ffmpeg/Metadata/FfprobeMetadataMapper.cs b/listenarr.infrastructure/Ffmpeg/Metadata/FfprobeMetadataMapper.cs index ca75eb1d1..d702c3f37 100644 --- a/listenarr.infrastructure/Ffmpeg/Metadata/FfprobeMetadataMapper.cs +++ b/listenarr.infrastructure/Ffmpeg/Metadata/FfprobeMetadataMapper.cs @@ -15,6 +15,7 @@ * You should have received a copy of the GNU Affero General Public License * along with this program. If not, see . */ +using System.Globalization; using System.Text.Json; namespace Listenarr.Infrastructure.Ffmpeg.Metadata @@ -47,7 +48,7 @@ private static void ApplyFormat(AudioMetadata metadata, JsonElement fmt, string { if (fmt.TryGetProperty("duration", out var durEl) && durEl.ValueKind == JsonValueKind.String - && double.TryParse(durEl.GetString(), out var dur)) + && double.TryParse(durEl.GetString(), NumberStyles.Float, CultureInfo.InvariantCulture, out var dur)) { metadata.Duration = TimeSpan.FromSeconds(dur); } diff --git a/listenarr.infrastructure/Search/Providers/Torznab/TorznabNewznabValueParser.cs b/listenarr.infrastructure/Search/Providers/Torznab/TorznabNewznabValueParser.cs index f36f91e10..c07865a6c 100644 --- a/listenarr.infrastructure/Search/Providers/Torznab/TorznabNewznabValueParser.cs +++ b/listenarr.infrastructure/Search/Providers/Torznab/TorznabNewznabValueParser.cs @@ -16,6 +16,7 @@ * along with this program. If not, see . */ +using System.Globalization; using System.Text.RegularExpressions; namespace Listenarr.Infrastructure.Search.Providers.Torznab; @@ -37,10 +38,12 @@ public static long ParseSize(string sizeStr) if (!match.Success) return 0; - if (!double.TryParse(match.Groups[1].Value, out var size)) + if (!double.TryParse(match.Groups[1].Value, NumberStyles.Float, CultureInfo.InvariantCulture, out var size)) return 0; - var unit = match.Groups[2].Value.ToUpper(); + // ToUpperInvariant, not ToUpper: under tr-TR the 'i' of "GiB" uppercases to 'I' with a + // dot (U+0130), no binary arm matches, and the release is recorded as zero bytes. + var unit = match.Groups[2].Value.ToUpperInvariant(); return unit switch { "TIB" => (long)(size * 1024 * 1024 * 1024 * 1024), diff --git a/tests/Features/Common/MachineFormatCultureParsingTests.cs b/tests/Features/Common/MachineFormatCultureParsingTests.cs new file mode 100644 index 000000000..84d23912d --- /dev/null +++ b/tests/Features/Common/MachineFormatCultureParsingTests.cs @@ -0,0 +1,355 @@ +/* + * Listenarr - Audiobook Management System + * Copyright (C) 2024-2026 Listenarr Contributors + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as published + * by the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ +using System.Globalization; +using System.Text.Json; +using Listenarr.Application.Search.Indexers.Torznab; +using Listenarr.Infrastructure.Ffmpeg.Metadata; +using Listenarr.Tests.Common; + +namespace Listenarr.Tests.Features.Common +{ + /// + /// ffprobe, MyAnonamouse, SABnzbd and Torznab all emit numbers in machine format: '.' is the + /// decimal separator, whatever locale the emitting process runs under. Parsing those strings + /// with the ambient culture reads them as a different number, or fails to read them at all. + /// + /// A default container runs under the invariant culture and is unaffected. It takes a real + /// culture reaching the process, which happens when LANG or LC_ALL is set, and on a desktop + /// install that inherits the operating system's locale: + /// + /// de-DE: '.' is the GROUP separator, so "43200.250" parsed as 43200250. + /// fr-FR: ',' is the decimal separator, so "43200.250" did not parse at all. + /// + [Trait("Name", "MachineFormatCultureParsingTests")] + [Trait("Category", "Unit")] + public class MachineFormatCultureParsingTests : BaseTests + { + // '' is the invariant culture: what a container with no LANG gets. + // de-DE treats '.' as the group separator; fr-FR treats ',' as the decimal separator. + // tr-TR is here for the case folding rather than the separator: 'i' uppercases to 'I' + // with a dot (U+0130), so "GiB" under ToUpper() matches no binary unit arm. + public static TheoryData ServerCultures => new() { "", "en-US", "de-DE", "fr-FR", "tr-TR" }; + + private static void InCulture(string culture, Action body) + { + var originalCulture = CultureInfo.CurrentCulture; + try + { + CultureInfo.CurrentCulture = new CultureInfo(culture); + body(); + } + finally + { + CultureInfo.CurrentCulture = originalCulture; + } + } + + private static AudioMetadata MapFfprobeDuration(string duration) + { + using var document = JsonDocument.Parse( + "{\"format\":{\"duration\":\"" + duration + "\"}}"); + return FfprobeMetadataMapper.Map(document.RootElement, "book.m4b"); + } + + // FfprobeMetadataMapper.cs:48-52. ffprobe emits format.duration as a string and always + // uses '.'. Under de-DE a twelve hour book was recorded as 43,200,250 seconds. + [Theory] + [MemberData(nameof(ServerCultures))] + public void FfprobeDuration_ParsesTheSameUnderEveryServerCulture(string culture) + { + InCulture(culture, () => + { + Assert.Equal(TimeSpan.FromSeconds(43200.250), MapFfprobeDuration("43200.250").Duration); + Assert.Equal(TimeSpan.FromSeconds(3600.5), MapFfprobeDuration("3600.5").Duration); + }); + } + + // MyAnonamouseSizeParser.cs:49. The description carries a formatted size and no byte + // count, so the size is whatever this parse makes of "1.5". + [Theory] + [MemberData(nameof(ServerCultures))] + public void MyAnonamouseFormattedSize_ParsesTheSameUnderEveryServerCulture(string culture) + { + InCulture(culture, () => + { + var size = MyAnonamouseSizeParser.ExtractFromDescription( + "Total Size: 1.5 GB", + Mock.Of()); + + Assert.Equal(1610612736L, size); + }); + } + + // MyAnonamouseSizeParser.cs:36. Same parse on the branch that has a byte count, reached + // when the byte count itself is not a usable long. Harder to hit than the branch above, + // and pinned for the same reason. + [Theory] + [MemberData(nameof(ServerCultures))] + public void MyAnonamouseSizeBesideUnusableByteCount_ParsesTheSameUnderEveryServerCulture(string culture) + { + InCulture(culture, () => + { + var size = MyAnonamouseSizeParser.ExtractFromDescription( + "Total Size: 1.5 GB (99999999999999999999999 bytes)", + Mock.Of()); + + Assert.Equal(1610612736L, size); + }); + } + + // TorznabNewznabValueParser.cs:40. The regex pulls "1.5" out of "1.5 GB" and the bare + // parse then decides whether the release is 1.5 GB, 15 GB, or zero. + [Theory] + [MemberData(nameof(ServerCultures))] + public void TorznabSize_ParsesTheSameUnderEveryServerCulture(string culture) + { + InCulture(culture, () => + { + Assert.Equal(1610612736L, TorznabNewznabValueParser.ParseSize("1.5 GB")); + Assert.Equal(1610612736L, TorznabNewznabValueParser.ParseSize("1.5 GiB")); + }); + } + + // SabnzbdResponseMapper.cs:201. SABnzbd reports the queue speed as a formatted string. + [Theory] + [MemberData(nameof(ServerCultures))] + public void SabnzbdSpeed_ParsesTheSameUnderEveryServerCulture(string culture) + { + InCulture(culture, () => + { + Assert.Equal(1.5 * 1024 * 1024, SabnzbdResponseMapper.ParseSpeed("1.5 M")); + }); + } + + // SabnzbdResponseMapper.cs:320. GetDouble reads mb/mbleft/percentage out of a queue slot. + // ParseJsonDouble at :224 already pins the same read; this is its unpinned twin. + [Theory] + [MemberData(nameof(ServerCultures))] + public void SabnzbdQueueSlotSize_ParsesTheSameUnderEveryServerCulture(string culture) + { + InCulture(culture, () => + { + using var document = JsonDocument.Parse( + """ + { + "nzo_id": "SABnzbd_nzo_test", + "filename": "Some Release", + "status": "Downloading", + "cat": "audiobooks", + "mb": "1.5", + "mbleft": "0.5", + "percentage": "66.6" + } + """); + + var item = SabnzbdResponseMapper.MapQueueSlotToQueueItem( + new DownloadClientConfiguration { Name = "sab", Type = "sabnzbd" }, + document.RootElement, + configuredCategory: string.Empty, + speed: 0); + + Assert.NotNull(item); + Assert.Equal((long)(1.5 * 1024 * 1024), item!.Size); + Assert.Equal((long)(1.0 * 1024 * 1024), item.Downloaded); + Assert.Equal(66.6, item.Progress); + }); + } + + // SabnzbdResponseMapper.cs:224, already pinned before this change. Asserted so it stays + // pinned, and so the expected behaviour of the sites above is stated against a site that + // was already right. + [Theory] + [MemberData(nameof(ServerCultures))] + public void SabnzbdJsonDouble_StaysPinnedUnderEveryServerCulture(string culture) + { + InCulture(culture, () => + { + using var document = JsonDocument.Parse("\"1.5\""); + Assert.Equal(1.5, SabnzbdResponseMapper.ParseJsonDouble(document.RootElement)); + }); + } + + // -------------------------------------------------------------------------- + // Case folding. Pinning the number parse and leaving the unit match to the + // ambient culture fixes half of the same function. + // -------------------------------------------------------------------------- + + // TorznabNewznabValueParser.cs:46. Under tr-TR the 'i' of "GiB" uppercases to U+0130, + // no binary arm of the switch matches, and the release is recorded as zero bytes: + // a larger error than any of the six number parses this PR pins. + [Theory] + [MemberData(nameof(ServerCultures))] + public void TorznabBinaryUnits_MatchUnderEveryServerCulture(string culture) + { + InCulture(culture, () => + { + Assert.Equal(1610612736L, TorznabNewznabValueParser.ParseSize("1.5 GiB")); + Assert.Equal(1572864L, TorznabNewznabValueParser.ParseSize("1.5 MiB")); + Assert.Equal(1536L, TorznabNewznabValueParser.ParseSize("1.5 KiB")); + Assert.Equal(1649267441664L, TorznabNewznabValueParser.ParseSize("1.5 TiB")); + + // Not zero: the failure mode being closed is the unmatched switch arm. + Assert.NotEqual(0L, TorznabNewznabValueParser.ParseSize("1.5 GiB")); + }); + } + + // TorznabResponseParser.SizeParsing.cs:32, the other Torznab size parser. Its number + // parse was already invariant on canary, so only the unit match was exposed; under + // tr-TR "1.5 GiB" fell through the switch to the (long)value default, i.e. 1 byte. + [Theory] + [MemberData(nameof(ServerCultures))] + public void TorznabResponseParserBinaryUnits_MatchUnderEveryServerCulture(string culture) + { + InCulture(culture, () => + { + using var httpClient = new HttpClient(); + var parser = new TorznabResponseParser(httpClient, Mock.Of()); + + Assert.Equal(1610612736L, parser.ParseSizeString("1.5 GiB")); + Assert.Equal(1572864L, parser.ParseSizeString("1.5 MiB")); + + // The decimal units were never at risk and are asserted as the control. + Assert.Equal(1500000000L, parser.ParseSizeString("1.5 GB")); + }); + } + + // -------------------------------------------------------------------------- + // NumberStyles.Float across the batch: a value that is not machine format must + // fail the parse rather than come back as a different number. + // -------------------------------------------------------------------------- + + // SabnzbdResponseMapper.cs:204 and :325 were pinned with NumberStyles.Any, which + // carries AllowThousands, AllowCurrencySymbol and AllowParentheses: under the + // invariant culture "1,5" reads as 15 and "(1.5)" as -1.5. Float rejects both. + [Theory] + [MemberData(nameof(ServerCultures))] + public void SabnzbdSpeed_RejectsAValueThatIsNotMachineFormat(string culture) + { + InCulture(culture, () => + { + Assert.Equal(0, SabnzbdResponseMapper.ParseSpeed("1,5 M")); + Assert.Equal(0, SabnzbdResponseMapper.ParseSpeed("(1.5) M")); + + // Control: the machine-format value still parses. + Assert.Equal(1.5 * 1024 * 1024, SabnzbdResponseMapper.ParseSpeed("1.5 M")); + }); + } + + [Theory] + [MemberData(nameof(ServerCultures))] + public void SabnzbdQueueSlotSize_RejectsAValueThatIsNotMachineFormat(string culture) + { + InCulture(culture, () => + { + using var document = JsonDocument.Parse( + """ + { + "nzo_id": "SABnzbd_nzo_test", + "filename": "Some Release", + "status": "Downloading", + "cat": "audiobooks", + "mb": "1,5", + "mbleft": "0.5", + "percentage": "66.6" + } + """); + + var item = SabnzbdResponseMapper.MapQueueSlotToQueueItem( + new DownloadClientConfiguration { Name = "sab", Type = "sabnzbd" }, + document.RootElement, + configuredCategory: string.Empty, + speed: 0); + + Assert.NotNull(item); + + // Zero, not 15 MB. A size that cannot be read is better than one read as ten + // times itself, which is what NumberStyles.Any returned here. + Assert.Equal(0L, item!.Size); + }); + } + + // -------------------------------------------------------------------------- + // MyAnonamousePublishDateParser.cs:77, :82 and :95. No damage was measured at + // these three: the ages seen in practice are whole numbers, which read the same + // under every culture. They are pinned for uniformity, and asserted so the + // uniformity is a property of the build rather than of the reviewer's memory. + // -------------------------------------------------------------------------- + + private static DateTime? ParsePublishDate(string json) + { + using var document = JsonDocument.Parse(json); + return MyAnonamousePublishDateParser.Parse( + document.RootElement, + "Some Release", + Mock.Of()); + } + + [Theory] + [MemberData(nameof(ServerCultures))] + public void MyAnonamouseAgeHours_ParsesTheSameUnderEveryServerCulture(string culture) + { + InCulture(culture, () => + { + var withFraction = ParsePublishDate("{\"ageHours\":\"2.5\"}"); + var whole = ParsePublishDate("{\"ageHours\":\"2\"}"); + + Assert.NotNull(withFraction); + Assert.NotNull(whole); + + // Half an hour apart, not twenty five hours or nothing at all. + var gap = whole!.Value - withFraction!.Value; + Assert.InRange(gap.TotalMinutes, 29, 31); + }); + } + + [Theory] + [MemberData(nameof(ServerCultures))] + public void MyAnonamouseAgeMinutes_ParsesTheSameUnderEveryServerCulture(string culture) + { + InCulture(culture, () => + { + var withFraction = ParsePublishDate("{\"ageMinutes\":\"90.5\"}"); + var whole = ParsePublishDate("{\"ageMinutes\":\"90\"}"); + + Assert.NotNull(withFraction); + Assert.NotNull(whole); + + var gap = whole!.Value - withFraction!.Value; + Assert.InRange(gap.TotalSeconds, 29, 31); + }); + } + + [Theory] + [MemberData(nameof(ServerCultures))] + public void MyAnonamouseAge_ParsesTheSameUnderEveryServerCulture(string culture) + { + InCulture(culture, () => + { + // 1.5 is below the 48 hour threshold, so it is read as hours. + var withFraction = ParsePublishDate("{\"age\":\"1.5\"}"); + var whole = ParsePublishDate("{\"age\":\"1\"}"); + + Assert.NotNull(withFraction); + Assert.NotNull(whole); + + var gap = whole!.Value - withFraction!.Value; + Assert.InRange(gap.TotalMinutes, 29, 31); + }); + } + } +}