From b15fe3c985257ea920b79b34405b2ef903fcc30c Mon Sep 17 00:00:00 2001 From: ttu Date: Sat, 25 Jul 2026 23:32:56 +0300 Subject: [PATCH 1/2] fix: keep the precision of high-precision decimals MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Newtonsoft materializes non-integral JSON numbers as double by default, which silently corrupted any value with more significant digits than a double holds. There were two independent loss points: JObject.Parse on every read, and the ExpandoObject round-trip _toJsonFunc uses to apply camelCase naming on write. decimal.MaxValue was worse than lossy — it was written back as 7.922816251426434E+28, after which the item no longer deserialized at all. JsonParser now parses with FloatParseHandling.Decimal on both paths. Decimal covers a much narrower range than double, so a document holding a value outside it is parsed the old way instead: too large throws and is caught, too small would round to zero without any error, so those documents are recognized from the text before parsing. The dynamic API is unchanged. GetItem normalizes a decimal back to double, because dynamic values have always been doubles and a dynamic decimal cannot even be compared to a double literal. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01VjQuyQX3v7BsMX1uURt8zU --- CHANGELOG.md | 1 + .../NumericEdgeCaseTests.cs | 76 +++++++++--- JsonFlatFileDataStore/CommitActionHandler.cs | 2 +- JsonFlatFileDataStore/DataStore.cs | 18 ++- JsonFlatFileDataStore/JsonParser.cs | 112 ++++++++++++++++++ 5 files changed, 188 insertions(+), 21 deletions(-) create mode 100644 JsonFlatFileDataStore/JsonParser.cs diff --git a/CHANGELOG.md b/CHANGELOG.md index f80f0ad..71a3b04 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,7 @@ * FIXED: Retry JSON parse on read to tolerate concurrent partial-file writes * FIXED: A failing commit action no longer hangs other callers in the same batch * FIXED: Collection key in file not matching configured case is now matched case-insensitively, instead of reading empty and duplicating the key on save +* FIXED: High-precision decimals are no longer corrupted on read and write, e.g. decimal.MaxValue previously made the item unreadable ### [2.4.2] - 2023-06-25 * FIXED: Duplicate collection data to JSON on save when configured case was not used with collection name diff --git a/JsonFlatFileDataStore.Test/NumericEdgeCaseTests.cs b/JsonFlatFileDataStore.Test/NumericEdgeCaseTests.cs index 81b6d87..31b7fb2 100644 --- a/JsonFlatFileDataStore.Test/NumericEdgeCaseTests.cs +++ b/JsonFlatFileDataStore.Test/NumericEdgeCaseTests.cs @@ -23,10 +23,6 @@ public async Task Decimal_RoundTrip_PreservesPrecision() var collection = store.GetCollection("decModel"); await collection.InsertOneAsync(new DecimalModel { Id = 1, Price = 19.95m, OptionalAmount = 0.0000001m }); await collection.InsertOneAsync(new DecimalModel { Id = 2, Price = 9999999999.99m, OptionalAmount = null }); - // Note: very large decimals lose precision on the current Newtonsoft path because - // the serializer routes data through ExpandoObject (JSON number → double → back). - // decimal.MaxValue specifically also fails to deserialize. Use a value within - // double precision range to assert lossless round-trip. await collection.InsertOneAsync(new DecimalModel { Id = 3, Price = -0.01m, OptionalAmount = 12345678.99m }); store.Dispose(); @@ -45,27 +41,79 @@ public async Task Decimal_RoundTrip_PreservesPrecision() UTHelpers.Down(path); } + [Theory] + [InlineData(true)] + [InlineData(false)] + public async Task Decimal_BeyondDoublePrecision_RoundTrip(bool useLowerCamelCase) + { + // Values with more significant digits than a double can hold. Both the read path and the + // camelCase write path used to materialize them as double, which corrupted them. + var path = UTHelpers.GetFullFilePath($"DecPrecision_{useLowerCamelCase}_{DateTime.UtcNow.Ticks}"); + var store = new DataStore(path, useLowerCamelCase); + + var collection = store.GetCollection("decModel"); + await collection.InsertOneAsync(new DecimalModel { Id = 1, Price = 123456789012345678.5m }); + await collection.InsertOneAsync(new DecimalModel { Id = 2, Price = 79228162514264337593543950m }); + await collection.InsertOneAsync(new DecimalModel { Id = 3, Price = decimal.MaxValue }); + + store.Dispose(); + + var store2 = new DataStore(path, useLowerCamelCase); + var items = store2.GetCollection("decModel").AsQueryable().OrderBy(e => e.Id).ToList(); + + Assert.Equal(123456789012345678.5m, items[0].Price); + Assert.Equal(79228162514264337593543950m, items[1].Price); + Assert.Equal(decimal.MaxValue, items[2].Price); + + store2.Dispose(); + UTHelpers.Down(path); + } + [Fact] - public async Task Decimal_LargeValue_LosesPrecision_BehaviorPinned() + public async Task Decimal_UpdatedAfterReload_KeepsPrecision() { - // Documented limitation of the current Newtonsoft path: very large decimal values - // are serialized via JObject → ExpandoObject (which uses double internally), - // so significant digits beyond double precision are lost on round trip. - // Migration note: STJ may behave differently here — expected to either round-trip - // exactly or fail explicitly. Update this test once the migration lands. - var path = UTHelpers.GetFullFilePath($"DecLoss_{DateTime.UtcNow.Ticks}"); + // The reload path parses the file again, so precision must survive a write that happens + // after the store has read its own output. + var path = UTHelpers.GetFullFilePath($"DecUpdate_{DateTime.UtcNow.Ticks}"); var store = new DataStore(path); var collection = store.GetCollection("decModel"); - await collection.InsertOneAsync(new DecimalModel { Id = 1, Price = 79228162514264337593543950m }); + await collection.InsertOneAsync(new DecimalModel { Id = 1, Price = 123456789012345678.5m }); + await collection.UpdateOneAsync(1, new { OptionalAmount = 12345678901234567890.5m }); store.Dispose(); var store2 = new DataStore(path); var item = store2.GetCollection("decModel").AsQueryable().First(); - // Precision is lost — assert the value differs from the input. - Assert.NotEqual(79228162514264337593543950m, item.Price); + Assert.Equal(123456789012345678.5m, item.Price); + Assert.Equal(12345678901234567890.5m, item.OptionalAmount); + + store2.Dispose(); + UTHelpers.Down(path); + } + + [Theory] + [InlineData(double.MaxValue)] + [InlineData(double.MinValue)] + [InlineData(double.Epsilon)] + [InlineData(1e-30)] + public async Task Double_OutsideDecimalRange_RoundTrip(double value) + { + // Values outside decimal's range must not be read as one: too large throws, too small + // rounds to zero without any error. Both fall back to double parsing. + var path = UTHelpers.GetFullFilePath($"DoubleRange_{value}_{DateTime.UtcNow.Ticks}"); + var store = new DataStore(path); + + var collection = store.GetCollection("movie"); + await collection.InsertOneAsync(new Movie { Name = "Extreme", Rating = value }); + + store.Dispose(); + + var store2 = new DataStore(path); + var movie = store2.GetCollection("movie").AsQueryable().First(); + + Assert.Equal(value, movie.Rating); store2.Dispose(); UTHelpers.Down(path); diff --git a/JsonFlatFileDataStore/CommitActionHandler.cs b/JsonFlatFileDataStore/CommitActionHandler.cs index d16b8a6..4cd8a21 100644 --- a/JsonFlatFileDataStore/CommitActionHandler.cs +++ b/JsonFlatFileDataStore/CommitActionHandler.cs @@ -47,7 +47,7 @@ internal static void HandleStoreCommitActions(CancellationToken token, BlockingC { try { - var (actionSuccess, updatedJson) = action.HandleAction(JObject.Parse(jsonText)); + var (actionSuccess, updatedJson) = action.HandleAction(JsonParser.Parse(jsonText)); callbacks.Enqueue((action, actionSuccess)); diff --git a/JsonFlatFileDataStore/DataStore.cs b/JsonFlatFileDataStore/DataStore.cs index ad68f27..865693e 100644 --- a/JsonFlatFileDataStore/DataStore.cs +++ b/JsonFlatFileDataStore/DataStore.cs @@ -45,7 +45,7 @@ public DataStore(string path, bool useLowerCamelCase = true, string keyProperty { // Serializing JObject ignores SerializerSettings, so we have to first deserialize to ExpandoObject and then serialize // http://json.codeplex.com/workitem/23853 - var jObject = JsonConvert.DeserializeObject(data.ToString()); + var jObject = JsonParser.ToExpandoObject(data.ToString()); return JsonConvert.SerializeObject(jObject, usedFormatting, _serializerSettings); }) : (s => s.ToString(usedFormatting)); @@ -82,7 +82,7 @@ public DataStore(string path, bool useLowerCamelCase = true, string keyProperty { lock (_jsonData) { - _jsonData = JObject.Parse(jsonText); + _jsonData = JsonParser.Parse(jsonText); } return FileAccess.WriteJsonToFile(_filePath, _encryptJson, jsonText); @@ -110,7 +110,7 @@ public void UpdateAll(string jsonData) { lock (_jsonData) { - _jsonData = JObject.Parse(jsonData); + _jsonData = JsonParser.Parse(jsonData); } FileAccess.WriteJsonToFile(_filePath, _encryptJson, jsonData); @@ -438,16 +438,22 @@ private dynamic SingleDynamicItemReadConverter(JToken e) return JsonConvert.DeserializeObject(content, _converter) as dynamic; case var arrayToken when e.Type == JTokenType.Array: - return e.ToObject>(); + return e.ToObject>().Select(NormalizeFloatValue).ToList(); case JValue jv when e is JValue: - return jv.Value; + return NormalizeFloatValue(jv.Value); default: return e.ToObject(); } } + // The document keeps non-integral numbers as decimal so their precision survives a round trip, + // but the dynamic API has always handed them out as double — and a dynamic decimal can not even + // be compared to a double literal. Objects are unaffected: they are read through + // ExpandoObject, which materializes them as double regardless. + private static object NormalizeFloatValue(object value) => value is decimal d ? (double)d : value; + private string GetJsonTextFromFile() => FileAccess.ReadJsonFromFile(_filePath, _encryptJson, _decryptJson); private JObject GetJsonObjectFromFile() @@ -463,7 +469,7 @@ private JObject GetJsonObjectFromFile() var jsonText = GetJsonTextFromFile(); try { - return JObject.Parse(jsonText); + return JsonParser.Parse(jsonText); } catch (JsonReaderException) when (attempt < maxAttempts - 1) { diff --git a/JsonFlatFileDataStore/JsonParser.cs b/JsonFlatFileDataStore/JsonParser.cs new file mode 100644 index 0000000..83c29f2 --- /dev/null +++ b/JsonFlatFileDataStore/JsonParser.cs @@ -0,0 +1,112 @@ +using System; +using System.Dynamic; +using System.IO; +using Newtonsoft.Json; +using Newtonsoft.Json.Linq; + +namespace JsonFlatFileDataStore; + +/// +/// JSON parsing that keeps the precision of non-integral numbers. +/// +/// Newtonsoft materializes them as double by default, which silently corrupts any value with more +/// significant digits than a double can hold — and a value such as decimal.MaxValue is then written +/// back in exponent notation, after which it no longer deserializes into a decimal property at all. +/// +/// Reading numbers as decimal avoids both, but decimal covers a much narrower range than double. +/// A document holding a value outside that range is parsed the old way instead: too large throws +/// and is caught, too small would round to zero without any error, so those documents are +/// recognized from the text before parsing. +/// +internal static class JsonParser +{ + // Smallest positive decimal is 1e-28, so an exponent at or past that can lose the value + private const int UnsafeNegativeExponent = 28; + + private static readonly string _unsafeSmallLiteral = "0." + new string('0', UnsafeNegativeExponent); + + private static readonly JsonSerializerSettings _decimalSettings = new JsonSerializerSettings + { FloatParseHandling = FloatParseHandling.Decimal }; + + internal static JObject Parse(string jsonText) + { + if (!MayLoseDecimalPrecision(jsonText)) + { + try + { + return LoadWithDecimalNumbers(jsonText); + } + catch (JsonReaderException) + { + // Either a number too large for decimal or genuinely invalid JSON. Parsing again + // with the default handling either succeeds or throws the error the caller expects. + } + } + + return JObject.Parse(jsonText); + } + + internal static ExpandoObject ToExpandoObject(string jsonText) + { + if (!MayLoseDecimalPrecision(jsonText)) + { + try + { + return JsonConvert.DeserializeObject(jsonText, _decimalSettings); + } + catch (JsonReaderException) + { + } + } + + return JsonConvert.DeserializeObject(jsonText); + } + + private static JObject LoadWithDecimalNumbers(string jsonText) + { + using (var stringReader = new StringReader(jsonText)) + using (var jsonReader = new JsonTextReader(stringReader) { FloatParseHandling = FloatParseHandling.Decimal }) + { + var jObject = JObject.Load(jsonReader); + + // JObject.Parse rejects trailing content after the object, so do the same + while (jsonReader.Read()) + { + if (jsonReader.TokenType != JsonToken.Comment) + throw new JsonReaderException("Additional text found in JSON string after parsing content."); + } + + return jObject; + } + } + + /// + /// True when the text contains a number so small that reading it as a decimal would round it + /// towards zero. Text inside strings is not skipped: a false positive only means the document + /// is read the way it always was, whereas a missed number would be silently destroyed. + /// + private static bool MayLoseDecimalPrecision(string jsonText) + { + for (var i = 0; i < jsonText.Length - 2; i++) + { + var current = jsonText[i]; + + if ((current != 'e' && current != 'E') || jsonText[i + 1] != '-') + continue; + + var exponent = 0; + var digitIndex = i + 2; + + for (; digitIndex < jsonText.Length && char.IsDigit(jsonText[digitIndex]); digitIndex++) + { + exponent = exponent * 10 + (jsonText[digitIndex] - '0'); + + if (exponent >= UnsafeNegativeExponent) + return true; + } + } + + // Same magnitude written without an exponent, which only a hand-authored file contains + return jsonText.IndexOf(_unsafeSmallLiteral, StringComparison.Ordinal) >= 0; + } +} \ No newline at end of file From 0fe59bb36ae2304846bab855654ba18acb0ae3d6 Mon Sep 17 00:00:00 2001 From: ttu Date: Sun, 26 Jul 2026 08:28:04 +0300 Subject: [PATCH 2/2] fix: normalize nested decimals and accept exact e-28 values Two gaps from review of the decimal-precision fix: A decimal reached dynamic callers through a nested array or object, because only the top level of a single-item array read was normalized. The array is now read through its JSON text like an object is, so every number inside it is materialized as double. A literal such as 1e-28 pushed the whole document onto the double path even though decimal represents it exactly, which corrupted a high-precision decimal stored next to it. The exponent is now weighed together with the digits the mantissa has after the point, so only values past decimal's 28-digit scale fall back. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01VjQuyQX3v7BsMX1uURt8zU --- .../NumericEdgeCaseTests.cs | 80 +++++++++++++++++++ JsonFlatFileDataStore/DataStore.cs | 9 ++- JsonFlatFileDataStore/JsonParser.cs | 26 +++++- 3 files changed, 108 insertions(+), 7 deletions(-) diff --git a/JsonFlatFileDataStore.Test/NumericEdgeCaseTests.cs b/JsonFlatFileDataStore.Test/NumericEdgeCaseTests.cs index 31b7fb2..d982e95 100644 --- a/JsonFlatFileDataStore.Test/NumericEdgeCaseTests.cs +++ b/JsonFlatFileDataStore.Test/NumericEdgeCaseTests.cs @@ -1,3 +1,6 @@ +using System.Collections; +using Newtonsoft.Json.Linq; + namespace JsonFlatFileDataStore.Test; /// @@ -69,6 +72,81 @@ public async Task Decimal_BeyondDoublePrecision_RoundTrip(bool useLowerCamelCase UTHelpers.Down(path); } + [Fact] + public async Task Decimal_DocumentWithSmallestRepresentableValue_KeepsPrecision() + { + // 1e-28 is exactly representable as a decimal, so it must not push the document onto the + // double path and corrupt the high-precision decimal stored next to it. + var path = UTHelpers.GetFullFilePath($"DecTinyNeighbour_{DateTime.UtcNow.Ticks}"); + var store = new DataStore(path); + + await store.GetCollection("decModel").InsertOneAsync(new DecimalModel { Id = 1, Price = decimal.MaxValue }); + await store.GetCollection("movie").InsertOneAsync(new Movie { Name = "Tiny", Rating = 1e-28 }); + + store.Dispose(); + + var store2 = new DataStore(path); + + Assert.Equal(decimal.MaxValue, store2.GetCollection("decModel").AsQueryable().First().Price); + Assert.Equal(1e-28, store2.GetCollection("movie").AsQueryable().First().Rating); + + store2.Dispose(); + UTHelpers.Down(path); + } + + [Fact] + public void GetItem_NestedContainers_DoNotExposeDecimal() + { + // Dynamic values have always been doubles and a dynamic decimal can not even be compared + // to a double literal, so no decimal may leak out through a nested array or object either. + var path = UTHelpers.GetFullFilePath($"NestedDyn_{DateTime.UtcNow.Ticks}"); + var store = new DataStore(path); + + store.InsertItem("nested", new object[] { new[] { 1.5, 2.5 }, new { inner = 3.5 } }); + store.Dispose(); + + var store2 = new DataStore(path); + var item = store2.GetItem("nested"); + + Assert.Empty(FindDecimals(item)); + + store2.Dispose(); + UTHelpers.Down(path); + } + + private static IEnumerable FindDecimals(object value) + { + switch (value) + { + case decimal: + yield return value; + break; + + case JValue jValue: + foreach (var found in FindDecimals(jValue.Value)) + yield return found; + break; + + case JObject jObject: + foreach (var found in jObject.Properties().SelectMany(p => FindDecimals(p.Value))) + yield return found; + break; + + case IEnumerable> expando: + foreach (var found in expando.SelectMany(p => FindDecimals(p.Value))) + yield return found; + break; + + case IEnumerable sequence when value is not string: + foreach (var element in sequence) + { + foreach (var found in FindDecimals(element)) + yield return found; + } + break; + } + } + [Fact] public async Task Decimal_UpdatedAfterReload_KeepsPrecision() { @@ -98,6 +176,8 @@ public async Task Decimal_UpdatedAfterReload_KeepsPrecision() [InlineData(double.MinValue)] [InlineData(double.Epsilon)] [InlineData(1e-30)] + [InlineData(1.5e-28)] + [InlineData(1e-28)] public async Task Double_OutsideDecimalRange_RoundTrip(double value) { // Values outside decimal's range must not be read as one: too large throws, too small diff --git a/JsonFlatFileDataStore/DataStore.cs b/JsonFlatFileDataStore/DataStore.cs index 865693e..869d34d 100644 --- a/JsonFlatFileDataStore/DataStore.cs +++ b/JsonFlatFileDataStore/DataStore.cs @@ -438,7 +438,10 @@ private dynamic SingleDynamicItemReadConverter(JToken e) return JsonConvert.DeserializeObject(content, _converter) as dynamic; case var arrayToken when e.Type == JTokenType.Array: - return e.ToObject>().Select(NormalizeFloatValue).ToList(); + // Read through the JSON text like an object rather than converting the tokens, so + // that no decimal survives anywhere inside a nested array or object either + var arrayContent = string.Format(CultureInfo.InvariantCulture, "{0}", arrayToken); + return JsonConvert.DeserializeObject>(arrayContent); case JValue jv when e is JValue: return NormalizeFloatValue(jv.Value); @@ -450,8 +453,8 @@ private dynamic SingleDynamicItemReadConverter(JToken e) // The document keeps non-integral numbers as decimal so their precision survives a round trip, // but the dynamic API has always handed them out as double — and a dynamic decimal can not even - // be compared to a double literal. Objects are unaffected: they are read through - // ExpandoObject, which materializes them as double regardless. + // be compared to a double literal. Objects and arrays are read through their JSON text, which + // materializes every number in them as double regardless. private static object NormalizeFloatValue(object value) => value is decimal d ? (double)d : value; private string GetJsonTextFromFile() => FileAccess.ReadJsonFromFile(_filePath, _encryptJson, _decryptJson); diff --git a/JsonFlatFileDataStore/JsonParser.cs b/JsonFlatFileDataStore/JsonParser.cs index 83c29f2..3157cfc 100644 --- a/JsonFlatFileDataStore/JsonParser.cs +++ b/JsonFlatFileDataStore/JsonParser.cs @@ -20,10 +20,10 @@ namespace JsonFlatFileDataStore; /// internal static class JsonParser { - // Smallest positive decimal is 1e-28, so an exponent at or past that can lose the value - private const int UnsafeNegativeExponent = 28; + // Decimal keeps at most 28 digits after the point + private const int MaxDecimalScale = 28; - private static readonly string _unsafeSmallLiteral = "0." + new string('0', UnsafeNegativeExponent); + private static readonly string _unsafeSmallLiteral = "0." + new string('0', MaxDecimalScale); private static readonly JsonSerializerSettings _decimalSettings = new JsonSerializerSettings { FloatParseHandling = FloatParseHandling.Decimal }; @@ -101,12 +101,30 @@ private static bool MayLoseDecimalPrecision(string jsonText) { exponent = exponent * 10 + (jsonText[digitIndex] - '0'); - if (exponent >= UnsafeNegativeExponent) + if (exponent > MaxDecimalScale) return true; } + + // The exponent alone does not decide it: the digits the mantissa already has after the + // point shift the value further down. 1e-28 is exact, 1.5e-28 is not. + if (digitIndex > i + 2 && exponent + CountFractionDigits(jsonText, i) > MaxDecimalScale) + return true; } // Same magnitude written without an exponent, which only a hand-authored file contains return jsonText.IndexOf(_unsafeSmallLiteral, StringComparison.Ordinal) >= 0; } + + /// + /// Number of digits the mantissa ending at has after the point. + /// + private static int CountFractionDigits(string jsonText, int exponentIndex) + { + var index = exponentIndex - 1; + + while (index >= 0 && char.IsDigit(jsonText[index])) + index--; + + return index >= 0 && jsonText[index] == '.' ? exponentIndex - 1 - index : 0; + } } \ No newline at end of file