fix: keep the precision of high-precision decimals

CHANGELOG.md

@@ -5,6 +5,7 @@
 * FIXED: Retry JSON parse on read to tolerate concurrent partial-file writes
 * FIXED: A failing commit action no longer hangs other callers in the same batch
 * FIXED: Collection key in file not matching configured case is now matched case-insensitively, instead of reading empty and duplicating the key on save
+* FIXED: High-precision decimals are no longer corrupted on read and write, e.g. decimal.MaxValue previously made the item unreadable
 
 ### [2.4.2] - 2023-06-25
 * FIXED: Duplicate collection data to JSON on save when configured case was not used with collection name

JsonFlatFileDataStore.Test/NumericEdgeCaseTests.cs

@@ -1,3 +1,6 @@
+using System.Collections;
+using Newtonsoft.Json.Linq;
+
 namespace JsonFlatFileDataStore.Test;
 
 /// <summary>
@@ -23,10 +26,6 @@ public async Task Decimal_RoundTrip_PreservesPrecision()
         var collection = store.GetCollection<DecimalModel>("decModel");
         await collection.InsertOneAsync(new DecimalModel { Id = 1, Price = 19.95m, OptionalAmount = 0.0000001m });
         await collection.InsertOneAsync(new DecimalModel { Id = 2, Price = 9999999999.99m, OptionalAmount = null });
-        // Note: very large decimals lose precision on the current Newtonsoft path because
-        // the serializer routes data through ExpandoObject (JSON number → double → back).
-        // decimal.MaxValue specifically also fails to deserialize. Use a value within
-        // double precision range to assert lossless round-trip.
         await collection.InsertOneAsync(new DecimalModel { Id = 3, Price = -0.01m, OptionalAmount = 12345678.99m });
 
         store.Dispose();
@@ -45,27 +44,156 @@ public async Task Decimal_RoundTrip_PreservesPrecision()
         UTHelpers.Down(path);
     }
 
+    [Theory]
+    [InlineData(true)]
+    [InlineData(false)]
+    public async Task Decimal_BeyondDoublePrecision_RoundTrip(bool useLowerCamelCase)
+    {
+        // Values with more significant digits than a double can hold. Both the read path and the
+        // camelCase write path used to materialize them as double, which corrupted them.
+        var path = UTHelpers.GetFullFilePath($"DecPrecision_{useLowerCamelCase}_{DateTime.UtcNow.Ticks}");
+        var store = new DataStore(path, useLowerCamelCase);
+
+        var collection = store.GetCollection<DecimalModel>("decModel");
+        await collection.InsertOneAsync(new DecimalModel { Id = 1, Price = 123456789012345678.5m });
+        await collection.InsertOneAsync(new DecimalModel { Id = 2, Price = 79228162514264337593543950m });
+        await collection.InsertOneAsync(new DecimalModel { Id = 3, Price = decimal.MaxValue });
+
+        store.Dispose();
+
+        var store2 = new DataStore(path, useLowerCamelCase);
+        var items = store2.GetCollection<DecimalModel>("decModel").AsQueryable().OrderBy(e => e.Id).ToList();
+
+        Assert.Equal(123456789012345678.5m, items[0].Price);
+        Assert.Equal(79228162514264337593543950m, items[1].Price);
+        Assert.Equal(decimal.MaxValue, items[2].Price);
+
+        store2.Dispose();
+        UTHelpers.Down(path);
+    }
+
+    [Fact]
+    public async Task Decimal_DocumentWithSmallestRepresentableValue_KeepsPrecision()
+    {
+        // 1e-28 is exactly representable as a decimal, so it must not push the document onto the
+        // double path and corrupt the high-precision decimal stored next to it.
+        var path = UTHelpers.GetFullFilePath($"DecTinyNeighbour_{DateTime.UtcNow.Ticks}");
+        var store = new DataStore(path);
+
+        await store.GetCollection<DecimalModel>("decModel").InsertOneAsync(new DecimalModel { Id = 1, Price = decimal.MaxValue });
+        await store.GetCollection<Movie>("movie").InsertOneAsync(new Movie { Name = "Tiny", Rating = 1e-28 });
+
+        store.Dispose();
+
+        var store2 = new DataStore(path);
+
+        Assert.Equal(decimal.MaxValue, store2.GetCollection<DecimalModel>("decModel").AsQueryable().First().Price);
+        Assert.Equal(1e-28, store2.GetCollection<Movie>("movie").AsQueryable().First().Rating);
+
+        store2.Dispose();
+        UTHelpers.Down(path);
+    }
+
     [Fact]
-    public async Task Decimal_LargeValue_LosesPrecision_BehaviorPinned()
+    public void GetItem_NestedContainers_DoNotExposeDecimal()
     {
-        // Documented limitation of the current Newtonsoft path: very large decimal values
-        // are serialized via JObject → ExpandoObject (which uses double internally),
-        // so significant digits beyond double precision are lost on round trip.
-        // Migration note: STJ may behave differently here — expected to either round-trip
-        // exactly or fail explicitly. Update this test once the migration lands.
-        var path = UTHelpers.GetFullFilePath($"DecLoss_{DateTime.UtcNow.Ticks}");
+        // Dynamic values have always been doubles and a dynamic decimal can not even be compared
+        // to a double literal, so no decimal may leak out through a nested array or object either.
+        var path = UTHelpers.GetFullFilePath($"NestedDyn_{DateTime.UtcNow.Ticks}");
+        var store = new DataStore(path);
+
+        store.InsertItem("nested", new object[] { new[] { 1.5, 2.5 }, new { inner = 3.5 } });
+        store.Dispose();
+
+        var store2 = new DataStore(path);
+        var item = store2.GetItem("nested");
+
+        Assert.Empty(FindDecimals(item));
+
+        store2.Dispose();
+        UTHelpers.Down(path);
+    }
+
+    private static IEnumerable<object> FindDecimals(object value)
+    {
+        switch (value)
+        {
+            case decimal:
+                yield return value;
+                break;
+
+            case JValue jValue:
+                foreach (var found in FindDecimals(jValue.Value))
+                    yield return found;
+                break;
+
+            case JObject jObject:
+                foreach (var found in jObject.Properties().SelectMany(p => FindDecimals(p.Value)))
+                    yield return found;
+                break;
+
+            case IEnumerable<KeyValuePair<string, object>> expando:
+                foreach (var found in expando.SelectMany(p => FindDecimals(p.Value)))
+                    yield return found;
+                break;
+
+            case IEnumerable sequence when value is not string:
+                foreach (var element in sequence)
+                {
+                    foreach (var found in FindDecimals(element))
+                        yield return found;
+                }
+                break;
+        }
+    }
+
+    [Fact]
+    public async Task Decimal_UpdatedAfterReload_KeepsPrecision()
+    {
+        // The reload path parses the file again, so precision must survive a write that happens
+        // after the store has read its own output.
+        var path = UTHelpers.GetFullFilePath($"DecUpdate_{DateTime.UtcNow.Ticks}");
         var store = new DataStore(path);
 
         var collection = store.GetCollection<DecimalModel>("decModel");
-        await collection.InsertOneAsync(new DecimalModel { Id = 1, Price = 79228162514264337593543950m });
+        await collection.InsertOneAsync(new DecimalModel { Id = 1, Price = 123456789012345678.5m });
+        await collection.UpdateOneAsync(1, new { OptionalAmount = 12345678901234567890.5m });
 
         store.Dispose();
 
         var store2 = new DataStore(path);
         var item = store2.GetCollection<DecimalModel>("decModel").AsQueryable().First();
 
-        // Precision is lost — assert the value differs from the input.
-        Assert.NotEqual(79228162514264337593543950m, item.Price);
+        Assert.Equal(123456789012345678.5m, item.Price);
+        Assert.Equal(12345678901234567890.5m, item.OptionalAmount);
+
+        store2.Dispose();
+        UTHelpers.Down(path);
+    }
+
+    [Theory]
+    [InlineData(double.MaxValue)]
+    [InlineData(double.MinValue)]
+    [InlineData(double.Epsilon)]
+    [InlineData(1e-30)]
+    [InlineData(1.5e-28)]
+    [InlineData(1e-28)]
+    public async Task Double_OutsideDecimalRange_RoundTrip(double value)
+    {
+        // Values outside decimal's range must not be read as one: too large throws, too small
+        // rounds to zero without any error. Both fall back to double parsing.
+        var path = UTHelpers.GetFullFilePath($"DoubleRange_{value}_{DateTime.UtcNow.Ticks}");
+        var store = new DataStore(path);
+
+        var collection = store.GetCollection<Movie>("movie");
+        await collection.InsertOneAsync(new Movie { Name = "Extreme", Rating = value });
+
+        store.Dispose();
+
+        var store2 = new DataStore(path);
+        var movie = store2.GetCollection<Movie>("movie").AsQueryable().First();
+
+        Assert.Equal(value, movie.Rating);
 
         store2.Dispose();
         UTHelpers.Down(path);

JsonFlatFileDataStore/CommitActionHandler.cs

@@ -47,7 +47,7 @@ internal static void HandleStoreCommitActions(CancellationToken token, BlockingC
             {
                 try
                 {
-                    var (actionSuccess, updatedJson) = action.HandleAction(JObject.Parse(jsonText));
+                    var (actionSuccess, updatedJson) = action.HandleAction(JsonParser.Parse(jsonText));
 
                     callbacks.Enqueue((action, actionSuccess));
 

JsonFlatFileDataStore/DataStore.cs

@@ -45,7 +45,7 @@ public DataStore(string path, bool useLowerCamelCase = true, string keyProperty
             {
                 // Serializing JObject ignores SerializerSettings, so we have to first deserialize to ExpandoObject and then serialize
                 // http://json.codeplex.com/workitem/23853
-                var jObject = JsonConvert.DeserializeObject<ExpandoObject>(data.ToString());
+                var jObject = JsonParser.ToExpandoObject(data.ToString());
                 return JsonConvert.SerializeObject(jObject, usedFormatting, _serializerSettings);
             })
             : (s => s.ToString(usedFormatting));
@@ -82,7 +82,7 @@ public DataStore(string path, bool useLowerCamelCase = true, string keyProperty
                 {
                     lock (_jsonData)
                     {
-                        _jsonData = JObject.Parse(jsonText);
+                        _jsonData = JsonParser.Parse(jsonText);
                     }
 
                     return FileAccess.WriteJsonToFile(_filePath, _encryptJson, jsonText);
@@ -110,7 +110,7 @@ public void UpdateAll(string jsonData)
     {
         lock (_jsonData)
         {
-            _jsonData = JObject.Parse(jsonData);
+            _jsonData = JsonParser.Parse(jsonData);
         }
 
         FileAccess.WriteJsonToFile(_filePath, _encryptJson, jsonData);
@@ -438,16 +438,25 @@ private dynamic SingleDynamicItemReadConverter(JToken e)
                 return JsonConvert.DeserializeObject<ExpandoObject>(content, _converter) as dynamic;
 
             case var arrayToken when e.Type == JTokenType.Array:
-                return e.ToObject<List<object>>();
+                // Read through the JSON text like an object rather than converting the tokens, so
+                // that no decimal survives anywhere inside a nested array or object either
+                var arrayContent = string.Format(CultureInfo.InvariantCulture, "{0}", arrayToken);
+                return JsonConvert.DeserializeObject<List<object>>(arrayContent);
 
             case JValue jv when e is JValue:
-                return jv.Value;
+                return NormalizeFloatValue(jv.Value);
 
             default:
                 return e.ToObject<object>();
         }
     }
 
+    // The document keeps non-integral numbers as decimal so their precision survives a round trip,
+    // but the dynamic API has always handed them out as double — and a dynamic decimal can not even
+    // be compared to a double literal. Objects and arrays are read through their JSON text, which
+    // materializes every number in them as double regardless.
+    private static object NormalizeFloatValue(object value) => value is decimal d ? (double)d : value;
+
     private string GetJsonTextFromFile() => FileAccess.ReadJsonFromFile(_filePath, _encryptJson, _decryptJson);
 
     private JObject GetJsonObjectFromFile()
@@ -463,7 +472,7 @@ private JObject GetJsonObjectFromFile()
             var jsonText = GetJsonTextFromFile();
             try
             {
-                return JObject.Parse(jsonText);
+                return JsonParser.Parse(jsonText);
             }
             catch (JsonReaderException) when (attempt < maxAttempts - 1)
             {

JsonFlatFileDataStore/JsonParser.cs

@@ -0,0 +1,130 @@
+using System;
+using System.Dynamic;
+using System.IO;
+using Newtonsoft.Json;
+using Newtonsoft.Json.Linq;
+
+namespace JsonFlatFileDataStore;
+
+/// <summary>
+/// JSON parsing that keeps the precision of non-integral numbers.
+///
+/// Newtonsoft materializes them as double by default, which silently corrupts any value with more
+/// significant digits than a double can hold — and a value such as decimal.MaxValue is then written
+/// back in exponent notation, after which it no longer deserializes into a decimal property at all.
+///
+/// Reading numbers as decimal avoids both, but decimal covers a much narrower range than double.
+/// A document holding a value outside that range is parsed the old way instead: too large throws
+/// and is caught, too small would round to zero without any error, so those documents are
+/// recognized from the text before parsing.
+/// </summary>
+internal static class JsonParser
+{
+    // Decimal keeps at most 28 digits after the point
+    private const int MaxDecimalScale = 28;
+
+    private static readonly string _unsafeSmallLiteral = "0." + new string('0', MaxDecimalScale);
+
+    private static readonly JsonSerializerSettings _decimalSettings = new JsonSerializerSettings
+    { FloatParseHandling = FloatParseHandling.Decimal };
+
+    internal static JObject Parse(string jsonText)
+    {
+        if (!MayLoseDecimalPrecision(jsonText))
+        {
+            try
+            {
+                return LoadWithDecimalNumbers(jsonText);
+            }
+            catch (JsonReaderException)
+            {
+                // Either a number too large for decimal or genuinely invalid JSON. Parsing again
+                // with the default handling either succeeds or throws the error the caller expects.
+            }
+        }
+
+        return JObject.Parse(jsonText);
+    }
+
+    internal static ExpandoObject ToExpandoObject(string jsonText)
+    {
+        if (!MayLoseDecimalPrecision(jsonText))
+        {
+            try
+            {
+                return JsonConvert.DeserializeObject<ExpandoObject>(jsonText, _decimalSettings);
+            }
+            catch (JsonReaderException)
+            {
+            }
+        }
+
+        return JsonConvert.DeserializeObject<ExpandoObject>(jsonText);
+    }
+
+    private static JObject LoadWithDecimalNumbers(string jsonText)
+    {
+        using (var stringReader = new StringReader(jsonText))
+        using (var jsonReader = new JsonTextReader(stringReader) { FloatParseHandling = FloatParseHandling.Decimal })
+        {
+            var jObject = JObject.Load(jsonReader);
+
+            // JObject.Parse rejects trailing content after the object, so do the same
+            while (jsonReader.Read())
+            {
+                if (jsonReader.TokenType != JsonToken.Comment)
+                    throw new JsonReaderException("Additional text found in JSON string after parsing content.");
+            }
+
+            return jObject;
+        }
+    }
+
+    /// <summary>
+    /// True when the text contains a number so small that reading it as a decimal would round it
+    /// towards zero. Text inside strings is not skipped: a false positive only means the document
+    /// is read the way it always was, whereas a missed number would be silently destroyed.
+    /// </summary>
+    private static bool MayLoseDecimalPrecision(string jsonText)
+    {
+        for (var i = 0; i < jsonText.Length - 2; i++)
+        {
+            var current = jsonText[i];
+
+            if ((current != 'e' && current != 'E') || jsonText[i + 1] != '-')
+                continue;
+
+            var exponent = 0;
+            var digitIndex = i + 2;
+
+            for (; digitIndex < jsonText.Length && char.IsDigit(jsonText[digitIndex]); digitIndex++)
+            {
+                exponent = exponent * 10 + (jsonText[digitIndex] - '0');
+
+                if (exponent > MaxDecimalScale)
+                    return true;
+            }
+
+            // The exponent alone does not decide it: the digits the mantissa already has after the
+            // point shift the value further down. 1e-28 is exact, 1.5e-28 is not.
+            if (digitIndex > i + 2 && exponent + CountFractionDigits(jsonText, i) > MaxDecimalScale)
+                return true;
+        }
+
+        // Same magnitude written without an exponent, which only a hand-authored file contains
+        return jsonText.IndexOf(_unsafeSmallLiteral, StringComparison.Ordinal) >= 0;
+    }
+
+    /// <summary>
+    /// Number of digits the mantissa ending at <paramref name="exponentIndex"/> has after the point.
+    /// </summary>
+    private static int CountFractionDigits(string jsonText, int exponentIndex)
+    {
+        var index = exponentIndex - 1;
+
+        while (index >= 0 && char.IsDigit(jsonText[index]))
+            index--;
+
+        return index >= 0 && jsonText[index] == '.' ? exponentIndex - 1 - index : 0;
+    }
+}