v3.7.3.3: fix decimal sep bug, revert to .NET 4.0

Resolve #17 with parsing of floating-point numbers in cultures where the decimal separator is `,` and not the `.` used in the USA. To preserve backwards compatibility with older versions of Windows, I'm reverting to .NET 4.0 and the old GitHub action. If you are trying to work with the package and you find that you can't build it using Visual Studio 2022 (for the record, I use VS 2022 and have no trouble), see this discussion: https://stackoverflow.com/a/71135859/18441560
molsonkiko · Oct 24, 2022 · 9f49838 · 9f49838
1 parent cb7db72
commit 9f49838
Show file tree

Hide file tree

Showing 14 changed files with 66 additions and 51 deletions.
diff --git a/.github/workflows/CI_build.yml b/.github/workflows/CI_build.yml
@@ -4,7 +4,7 @@ on: [push, pull_request]
 
 jobs:
   build:
-    runs-on: windows-2022
+    runs-on: windows-2019
     strategy:
       max-parallel: 4
       matrix:
@@ -16,7 +16,7 @@ jobs:
       uses: actions/checkout@v3
 
     - name: Add msbuild to PATH
-      uses: microsoft/setup-msbuild@v1.1.3
+      uses: microsoft/setup-msbuild@v1.0.3
 
     - name: MSBuild of solution
       run: msbuild JsonToolsNppPlugin/JsonToolsNppPlugin.sln /p:configuration="${{ matrix.build_configuration }}" /p:platform="${{ matrix.build_platform }}" /m /verbosity:minimal

diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -51,12 +51,13 @@ and this project adheres to [Semantic Versioning](http://semver.org/).
 - Improve how well the caret tracks the node selected in the query tree, after a query that selects a subset of nodes. The iterables have their line number set to 0.
 - Get rid of __ALL__ dinging sounds from the forms, including the `TreeView` control in the TreeViewer.
 
-## [3.7.3.1] (unreleased) - 2022-MM-DD
+## [3.7.3.3] - 2022-10-24
 
 ### Fixed
 
 1. Changed queries produced by [find/replace form](/docs/README.md#find-and-replace-form) to make it more robust and easy to use.
 2. Eliminated potential errors when using the [remove files](/docs/README.md#clearing-selected-files) button on the JSON from files and APIs form.
+3. Resolve [Issue #17](https://github.com/molsonkiko/JsonToolsNppPlugin/issues/17) with parsing of floating-point numbers in cultures where the decimal separator is `,` and not the `.` used in the USA.
 
 ## [3.7.2.1] - 2022-10-20
 

diff --git a/JsonToolsNppPlugin/JSONTools/JNode.cs b/JsonToolsNppPlugin/JSONTools/JNode.cs
@@ -6,6 +6,7 @@
 using System.Text;
 using System.Text.RegularExpressions;
 using System.Collections.Generic; // for dictionary, list
+using System.Globalization;
 
 namespace JSON_Tools.JSON_Tools
 {
@@ -219,6 +220,13 @@ public JNode()
             ['\f'] = "\\f",
         };
 
+        // in some places like Germany they use ',' as the normal decimal separator.
+        // need to override this to ensure that we parse JSON correctly
+        public static readonly NumberFormatInfo DOT_DECIMAL_SEP = new NumberFormatInfo
+        {
+            NumberDecimalSeparator = "."
+        };
+
         private const string HEX_CHARS = "0123456789abcdef";
         /// <summary>
         /// Return the hexadecimal string representation of an integer
@@ -299,10 +307,10 @@ public virtual string ToString(bool sort_keys = true, string key_value_sep = ":
                     if (v == Math.Round(v))
                     {
                         // add ending ".0" to distinguish doubles equal to integers from actual integers
-                        return v.ToString() + ".0";
+                        return v.ToString(DOT_DECIMAL_SEP) + ".0";
                     }
 
-                    return v.ToString();
+                    return v.ToString(DOT_DECIMAL_SEP);
                 }
                 case Dtype.INT: return Convert.ToInt64(value).ToString();
                 case Dtype.NULL: return "null";

diff --git a/JsonToolsNppPlugin/JSONTools/JsonParser.cs b/JsonToolsNppPlugin/JSONTools/JsonParser.cs
@@ -3,6 +3,7 @@
 */
 using System;
 using System.Collections.Generic;
+using System.Globalization;
 using System.Text;
 using System.Text.RegularExpressions;
 using JSON_Tools.Utils;
@@ -455,7 +456,7 @@ public JNode ParseNumber(string q)
                 else if (c == '/')
                 {
                     // fractions are part of the JSON language specification
-                    double numer = double.Parse(sb.ToString());
+                    double numer = double.Parse(sb.ToString(), JNode.DOT_DECIMAL_SEP);
                     JNode denom_node;
                     ii++;
                     denom_node = ParseNumber(q);
@@ -471,7 +472,7 @@ public JNode ParseNumber(string q)
             {
                 return new JNode(long.Parse(sb.ToString()), Dtype.INT, line_num);
             }
-            return new JNode(double.Parse(sb.ToString()), Dtype.FLOAT, line_num);
+            return new JNode(double.Parse(sb.ToString(), JNode.DOT_DECIMAL_SEP), Dtype.FLOAT, line_num);
         }
 
         /// <summary>

diff --git a/JsonToolsNppPlugin/JSONTools/RemesPathFunctions.cs b/JsonToolsNppPlugin/JSONTools/RemesPathFunctions.cs
@@ -1476,7 +1476,7 @@ public static JNode ToFloat(JNode[] args)
             JNode val = args[0];
             if (val.type == Dtype.STR)
             {
-                return new JNode(double.Parse((string)val.value), Dtype.FLOAT, 0);
+                return new JNode(double.Parse((string)val.value, JNode.DOT_DECIMAL_SEP), Dtype.FLOAT, 0);
             }
             return new JNode(Convert.ToDouble(val.value), Dtype.FLOAT, 0);
         }

diff --git a/JsonToolsNppPlugin/JSONTools/RemesPathLexer.cs b/JsonToolsNppPlugin/JSONTools/RemesPathLexer.cs
@@ -135,7 +135,7 @@ public JNode ParseNumber(string q)
             {
                 return new JNode(long.Parse(sb.ToString()), Dtype.INT, 0);
             }
-            return new JNode(double.Parse(sb.ToString()), Dtype.FLOAT, 0);
+            return new JNode(double.Parse(sb.ToString(), JNode.DOT_DECIMAL_SEP), Dtype.FLOAT, 0);
         }
 
         public JNode ParseQuotedString(string q)

diff --git a/JsonToolsNppPlugin/JSONTools/YamlDumper.cs b/JsonToolsNppPlugin/JSONTools/YamlDumper.cs
@@ -98,7 +98,14 @@ private string EscapeBackslashKey(string s)
 
         private string YamlKeyRepr(string k)
         {
-            if (double.TryParse(k, out double d))
+            bool is_float_str = false;
+            try
+            {
+                double.Parse(k, JNode.DOT_DECIMAL_SEP);
+                is_float_str = true;
+            }
+            catch { }
+            if (is_float_str)
             {
                 // k is a string representing a number; we need to enquote it so that
                 // a YAML parser will recognize that it is not actually a number.
@@ -115,46 +122,43 @@ private string YamlKeyRepr(string k)
 
         private string YamlValRepr(JNode v)
         {
-            if (v.type == Dtype.NULL)
+            if (v.value == null)
             {
                 return "null";
             }
-            object val = v.value;
-            string strv = val.ToString();
-            if (v.type == Dtype.INT || v.type == Dtype.BOOL)
+            string strv = v.value.ToString();
+            if (v.type == Dtype.STR)
             {
-                return strv;
+                try
+                {
+                    _ = double.Parse(strv, JNode.DOT_DECIMAL_SEP);
+                    return $"'{strv}'";
+                    // enquote stringified numbers to avoid confusion with actual numbers
+                }
+                catch { }
+                if (StartsOrEndsWith(strv, " "))
+                {
+                    return $"\"{strv}\"";
+                }
+                Regex backslash = new Regex("([\\\\:\"'\r\t\n\f\b])");
+                if (backslash.IsMatch(strv))
+                {
+                    return EscapeBackslash(strv);
+                }
             }
-            if (double.TryParse(strv, out double d))
+            else if (v.type == Dtype.FLOAT)
             {
                 // k is a number
+                double d = (double)v.value;
                 if (d == NanInf.inf) return ".inf";
                 if (d == NanInf.neginf) return "-.inf";
                 if (double.IsNaN(d))
                 {
                     return ".nan";
                 }
-                if (v.type == Dtype.STR)
-                {
-                    // enquote numstrings to prevent confusion
-                    return "'" + strv + "'";
-                }
-                if (v.type == Dtype.FLOAT && double.Parse(strv) % 1 == 0)
-                {
-                    // ensure that floats equal to ints are rendered as floats
-                    return strv + ".0";
-                }
-                return strv;
-            }
-            if (StartsOrEndsWith(strv, " "))
-            {
-                return '"' + strv + '"';
-            }
-            Regex backslash = new Regex("([\\\\:\"'\r\t\n\f\b])");
-            if (backslash.IsMatch(strv))
-            {
-                //Console.WriteLine("has backslash");
-                return EscapeBackslash(strv);
+                if (d % 1 == 0)
+                    return $"{d}.0"; // add .0 at end of floats that are equal to ints
+                return d.ToString(JNode.DOT_DECIMAL_SEP);
             }
             return strv;
         }

diff --git a/JsonToolsNppPlugin/JsonToolsNppPlugin.csproj b/JsonToolsNppPlugin/JsonToolsNppPlugin.csproj
@@ -10,7 +10,7 @@
     <AppDesignerFolder>Properties</AppDesignerFolder>
     <RootNamespace>NppPluginNET</RootNamespace>
     <AssemblyName>JsonTools</AssemblyName>
-    <TargetFrameworkVersion>v4.8</TargetFrameworkVersion>
+    <TargetFrameworkVersion>v4.0</TargetFrameworkVersion>
     <OldToolsVersion>3.5</OldToolsVersion>
     <ProjectGuid>{EB8FC3A3-93E8-457B-B281-FAFA5119611A}</ProjectGuid>
   </PropertyGroup>

diff --git a/JsonToolsNppPlugin/Properties/AssemblyInfo.cs b/JsonToolsNppPlugin/Properties/AssemblyInfo.cs
@@ -29,5 +29,5 @@
 //      Build Number
 //      Revision
 //
-[assembly: AssemblyVersion("3.7.3.1")]
-[assembly: AssemblyFileVersion("3.7.3.1")]
+[assembly: AssemblyVersion("3.7.3.3")]
+[assembly: AssemblyFileVersion("3.7.3.3")]
diff --git a/JsonToolsNppPlugin/Release_x64.zip b/JsonToolsNppPlugin/Release_x64.zip
diff --git a/JsonToolsNppPlugin/Release_x86.zip b/JsonToolsNppPlugin/Release_x86.zip
diff --git a/JsonToolsNppPlugin/Tests/Benchmarker.cs b/JsonToolsNppPlugin/Tests/Benchmarker.cs
@@ -125,7 +125,7 @@ public static void Benchmark(string query, string fname, int num_trials = 8)
             var query_times_str = new string[query_times.Length];
             for (int ii = 0; ii < query_times.Length; ii++)
             {
-                query_times_str[ii] = Math.Round(query_times[ii] / 1e4, 3).ToString();
+                query_times_str[ii] = Math.Round(query_times[ii] / 1e4, 3).ToString(JNode.DOT_DECIMAL_SEP);
             }
             Npp.AddLine($"Query times (ms): {String.Join(", ", query_times_str)}");
             string result_preview = result.ToString().Slice(":300") + "\n...";

diff --git a/JsonToolsNppPlugin/Tests/JsonGrepperTests.cs b/JsonToolsNppPlugin/Tests/JsonGrepperTests.cs
@@ -1,5 +1,6 @@
 using System;
 using System.Collections.Generic;
+using System.Globalization;
 using System.IO;
 using JSON_Tools.JSON_Tools;
 using JSON_Tools.Utils;

diff --git a/most recent errors.txt b/most recent errors.txt
@@ -170,11 +170,11 @@ Performance tests for JsonParser and RemesPath (arithmetic)
 
 Preview of json: [{"A": "Ky'c^g#~)0", "a": 1850111954, "b": 9318359041, "B": "Oyi:/ xxe2", "C": "sKCSa_^7Gg", "c": 7974777124, "d": 2670309238, "D": "0d_K)HmX!.", "E": ".uM*Z{0EJ_", "e": 6958410336, "f": 8050244728, "F": "1%SG_A!xB\t", "g": 3799657125, "G": "il1^k\\\nat*", "H": {"a": 6079042826, "b": 7292804611, "c"
 ...
-To convert JSON string of size 975068 into JNode took 50.155 +/- 9.712 ms over 14 trials
-Load times (ms): 42, 47, 45, 40, 54, 46, 58, 60, 77, 46, 39, 51, 47, 44
-Compiling query "@[@[:].a * @[:].q < @[:].e]" into took 0.007 +/- 0.011 ms over 14 trials
-To run pre-compiled query "@[@[:].a * @[:].q < @[:].e]" on JNode from JSON of size 975068 into took 0.344 +/- 0.148 ms over 14 trials
-Query times (ms): 0.418, 0.324, 0.435, 0.447, 0.331, 0.755, 0.231, 0.227, 0.226, 0.241, 0.494, 0.225, 0.236, 0.225
+To convert JSON string of size 975068 into JNode took 53.747 +/- 7.352 ms over 14 trials
+Load times (ms): 54, 54, 46, 53, 46, 50, 53, 51, 63, 50, 47, 50, 75, 53
+Compiling query "@[@[:].a * @[:].q < @[:].e]" into took 0.005 +/- 0.005 ms over 14 trials
+To run pre-compiled query "@[@[:].a * @[:].q < @[:].e]" on JNode from JSON of size 975068 into took 1.169 +/- 2.671 ms over 14 trials
+Query times (ms): 0.451, 1.006, 0.244, 10.753, 0.428, 0.407, 0.292, 0.223, 1.001, 0.249, 0.231, 0.229, 0.222, 0.628
 Preview of result: [{"A": "Ky'c^g#~)0", "a": 1850111954, "b": 9318359041, "B": "Oyi:/ xxe2", "C": "sKCSa_^7Gg", "c": 7974777124, "d": 2670309238, "D": "0d_K)HmX!.", "E": ".uM*Z{0EJ_", "e": 6958410336, "f": 8050244728, "F": "1%SG_A!xB\t", "g": 3799657125, "G": "il1^k\\\nat*", "H": {"a": 6079042826, "b": 7292804611, "c"
 ...
 =========================
@@ -183,10 +183,10 @@ Performance tests for JsonParser and RemesPath (string operations)
 
 Preview of json: [{"A": "Ky'c^g#~)0", "a": 1850111954, "b": 9318359041, "B": "Oyi:/ xxe2", "C": "sKCSa_^7Gg", "c": 7974777124, "d": 2670309238, "D": "0d_K)HmX!.", "E": ".uM*Z{0EJ_", "e": 6958410336, "f": 8050244728, "F": "1%SG_A!xB\t", "g": 3799657125, "G": "il1^k\\\nat*", "H": {"a": 6079042826, "b": 7292804611, "c"
 ...
-To convert JSON string of size 975068 into JNode took 51.64 +/- 9.921 ms over 14 trials
-Load times (ms): 63, 70, 39, 41, 61, 39, 42, 44, 57, 49, 56, 59, 39, 55
-Compiling query "@[@[:].z =~ `(?i)[a-z]{5}`]" into took 0.006 +/- 0.016 ms over 14 trials
-To run pre-compiled query "@[@[:].z =~ `(?i)[a-z]{5}`]" on JNode from JSON of size 975068 into took 0.628 +/- 0.323 ms over 14 trials
-Query times (ms): 0.594, 1.378, 0.502, 0.487, 0.476, 0.483, 0.473, 0.474, 0.475, 0.475, 0.474, 0.474, 0.578, 1.45
+To convert JSON string of size 975068 into JNode took 49.997 +/- 9.209 ms over 14 trials
+Load times (ms): 57, 52, 41, 42, 61, 40, 56, 41, 64, 41, 44, 65, 40, 52
+Compiling query "@[@[:].z =~ `(?i)[a-z]{5}`]" into took 0.003 +/- 0.004 ms over 14 trials
+To run pre-compiled query "@[@[:].z =~ `(?i)[a-z]{5}`]" on JNode from JSON of size 975068 into took 0.594 +/- 0.238 ms over 14 trials
+Query times (ms): 0.677, 1.167, 0.502, 0.488, 0.477, 0.479, 0.481, 0.479, 1.158, 0.485, 0.486, 0.475, 0.473, 0.484
 Preview of result: [{"A": "\n]o1VQ5t6g", "a": 4710024278, "b": 3268860721, "B": "g4Y7+ew^.v", "C": "<E_7XL7YS`", "c": 4921465277, "d": 9420665097, "D": "Q&S>NK<OOn", "E": "M?6Ll1W\nFM", "e": 4146283970, "f": 8384193493, "F": "z[jPvslL\tc", "g": 1578133296, "G": "m'M4h,`|Wk", "H": {"a": 5184250383, "b": 5337791147, "c"
 ...