diff --git a/.github/workflows/CI_build.yml b/.github/workflows/CI_build.yml index 901b14a..f4103e4 100644 --- a/.github/workflows/CI_build.yml +++ b/.github/workflows/CI_build.yml @@ -4,7 +4,7 @@ on: [push, pull_request] jobs: build: - runs-on: windows-2022 + runs-on: windows-2019 strategy: max-parallel: 4 matrix: @@ -16,7 +16,7 @@ jobs: uses: actions/checkout@v3 - name: Add msbuild to PATH - uses: microsoft/setup-msbuild@v1.1.3 + uses: microsoft/setup-msbuild@v1.0.3 - name: MSBuild of solution run: msbuild JsonToolsNppPlugin/JsonToolsNppPlugin.sln /p:configuration="${{ matrix.build_configuration }}" /p:platform="${{ matrix.build_platform }}" /m /verbosity:minimal diff --git a/CHANGELOG.md b/CHANGELOG.md index 8766720..c2364fb 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -51,12 +51,13 @@ and this project adheres to [Semantic Versioning](http://semver.org/). - Improve how well the caret tracks the node selected in the query tree, after a query that selects a subset of nodes. The iterables have their line number set to 0. - Get rid of __ALL__ dinging sounds from the forms, including the `TreeView` control in the TreeViewer. -## [3.7.3.1] (unreleased) - 2022-MM-DD +## [3.7.3.3] - 2022-10-24 ### Fixed 1. Changed queries produced by [find/replace form](/docs/README.md#find-and-replace-form) to make it more robust and easy to use. 2. Eliminated potential errors when using the [remove files](/docs/README.md#clearing-selected-files) button on the JSON from files and APIs form. +3. Resolve [Issue #17](https://github.com/molsonkiko/JsonToolsNppPlugin/issues/17) with parsing of floating-point numbers in cultures where the decimal separator is `,` and not the `.` used in the USA. ## [3.7.2.1] - 2022-10-20 diff --git a/JsonToolsNppPlugin/JSONTools/JNode.cs b/JsonToolsNppPlugin/JSONTools/JNode.cs index 62a6c87..aaffd39 100644 --- a/JsonToolsNppPlugin/JSONTools/JNode.cs +++ b/JsonToolsNppPlugin/JSONTools/JNode.cs @@ -6,6 +6,7 @@ using System.Text; using System.Text.RegularExpressions; using System.Collections.Generic; // for dictionary, list +using System.Globalization; namespace JSON_Tools.JSON_Tools { @@ -219,6 +220,13 @@ public JNode() ['\f'] = "\\f", }; + // in some places like Germany they use ',' as the normal decimal separator. + // need to override this to ensure that we parse JSON correctly + public static readonly NumberFormatInfo DOT_DECIMAL_SEP = new NumberFormatInfo + { + NumberDecimalSeparator = "." + }; + private const string HEX_CHARS = "0123456789abcdef"; /// /// Return the hexadecimal string representation of an integer @@ -299,10 +307,10 @@ public virtual string ToString(bool sort_keys = true, string key_value_sep = ": if (v == Math.Round(v)) { // add ending ".0" to distinguish doubles equal to integers from actual integers - return v.ToString() + ".0"; + return v.ToString(DOT_DECIMAL_SEP) + ".0"; } - return v.ToString(); + return v.ToString(DOT_DECIMAL_SEP); } case Dtype.INT: return Convert.ToInt64(value).ToString(); case Dtype.NULL: return "null"; diff --git a/JsonToolsNppPlugin/JSONTools/JsonParser.cs b/JsonToolsNppPlugin/JSONTools/JsonParser.cs index 129c583..cab4f9b 100644 --- a/JsonToolsNppPlugin/JSONTools/JsonParser.cs +++ b/JsonToolsNppPlugin/JSONTools/JsonParser.cs @@ -3,6 +3,7 @@ */ using System; using System.Collections.Generic; +using System.Globalization; using System.Text; using System.Text.RegularExpressions; using JSON_Tools.Utils; @@ -455,7 +456,7 @@ public JNode ParseNumber(string q) else if (c == '/') { // fractions are part of the JSON language specification - double numer = double.Parse(sb.ToString()); + double numer = double.Parse(sb.ToString(), JNode.DOT_DECIMAL_SEP); JNode denom_node; ii++; denom_node = ParseNumber(q); @@ -471,7 +472,7 @@ public JNode ParseNumber(string q) { return new JNode(long.Parse(sb.ToString()), Dtype.INT, line_num); } - return new JNode(double.Parse(sb.ToString()), Dtype.FLOAT, line_num); + return new JNode(double.Parse(sb.ToString(), JNode.DOT_DECIMAL_SEP), Dtype.FLOAT, line_num); } /// diff --git a/JsonToolsNppPlugin/JSONTools/RemesPathFunctions.cs b/JsonToolsNppPlugin/JSONTools/RemesPathFunctions.cs index c6094ca..ac52fdd 100644 --- a/JsonToolsNppPlugin/JSONTools/RemesPathFunctions.cs +++ b/JsonToolsNppPlugin/JSONTools/RemesPathFunctions.cs @@ -1476,7 +1476,7 @@ public static JNode ToFloat(JNode[] args) JNode val = args[0]; if (val.type == Dtype.STR) { - return new JNode(double.Parse((string)val.value), Dtype.FLOAT, 0); + return new JNode(double.Parse((string)val.value, JNode.DOT_DECIMAL_SEP), Dtype.FLOAT, 0); } return new JNode(Convert.ToDouble(val.value), Dtype.FLOAT, 0); } diff --git a/JsonToolsNppPlugin/JSONTools/RemesPathLexer.cs b/JsonToolsNppPlugin/JSONTools/RemesPathLexer.cs index 108ed19..770562c 100644 --- a/JsonToolsNppPlugin/JSONTools/RemesPathLexer.cs +++ b/JsonToolsNppPlugin/JSONTools/RemesPathLexer.cs @@ -135,7 +135,7 @@ public JNode ParseNumber(string q) { return new JNode(long.Parse(sb.ToString()), Dtype.INT, 0); } - return new JNode(double.Parse(sb.ToString()), Dtype.FLOAT, 0); + return new JNode(double.Parse(sb.ToString(), JNode.DOT_DECIMAL_SEP), Dtype.FLOAT, 0); } public JNode ParseQuotedString(string q) diff --git a/JsonToolsNppPlugin/JSONTools/YamlDumper.cs b/JsonToolsNppPlugin/JSONTools/YamlDumper.cs index 7013fdf..d1b7f59 100644 --- a/JsonToolsNppPlugin/JSONTools/YamlDumper.cs +++ b/JsonToolsNppPlugin/JSONTools/YamlDumper.cs @@ -98,7 +98,14 @@ private string EscapeBackslashKey(string s) private string YamlKeyRepr(string k) { - if (double.TryParse(k, out double d)) + bool is_float_str = false; + try + { + double.Parse(k, JNode.DOT_DECIMAL_SEP); + is_float_str = true; + } + catch { } + if (is_float_str) { // k is a string representing a number; we need to enquote it so that // a YAML parser will recognize that it is not actually a number. @@ -115,46 +122,43 @@ private string YamlKeyRepr(string k) private string YamlValRepr(JNode v) { - if (v.type == Dtype.NULL) + if (v.value == null) { return "null"; } - object val = v.value; - string strv = val.ToString(); - if (v.type == Dtype.INT || v.type == Dtype.BOOL) + string strv = v.value.ToString(); + if (v.type == Dtype.STR) { - return strv; + try + { + _ = double.Parse(strv, JNode.DOT_DECIMAL_SEP); + return $"'{strv}'"; + // enquote stringified numbers to avoid confusion with actual numbers + } + catch { } + if (StartsOrEndsWith(strv, " ")) + { + return $"\"{strv}\""; + } + Regex backslash = new Regex("([\\\\:\"'\r\t\n\f\b])"); + if (backslash.IsMatch(strv)) + { + return EscapeBackslash(strv); + } } - if (double.TryParse(strv, out double d)) + else if (v.type == Dtype.FLOAT) { // k is a number + double d = (double)v.value; if (d == NanInf.inf) return ".inf"; if (d == NanInf.neginf) return "-.inf"; if (double.IsNaN(d)) { return ".nan"; } - if (v.type == Dtype.STR) - { - // enquote numstrings to prevent confusion - return "'" + strv + "'"; - } - if (v.type == Dtype.FLOAT && double.Parse(strv) % 1 == 0) - { - // ensure that floats equal to ints are rendered as floats - return strv + ".0"; - } - return strv; - } - if (StartsOrEndsWith(strv, " ")) - { - return '"' + strv + '"'; - } - Regex backslash = new Regex("([\\\\:\"'\r\t\n\f\b])"); - if (backslash.IsMatch(strv)) - { - //Console.WriteLine("has backslash"); - return EscapeBackslash(strv); + if (d % 1 == 0) + return $"{d}.0"; // add .0 at end of floats that are equal to ints + return d.ToString(JNode.DOT_DECIMAL_SEP); } return strv; } diff --git a/JsonToolsNppPlugin/JsonToolsNppPlugin.csproj b/JsonToolsNppPlugin/JsonToolsNppPlugin.csproj index 4f26b8f..c0cfb4a 100644 --- a/JsonToolsNppPlugin/JsonToolsNppPlugin.csproj +++ b/JsonToolsNppPlugin/JsonToolsNppPlugin.csproj @@ -10,7 +10,7 @@ Properties NppPluginNET JsonTools - v4.8 + v4.0 3.5 {EB8FC3A3-93E8-457B-B281-FAFA5119611A} diff --git a/JsonToolsNppPlugin/Properties/AssemblyInfo.cs b/JsonToolsNppPlugin/Properties/AssemblyInfo.cs index 5578cf0..2d0f71d 100644 --- a/JsonToolsNppPlugin/Properties/AssemblyInfo.cs +++ b/JsonToolsNppPlugin/Properties/AssemblyInfo.cs @@ -29,5 +29,5 @@ // Build Number // Revision // -[assembly: AssemblyVersion("3.7.3.1")] -[assembly: AssemblyFileVersion("3.7.3.1")] +[assembly: AssemblyVersion("3.7.3.3")] +[assembly: AssemblyFileVersion("3.7.3.3")] diff --git a/JsonToolsNppPlugin/Release_x64.zip b/JsonToolsNppPlugin/Release_x64.zip index fb36e70..d502f23 100644 Binary files a/JsonToolsNppPlugin/Release_x64.zip and b/JsonToolsNppPlugin/Release_x64.zip differ diff --git a/JsonToolsNppPlugin/Release_x86.zip b/JsonToolsNppPlugin/Release_x86.zip index a5ce7ae..9ddbe5b 100644 Binary files a/JsonToolsNppPlugin/Release_x86.zip and b/JsonToolsNppPlugin/Release_x86.zip differ diff --git a/JsonToolsNppPlugin/Tests/Benchmarker.cs b/JsonToolsNppPlugin/Tests/Benchmarker.cs index 8a07f30..50cf3fd 100644 --- a/JsonToolsNppPlugin/Tests/Benchmarker.cs +++ b/JsonToolsNppPlugin/Tests/Benchmarker.cs @@ -125,7 +125,7 @@ public static void Benchmark(string query, string fname, int num_trials = 8) var query_times_str = new string[query_times.Length]; for (int ii = 0; ii < query_times.Length; ii++) { - query_times_str[ii] = Math.Round(query_times[ii] / 1e4, 3).ToString(); + query_times_str[ii] = Math.Round(query_times[ii] / 1e4, 3).ToString(JNode.DOT_DECIMAL_SEP); } Npp.AddLine($"Query times (ms): {String.Join(", ", query_times_str)}"); string result_preview = result.ToString().Slice(":300") + "\n..."; diff --git a/JsonToolsNppPlugin/Tests/JsonGrepperTests.cs b/JsonToolsNppPlugin/Tests/JsonGrepperTests.cs index 1cab6dd..f00bd5c 100644 --- a/JsonToolsNppPlugin/Tests/JsonGrepperTests.cs +++ b/JsonToolsNppPlugin/Tests/JsonGrepperTests.cs @@ -1,5 +1,6 @@ using System; using System.Collections.Generic; +using System.Globalization; using System.IO; using JSON_Tools.JSON_Tools; using JSON_Tools.Utils; diff --git a/most recent errors.txt b/most recent errors.txt index 59f5b6d..2bb56de 100644 --- a/most recent errors.txt +++ b/most recent errors.txt @@ -170,11 +170,11 @@ Performance tests for JsonParser and RemesPath (arithmetic) Preview of json: [{"A": "Ky'c^g#~)0", "a": 1850111954, "b": 9318359041, "B": "Oyi:/ xxe2", "C": "sKCSa_^7Gg", "c": 7974777124, "d": 2670309238, "D": "0d_K)HmX!.", "E": ".uM*Z{0EJ_", "e": 6958410336, "f": 8050244728, "F": "1%SG_A!xB\t", "g": 3799657125, "G": "il1^k\\\nat*", "H": {"a": 6079042826, "b": 7292804611, "c" ... -To convert JSON string of size 975068 into JNode took 50.155 +/- 9.712 ms over 14 trials -Load times (ms): 42, 47, 45, 40, 54, 46, 58, 60, 77, 46, 39, 51, 47, 44 -Compiling query "@[@[:].a * @[:].q < @[:].e]" into took 0.007 +/- 0.011 ms over 14 trials -To run pre-compiled query "@[@[:].a * @[:].q < @[:].e]" on JNode from JSON of size 975068 into took 0.344 +/- 0.148 ms over 14 trials -Query times (ms): 0.418, 0.324, 0.435, 0.447, 0.331, 0.755, 0.231, 0.227, 0.226, 0.241, 0.494, 0.225, 0.236, 0.225 +To convert JSON string of size 975068 into JNode took 53.747 +/- 7.352 ms over 14 trials +Load times (ms): 54, 54, 46, 53, 46, 50, 53, 51, 63, 50, 47, 50, 75, 53 +Compiling query "@[@[:].a * @[:].q < @[:].e]" into took 0.005 +/- 0.005 ms over 14 trials +To run pre-compiled query "@[@[:].a * @[:].q < @[:].e]" on JNode from JSON of size 975068 into took 1.169 +/- 2.671 ms over 14 trials +Query times (ms): 0.451, 1.006, 0.244, 10.753, 0.428, 0.407, 0.292, 0.223, 1.001, 0.249, 0.231, 0.229, 0.222, 0.628 Preview of result: [{"A": "Ky'c^g#~)0", "a": 1850111954, "b": 9318359041, "B": "Oyi:/ xxe2", "C": "sKCSa_^7Gg", "c": 7974777124, "d": 2670309238, "D": "0d_K)HmX!.", "E": ".uM*Z{0EJ_", "e": 6958410336, "f": 8050244728, "F": "1%SG_A!xB\t", "g": 3799657125, "G": "il1^k\\\nat*", "H": {"a": 6079042826, "b": 7292804611, "c" ... ========================= @@ -183,10 +183,10 @@ Performance tests for JsonParser and RemesPath (string operations) Preview of json: [{"A": "Ky'c^g#~)0", "a": 1850111954, "b": 9318359041, "B": "Oyi:/ xxe2", "C": "sKCSa_^7Gg", "c": 7974777124, "d": 2670309238, "D": "0d_K)HmX!.", "E": ".uM*Z{0EJ_", "e": 6958410336, "f": 8050244728, "F": "1%SG_A!xB\t", "g": 3799657125, "G": "il1^k\\\nat*", "H": {"a": 6079042826, "b": 7292804611, "c" ... -To convert JSON string of size 975068 into JNode took 51.64 +/- 9.921 ms over 14 trials -Load times (ms): 63, 70, 39, 41, 61, 39, 42, 44, 57, 49, 56, 59, 39, 55 -Compiling query "@[@[:].z =~ `(?i)[a-z]{5}`]" into took 0.006 +/- 0.016 ms over 14 trials -To run pre-compiled query "@[@[:].z =~ `(?i)[a-z]{5}`]" on JNode from JSON of size 975068 into took 0.628 +/- 0.323 ms over 14 trials -Query times (ms): 0.594, 1.378, 0.502, 0.487, 0.476, 0.483, 0.473, 0.474, 0.475, 0.475, 0.474, 0.474, 0.578, 1.45 +To convert JSON string of size 975068 into JNode took 49.997 +/- 9.209 ms over 14 trials +Load times (ms): 57, 52, 41, 42, 61, 40, 56, 41, 64, 41, 44, 65, 40, 52 +Compiling query "@[@[:].z =~ `(?i)[a-z]{5}`]" into took 0.003 +/- 0.004 ms over 14 trials +To run pre-compiled query "@[@[:].z =~ `(?i)[a-z]{5}`]" on JNode from JSON of size 975068 into took 0.594 +/- 0.238 ms over 14 trials +Query times (ms): 0.677, 1.167, 0.502, 0.488, 0.477, 0.479, 0.481, 0.479, 1.158, 0.485, 0.486, 0.475, 0.473, 0.484 Preview of result: [{"A": "\n]o1VQ5t6g", "a": 4710024278, "b": 3268860721, "B": "g4Y7+ew^.v", "C": "NK