Modify time-series limits

This fits with internal limits, and it now more or less matches the limits for assets and data-sets.
cognitedata · Aug 31, 2020 · a0a83e4 · a0a83e4
1 parent 528d066
commit a0a83e4
Show file tree

Hide file tree

Showing 2 changed files with 11 additions and 31 deletions.
diff --git a/Cognite.Extensions/Sanitation.cs b/Cognite.Extensions/Sanitation.cs
@@ -21,9 +21,10 @@ public static class Sanitation
         public const int TimeSeriesNameMax = 255;
         public const int TimeSeriesDescriptionMax = 1000;
         public const int TimeSeriesUnitMax = 32;
-        public const int TimeSeriesMetadataMaxPerKey = 32;
-        public const int TimeSeriesMetadataMaxPerValue = 512;
-        public const int TimeSeriesMetadataMaxPairs = 16;
+        public const int TimeSeriesMetadataMaxPerKey = 128;
+        public const int TimeSeriesMetadataMaxPerValue = 10000;
+        public const int TimeSeriesMetadataMaxBytes = 10000;
+        public const int TimeSeriesMetadataMaxPairs = 256;
 
         public const int EventTypeMax = 64;
         public const int EventDescriptionMax = 500;
@@ -138,28 +139,6 @@ public static Dictionary<string, string> SanitizeMetadata(this Dictionary<string
                 .ToDictionarySafe(pair => pair.Item1, pair => pair.Item2);
         }
 
-        /// <summary>
-        /// Sanitize a string, string metadata dictionary by limiting the number of UTF8 bytes per key
-        /// and value, as well as the total number of key, value pairs.
-        /// </summary>
-        /// <param name="data">Metadata to limit</param>
-        /// <param name="maxPerKey">Maximum number of bytes per key</param>
-        /// <param name="maxKeys">Maximum number of keys</param>
-        /// <param name="maxPerValue">Maximum number of bytes per value</param>
-        /// <returns></returns>
-        public static Dictionary<string, string> SanitizeMetadata(this Dictionary<string, string> data,
-            int maxPerKey,
-            int maxKeys,
-            int maxPerValue)
-        {
-            if (data == null || !data.Any()) return data;
-            return data
-                .Where(kvp => kvp.Key != null)
-                .Select(kvp => (kvp.Key.LimitUtf8ByteCount(maxPerKey), kvp.Value.LimitUtf8ByteCount(maxPerValue)))
-                .Take(maxKeys)
-                .ToDictionarySafe(pair => pair.Item1, pair => pair.Item2);
-        }
-
         /// <summary>
         /// Sanitize an AssetCreate so that it can be safely sent to CDF.
         /// Requests may still fail due to conflicts or missing ids.
@@ -195,7 +174,8 @@ public static void Sanitize(this TimeSeriesCreate ts)
             if (ts.AssetId < 1) ts.AssetId = null;
             ts.Description = ts.Description?.Truncate(TimeSeriesDescriptionMax);
             if (ts.DataSetId < 1) ts.DataSetId = null;
-            ts.Metadata = ts.Metadata?.SanitizeMetadata(TimeSeriesMetadataMaxPerKey, TimeSeriesMetadataMaxPairs, TimeSeriesMetadataMaxPerValue);
+            ts.Metadata = ts.Metadata?.SanitizeMetadata(TimeSeriesMetadataMaxPerKey, TimeSeriesMetadataMaxPairs,
+                TimeSeriesMetadataMaxPerValue, TimeSeriesMetadataMaxBytes);
             ts.Unit = ts.Unit?.Truncate(TimeSeriesUnitMax);
             ts.LegacyName = ts.LegacyName?.Truncate(ExternalIdMax);
         }

diff --git a/ExtractorUtils.Test/SanitationTest.cs b/ExtractorUtils.Test/SanitationTest.cs
@@ -55,7 +55,7 @@ public void TestSanitizeTimeSeries()
                 AssetId = -1239,
                 LegacyName = new string('æ', 300),
                 Metadata = Enumerable.Range(0, 100)
-                    .ToDictionary(i => $"key{i.ToString("000")}{new string('æ', 100)}", i => new string('æ', 600)),
+                    .ToDictionary(i => $"key{i.ToString("000")}{new string('æ', 100)}", i => new string('æ', 200)),
                 Name = new string('æ', 300),
                 Unit = new string('æ', 200)
             };
@@ -67,10 +67,10 @@ public void TestSanitizeTimeSeries()
             Assert.Null(ts.DataSetId);
             Assert.Null(ts.AssetId);
             Assert.Equal(new string('æ', 255), ts.LegacyName);
-            Assert.Equal(16, ts.Metadata.Count);
-            // 32-6 = 26, 26/2 = 13, 13+6 = 19.
-            Assert.All(ts.Metadata, kvp => Assert.Equal(19, kvp.Key.Length));
-            Assert.All(ts.Metadata, kvp => Assert.Equal(256, kvp.Value.Length));
+            Assert.Equal(18, ts.Metadata.Count);
+            // 'æ' is 2 bytes, key{i} will be 6 bytes, so 128-6 = 122, 122/2 = 61, 61 + 6 = 67
+            Assert.All(ts.Metadata, kvp => Assert.Equal(67, kvp.Key.Length));
+            Assert.All(ts.Metadata, kvp => Assert.Equal(new string('æ', 200), kvp.Value));
             Assert.Equal(new string('æ', 255), ts.Name);
             Assert.Equal(new string('æ', 32), ts.Unit);
         }