Skip to content

Commit

Permalink
Utilities to insert data points (#17)
Browse files Browse the repository at this point in the history 
* Add EnsureTimeSeries extension method

* Add test utilities

* Add unit tests

* Try to fix test coverage

* Remove report path

* Add test for multiple duplicate resposes

* Add CogniteDestination

* small fix

* Add InsertDataPoints

* Separate in two methods: GetOrCreateTimeSeries and EnsureTimeSeriesExists

* Add missing comments

* Trim data points values

* Add tests

* Add comments
  • Loading branch information
@lprovensi
lprovensi authored Apr 27, 2020
1 parent 2837fe6 commit c7fa572
Show file tree
Hide file tree
Showing 8 changed files with 434 additions and 12 deletions.
108 changes: 108 additions & 0 deletions ExtractorUtils.Test/CogniteTest.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -13,6 +13,7 @@
using Xunit;
using Newtonsoft.Json;
using System.Net.Http.Headers;
using Com.Cognite.V1.Timeseries.Proto;

namespace ExtractorUtils.Test
{
Expand Down Expand Up @@ -343,6 +344,66 @@ public async Task TestEnsureTimeSeries(params string[] ids)
System.IO.File.Delete(path);
}

private static Dictionary<long, List<DataPoint>> _createdDataPoints = new Dictionary<long, List<DataPoint>>();

[Fact]
public async Task TestInsertDataPoints()
{
string path = "test-insert-data-points-config.yml";
string[] lines = { "version: 2",
"logger:",
" console:",
" level: verbose",
"cognite:",
$" project: {_project}",
$" api-key: {_apiKey}",
$" host: {_host}",
" cdf-chunking:",
" data-points: 4",
" data-point-time-series: 2",
" cdf-throttling:",
" data-points: 2" };
System.IO.File.WriteAllLines(path, lines);

var mocks = TestUtilities.GetMockedHttpClientFactory(mockInsertDataPointsAsync);
var mockHttpMessageHandler = mocks.handler;
var mockFactory = mocks.factory;

// Setup services
var services = new ServiceCollection();
services.AddSingleton<IHttpClientFactory>(mockFactory.Object); // inject the mock factory
services.AddConfig<BaseConfig>(path, 2);
services.AddLogger();
services.AddCogniteClient("testApp");
using (var provider = services.BuildServiceProvider()) {
var cogniteDestination = provider.GetRequiredService<CogniteDestination>();

double[] doublePoints = { 0.0, 1.1, 2.2, double.NaN, 3.3, 4.4, double.NaN, 5.5, double.NegativeInfinity };
string[] stringPoints = { "0", null, "1", new string('!', CogniteUtils.StringLengthMax), new string('2', CogniteUtils.StringLengthMax + 1), "3"};

var datapoints = new Dictionary<Identity, IEnumerable<DataPoint>>() {
{ new Identity(1), doublePoints.Select(d => new DataPoint(DateTime.UtcNow, d))},
{ new Identity(2), stringPoints.Select(s => new DataPoint(DateTime.UtcNow, s))},
{ new Identity(3), new DataPoint[] { } }
};
_createdDataPoints.Clear();
await cogniteDestination.InsertDataPointsAsync(
datapoints,
CancellationToken.None
);
Assert.False(_createdDataPoints.ContainsKey(3));
Assert.Equal(6, _createdDataPoints[1].Count());
Assert.Empty(_createdDataPoints[1]
.Where(dp => dp.NumericValue == null || dp.NumericValue == double.NaN || dp.NumericValue == double.NegativeInfinity));
Assert.Equal(5, _createdDataPoints[2].Count());
Assert.Empty(_createdDataPoints[2]
.Where(dp => dp.StringValue == null || dp.StringValue.Length > CogniteUtils.StringLengthMax));

}

System.IO.File.Delete(path);
}

private static Dictionary<string, int> _ensuredTimeSeries = new Dictionary<string, int>();

private static async Task<HttpResponseMessage> mockEnsureTimeSeriesSendAsync(HttpRequestMessage message , CancellationToken token) {
Expand Down Expand Up @@ -448,5 +509,52 @@ private static async Task<HttpResponseMessage> mockEnsureTimeSeriesSendAsync(Htt
return response;
}

private static async Task<HttpResponseMessage> mockInsertDataPointsAsync(HttpRequestMessage message , CancellationToken token) {
var uri = message.RequestUri.ToString();
var responseBody = "{ }";
var statusCode = HttpStatusCode.OK;

Assert.Contains($"{_project}/timeseries/data", uri);

var bytes = await message.Content.ReadAsByteArrayAsync();
var data = DataPointInsertionRequest.Parser.ParseFrom(bytes);
Assert.True(data.Items.Count <= 2); // data-points-time-series chunk size
Assert.True(data.Items
.Select(i => i.DatapointTypeCase == DataPointInsertionItem.DatapointTypeOneofCase.NumericDatapoints ?
i.NumericDatapoints.Datapoints.Count : i.StringDatapoints.Datapoints.Count)
.Sum() <= 4); // data-points chunk size

foreach (var item in data.Items)
{
if (!_createdDataPoints.TryGetValue(item.Id, out List<DataPoint> dps))
{
dps = new List<DataPoint>();
_createdDataPoints.TryAdd(item.Id, dps);
}
if (item.NumericDatapoints != null)
{
foreach (var dp in item.NumericDatapoints.Datapoints)
{
dps.Add(new DataPoint(CogniteTime.FromUnixTimeMilliseconds(dp.Timestamp), dp.Value));
}
}
else if (item.StringDatapoints != null)
{
foreach (var dp in item.StringDatapoints?.Datapoints)
{
dps.Add(new DataPoint(CogniteTime.FromUnixTimeMilliseconds(dp.Timestamp), dp.Value));
}
}
}

var response = new HttpResponseMessage
{
StatusCode = statusCode,
Content = new StringContent(responseBody)
};
response.Content.Headers.ContentType = new MediaTypeHeaderValue("application/json");
response.Headers.Add("x-request-id", "1");
return response;
}
}
}
18 changes: 9 additions & 9 deletions ExtractorUtils.Test/CogniteTimeTest.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -11,7 +11,7 @@ public static class CogniteTimeTest
public static void TestToFormattedString(
long ms, string expected)
{
var time = CogniteTime.FromMilliseconds(ms);
var time = CogniteTime.FromUnixTimeMilliseconds(ms);
Assert.Equal(expected, time.ToISOString());
}

Expand All @@ -30,37 +30,37 @@ public static void TestMaxValueToString()
[Fact]
public static void TestMax()
{
var t1 = CogniteTime.FromMilliseconds(1000);
var t2 = CogniteTime.FromMilliseconds(2000);
var t1 = CogniteTime.FromUnixTimeMilliseconds(1000);
var t2 = CogniteTime.FromUnixTimeMilliseconds(2000);
Assert.Equal(t2, CogniteTime.Max(t1, t2));
}

[Fact]
public static void TestMin()
{
var t1 = CogniteTime.FromMilliseconds(1000);
var t2 = CogniteTime.FromMilliseconds(2000);
var t1 = CogniteTime.FromUnixTimeMilliseconds(1000);
var t2 = CogniteTime.FromUnixTimeMilliseconds(2000);
Assert.Equal(t1, CogniteTime.Min(t1, t2));
}
[Fact]
public static void TestMaxValue()
{
var msMax = DateTime.MaxValue.ToUnixTimeMilliseconds();
var dMax = CogniteTime.FromMilliseconds(msMax);
var dMax = CogniteTime.FromUnixTimeMilliseconds(msMax);
Assert.Equal("9999-12-31 23:59:59.999", dMax.ToISOString());
Assert.Equal(DateTime.MaxValue.Ticks / 10_000, dMax.Ticks / 10_000);
}

[Fact]
public static void TestFromMilliseconds()
{
var d1 = CogniteTime.FromMilliseconds(0);
var d1 = CogniteTime.FromUnixTimeMilliseconds(0);
Assert.Equal(CogniteTime.DateTimeEpoch, d1);

Assert.Throws<ArgumentOutOfRangeException>(() => CogniteTime.FromMilliseconds(-1));
Assert.Throws<ArgumentOutOfRangeException>(() => CogniteTime.FromUnixTimeMilliseconds(-1));

var outOfRangeValue = DateTime.MaxValue.ToUnixTimeMilliseconds() + 1;
Assert.Throws<ArgumentOutOfRangeException>(() => CogniteTime.FromMilliseconds(outOfRangeValue));
Assert.Throws<ArgumentOutOfRangeException>(() => CogniteTime.FromUnixTimeMilliseconds(outOfRangeValue));
}

[Fact]
Expand Down
53 changes: 53 additions & 0 deletions ExtractorUtils.Test/CogniteUtilsTest.cs
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,53 @@
using System;
using System.Linq;
using Xunit;

namespace ExtractorUtils.Test
{
public class CogniteUtilsTest
{
private static readonly DataPoint[] _points = {
new DataPoint(DateTime.UtcNow , 0),
new DataPoint(DateTime.UtcNow, 100), // 1
new DataPoint(DateTime.UtcNow, -100), // 2
new DataPoint(DateTime.UtcNow, -101), // 3
new DataPoint(DateTime.UtcNow, 101), // 4
new DataPoint(DateTime.UtcNow, -1e100), // 5
new DataPoint(DateTime.UtcNow, 1e100), // 6
new DataPoint(DateTime.UtcNow, -1e101), // 7
new DataPoint(DateTime.UtcNow, 1e101), // 8
new DataPoint(DateTime.UtcNow, double.NaN), // 9
new DataPoint(DateTime.UtcNow, double.PositiveInfinity), // 10
new DataPoint(DateTime.UtcNow, double.NegativeInfinity), // 11
};

private static readonly DataPoint[] _stringPoints = {
new DataPoint(DateTime.UtcNow, "Some String Value"),
new DataPoint(DateTime.UtcNow, String.Empty),
new DataPoint(DateTime.UtcNow, new string(Enumerable.Repeat('a', 255).ToArray())),
new DataPoint(DateTime.UtcNow, new string(Enumerable.Repeat('b', 256).ToArray())),
};

[Fact]
public static void TestTrimDoubles()
{
var values = _points.TrimValues().ToList();
Assert.Equal(_points.Length - 3, values.Count()); // except invalid ones
Assert.True(values.All(p => p.NumericValue <= CogniteUtils.NumericValueMax));
Assert.True(values.All(p => p.NumericValue >= CogniteUtils.NumericValueMin));
Assert.True(!values.Any(p => double.IsNaN(p.NumericValue.Value)));
}


[Fact]
public static void TestTrimStrings()
{
var values = _stringPoints.TrimValues().ToList();
Assert.Equal(_stringPoints.Count(), values.Count());
Assert.True(values.All(p => p.StringValue.Length <= CogniteUtils.StringLengthMax));
var payloads = values.Select(v => v.StringValue).ToList();
Assert.True(_stringPoints.Select(v => v.StringValue).All(v => payloads.Any(p => v.StartsWith(p))));
}

}
}
Loading

0 comments on commit c7fa572

Please sign in to comment.