Skip to content

Commit

Permalink
Functionality to delete data points (#35)
Browse files Browse the repository at this point in the history 
* Functionality to delete data points

* Test time range

* Delete response + tests

* Add missing comments
  • Loading branch information
@lprovensi
lprovensi authored Jun 15, 2020
1 parent eff180a commit f72c612
Show file tree
Hide file tree
Showing 8 changed files with 838 additions and 187 deletions.
189 changes: 2 additions & 187 deletions ExtractorUtils.Test/CogniteTest.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -286,7 +286,7 @@ public async Task TestEnsureTimeSeries(params string[] ids)
" time-series: 2" };
System.IO.File.WriteAllLines(path, lines);

var mocks = TestUtilities.GetMockedHttpClientFactory(mockEnsureTimeSeriesSendAsync);
var mocks = TestUtilities.GetMockedHttpClientFactory(MockEnsureTimeSeriesSendAsync);
var mockHttpMessageHandler = mocks.handler;
var mockFactory = mocks.factory;

Expand Down Expand Up @@ -421,194 +421,10 @@ public async Task TestEnsureAssets(params string[] ids)
System.IO.File.Delete(path);
}

[Fact]
public async Task TestInsertDataPoints()
{
string path = "test-insert-data-points-config.yml";
string[] lines = { "version: 2",
"logger:",
" console:",
" level: verbose",
"cognite:",
$" project: {_project}",
$" api-key: {_apiKey}",
$" host: {_host}",
" cdf-chunking:",
" data-points: 4",
" data-point-time-series: 2",
" cdf-throttling:",
" data-points: 2" };
System.IO.File.WriteAllLines(path, lines);

var mocks = TestUtilities.GetMockedHttpClientFactory(mockInsertDataPointsAsync);
var mockHttpMessageHandler = mocks.handler;
var mockFactory = mocks.factory;

// Setup services
var services = new ServiceCollection();
services.AddSingleton<IHttpClientFactory>(mockFactory.Object); // inject the mock factory
services.AddConfig<BaseConfig>(path, 2);
services.AddLogger();
services.AddCogniteClient("testApp");
using (var provider = services.BuildServiceProvider()) {
var cogniteDestination = provider.GetRequiredService<CogniteDestination>();

double[] doublePoints = { 0.0, 1.1, 2.2, double.NaN, 3.3, 4.4, double.NaN, 5.5, double.NegativeInfinity };
string[] stringPoints = { "0", null, "1", new string('!', CogniteUtils.StringLengthMax), new string('2', CogniteUtils.StringLengthMax + 1), "3"};

var datapoints = new Dictionary<Identity, IEnumerable<Datapoint>>() {
{ new Identity("A"), new Datapoint[] { new Datapoint(DateTime.UtcNow, "1")}},
{ new Identity("A"), new Datapoint[] { new Datapoint(DateTime.UtcNow, "2")}},
{ new Identity(1), doublePoints.Select(d => new Datapoint(DateTime.UtcNow, d))},
{ new Identity(2), stringPoints.Select(s => new Datapoint(DateTime.UtcNow, s))},
{ new Identity(3), new Datapoint[] { } },
{ new Identity(4), new Datapoint[] { new Datapoint(CogniteTime.DateTimeEpoch, 1), new Datapoint(DateTime.MaxValue, 1)}}
};
_createdDataPoints.Clear();
await cogniteDestination.InsertDataPointsAsync(
datapoints,
CancellationToken.None);
Assert.False(_createdDataPoints.ContainsKey(3 + "")); // No data points
Assert.False(_createdDataPoints.ContainsKey(4 + "")); // Invalid timestamps
Assert.Equal(6, _createdDataPoints[1 + ""].Count());
Assert.Equal(2, _createdDataPoints["A"].Count());
Assert.Empty(_createdDataPoints[1 + ""]
.Where(dp => dp.NumericValue == null || dp.NumericValue == double.NaN || dp.NumericValue == double.NegativeInfinity));
Assert.Equal(5, _createdDataPoints[2 + ""].Count());
Assert.Empty(_createdDataPoints[2 + ""]
.Where(dp => dp.StringValue == null || dp.StringValue.Length > CogniteUtils.StringLengthMax));

_createdDataPoints.Clear();
datapoints = new Dictionary<Identity, IEnumerable<Datapoint>>() {
{ new Identity("idMissing1"), new Datapoint[] { new Datapoint(DateTime.UtcNow, "1")}},
{ new Identity("idNumeric1"), new Datapoint[] { new Datapoint(DateTime.UtcNow, 1)}},
{ new Identity("idNumeric2"), new Datapoint[] { new Datapoint(DateTime.UtcNow, 1)}},
{ new Identity(-1), doublePoints.Select(d => new Datapoint(DateTime.UtcNow, d)).Take(2)},
{ new Identity("idMismatchedString1"), new Datapoint[] { new Datapoint(DateTime.UtcNow, 1)}},
{ new Identity("idString1"), new Datapoint[] { new Datapoint(DateTime.UtcNow, "1")}},
{ new Identity("idMismatched2"), new Datapoint[] { new Datapoint(DateTime.UtcNow, "1")}}
};
var errors = await cogniteDestination.InsertDataPointsIgnoreErrorsAsync(
datapoints,
CancellationToken.None);
var comparer = new IdentityComparer();
Assert.Contains(new Identity("idMissing1"), errors.IdsNotFound, comparer);
Assert.Contains(new Identity(-1), errors.IdsNotFound, comparer);
Assert.Contains(new Identity("idMismatchedString1"), errors.IdsWithMismatchedData, comparer);
Assert.Contains(new Identity("idMismatched2"), errors.IdsWithMismatchedData, comparer);
Assert.Single(_createdDataPoints["idNumeric1"]);
Assert.Single(_createdDataPoints["idNumeric2"]);
Assert.Single(_createdDataPoints["idString1"]);
}

System.IO.File.Delete(path);
}



#region mock
private static Dictionary<string, List<Datapoint>> _createdDataPoints = new Dictionary<string, List<Datapoint>>();

private static async Task<HttpResponseMessage> mockInsertDataPointsAsync(HttpRequestMessage message , CancellationToken token) {
var uri = message.RequestUri.ToString();

if (uri.Contains("/timeseries/byids"))
{
return await mockEnsureTimeSeriesSendAsync(message, token);
}
Assert.Contains($"{_project}/timeseries/data", uri);

var responseBody = "{ }";
var statusCode = HttpStatusCode.OK;
var bytes = await message.Content.ReadAsByteArrayAsync();
var data = DataPointInsertionRequest.Parser.ParseFrom(bytes);
Assert.True(data.Items.Count <= 2); // data-points-time-series chunk size
Assert.True(data.Items
.Select(i => i.DatapointTypeCase == DataPointInsertionItem.DatapointTypeOneofCase.NumericDatapoints ?
i.NumericDatapoints.Datapoints.Count : i.StringDatapoints.Datapoints.Count)
.Sum() <= 4); // data-points chunk size

dynamic missingResponse = new ExpandoObject();
missingResponse.error = new ExpandoObject();
missingResponse.error.missing = new List<ExpandoObject>();
missingResponse.error.code = 400;
missingResponse.error.message = "Time series ids not found";

dynamic mismatchedResponse = new ExpandoObject();
mismatchedResponse.error = new ExpandoObject();
mismatchedResponse.error.code = 400;
mismatchedResponse.error.message = "";

foreach (var item in data.Items)
{
if (item.Id < 0 || item.ExternalId.StartsWith("idMissing"))
{
dynamic id = new ExpandoObject();
if (!string.IsNullOrEmpty(item.ExternalId)) id.externalId = item.ExternalId;
else id.id = item.Id;
missingResponse.error.missing.Add(id);
}
else if (item.ExternalId.StartsWith("idMismatched"))
{
if (item.NumericDatapoints != null)
{
mismatchedResponse.error.message = "Expected string value for datapoint";
}
else {
mismatchedResponse.error.message = "Expected numeric value for datapoint";
}
break;
}
}
if (!string.IsNullOrEmpty(mismatchedResponse.error.message))
{
statusCode = HttpStatusCode.BadRequest;
responseBody = JsonConvert.SerializeObject(mismatchedResponse);
}
else if (missingResponse.error.missing.Count > 0) {
statusCode = HttpStatusCode.BadRequest;
responseBody = JsonConvert.SerializeObject(missingResponse);
}
else
{
foreach (var item in data.Items)
{
var sId = string.IsNullOrEmpty(item.ExternalId) ? item.Id + "" : item.ExternalId;
if (!_createdDataPoints.TryGetValue(sId, out List<Datapoint> dps))
{
dps = new List<Datapoint>();
_createdDataPoints.TryAdd(sId, dps);
}
if (item.NumericDatapoints != null)
{
foreach (var dp in item.NumericDatapoints.Datapoints)
{
dps.Add(new Datapoint(CogniteTime.FromUnixTimeMilliseconds(dp.Timestamp), dp.Value));
}
}
else if (item.StringDatapoints != null)
{
foreach (var dp in item.StringDatapoints?.Datapoints)
{
dps.Add(new Datapoint(CogniteTime.FromUnixTimeMilliseconds(dp.Timestamp), dp.Value));
}
}
}
}

var response = new HttpResponseMessage
{
StatusCode = statusCode,
Content = new StringContent(responseBody)
};
response.Content.Headers.ContentType = new MediaTypeHeaderValue("application/json");
response.Headers.Add("x-request-id", "1");
return response;
}

private static Dictionary<string, int> _ensuredTimeSeries = new Dictionary<string, int>();

private static async Task<HttpResponseMessage> mockEnsureTimeSeriesSendAsync(
internal static async Task<HttpResponseMessage> MockEnsureTimeSeriesSendAsync(
HttpRequestMessage message,
CancellationToken token)
{
Expand All @@ -620,7 +436,6 @@ private static async Task<HttpResponseMessage> mockEnsureTimeSeriesSendAsync(
var ids = JsonConvert.DeserializeObject<dynamic>(content);
IEnumerable<dynamic> items = ids.items;


if (uri.Contains("/timeseries/byids"))
{
dynamic missingData = new ExpandoObject();
Expand Down
62 changes: 62 additions & 0 deletions ExtractorUtils.Test/CogniteTimeTest.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -77,5 +77,67 @@ public static void TestNanosecondsSinceEpoch()
Assert.Throws<ArgumentException>(() => CogniteTime.NanosecondsSinceEpoch(invalid2));
}

[Fact]
public static void TestTimeRangeEquality()
{
Assert.Equal(new TimeRange(DateTime.MaxValue, CogniteTime.DateTimeEpoch), TimeRange.Empty);
Assert.True(new TimeRange(DateTime.MaxValue, CogniteTime.DateTimeEpoch) == TimeRange.Empty);
Assert.Equal((new TimeRange(DateTime.MaxValue, CogniteTime.DateTimeEpoch)).GetHashCode(), TimeRange.Empty.GetHashCode());

TimeRange r1 = new TimeRange(new DateTime(2000, 01, 01), new DateTime(2010, 01, 01));
TimeRange r2 = new TimeRange(new DateTime(2005, 01, 01), new DateTime(2010, 01, 01));

Assert.NotEqual(r1, r2);
Assert.True(r1 != r2);
Assert.NotEqual(r1.GetHashCode(), r2.GetHashCode());

r2 = r2.Extend(r1);
Assert.Equal(r1, r2);
Assert.Equal(r1.GetHashCode(), r2.GetHashCode());
}

[Fact]
public static void TestTimeRangeContains()
{
TimeRange r1 = new TimeRange(new DateTime(2000, 01, 01), new DateTime(2010, 01, 01));
DateTime d1 = new DateTime(1990, 01, 01);
DateTime d2 = new DateTime(2005, 01, 01);
DateTime d3 = new DateTime(2020, 01, 01);

Assert.False(r1.Contains(d1));
Assert.True(r1.Contains(d2));
Assert.False(r1.Contains(d3));

Assert.True(r1.Before(d1));
Assert.False(r1.Before(d2));
Assert.False(r1.Before(d3));

Assert.False(r1.After(d1));
Assert.False(r1.After(d2));
Assert.True(r1.After(d3));
}

[Fact]
public static void TestTimeRangeExtend()
{
TimeRange r1 = TimeRange.Empty;
Assert.True(r1.IsEmpty);
DateTime d1 = new DateTime(1990, 01, 01);
DateTime d2 = new DateTime(2005, 01, 01);
DateTime d3 = new DateTime(2010, 01, 01);
DateTime d4 = new DateTime(2020, 01, 01);

var r2 = r1.Extend(d2, d3);
Assert.False(r2.IsEmpty);

var r3 = new TimeRange(d3, d2);
Assert.True(r3.IsEmpty);

var r4 = r3.Extend(r2);
Assert.Equal(r4, r2);

var r5 = r4.Extend(d1, d4);
Assert.Equal(new TimeRange(d1, d4), r5);
}
}
}
Loading

0 comments on commit f72c612

Please sign in to comment.