Skip to content

Commit

Permalink
Revert "Store traded securities' subscriptions in results (#7567)" (#…
Browse files Browse the repository at this point in the history 
…7578)

This reverts commit 145ead7.
  • Loading branch information
@Martin-Molinero
Martin-Molinero authored Nov 16, 2023
1 parent 428d620 commit e983383
Show file tree
Hide file tree
Showing 8 changed files with 24 additions and 410 deletions.
2 changes: 1 addition & 1 deletion Algorithm.Python/QuantConnect.Algorithm.Python.csproj
View file
Original file line number Diff line number Diff line change
Expand Up @@ -56,7 +56,7 @@
<Content Include="CustomPartialFillModelAlgorithm.py" />
<Content Include="AddRiskManagementAlgorithm.py" />
<Content Include="AddUniverseSelectionModelAlgorithm.py" />
<Content Include="IndicatorWithRenkoBarsRegressionAlgorithm.py" />
<Compile Include="IndicatorWithRenkoBarsRegressionAlgorithm.py" />
<Content Include="Alphas\ContingentClaimsAnalysisDefaultPredictionAlpha.py" />
<Content Include="Alphas\GasAndCrudeOilEnergyCorrelationAlpha.py" />
<Content Include="Alphas\GlobalEquityMeanReversionIBSAlpha.py" />
Expand Down
117 changes: 19 additions & 98 deletions Common/Data/DataMonitor.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -16,13 +16,10 @@
using System;
using System.Collections.Generic;
using System.IO;
using System.Linq;
using System.Threading;
using Newtonsoft.Json;
using QuantConnect.Configuration;
using QuantConnect.Data.Serialization;
using QuantConnect.Interfaces;
using QuantConnect.Orders;
using QuantConnect.Util;

namespace QuantConnect.Data
Expand All @@ -34,8 +31,8 @@ public class DataMonitor : IDataMonitor
{
private bool _exited;

private Lazy<TextWriter> _succeededDataRequestsWriter;
private Lazy<TextWriter> _failedDataRequestsWriter;
private TextWriter _succeededDataRequestsWriter;
private TextWriter _failedDataRequestsWriter;

private long _succeededDataRequestsCount;
private long _failedDataRequestsCount;
Expand All @@ -56,34 +53,16 @@ public class DataMonitor : IDataMonitor

private readonly object _threadLock = new();

private SubscriptionManager _subscriptionManager;
private readonly Dictionary<Symbol, List<SubscriptionDataConfig>> _tradedSecuritiesSubscriptions = new();

/// <summary>
/// Time provider instance used to get the current time (for testing purposes)
/// </summary>
protected ITimeProvider TimeProvider { get; set; }

/// <summary>
/// Initializes a new instance of the <see cref="DataMonitor"/> class
/// </summary>
public DataMonitor()
{
TimeProvider = new RealTimeProvider();
_resultsDestinationFolder = Config.Get("results-destination-folder", Directory.GetCurrentDirectory());
_succeededDataRequestsFileName = GetFilePath("succeeded-data-requests.txt");
_failedDataRequestsFileName = GetFilePath("failed-data-requests.txt");
}

/// <summary>
/// Set the <see cref="SubscriptionManager"/> instance to use to select the traded subscriptions
/// </summary>
/// <param name="subscriptionManager">The subscription manager to use</param>
public void SetSubscriptionManager(SubscriptionManager subscriptionManager)
{
_subscriptionManager = subscriptionManager;
}

/// <summary>
/// Event handler for the <see cref="IDataProvider.NewDataRequest"/> event
/// </summary>
Expand All @@ -108,7 +87,7 @@ public void OnNewDataRequest(object sender, DataProviderNewDataRequestEventArgs

if (e.Succeded)
{
WriteLineToFile(_succeededDataRequestsWriter.Value, path, _succeededDataRequestsFileName);
WriteLineToFile(_succeededDataRequestsWriter, path, _succeededDataRequestsFileName);
Interlocked.Increment(ref _succeededDataRequestsCount);
if (isUniverseData)
{
Expand All @@ -117,7 +96,7 @@ public void OnNewDataRequest(object sender, DataProviderNewDataRequestEventArgs
}
else
{
WriteLineToFile(_failedDataRequestsWriter.Value, path, _failedDataRequestsFileName);
WriteLineToFile(_failedDataRequestsWriter, path, _failedDataRequestsFileName);
Interlocked.Increment(ref _failedDataRequestsCount);
if (isUniverseData)
{
Expand All @@ -131,64 +110,26 @@ public void OnNewDataRequest(object sender, DataProviderNewDataRequestEventArgs
}
}

/// <summary>
/// Registers a new order event to select the trade subscriptions
/// </summary>
public void OnOrderEvent(object sender, OrderEvent orderEvent)
{
if (_exited)
{
return;
}

if (!_tradedSecuritiesSubscriptions.ContainsKey(orderEvent.Symbol))
{
_tradedSecuritiesSubscriptions[orderEvent.Symbol] = _subscriptionManager.SubscriptionDataConfigService
.GetSubscriptionDataConfigs(orderEvent.Symbol, includeInternalConfigs: false)
.ToList();
}
}

/// <summary>
/// Terminates the data monitor generating a final report
/// </summary>
public void Exit()
{
if (_exited)
if (_exited || _requestRateCalculationThread == null)
{
return;
}
_exited = true;

if (_requestRateCalculationThread != null)
{
_requestRateCalculationThread.StopSafely(TimeSpan.FromSeconds(5), _cancellationTokenSource);
_requestRateCalculationThread.StopSafely(TimeSpan.FromSeconds(5), _cancellationTokenSource);
_succeededDataRequestsWriter?.Close();
_failedDataRequestsWriter?.Close();

var succeededDataRequestWriterWasCreated = _succeededDataRequestsWriter?.IsValueCreated ?? false;
if (succeededDataRequestWriterWasCreated)
{
_succeededDataRequestsWriter.Value.Close();
}
var failedDataRequestsWriterWasCreated = _failedDataRequestsWriter?.IsValueCreated ?? false;
if (failedDataRequestsWriterWasCreated)
{
_failedDataRequestsWriter.Value.Close();
}

StoreDataMonitorReport(GenerateReport());

if (failedDataRequestsWriterWasCreated)
{
_succeededDataRequestsWriter.Value.DisposeSafely();
}
if (failedDataRequestsWriterWasCreated)
{
_failedDataRequestsWriter.Value.DisposeSafely();
}
_cancellationTokenSource.DisposeSafely();
}
StoreDataMonitorReport(GenerateReport());

StoreTradedSubscriptions();
_succeededDataRequestsWriter.DisposeSafely();
_failedDataRequestsWriter.DisposeSafely();
_cancellationTokenSource.DisposeSafely();
}

public void Dispose()
Expand Down Expand Up @@ -222,8 +163,8 @@ private void Initialize()
return;
}
// we create the files on demand
_succeededDataRequestsWriter = new(() => OpenStream(_succeededDataRequestsFileName));
_failedDataRequestsWriter = new(() => OpenStream(_failedDataRequestsFileName));
_succeededDataRequestsWriter = OpenStream(_succeededDataRequestsFileName);
_failedDataRequestsWriter = OpenStream(_failedDataRequestsFileName);

_cancellationTokenSource = new CancellationTokenSource();

Expand Down Expand Up @@ -268,13 +209,13 @@ private void ComputeFileRequestFrequency()
{
// First time we calculate the request rate.
// We don't have a previous value to compare to so we just store the current value.
_lastRequestRateCalculationTime = TimeProvider.GetUtcNow();
_lastRequestRateCalculationTime = DateTime.UtcNow;
_prevRequestsCount = requestsCount;
return;
}

var requestsCountDelta = requestsCount - _prevRequestsCount;
var now = TimeProvider.GetUtcNow();
var now = DateTime.UtcNow;
var timeDelta = now - _lastRequestRateCalculationTime;

_requestRates.Add(Math.Round(requestsCountDelta / timeDelta.TotalSeconds));
Expand All @@ -286,7 +227,7 @@ private void ComputeFileRequestFrequency()
/// Stores the data monitor report
/// </summary>
/// <param name="report">The data monitor report to be stored<param>
protected void StoreDataMonitorReport(DataMonitorReport report)
private void StoreDataMonitorReport(DataMonitorReport report)
{
if (report == null)
{
Expand All @@ -298,30 +239,10 @@ protected void StoreDataMonitorReport(DataMonitorReport report)
File.WriteAllText(path, data);
}

/// <summary>
/// Stores the traded securities subscription configurations
/// </summary>
protected void StoreTradedSubscriptions()
{
var configs = Enumerable.Empty<SerializedSubscriptionDataConfig>();
if (_subscriptionManager != null)
{
configs = _tradedSecuritiesSubscriptions.Values.Select(x => new SerializedSubscriptionDataConfig(x
// Get the highest resolution config for each symbol
.GroupBy(config => config.Resolution)
.OrderBy(grouping => grouping.Key)
.First()));
}

var path = GetFilePath("traded-securities-subscriptions.json");
var data = JsonConvert.SerializeObject(configs, Formatting.None);
File.WriteAllText(path, data);
}

protected string GetFilePath(string filename)
private string GetFilePath(string filename)
{
var baseFilename = Path.GetFileNameWithoutExtension(filename);
var timestamp = TimeProvider.GetUtcNow().ToStringInvariant("yyyyMMddHHmmssfff");
var timestamp = DateTime.UtcNow.ToStringInvariant("yyyyMMddHHmmssfff");
var extension = Path.GetExtension(filename);
return Path.Combine(_resultsDestinationFolder, $"{baseFilename}-{timestamp}{extension}");
}
Expand Down
133 changes: 0 additions & 133 deletions Common/Data/Serialization/SerializedSubscriptionDataConfig.cs
View file

This file was deleted.

Loading

0 comments on commit e983383

Please sign in to comment.