Skip to content

Commit

Permalink
Infer data type from configs in bar count history requests (#8344)
Browse files Browse the repository at this point in the history 
* Infer data type from configs in bar count history requests

* Infer data type from configs in bar count history requests

* Minor changes

* Minor fix

* Cleanup
  • Loading branch information
@jhonabreul
jhonabreul authored Sep 26, 2024
1 parent a1bb907 commit ad5aa26
Show file tree
Hide file tree
Showing 2 changed files with 185 additions and 70 deletions.
105 changes: 72 additions & 33 deletions Algorithm/QCAlgorithm.History.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -177,7 +177,7 @@ private bool TryGetWarmupHistoryStartTime(out DateTime result)
var startTimeUtc = CreateBarCountHistoryRequests(symbols, _warmupBarCount.Value, Settings.WarmupResolution)
.DefaultIfEmpty()
.Min(request => request == null ? default : request.StartTimeUtc);
if(startTimeUtc != default)
if (startTimeUtc != default)
{
result = startTimeUtc.ConvertFromUtc(TimeZone);
return true;
Expand Down Expand Up @@ -353,7 +353,7 @@ public IEnumerable<BaseDataCollection> History(Universe universe, DateTime start
/// <returns>An enumerable of slice containing the requested historical data</returns>
[DocumentationAttribute(HistoricalData)]
public IEnumerable<DataDictionary<T>> History<T>(TimeSpan span, Resolution? resolution = null, bool? fillForward = null,
bool? extendedMarketHours = null, DataMappingMode? dataMappingMode = null, DataNormalizationMode ? dataNormalizationMode = null,
bool? extendedMarketHours = null, DataMappingMode? dataMappingMode = null, DataNormalizationMode? dataNormalizationMode = null,
int? contractDepthOffset = null)
where T : IBaseData
{
Expand Down Expand Up @@ -518,7 +518,7 @@ public IEnumerable<T> History<T>(Symbol symbol, int periods, Resolution? resolut
{
resolution = GetResolution(symbol, resolution, typeof(T));
CheckPeriodBasedHistoryRequestResolution(new[] { symbol }, resolution, typeof(T));
var requests = CreateBarCountHistoryRequests(new [] { symbol }, typeof(T), periods, resolution, fillForward, extendedMarketHours,
var requests = CreateBarCountHistoryRequests(new[] { symbol }, typeof(T), periods, resolution, fillForward, extendedMarketHours,
dataMappingMode, dataNormalizationMode, contractDepthOffset);
return GetDataTypedHistory<T>(requests, symbol);
}
Expand Down Expand Up @@ -948,9 +948,19 @@ protected IEnumerable<HistoryRequest> CreateDateRangeHistoryRequests(IEnumerable
Resolution? resolution = null, bool? fillForward = null, bool? extendedMarketHours = null, DataMappingMode? dataMappingMode = null,
DataNormalizationMode? dataNormalizationMode = null, int? contractDepthOffset = null)
{
var arrayOfSymbols = symbols.ToArray();
return CreateDateRangeHistoryRequests(symbols, Extensions.GetCustomDataTypeFromSymbols(arrayOfSymbols) ?? typeof(BaseData), startAlgoTz, endAlgoTz, resolution, fillForward, extendedMarketHours,
dataMappingMode, dataNormalizationMode, contractDepthOffset);
// Materialize the symbols to avoid multiple enumeration
var symbolsArray = symbols.ToArray();
return CreateDateRangeHistoryRequests(
symbolsArray,
Extensions.GetCustomDataTypeFromSymbols(symbolsArray),
startAlgoTz,
endAlgoTz,
resolution,
fillForward,
extendedMarketHours,
dataMappingMode,
dataNormalizationMode,
contractDepthOffset);
}

/// <summary>
Expand Down Expand Up @@ -982,8 +992,18 @@ private IEnumerable<HistoryRequest> CreateBarCountHistoryRequests(IEnumerable<Sy
bool? fillForward = null, bool? extendedMarketHours = null, DataMappingMode? dataMappingMode = null,
DataNormalizationMode? dataNormalizationMode = null, int? contractDepthOffset = null)
{
return CreateBarCountHistoryRequests(symbols, typeof(BaseData), periods, resolution, fillForward, extendedMarketHours, dataMappingMode,
dataNormalizationMode, contractDepthOffset);
// Materialize the symbols to avoid multiple enumeration
var symbolsArray = symbols.ToArray();
return CreateBarCountHistoryRequests(
symbolsArray,
Extensions.GetCustomDataTypeFromSymbols(symbolsArray),
periods,
resolution,
fillForward,
extendedMarketHours,
dataMappingMode,
dataNormalizationMode,
contractDepthOffset);
}

/// <summary>
Expand All @@ -995,20 +1015,26 @@ private IEnumerable<HistoryRequest> CreateBarCountHistoryRequests(IEnumerable<Sy
{
return symbols.Where(HistoryRequestValid).SelectMany(symbol =>
{
var res = GetResolution(symbol, resolution, requestedType);
var exchange = GetExchangeHours(symbol, requestedType);
// Match or create configs for the symbol
var configs = GetMatchingSubscriptions(symbol, requestedType, resolution).ToList();
if (configs.Count == 0)
{
return Enumerable.Empty<HistoryRequest>();
}

var config = configs.First();
var start = _historyRequestFactory.GetStartTimeAlgoTz(symbol, periods, res, exchange, config.DataTimeZone, config.Type, extendedMarketHours);
var end = Time;

return configs.Select(config => _historyRequestFactory.CreateHistoryRequest(config, start, end, exchange, res, fillForward,
extendedMarketHours, dataMappingMode, dataNormalizationMode, contractDepthOffset));
return configs.Select(config =>
{
// If no requested type was passed, use the config type to get the resolution (if not provided) and the exchange hours
var type = requestedType ?? config.Type;
var res = GetResolution(symbol, resolution, type);
var exchange = GetExchangeHours(symbol, type);
var start = _historyRequestFactory.GetStartTimeAlgoTz(symbol, periods, res, exchange, config.DataTimeZone,
config.Type, extendedMarketHours);
var end = Time;

return _historyRequestFactory.CreateHistoryRequest(config, start, end, exchange, res, fillForward,
extendedMarketHours, dataMappingMode, dataNormalizationMode, contractDepthOffset);
});
});
}

Expand Down Expand Up @@ -1045,7 +1071,7 @@ private IEnumerable<SubscriptionDataConfig> GetMatchingSubscriptions(Symbol symb

// if we have any user defined subscription configuration we use it, else we use internal ones if any
List<SubscriptionDataConfig> configs = null;
if(userConfig.Count != 0)
if (userConfig.Count != 0)
{
configs = userConfig;
}
Expand Down Expand Up @@ -1078,13 +1104,14 @@ private IEnumerable<SubscriptionDataConfig> GetMatchingSubscriptions(Symbol symb
}
else
{
var entry = MarketHoursDatabase.GetEntry(symbol, new []{ type });
resolution = GetResolution(symbol, resolution, type);

if (!LeanData.IsCommonLeanDataType(type) && !type.IsAbstract)
// If type was specified and not a lean data type and also not abstract, we create a new subscription
if (type != null && !LeanData.IsCommonLeanDataType(type) && !type.IsAbstract)
{
// we already know it's not a common lean data type
var isCustom = Extensions.IsCustomDataType(symbol, type);
var entry = MarketHoursDatabase.GetEntry(symbol, new[] { type });

// we were giving a specific type let's fetch it
return new[] { new SubscriptionDataConfig(
Expand All @@ -1105,19 +1132,26 @@ private IEnumerable<SubscriptionDataConfig> GetMatchingSubscriptions(Symbol symb
return SubscriptionManager
.LookupSubscriptionConfigDataTypes(symbol.SecurityType, resolution.Value, symbol.IsCanonical())
.Where(tuple => SubscriptionDataConfigTypeFilter(type, tuple.Item1))
.Select(x => new SubscriptionDataConfig(
x.Item1,
symbol,
resolution.Value,
entry.DataTimeZone,
entry.ExchangeHours.TimeZone,
UniverseSettings.FillForward,
UniverseSettings.ExtendedMarketHours,
true,
false,
x.Item2,
true,
UniverseSettings.GetUniverseNormalizationModeOrDefault(symbol.SecurityType)))
.Select(x =>
{
var configType = x.Item1;
// Use the config type to get an accurate mhdb entry
var entry = MarketHoursDatabase.GetEntry(symbol, new[] { configType });

return new SubscriptionDataConfig(
configType,
symbol,
resolution.Value,
entry.DataTimeZone,
entry.ExchangeHours.TimeZone,
UniverseSettings.FillForward,
UniverseSettings.ExtendedMarketHours,
true,
false,
x.Item2,
true,
UniverseSettings.GetUniverseNormalizationModeOrDefault(symbol.SecurityType));
})
// lets make sure to respect the order of the data types, if used on a history request will affect outcome when using pushthrough for example
.OrderByDescending(config => GetTickTypeOrder(config.SecurityType, config.TickType));
}
Expand All @@ -1130,6 +1164,11 @@ private IEnumerable<SubscriptionDataConfig> GetMatchingSubscriptions(Symbol symb
/// This is useful to filter OpenInterest by default from history requests unless it's explicitly requested</remarks>
private bool SubscriptionDataConfigTypeFilter(Type targetType, Type configType)
{
if (targetType == null)
{
return configType != typeof(OpenInterest);
}

var targetIsGenericType = targetType == typeof(BaseData);

return targetType.IsAssignableFrom(configType) && (!targetIsGenericType || configType != typeof(OpenInterest));
Expand Down Expand Up @@ -1188,7 +1227,7 @@ private Resolution GetResolution(Symbol symbol, Resolution? resolution, Type typ
}
else
{
if(resolution != null)
if (resolution != null)
{
return resolution.Value;
}
Expand Down
Loading

0 comments on commit ad5aa26

Please sign in to comment.