QuantBook Universe Selection (#7587)

* QuantBook Universe Selection

- QuantBook universe selection helper method. Adding new unit tests.
- Universe selection data sets improvements

* QuantBook API renames

Algorithm.CSharp/FundamentalRegressionAlgorithm.cs

@@ -15,12 +15,12 @@
 
 using System;
 using System.Linq;
+using QuantConnect.Data;
+using QuantConnect.Securities;
 using QuantConnect.Interfaces;
-using QuantConnect.Data.Market;
 using System.Collections.Generic;
 using QuantConnect.Data.Fundamental;
 using QuantConnect.Data.UniverseSelection;
-using QuantConnect.Data;
 
 namespace QuantConnect.Algorithm.CSharp
 {
@@ -40,6 +40,9 @@ public override void Initialize()
             SetStartDate(2014, 03, 25);
             SetEndDate(2014, 04, 07);
 
+            // before we add any symbol
+            AssertFundamentalUniverseData();
+
             AddEquity("SPY");
             AddEquity("AAPL");
 
@@ -81,23 +84,28 @@ public override void Initialize()
                     throw new Exception($"Unexpected {ticker} fundamental data");
                 }
             }
+            AssertFundamentalUniverseData();
 
+            AddUniverse(FundamentalSelectionFunction);
+        }
+
+        private void AssertFundamentalUniverseData()
+        {
             // Request historical fundamental data for all symbols
             var history2 = History<Fundamentals>(new TimeSpan(1, 0, 0, 0)).ToList();
             if (history2.Count != 1)
             {
-                throw new Exception($"Unexpected {nameof(Fundamentals)} history count {history.Count}! Expected 1");
+                throw new Exception($"Unexpected {nameof(Fundamentals)} history count {history2.Count}! Expected 1");
             }
-            if (history2[0].Single().Value.Data.Count < 7000)
+            var data = history2[0].Single().Value.Data;
+            if (data.Count < 7000)
             {
-                throw new Exception($"Unexpected {nameof(Fundamentals)} data count {history.Count}! Expected > 7000");
+                throw new Exception($"Unexpected {nameof(Fundamentals)} data count {data.Count}! Expected > 7000");
             }
-            if (history2[0].Single().Value.Data.Any(x => x.GetType() != typeof(Fundamental)))
+            if (data.Any(x => x.GetType() != typeof(Fundamental)))
             {
                 throw new Exception($"Unexpected {nameof(Fundamentals)} data type!");
             }
-
-            AddUniverse(FundamentalSelectionFunction);
         }
 
         // sort the data by daily dollar volume and take the top 'NumberOfSymbolsCoarse'
@@ -165,7 +173,7 @@ public override void OnSecuritiesChanged(SecurityChanges changes)
         /// <summary>
         /// Data Points count of the algorithm history
         /// </summary>
-        public virtual int AlgorithmHistoryDataPoints => 3;
+        public virtual int AlgorithmHistoryDataPoints => 4;
 
         /// <summary>
         /// This is used by the regression test system to indicate what the expected statistics are from running the algorithm

Algorithm.Python/FundamentalRegressionAlgorithm.py

@@ -28,6 +28,9 @@ def Initialize(self):
 
         self.UniverseSettings.Resolution = Resolution.Daily
 
+        # before we add any symbol
+        self.AssertFundamentalUniverseData();
+
         self.AddEquity("SPY")
         self.AddEquity("AAPL")
 
@@ -53,6 +56,14 @@ def Initialize(self):
             if data["value"][0] == 0:
                 raise ValueError(f"Unexpected {data} fundamental data")
 
+        self.AssertFundamentalUniverseData();
+
+        self.AddUniverse(self.SelectionFunction)
+
+        self.changes = None
+        self.numberOfSymbolsFundamental = 2
+
+    def AssertFundamentalUniverseData(self):
         # Request historical fundamental data for all symbols
         history2 = self.History(Fundamentals, TimeSpan(1, 0, 0, 0))
         if len(history2) != 1:
@@ -64,11 +75,6 @@ def Initialize(self):
             if type(fundamental) is not Fundamental:
                 raise ValueError(f"Unexpected Fundamentals data type! {fundamental}")
 
-        self.AddUniverse(self.SelectionFunction)
-
-        self.changes = None
-        self.numberOfSymbolsFundamental = 2
-
     # return a list of three fixed symbol objects
     def SelectionFunction(self, fundamental):
         # sort descending by daily dollar volume

Algorithm/QCAlgorithm.History.cs

@@ -823,24 +823,11 @@ private IEnumerable<Slice> History(IEnumerable<HistoryRequest> requests, DateTim
             return history;
         }
 
-        private List<HistoryRequest> GetFilterestRequests(IEnumerable<HistoryRequest> requests)
+        private IEnumerable<HistoryRequest> GetFilterestRequests(IEnumerable<HistoryRequest> requests)
         {
-            List<HistoryRequest> result = new();
             var sentMessage = false;
-            // filter out any universe securities that may have made it this far
-            Dictionary<Type, HistoryRequest> baseDataCollectionTypes = null;
             foreach (var request in requests.Where(hr => HistoryRequestValid(hr.Symbol)))
             {
-                if (request.DataType.IsAssignableTo(typeof(BaseDataCollection)))
-                {
-                    baseDataCollectionTypes ??= new();
-                    if (!baseDataCollectionTypes.TryAdd(request.DataType, request))
-                    {
-                        // for base data collection types we allow a single history request at the time, these are universe types which return all available data
-                        continue;
-                    }
-                }
-
                 // prevent future requests
                 if (request.EndTimeUtc > UtcTime)
                 {
@@ -851,11 +838,11 @@ private List<HistoryRequest> GetFilterestRequests(IEnumerable<HistoryRequest> re
                         startTimeUtc = request.EndTimeUtc;
                     }
 
-                    result.Add(new HistoryRequest(startTimeUtc, endTimeUtc,
+                    yield return new HistoryRequest(startTimeUtc, endTimeUtc,
                         request.DataType, request.Symbol, request.Resolution, request.ExchangeHours,
                         request.DataTimeZone, request.FillForwardResolution, request.IncludeExtendedMarketHours,
                         request.IsCustomData, request.DataNormalizationMode, request.TickType, request.DataMappingMode,
-                        request.ContractDepthOffset));
+                        request.ContractDepthOffset);
 
                     if (!sentMessage)
                     {
@@ -865,17 +852,15 @@ private List<HistoryRequest> GetFilterestRequests(IEnumerable<HistoryRequest> re
                 }
                 else
                 {
-                    result.Add(request);
+                    yield return request;
                 }
             }
-
-            return result;
         }
 
         /// <summary>
         /// Helper method to create history requests from a date range
         /// </summary>
-        private IEnumerable<HistoryRequest> CreateDateRangeHistoryRequests(IEnumerable<Symbol> symbols, DateTime startAlgoTz, DateTime endAlgoTz,
+        protected IEnumerable<HistoryRequest> CreateDateRangeHistoryRequests(IEnumerable<Symbol> symbols, DateTime startAlgoTz, DateTime endAlgoTz,
             Resolution? resolution = null, bool? fillForward = null, bool? extendedMarketHours = null, DataMappingMode? dataMappingMode = null,
             DataNormalizationMode? dataNormalizationMode = null, int? contractDepthOffset = null)
         {
@@ -886,11 +871,11 @@ private IEnumerable<HistoryRequest> CreateDateRangeHistoryRequests(IEnumerable<S
         /// <summary>
         /// Helper method to create history requests from a date range with custom data type
         /// </summary>
-        private IEnumerable<HistoryRequest> CreateDateRangeHistoryRequests(IEnumerable<Symbol> symbols, Type requestedType, DateTime startAlgoTz, DateTime endAlgoTz,
+        protected IEnumerable<HistoryRequest> CreateDateRangeHistoryRequests(IEnumerable<Symbol> symbols, Type requestedType, DateTime startAlgoTz, DateTime endAlgoTz,
             Resolution? resolution = null, bool? fillForward = null, bool? extendedMarketHours = null, DataMappingMode? dataMappingMode = null,
             DataNormalizationMode? dataNormalizationMode = null, int? contractDepthOffset = null)
         {
-            return symbols.Where(HistoryRequestValid).SelectMany(x =>
+            return GetSymbolsForType(symbols, requestedType).Where(HistoryRequestValid).SelectMany(x =>
             {
                 var requests = new List<HistoryRequest>();
 
@@ -923,7 +908,7 @@ private IEnumerable<HistoryRequest> CreateBarCountHistoryRequests(IEnumerable<Sy
             Resolution? resolution = null, bool? fillForward = null, bool? extendedMarketHours = null, DataMappingMode? dataMappingMode = null,
             DataNormalizationMode? dataNormalizationMode = null, int? contractDepthOffset = null)
         {
-            return symbols.Where(HistoryRequestValid).SelectMany(symbol =>
+            return GetSymbolsForType(symbols, requestedType).Where(HistoryRequestValid).SelectMany(symbol =>
             {
                 var res = GetResolution(symbol, resolution, requestedType);
                 var exchange = GetExchangeHours(symbol, requestedType);
@@ -941,6 +926,16 @@ private IEnumerable<HistoryRequest> CreateBarCountHistoryRequests(IEnumerable<Sy
             });
         }
 
+        private IEnumerable<Symbol> GetSymbolsForType(IEnumerable<Symbol> symbols, Type requestedType)
+        {
+            if (requestedType.IsAssignableTo(typeof(BaseDataCollection)))
+            {
+                var instance = requestedType.GetBaseDataInstance();
+                return new [] { ((BaseDataCollection)instance).UniverseSymbol() };
+            }
+            return symbols;
+        }
+
         private int GetTickTypeOrder(SecurityType securityType, TickType tickType)
         {
             return SubscriptionManager.AvailableDataTypes[securityType].IndexOf(tickType);

Algorithm/QCAlgorithm.Python.cs

@@ -1466,7 +1466,7 @@ private PythonIndicator WrapPythonIndicator(PyObject pyObject)
             return pythonIndicator;
         }
 
-        private PyObject GetDataFrame(IEnumerable<Slice> data, Type dataType = null)
+        protected PyObject GetDataFrame(IEnumerable<Slice> data, Type dataType = null)
         {
             var memoizingEnumerable = data as MemoizingEnumerable<Slice>;
             if (memoizingEnumerable != null)
@@ -1475,7 +1475,20 @@ private PyObject GetDataFrame(IEnumerable<Slice> data, Type dataType = null)
                 // the user will only have access to the final pandas data frame object
                 memoizingEnumerable.Enabled = false;
             }
-            return PandasConverter.GetDataFrame(data, dataType);
+            var history = PandasConverter.GetDataFrame(data, dataType);
+            if(dataType != null && dataType.IsAssignableTo(typeof(BaseDataCollection)))
+            {
+                // clear out the first symbol level since it doesn't make sense, it's the universe generic symbol
+                dynamic dynamic = history;
+                using (Py.GIL())
+                {
+                    if (!dynamic.empty)
+                    {
+                        dynamic.index = dynamic.index.droplevel("symbol");
+                    }
+                }
+            }
+            return history;
         }
     }
 }

Common/Data/Fundamental/Fundamentals.cs

@@ -82,16 +82,13 @@ public override BaseData Clone()
         }
 
         /// <summary>
-        /// Creates the symbol used for coarse fundamental data
+        /// Gets the default resolution for this data and security type
         /// </summary>
-        /// <param name="market">The market</param>
-        /// <returns>A coarse universe symbol for the specified market</returns>
-        public static Symbol CreateUniverseSymbol(string market)
+        /// <remarks>This is a method and not a property so that python
+        /// custom data types can override it</remarks>
+        public override Resolution DefaultResolution()
         {
-            market = market.ToLowerInvariant();
-            var ticker = $"qc-universe-fundamental-{market}-{Guid.NewGuid()}";
-            var sid = SecurityIdentifier.GenerateEquity(SecurityIdentifier.DefaultDate, ticker, market);
-            return new Symbol(sid, ticker);
+            return Resolution.Daily;
         }
     }
 }

Common/Data/Market/DataDictionary.cs

@@ -62,7 +62,6 @@ public DataDictionary(DateTime time)
         /// <summary>
         /// Gets or sets the time associated with this collection of data
         /// </summary>
-        [Obsolete("The DataDictionary<T> Time property is now obsolete. All algorithms should use algorithm.Time instead.")]
         public DateTime Time { get; set; }
 
         /// <summary>
@@ -300,4 +299,4 @@ public static void Add<T>(this DataDictionary<T> dictionary, T data)
             dictionary.Add(data.Symbol, data);
         }
     }
-}
+}

Common/Data/Slice.cs

@@ -358,7 +358,7 @@ public dynamic Get(Type type)
         /// Gets the data of the specified type.
         /// </summary>
         /// <remarks>Supports both C# and Python use cases</remarks>
-        protected static dynamic GetImpl(Type type, Slice instance)
+        protected dynamic GetImpl(Type type, Slice instance)
         {
             if (instance._dataByType == null)
             {
@@ -374,6 +374,7 @@ protected static dynamic GetImpl(Type type, Slice instance)
                 {
                     var dataDictionaryCache = GenericDataDictionary.Get(type, isPythonData: false);
                     dictionary = Activator.CreateInstance(dataDictionaryCache.GenericType);
+                    ((dynamic)dictionary).Time = Time;
 
                     foreach (var data in instance.Ticks)
                     {
@@ -430,6 +431,7 @@ protected static dynamic GetImpl(Type type, Slice instance)
 
                     var dataDictionaryCache = GenericDataDictionary.Get(type, isPythonData);
                     dictionary = Activator.CreateInstance(dataDictionaryCache.GenericType);
+                    ((dynamic)dictionary).Time = Time;
 
                     foreach (var data in instance._data.Value.Values)
                     {

Common/Data/UniverseSelection/BaseDataCollection.cs

@@ -112,6 +112,18 @@ public BaseDataCollection(DateTime time, DateTime endTime, Symbol symbol, List<B
             }
         }
 
+        /// <summary>
+        /// Creates the universe symbol
+        /// </summary>
+        /// <returns></returns>
+        public virtual Symbol UniverseSymbol()
+        {
+            var market = QuantConnect.Market.USA;
+            var ticker = $"universe-{GetType().Name}-{market}-{Guid.NewGuid()}";
+            var sid = SecurityIdentifier.GenerateEquity(SecurityIdentifier.DefaultDate, ticker, market);
+            return new Symbol(sid, ticker);
+        }
+
         /// <summary>
         /// Indicates whether this contains data that should be stored in the security cache
         /// </summary>

Common/Data/UniverseSelection/CoarseFundamentalUniverse.cs

@@ -39,7 +39,7 @@ public class CoarseFundamentalUniverse : Universe
         /// <param name="universeSettings">The settings used for new subscriptions generated by this universe</param>
         /// <param name="selector">Returns the symbols that should be included in the universe</param>
         public CoarseFundamentalUniverse(UniverseSettings universeSettings, Func<IEnumerable<CoarseFundamental>, IEnumerable<Symbol>> selector)
-            : base(CreateConfiguration(Fundamental.Fundamentals.CreateUniverseSymbol(QuantConnect.Market.USA)))
+            : base(CreateConfiguration(FundamentalUniverse.SymbolFactory.UniverseSymbol()))
         {
             _universeSettings = universeSettings;
             _selector = selector;
@@ -51,7 +51,7 @@ public CoarseFundamentalUniverse(UniverseSettings universeSettings, Func<IEnumer
         /// <param name="universeSettings">The settings used for new subscriptions generated by this universe</param>
         /// <param name="selector">Returns the symbols that should be included in the universe</param>
         public CoarseFundamentalUniverse(UniverseSettings universeSettings, PyObject selector)
-            : this(Fundamental.Fundamentals.CreateUniverseSymbol(QuantConnect.Market.USA), universeSettings, selector)
+            : this(FundamentalUniverse.SymbolFactory.UniverseSymbol(), universeSettings, selector)
         {
         }
 

Common/Data/UniverseSelection/FineFundamentalUniverse.cs

@@ -38,7 +38,7 @@ public class FineFundamentalUniverse : Universe
         /// <param name="universeSettings">The settings used for new subscriptions generated by this universe</param>
         /// <param name="selector">Returns the symbols that should be included in the universe</param>
         public FineFundamentalUniverse(UniverseSettings universeSettings, Func<IEnumerable<FineFundamental>, IEnumerable<Symbol>> selector)
-            : base(CreateConfiguration(Fundamental.Fundamentals.CreateUniverseSymbol(QuantConnect.Market.USA)))
+            : base(CreateConfiguration(FundamentalUniverse.SymbolFactory.UniverseSymbol()))
         {
             UniverseSettings = universeSettings;
             _selector = selector;

Common/Data/UniverseSelection/FundamentalUniverse.cs

@@ -25,6 +25,8 @@ namespace QuantConnect.Data.UniverseSelection
     /// </summary>
     public class FundamentalUniverse : Universe
     {
+        internal static readonly Fundamental.Fundamentals SymbolFactory = new();
+
         private readonly UniverseSettings _universeSettings;
         private readonly Func<IEnumerable<Fundamental.Fundamental>, IEnumerable<Symbol>> _selector;
 
@@ -39,7 +41,7 @@ public class FundamentalUniverse : Universe
         /// <param name="universeSettings">The settings used for new subscriptions generated by this universe</param>
         /// <param name="selector">Returns the symbols that should be included in the universe</param>
         public FundamentalUniverse(UniverseSettings universeSettings, Func<IEnumerable<Fundamental.Fundamental>, IEnumerable<Symbol>> selector)
-            : base(CreateConfiguration(Fundamental.Fundamentals.CreateUniverseSymbol(QuantConnect.Market.USA)))
+            : base(CreateConfiguration(SymbolFactory.UniverseSymbol()))
         {
             _universeSettings = universeSettings;
             _selector = selector;
@@ -51,7 +53,7 @@ public FundamentalUniverse(UniverseSettings universeSettings, Func<IEnumerable<F
         /// <param name="universeSettings">The settings used for new subscriptions generated by this universe</param>
         /// <param name="selector">Returns the symbols that should be included in the universe</param>
         public FundamentalUniverse(UniverseSettings universeSettings, PyObject selector)
-            : this(Fundamental.Fundamentals.CreateUniverseSymbol(QuantConnect.Market.USA), universeSettings, selector)
+            : this(SymbolFactory.UniverseSymbol(), universeSettings, selector)
         {
         }
 

Engine/DataFeeds/BaseDataCollectionAggregatorReader.cs

@@ -71,7 +71,7 @@ public override IEnumerable<BaseData> Read(SubscriptionDataSource source)
                     {
                         _collection = (BaseDataCollection)Activator.CreateInstance(_collectionType);
                         _collection.Time = point.Time;
-                        _collection.Symbol = point.Symbol;
+                        _collection.Symbol = Config.Symbol;
                         _collection.EndTime = point.EndTime;
                     }
                     // aggregate the data points