Async universe selection (#7556)

* Async universe selection

- Add support for async universe selection, which will happen ahead of
  time in the data stack for a performance improvement

* Thrown if using Coarse+Fine Asynchronous Universe selectioon

Algorithm.CSharp/AsynchronousUniverseRegressionAlgorithm.cs

@@ -0,0 +1,33 @@
+/*
+ * QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals.
+ * Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+*/
+
+namespace QuantConnect.Algorithm.CSharp
+{
+    /// <summary>
+    /// Example algorithm using the asynchronous universe selection functionality
+    /// </summary>
+    public class AsynchronousUniverseRegressionAlgorithm : FundamentalRegressionAlgorithm
+    {
+        /// <summary>
+        /// Initialise the data and resolution required, as well as the cash and start-end dates for your algorithm. All algorithms must initialized.
+        /// </summary>
+        public override void Initialize()
+        {
+            base.Initialize();
+
+            UniverseSettings.Asynchronous = true;
+        }
+    }
+}

Algorithm.CSharp/CoarseFineAsyncUniverseRegressionAlgorithm.cs

@@ -0,0 +1,98 @@
+/*
+ * QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals.
+ * Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+*/
+
+using System;
+using System.Linq;
+using QuantConnect.Interfaces;
+using System.Collections.Generic;
+using QuantConnect.Data.Fundamental;
+using QuantConnect.Data.UniverseSelection;
+using QuantConnect.Algorithm.Framework.Selection;
+
+namespace QuantConnect.Algorithm.CSharp
+{
+    /// <summary>
+    /// Regression algorithm asserting that using separate coarse & fine selection with async universe settings is not allowed
+    /// </summary>
+    public class CoarseFineAsyncUniverseRegressionAlgorithm : QCAlgorithm, IRegressionAlgorithmDefinition
+    {
+        /// <summary>
+        /// Initialise the data and resolution required, as well as the cash and start-end dates for your algorithm. All algorithms must initialized.
+        /// </summary>
+        public override void Initialize()
+        {
+            SetStartDate(2013, 10, 07);
+            SetEndDate(2013, 10, 11);
+
+            UniverseSettings.Asynchronous = true;
+
+            var threwException = false;
+            try
+            {
+                AddUniverse(CoarseSelectionFunction, FineSelectionFunction);
+            }
+            catch (ArgumentException)
+            {
+                threwException = true;
+                // expected
+            }
+
+            if (!threwException)
+            {
+                throw new Exception("Expected exception to be thrown for AddUniverse");
+            }
+
+            SetUniverseSelection(new FineFundamentalUniverseSelectionModel(CoarseSelectionFunction, FineSelectionFunction));
+        }
+
+        private IEnumerable<Symbol> CoarseSelectionFunction(IEnumerable<CoarseFundamental> coarse)
+        {
+            return Enumerable.Empty<Symbol>();
+        }
+
+        private IEnumerable<Symbol> FineSelectionFunction(IEnumerable<FineFundamental> fine)
+        {
+            return Enumerable.Empty<Symbol>();
+        }
+
+        /// <summary>
+        /// This is used by the regression test system to indicate if the open source Lean repository has the required data to run this algorithm.
+        /// </summary>
+        public bool CanRunLocally { get; } = true;
+
+        /// <summary>
+        /// This is used by the regression test system to indicate which languages this algorithm is written in.
+        /// </summary>
+        public Language[] Languages { get; } = { Language.CSharp, Language.Python };
+
+        /// <summary>
+        /// Data Points count of all timeslices of algorithm
+        /// </summary>
+        public long DataPoints => 0;
+
+        /// <summary>
+        /// Data Points count of the algorithm history
+        /// </summary>
+        public int AlgorithmHistoryDataPoints => 0;
+
+        /// <summary>
+        /// This is used by the regression test system to indicate what the expected statistics are from running the algorithm
+        /// </summary>
+        public Dictionary<string, string> ExpectedStatistics => new Dictionary<string, string>
+        {
+            {"OrderListHash", "d41d8cd98f00b204e9800998ecf8427e"}
+        };
+    }
+}

Algorithm.Framework/Selection/FineFundamentalUniverseSelectionModel.cs

@@ -1,4 +1,4 @@
-/*
+/*
  * QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals.
  * Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation.
  *
@@ -18,7 +18,6 @@
 using Python.Runtime;
 using QuantConnect.Data.Fundamental;
 using QuantConnect.Data.UniverseSelection;
-using QuantConnect.Securities;
 
 namespace QuantConnect.Algorithm.Framework.Selection
 {
@@ -81,4 +80,4 @@ public override IEnumerable<Symbol> SelectFine(QCAlgorithm algorithm, IEnumerabl
             return _fineSelector(fine);
         }
     }
-}
+}

Algorithm.Framework/Selection/FundamentalUniverseSelectionModel.cs

@@ -118,6 +118,10 @@ public override IEnumerable<Universe> CreateUniverses(QCAlgorithm algorithm)
                 var universe = CreateCoarseFundamentalUniverse(algorithm);
                 if (_filterFineData)
                 {
+                    if (universe.UniverseSettings.Asynchronous.HasValue && universe.UniverseSettings.Asynchronous.Value)
+                    {
+                        throw new ArgumentException("Asynchronous universe setting is not supported for coarse & fine selections, please use the new Fundamental single pass selection");
+                    }
 #pragma warning disable CS0618 // Type or member is obsolete
                     universe = new FineFundamentalFilteredUniverse(universe, fine => SelectFine(algorithm, fine));
 #pragma warning restore CS0618 // Type or member is obsolete

Algorithm.Framework/Selection/FundamentalUniverseSelectionModel.py

@@ -44,6 +44,8 @@ def CreateUniverses(self, algorithm):
         else:
             universe = self.CreateCoarseFundamentalUniverse(algorithm)
             if self.filterFineData:
+                if universe.UniverseSettings.Asynchronous:
+                    raise ValueError("Asynchronous universe setting is not supported for coarse & fine selections, please use the new Fundamental single pass selection")
                 universe = FineFundamentalFilteredUniverse(universe, lambda fine: self.SelectFine(algorithm, fine))
             return [universe]
 

Algorithm.Python/AsynchronousUniverseRegressionAlgorithm.py

@@ -0,0 +1,24 @@
+# QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals.
+# Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from AlgorithmImports import *
+from FundamentalRegressionAlgorithm import FundamentalRegressionAlgorithm
+
+### <summary>
+### Example algorithm using the asynchronous universe selection functionality
+### </summary>
+class AsynchronousUniverseRegressionAlgorithm(FundamentalRegressionAlgorithm):
+
+    def Initialize(self):
+        super().Initialize()
+        self.UniverseSettings.Asynchronous = True

Algorithm.Python/CoarseFineAsyncUniverseRegressionAlgorithm.py

@@ -0,0 +1,46 @@
+# QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals.
+# Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from AlgorithmImports import *
+
+### <summary>
+### Regression algorithm asserting that using separate coarse & fine selection with async universe settings is not allowed
+### </summary>
+class CoarseFineAsyncUniverseRegressionAlgorithm(QCAlgorithm):
+
+    def Initialize(self):
+        '''Initialise the data and resolution required, as well as the cash and start-end dates for your algorithm. All algorithms must initialized.'''
+
+        self.SetStartDate(2013, 10, 7)
+        self.SetEndDate(2013, 10, 11)
+
+        self.UniverseSettings.Asynchronous = True
+
+        threw_exception = False
+        try:
+            self.AddUniverse(self.CoarseSelectionFunction, self.FineSelectionFunction)
+        except:
+            # expected
+            threw_exception = True
+            pass
+
+        if not threw_exception:
+            raise ValueError("Expected exception to be thrown for AddUniverse")
+
+        self.SetUniverseSelection(FineFundamentalUniverseSelectionModel(self.CoarseSelectionFunction, self.FineSelectionFunction))
+
+    def CoarseSelectionFunction(self, coarse):
+        return []
+
+    def FineSelectionFunction(self, fine):
+        return []

Algorithm/QCAlgorithm.cs

@@ -1851,7 +1851,10 @@ public Security AddSecurity(Symbol symbol, Resolution? resolution = null, bool f
                 if (!UniverseManager.ContainsKey(symbol))
                 {
                     var canonicalConfig = configs.First();
-                    var settings = new UniverseSettings(canonicalConfig.Resolution, leverage, fillForward, extendedMarketHours, UniverseSettings.MinimumTimeInUniverse);
+                    var settings = new UniverseSettings(canonicalConfig.Resolution, leverage, fillForward, extendedMarketHours, UniverseSettings.MinimumTimeInUniverse)
+                    {
+                        Asynchronous = UniverseSettings.Asynchronous
+                    };
                     if (symbol.SecurityType.IsOption())
                     {
                         universe = new OptionChainUniverse((Option)security, settings);
@@ -1868,6 +1871,7 @@ public Security AddSecurity(Symbol symbol, Resolution? resolution = null, bool f
                             DataNormalizationMode = dataNormalizationMode ?? UniverseSettings.GetUniverseNormalizationModeOrDefault(symbol.SecurityType),
                             ContractDepthOffset = (int)contractOffset,
                             SubscriptionDataTypes = dataTypes,
+                            Asynchronous = UniverseSettings.Asynchronous
                         };
                         ContinuousContractUniverse.AddConfigurations(SubscriptionManager.SubscriptionDataConfigService, continuousUniverseSettings, security.Symbol);
 

Common/Data/UniverseSelection/BaseDataCollection.cs

@@ -36,7 +36,7 @@ public class BaseDataCollection : BaseData, IEnumerable<BaseData>
         /// <summary>
         /// Gets or sets the contracts selected by the universe
         /// </summary>
-        public IReadOnlyCollection<Symbol> FilteredContracts { get; set; }
+        public HashSet<Symbol> FilteredContracts { get; set; }
 
         /// <summary>
         /// Gets the data list
@@ -80,7 +80,7 @@ public BaseDataCollection(DateTime time, Symbol symbol, IEnumerable<BaseData> da
         /// <param name="data">The data to add to this collection</param>
         /// <param name="underlying">The associated underlying price data if any</param>
         /// <param name="filteredContracts">The contracts selected by the universe</param>
-        public BaseDataCollection(DateTime time, DateTime endTime, Symbol symbol, IEnumerable<BaseData> data = null, BaseData underlying = null, IReadOnlyCollection<Symbol> filteredContracts = null)
+        public BaseDataCollection(DateTime time, DateTime endTime, Symbol symbol, IEnumerable<BaseData> data = null, BaseData underlying = null, HashSet<Symbol> filteredContracts = null)
             : this(time, endTime, symbol, data != null ? data.ToList() : new List<BaseData>(), underlying, filteredContracts)
         {
         }
@@ -94,7 +94,7 @@ public BaseDataCollection(DateTime time, DateTime endTime, Symbol symbol, IEnume
         /// <param name="data">The data to add to this collection</param>
         /// <param name="underlying">The associated underlying price data if any</param>
         /// <param name="filteredContracts">The contracts selected by the universe</param>
-        public BaseDataCollection(DateTime time, DateTime endTime, Symbol symbol, List<BaseData> data, BaseData underlying, IReadOnlyCollection<Symbol> filteredContracts)
+        public BaseDataCollection(DateTime time, DateTime endTime, Symbol symbol, List<BaseData> data, BaseData underlying, HashSet<Symbol> filteredContracts)
         {
             Symbol = symbol;
             Time = time;

Common/Data/UniverseSelection/ContinuousContractUniverse.cs

@@ -41,6 +41,12 @@ public class ContinuousContractUniverse : Universe, ITimeTriggeredUniverse
         /// </summary>
         public override UniverseSettings UniverseSettings { get; }
 
+        /// <summary>
+        /// True if this universe filter can run async in the data stack
+        /// TODO: see IContinuousSecurity.Mapped
+        /// </summary>
+        public override bool Asynchronous => false;
+
         /// <summary>
         /// Creates a new instance
         /// </summary>
@@ -84,6 +90,7 @@ public override IEnumerable<Symbol> SelectSymbols(DateTime utcTime, BaseDataColl
 
             if (_currentSymbol != null)
             {
+                // TODO: this won't work with async universe selection
                 ((IContinuousSecurity)_security).Mapped = _currentSymbol;
                 yield return _currentSymbol;
             }

Common/Data/UniverseSelection/FineFundamentalFilteredUniverse.cs

@@ -38,6 +38,11 @@ public class FineFundamentalFilteredUniverse : SelectSymbolsUniverseDecorator
         public FineFundamentalFilteredUniverse(Universe universe, Func<IEnumerable<FineFundamental>, IEnumerable<Symbol>> fineSelector)
             : base(universe, universe.SelectSymbols)
         {
+            if (universe is CoarseFundamentalUniverse && universe.UniverseSettings.Asynchronous.HasValue && universe.UniverseSettings.Asynchronous.Value)
+            {
+                throw new ArgumentException("Asynchronous universe setting is not supported for coarse & fine selections, please use the new Fundamental single pass selection");
+            }
+
             FineFundamentalUniverse = new FineFundamentalUniverse(universe.UniverseSettings, fineSelector);
             FineFundamentalUniverse.SelectionChanged += (sender, args) => OnSelectionChanged(((SelectionEventArgs) args).CurrentSelection);
         }

Common/Data/UniverseSelection/FuturesChainUniverse.cs

@@ -17,7 +17,6 @@
 using System;
 using System.Collections.Generic;
 using System.Linq;
-using QuantConnect.Data.Market;
 using QuantConnect.Interfaces;
 using QuantConnect.Securities;
 using QuantConnect.Securities.Future;
@@ -32,6 +31,21 @@ public class FuturesChainUniverse : Universe
         private readonly UniverseSettings _universeSettings;
         private DateTime _cacheDate;
 
+        /// <summary>
+        /// True if this universe filter can run async in the data stack
+        /// </summary>
+        public override bool Asynchronous
+        {
+            get
+            {
+                if (UniverseSettings.Asynchronous.HasValue)
+                {
+                    return UniverseSettings.Asynchronous.Value;
+                }
+                return Future.ContractFilter.Asynchronous;
+            }
+        }
+
         /// <summary>
         /// Initializes a new instance of the <see cref="FuturesChainUniverse"/> class
         /// </summary>
@@ -67,7 +81,7 @@ public override UniverseSettings UniverseSettings
         public override IEnumerable<Symbol> SelectSymbols(DateTime utcTime, BaseDataCollection data)
         {
             // date change detection needs to be done in exchange time zone
-            var localEndTime = data.EndTime.ConvertFromUtc(Future.Exchange.TimeZone);
+            var localEndTime = utcTime.ConvertFromUtc(Future.Exchange.TimeZone);
             var exchangeDate = localEndTime.Date;
             if (_cacheDate == exchangeDate)
             {

Common/Data/UniverseSelection/OptionChainUniverse.cs

@@ -34,6 +34,21 @@ public class OptionChainUniverse : Universe
         private readonly Symbol[] _underlyingSymbol;
         private DateTime _cacheDate;
 
+        /// <summary>
+        /// True if this universe filter can run async in the data stack
+        /// </summary>
+        public override bool Asynchronous
+        {
+            get
+            {
+                if (UniverseSettings.Asynchronous.HasValue)
+                {
+                    return UniverseSettings.Asynchronous.Value;
+                }
+                return Option.ContractFilter.Asynchronous;
+            }
+        }
+
         /// <summary>
         /// Initializes a new instance of the <see cref="OptionChainUniverse"/> class
         /// </summary>
@@ -71,7 +86,7 @@ public override UniverseSettings UniverseSettings
         public override IEnumerable<Symbol> SelectSymbols(DateTime utcTime, BaseDataCollection data)
         {
             // date change detection needs to be done in exchange time zone
-            var localEndTime = data.EndTime.ConvertFromUtc(Option.Exchange.TimeZone);
+            var localEndTime = utcTime.ConvertFromUtc(Option.Exchange.TimeZone);
             var exchangeDate = localEndTime.Date;
             if (_cacheDate == exchangeDate)
             {