Fix delisting time handling in SubscriptionDataReader (#8470)

The delisting time is handled by the DateChangeTimeKeeper now, so there is no need for the SubscriptionDataReader to do any special logic for it.
This was preventing the new tradable date events to be emitted after the delisting date in some cases, like when the day after delisting is not tradable

Co-authored-by: Jhonathan Abreu <[email protected]>

Algorithm.CSharp/BasicTemplateContinuousFutureWithExtendedMarketAlgorithm.cs

@@ -123,7 +123,7 @@ public override void OnSecuritiesChanged(SecurityChanges changes)
         /// <summary>
         /// Data Points count of all timeslices of algorithm
         /// </summary>
-        public long DataPoints => 2217328;
+        public long DataPoints => 2217330;
 
         /// <summary>
         /// Data Points count of the algorithm history

Algorithm.CSharp/BasicTemplateFuturesWithExtendedMarketDailyAlgorithm.cs

@@ -36,7 +36,7 @@ public class BasicTemplateFuturesWithExtendedMarketDailyAlgorithm : BasicTemplat
         /// <summary>
         /// Data Points count of all timeslices of algorithm
         /// </summary>
-        public override long DataPoints => 14181;
+        public override long DataPoints => 14182;
 
         /// <summary>
         /// This is used by the regression test system to indicate what the expected statistics are from running the algorithm

Algorithm.CSharp/BasicTemplateFuturesWithExtendedMarketHourlyAlgorithm.cs

@@ -41,7 +41,7 @@ public class BasicTemplateFuturesWithExtendedMarketHourlyAlgorithm : BasicTempla
         /// <summary>
         /// Data Points count of all timeslices of algorithm
         /// </summary>
-        public override long DataPoints => 228935;
+        public override long DataPoints => 228941;
 
         /// <summary>
         /// This is used by the regression test system to indicate what the expected statistics are from running the algorithm

Algorithm.CSharp/OptionExerciseOnExpiryAndNonTradableDateRegressionAlgorithm.cs

@@ -0,0 +1,161 @@
+/*
+ * QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals.
+ * Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+*/
+
+using System;
+using System.Collections.Generic;
+using QuantConnect.Data;
+using QuantConnect.Orders;
+using QuantConnect.Interfaces;
+
+namespace QuantConnect.Algorithm.CSharp
+{
+    /// <summary>
+    /// Regression algorithm asserting that options are automatically exercised on expiry regardless on whether
+    /// the day after expiration is tradable or not.
+    /// This specific algorithm works with contracts added manually.
+    /// </summary>
+    public class OptionExerciseOnExpiryAndNonTradableDateRegressionAlgorithm : QCAlgorithm, IRegressionAlgorithmDefinition
+    {
+        private Symbol _spxOption1;
+        private Symbol _spxOption2;
+        private bool _tradedOptions;
+        private bool _exercisedOption1;
+        private bool _exercisedOption2;
+
+        public override void Initialize()
+        {
+            SetStartDate(2023, 6, 25);
+            SetEndDate(2023, 7, 10);
+
+            var spx = AddIndex("SPX").Symbol;
+
+            _spxOption1 = QuantConnect.Symbol.CreateOption(
+                spx,
+                "SPXW",
+                Market.USA,
+                OptionStyle.European,
+                OptionRight.Call,
+                4445m,
+                // Next day is tradable
+                new DateTime(2023, 6, 30));
+
+            _spxOption2 = QuantConnect.Symbol.CreateOption(
+                spx,
+                "SPXW",
+                Market.USA,
+                OptionStyle.European,
+                OptionRight.Call,
+                4445m,
+                // Next day is a holiday
+                new DateTime(2023, 7, 3));
+
+            InitializeOptions(spx, [_spxOption1, _spxOption2]);
+        }
+
+        protected virtual void InitializeOptions(Symbol underlying, Symbol[] options)
+        {
+            AddIndexOptionContract(_spxOption1);
+            AddIndexOptionContract(_spxOption2);
+        }
+
+        public override void OnData(Slice slice)
+        {
+            if (!Portfolio.Invested && !_tradedOptions)
+            {
+                Buy(_spxOption1, 1);
+                Buy(_spxOption2, 1);
+                _tradedOptions = true;
+            }
+        }
+
+        public override void OnOrderEvent(OrderEvent orderEvent)
+        {
+            Log(orderEvent.ToString());
+            if (Transactions.GetOrderById(orderEvent.OrderId) is OptionExerciseOrder order)
+            {
+                _exercisedOption1 |= order.Symbol == _spxOption1;
+                _exercisedOption2 |= order.Symbol == _spxOption2;
+            }
+        }
+
+        public override void OnEndOfAlgorithm()
+        {
+            if (!_exercisedOption1 || !_exercisedOption2)
+            {
+                throw new RegressionTestException("Expected both options to be exercised");
+            }
+        }
+
+        /// <summary>
+        /// This is used by the regression test system to indicate if the open source Lean repository has the required data to run this algorithm.
+        /// </summary>
+        public bool CanRunLocally { get; } = true;
+
+        /// <summary>
+        /// This is used by the regression test system to indicate which languages this algorithm is written in.
+        /// </summary>
+        public List<Language> Languages { get; } = new() { Language.CSharp };
+
+        /// <summary>
+        /// Data Points count of all timeslices of algorithm
+        /// </summary>
+        public virtual long DataPoints => 16638;
+
+        /// <summary>
+        /// Data Points count of the algorithm history
+        /// </summary>
+        public virtual int AlgorithmHistoryDataPoints => 0;
+
+        /// <summary>
+        /// Final status of the algorithm
+        /// </summary>
+        public AlgorithmStatus AlgorithmStatus => AlgorithmStatus.Completed;
+
+        /// <summary>
+        /// This is used by the regression test system to indicate what the expected statistics are from running the algorithm
+        /// </summary>
+        public Dictionary<string, string> ExpectedStatistics => new Dictionary<string, string>
+        {
+            {"Total Orders", "4"},
+            {"Average Win", "0%"},
+            {"Average Loss", "-0.16%"},
+            {"Compounding Annual Return", "31.165%"},
+            {"Drawdown", "0.300%"},
+            {"Expectancy", "0"},
+            {"Start Equity", "100000"},
+            {"End Equity", "101172"},
+            {"Net Profit", "1.172%"},
+            {"Sharpe Ratio", "4.049"},
+            {"Sortino Ratio", "0"},
+            {"Probabilistic Sharpe Ratio", "94.902%"},
+            {"Loss Rate", "0%"},
+            {"Win Rate", "100%"},
+            {"Profit-Loss Ratio", "0"},
+            {"Alpha", "0"},
+            {"Beta", "0"},
+            {"Annual Standard Deviation", "0.041"},
+            {"Annual Variance", "0.002"},
+            {"Information Ratio", "5.34"},
+            {"Tracking Error", "0.041"},
+            {"Treynor Ratio", "0"},
+            {"Total Fees", "$0.00"},
+            {"Estimated Strategy Capacity", "$8000.00"},
+            {"Lowest Capacity Asset", "SPXW Y9T7LPL1X0TQ|SPX 31"},
+            {"Portfolio Turnover", "0.02%"},
+            {"OrderListHash", "6d154a036e1268579b585278636c3a8e"}
+        };
+    }
+}

Algorithm.CSharp/OptionExerciseOnExpiryAndNonTradableDateWithOptionSelectionRegressionAlgorithm.cs

@@ -0,0 +1,45 @@
+/*
+ * QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals.
+ * Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+*/
+
+using System.Collections.Generic;
+
+namespace QuantConnect.Algorithm.CSharp
+{
+    /// <summary>
+    /// Regression algorithm asserting that options are automatically exercised on expiry regardless on whether
+    /// the day after expiration is tradable or not.
+    /// This specific algorithm works with contracts added by selection using the option security filter.
+    /// </summary>
+    public class OptionExerciseOnExpiryAndNonTradableDateWithOptionSelectionRegressionAlgorithm
+        : OptionExerciseOnExpiryAndNonTradableDateRegressionAlgorithm
+    {
+        protected override void InitializeOptions(Symbol underlying, Symbol[] options)
+        {
+            AddIndexOption(underlying, options[0].ID.Symbol)
+                .SetFilter(u => u.IncludeWeeklys().Contracts(contracts => options));
+        }
+
+        /// <summary>
+        /// Data Points count of all timeslices of algorithm
+        /// </summary>
+        public override long DataPoints => 16649;
+
+        /// <summary>
+        /// Data Points count of the algorithm history
+        /// </summary>
+        public override int AlgorithmHistoryDataPoints => 0;
+    }
+}

Data/indexoption/usa/universes/spxw/20230623.csv

@@ -0,0 +1,14 @@
+#symbol_id,symbol_value,open,high,low,close,volume,open_interest,implied_volatility,delta,gamma,vega,theta,rho
+SPX 31,SPX,4350.18,4366.55,4341.34,4348.23,0,,,,,,,
+SPXW Y9Q98X3UTQXA|SPX 31,SPXW  230630C04440000,2.8750,5.1500,0.7500,1.7000,893,1209,0.0956202,0.0675138,0.0022678,0.7862798,-0.5790110,0.0559743
+SPXW Y9T7L77D20EM|SPX 31,SPXW  230703C04440000,3.8500,5.5500,1.1500,2.5750,320,371,0.0875840,0.0893466,0.0025798,1.1545523,-0.5680520,0.1043011
+SPXW Y9Q99FHJORCE|SPX 31,SPXW  230630C04445000,2.5000,4.4000,0.6250,1.4000,881,819,0.0956979,0.0572521,0.0019925,0.6913961,-0.5082137,0.0474740
+SPXW Y9T7LPL1X0TQ|SPX 31,SPXW  230703C04445000,3.4000,5.8500,1.0000,2.2000,102,344,0.0878738,0.0780638,0.0023232,1.0431794,-0.5131396,0.0911432
+SPXW Y9Q98X40S2Z2|SPX 31,SPXW  230630C04450000,2.1250,2.4250,0.9500,1.1500,5880,11430,0.0957617,0.0482583,0.0017386,0.6037060,-0.4429590,0.0400223
+SPXW Y9T7L77J0CGE|SPX 31,SPXW  230703C04450000,2.9750,4.3000,0.8500,1.8750,262,751,0.0881580,0.0679586,0.0020826,0.9381456,-0.4614357,0.0793561
+SPXW 328U9ED323KB2|SPX 31,SPXW  230630P04440000,92.5000,97.3000,69.2500,90.1500,23,465,0.0956202,-0.9324862,0.0022678,0.7862798,0.0589765,-0.7946757
+SPXW 328X7QN6KBTSE|SPX 31,SPXW  230703P04440000,93.0000,97.2500,69.8000,90.5500,33,124,0.0875840,-0.9106534,0.0025798,1.1545523,0.0696726,-1.0939632
+SPXW 328U9EVGQYKQ6|SPX 31,SPXW  230630P04445000,97.1000,100.9000,73.2000,94.8500,14,254,0.0956979,-0.9427479,0.0019925,0.6913961,0.1304922,-0.8041339
+SPXW 328X7R5K96U7I|SPX 31,SPXW  230703P04445000,97.5000,101.6500,73.7500,95.1500,0,148,0.0878738,-0.9219362,0.0023232,1.0431794,0.1253032,-1.1084705
+SPXW 328U9ED381WCU|SPX 31,SPXW  230630P04450000,101.7000,106.3000,78.9500,99.5500,60,2058,0.0957617,-0.9517417,0.0017386,0.6037060,0.1964654,-0.8125435
+SPXW 328X7QN6QA5U6|SPX 31,SPXW  230703P04450000,101.9500,106.6500,78.6000,100.0000,9,62,0.0881580,-0.9320414,0.0020826,0.9381456,0.1777252,-1.1216071

Engine/DataFeeds/SubscriptionDataReader.cs

@@ -253,9 +253,6 @@ public void Initialize()
 
             _delistingDate = _config.Symbol.GetDelistingDate(_mapFile);
 
-            // adding a day so we stop at EOD
-            _delistingDate = _delistingDate.AddDays(1);
-
             _timeKeeper = new DateChangeTimeKeeper(_tradableDatesInDataTimeZone, _config, _exchangeHours, _delistingDate);
             _timeKeeper.NewExchangeDate += HandleNewTradableDate;
 
@@ -566,13 +563,6 @@ private bool TryGetNextDate(out DateTime date)
             {
                 date = _timeKeeper.DataTime.Date;
 
-                if (_pastDelistedDate || date > _delistingDate)
-                {
-                    // if we already passed our delisting date we stop
-                    _pastDelistedDate = true;
-                    break;
-                }
-
                 if (!_mapFile.HasData(date))
                 {
                     continue;
@@ -588,6 +578,11 @@ private bool TryGetNextDate(out DateTime date)
                 return true;
             }
 
+            if (_timeKeeper.ExchangeTime.Date > _delistingDate)
+            {
+                _pastDelistedDate = true;
+            }
+
             // no more tradeable dates, we've exhausted the enumerator
             date = DateTime.MaxValue.Date;
             return false;

Tests/Engine/DataFeeds/SubscriptionDataReaderTests.cs

@@ -17,12 +17,10 @@
 using System;
 using System.Collections.Generic;
 using System.IO;
-using NodaTime;
+using System.Linq;
 using NUnit.Framework;
 using QuantConnect.Data;
-using QuantConnect.Data.Auxiliary;
 using QuantConnect.Data.Market;
-using QuantConnect.Data.UniverseSelection;
 using QuantConnect.Interfaces;
 using QuantConnect.Lean.Engine.DataFeeds;
 using QuantConnect.Securities;
@@ -74,6 +72,58 @@ public void DoesNotEmitDataBeyondTradableDate(string data, bool shouldEmitSecond
 
         }
 
+        [TestCase(typeof(TradeBar))]
+        [TestCase(typeof(QuoteBar))]
+        [TestCase(typeof(OpenInterest))]
+        public void EmitsNewTradableDateWhenDateAfterDelistingIsNonTradable(Type dataType)
+        {
+            var start = new DateTime(2023, 06, 30);
+            var end = new DateTime(2023, 08, 01);
+
+            var symbol = Symbol.CreateOption(
+                Symbols.SPX,
+                "SPXW",
+                Market.USA,
+                OptionStyle.European,
+                OptionRight.Call,
+                4445m,
+                // Next day is a holiday
+                new DateTime(2023, 7, 3));
+
+            var entry = MarketHoursDatabase.FromDataFolder().GetEntry(symbol.ID.Market, symbol, symbol.SecurityType);
+            var config = new SubscriptionDataConfig(dataType,
+                symbol,
+                Resolution.Minute,
+                entry.DataTimeZone,
+                entry.ExchangeHours.TimeZone,
+                false,
+                false,
+                false);
+            using var testDataCacheProvider = new TestDataCacheProvider();
+            var request = new HistoryRequest(config, entry.ExchangeHours, start, end);
+            using var dataReader = new SubscriptionDataReader(config,
+                request,
+                TestGlobals.MapFileProvider,
+                TestGlobals.FactorFileProvider,
+                testDataCacheProvider,
+                TestGlobals.DataProvider,
+                null);
+
+            var expectedLastTradableDate = new DateTime(2023, 07, 05);
+            var lastTradableDate = default(DateTime);
+
+            dataReader.NewTradableDate += (sender, args) =>
+            {
+                lastTradableDate = args.Date;
+            };
+
+            while (dataReader.MoveNext())
+            {
+            }
+
+            Assert.AreEqual(expectedLastTradableDate, lastTradableDate);
+        }
+
         private class TestDataCacheProvider : IDataCacheProvider
         {
             private StreamWriter _writer;