Out of sample support tweaks (#7542)

* Add backtest job end date handling

* Fixes & rebases

* Refactor out of sample configuration handling

* Minor datetime format tweak

Common/AlgorithmConfiguration.cs

@@ -13,7 +13,10 @@
  * limitations under the License.
 */
 
+using System;
 using Newtonsoft.Json;
+using QuantConnect.Util;
+using QuantConnect.Packets;
 using QuantConnect.Interfaces;
 using QuantConnect.Brokerages;
 using System.Collections.Generic;
@@ -52,16 +55,46 @@ public class AlgorithmConfiguration
         [JsonProperty(PropertyName = "Parameters")]
         public IReadOnlyDictionary<string, string> Parameters;
 
+        /// <summary>
+        /// Backtest maximum end date
+        /// </summary>
+        [JsonProperty(PropertyName = "OutOfSampleMaxEndDate")]
+        public DateTime? OutOfSampleMaxEndDate;
+
+        /// <summary>
+        /// The backtest out of sample day count
+        /// </summary>
+        [JsonProperty(PropertyName = "OutOfSampleDays")]
+        public int OutOfSampleDays;
+
+        /// <summary>
+        /// The backtest start date
+        /// </summary>
+        [JsonProperty(PropertyName = "StartDate")]
+        [JsonConverter(typeof(DateTimeJsonConverter), DateFormat.UI)]
+        public DateTime StartDate;
+
+        /// <summary>
+        /// The backtest end date
+        /// </summary>
+        [JsonProperty(PropertyName = "EndDate")]
+        [JsonConverter(typeof(DateTimeJsonConverter), DateFormat.UI)]
+        public DateTime EndDate;
+
         /// <summary>
         /// Initializes a new instance of the <see cref="AlgorithmConfiguration"/> class
         /// </summary>
-        public AlgorithmConfiguration(string accountCurrency, BrokerageName brokerageName, AccountType accountType,
-            IReadOnlyDictionary<string, string> parameters)
+        public AlgorithmConfiguration(string accountCurrency, BrokerageName brokerageName, AccountType accountType, IReadOnlyDictionary<string, string> parameters,
+            DateTime startDate, DateTime endDate, DateTime? outOfSampleMaxEndDate, int outOfSampleDays = 0)
         {
+            OutOfSampleMaxEndDate = outOfSampleMaxEndDate;
+            OutOfSampleDays = outOfSampleDays;
             AccountCurrency = accountCurrency;
             BrokerageName = brokerageName;
             AccountType = accountType;
             Parameters = parameters;
+            StartDate = startDate;
+            EndDate = endDate;
         }
 
         /// <summary>
@@ -75,14 +108,19 @@ public AlgorithmConfiguration()
         /// Provides a convenience method for creating a <see cref="AlgorithmConfiguration"/> for a given algorithm.
         /// </summary>
         /// <param name="algorithm">Algorithm for which the configuration object is being created</param>
+        /// <param name="backtestNodePacket">The associated backtest node packet if any</param>
         /// <returns>A new AlgorithmConfiguration object for the specified algorithm</returns>
-        public static AlgorithmConfiguration Create(IAlgorithm algorithm)
+        public static AlgorithmConfiguration Create(IAlgorithm algorithm, BacktestNodePacket backtestNodePacket)
         {
             return new AlgorithmConfiguration(
                 algorithm.AccountCurrency,
                 BrokerageModel.GetBrokerageName(algorithm.BrokerageModel),
                 algorithm.BrokerageModel.AccountType,
-                algorithm.GetParameters());
+                algorithm.GetParameters(),
+                algorithm.StartDate,
+                algorithm.EndDate,
+                backtestNodePacket?.OutOfSampleMaxEndDate,
+                backtestNodePacket?.OutOfSampleDays ?? 0);
         }
     }
 }

Common/Api/OptimizationBacktest.cs

@@ -13,8 +13,9 @@
  * limitations under the License.
 */
 
-using System.Collections.Generic;
+using System;
 using Newtonsoft.Json;
+using System.Collections.Generic;
 using QuantConnect.Optimizer.Parameters;
 
 namespace QuantConnect.Api
@@ -65,6 +66,26 @@ public class OptimizationBacktest
         /// </summary>
         public int ExitCode { get; set; }
 
+        /// <summary>
+        /// Backtest maximum end date
+        /// </summary>
+        public DateTime? OutOfSampleMaxEndDate { get; set; }
+
+        /// <summary>
+        /// The backtest out of sample day count
+        /// </summary>
+        public int OutOfSampleDays { get; set; }
+
+        /// <summary>
+        /// The backtest start date
+        /// </summary>
+        public DateTime StartDate { get; set; }
+
+        /// <summary>
+        /// The backtest end date
+        /// </summary>
+        public DateTime EndDate { get; set; }
+
         /// <summary>
         /// Creates a new instance
         /// </summary>

Common/Api/OptimizationBacktestJsonConverter.cs

@@ -15,6 +15,7 @@
 
 using System;
 using System.Collections.Generic;
+using System.Globalization;
 using System.Linq;
 using Newtonsoft.Json;
 using Newtonsoft.Json.Linq;
@@ -72,6 +73,27 @@ public override void WriteJson(JsonWriter writer, object value, JsonSerializer s
                 writer.WriteValue(optimizationBacktest.ExitCode);
             }
 
+            if (optimizationBacktest.StartDate != default)
+            {
+                writer.WritePropertyName("startDate");
+                writer.WriteValue(optimizationBacktest.StartDate.ToStringInvariant(DateFormat.UI));
+            }
+
+            if (optimizationBacktest.EndDate != default)
+            {
+                writer.WritePropertyName("endDate");
+                writer.WriteValue(optimizationBacktest.EndDate.ToStringInvariant(DateFormat.UI));
+            }
+
+            if (optimizationBacktest.OutOfSampleMaxEndDate != null)
+            {
+                writer.WritePropertyName("outOfSampleMaxEndDate");
+                writer.WriteValue(optimizationBacktest.OutOfSampleMaxEndDate.ToStringInvariant(DateFormat.UI));
+
+                writer.WritePropertyName("outOfSampleDays");
+                writer.WriteValue(optimizationBacktest.OutOfSampleDays);
+            }
+
             if (!optimizationBacktest.Statistics.IsNullOrEmpty())
             {
                 writer.WritePropertyName("statistics");
@@ -128,6 +150,11 @@ public override object ReadJson(JsonReader reader, Type objectType, object exist
             var progress = jObject["progress"].Value<decimal>();
             var exitCode = jObject["exitCode"].Value<int>();
 
+            var outOfSampleDays = jObject["outOfSampleDays"]?.Value<int>() ?? default;
+            var startDate = jObject["startDate"]?.Value<DateTime>() ?? default;
+            var endDate = jObject["endDate"]?.Value<DateTime>() ?? default;
+            var outOfSampleMaxEndDate = jObject["outOfSampleMaxEndDate"]?.Value<DateTime>();
+
             var jStatistics = jObject["statistics"];
             var statistics = new Dictionary<string, string>
             {
@@ -168,7 +195,11 @@ public override object ReadJson(JsonReader reader, Type objectType, object exist
                 Progress = progress,
                 ExitCode = exitCode,
                 Statistics = statistics,
-                Equity = equity
+                Equity = equity,
+                EndDate = endDate,
+                StartDate = startDate,
+                OutOfSampleDays = outOfSampleDays,
+                OutOfSampleMaxEndDate = outOfSampleMaxEndDate,
             };
 
             return optimizationBacktest;

Common/Packets/BacktestNodePacket.cs

@@ -61,6 +61,18 @@ private static readonly string DefaultId
         [JsonProperty(PropertyName = "dtPeriodFinish")]
         public DateTime? PeriodFinish;
 
+        /// <summary>
+        /// Backtest maximum end date
+        /// </summary>
+        [JsonProperty(PropertyName = "dtOutOfSampleMaxEndDate")]
+        public DateTime? OutOfSampleMaxEndDate;
+
+        /// <summary>
+        /// The backtest out of sample day count
+        /// </summary>
+        [JsonProperty(PropertyName = "iOutOfSampleDays")]
+        public int OutOfSampleDays;
+
         /// <summary>
         /// Estimated number of trading days in this backtest task based on the start-end dates.
         /// </summary>

Engine/Results/BacktestingResultHandler.cs

@@ -353,7 +353,7 @@ protected void SendFinalResult()
                     result = new BacktestResultPacket(_job,
                         new BacktestResult(new BacktestResultParameters(charts, orders, profitLoss, statisticsResults.Summary, runtime,
                             statisticsResults.RollingPerformances, orderEvents, statisticsResults.TotalPerformance,
-                            AlgorithmConfiguration.Create(Algorithm), GetAlgorithmState(endTime))),
+                            AlgorithmConfiguration.Create(Algorithm, _job), GetAlgorithmState(endTime))),
                         Algorithm.EndDate, Algorithm.StartDate);
                 }
                 else

Engine/Results/LiveTradingResultHandler.cs

@@ -795,7 +795,7 @@ protected void SendFinalResult()
                     result = new LiveResultPacket(_job,
                         new LiveResult(new LiveResultParameters(charts, orders, profitLoss, new Dictionary<string, Holding>(),
                             Algorithm.Portfolio.CashBook, statisticsResults.Summary, runtime, GetOrderEventsToStore(),
-                            algorithmConfiguration: AlgorithmConfiguration.Create(Algorithm), state: endState)));
+                            algorithmConfiguration: AlgorithmConfiguration.Create(Algorithm, null), state: endState)));
                 }
                 else
                 {

Engine/Setup/BacktestingSetupHandler.cs

@@ -191,6 +191,20 @@ public bool Setup(SetupHandlerParameters parameters)
                         algorithm.SetEndDate(job.PeriodFinish.Value);
                     }
 
+                    if(job.OutOfSampleMaxEndDate.HasValue)
+                    {
+                        if(algorithm.EndDate > job.OutOfSampleMaxEndDate.Value)
+                        {
+                            Log.Trace($"BacktestingSetupHandler.Setup(): setting end date to {job.OutOfSampleMaxEndDate.Value:yyyyMMdd}");
+                            algorithm.SetEndDate(job.OutOfSampleMaxEndDate.Value);
+
+                            if (algorithm.StartDate > algorithm.EndDate)
+                            {
+                                algorithm.SetStartDate(algorithm.EndDate);
+                            }
+                        }
+                    }
+
                     // after we call initialize
                     BaseSetupHandler.LoadBacktestJobCashAmount(algorithm, job);
 

Optimizer/OptimizationNodePacket.cs

@@ -1,4 +1,4 @@
-/*
+/*
  * QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals.
  * Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation.
  *
@@ -13,6 +13,7 @@
  * limitations under the License.
 */
 
+using System;
 using System.Collections.Generic;
 using Newtonsoft.Json;
 using QuantConnect.Optimizer.Objectives;
@@ -97,6 +98,18 @@ public class OptimizationNodePacket : Packet
         [JsonProperty(PropertyName = "optimizationStrategySettings", TypeNameHandling = TypeNameHandling.All)]
         public OptimizationStrategySettings OptimizationStrategySettings;
 
+        /// <summary>
+        /// Backtest out of sample maximum end date
+        /// </summary>
+        [JsonProperty(PropertyName = "outOfSampleMaxEndDate")]
+        public DateTime? OutOfSampleMaxEndDate;
+
+        /// <summary>
+        /// The backtest out of sample day count
+        /// </summary>
+        [JsonProperty(PropertyName = "outOfSampleDays")]
+        public int OutOfSampleDays;
+
         /// <summary>
         /// Creates a new instance
         /// </summary>

Queues/JobQueue.cs

@@ -212,6 +212,14 @@ public AlgorithmNodePacket NextJob(out string location)
                 PythonVirtualEnvironment = Config.Get("python-venv"),
                 DeploymentTarget = DeploymentTarget.LocalPlatform,
             };
+
+            var outOfSampleMaxEndDate = Config.Get("out-of-sample-max-end-date");
+            if (!string.IsNullOrEmpty(outOfSampleMaxEndDate))
+            {
+                backtestJob.OutOfSampleMaxEndDate = Time.ParseDate(outOfSampleMaxEndDate);
+            }
+            backtestJob.OutOfSampleDays = Config.GetInt("out-of-sample-days");
+
             // Only set optimization id when backtest is for optimization
             if (!optimizationId.IsNullOrEmpty())
             {

Tests/Api/OptimizationBacktestJsonConverterTests.cs

@@ -13,6 +13,7 @@
  * limitations under the License.
 */
 
+using System;
 using System.Collections.Generic;
 using Newtonsoft.Json;
 using NUnit.Framework;
@@ -26,9 +27,12 @@ namespace QuantConnect.Tests.API
     public class OptimizationBacktestJsonConverterTests
     {
         private const string _validSerialization = "{\"name\":\"ImABacktestName\",\"id\":\"backtestId\",\"progress\":0.0,\"exitCode\":0," +
+            "\"startDate\":\"2023-01-01 00:00:00\",\"endDate\":\"2024-01-01 00:00:00\",\"outOfSampleMaxEndDate\":\"2024-01-01 00:00:00\",\"outOfSampleDays\":10,\"statistics\":[0.374,0.217,0.047,-4.51,2.86,-0.664,52.602,17.800,6300000.00,0.196,1.571,27.0,123.888,77.188,0.63,1.707,1390.49,180.0,0.233,-0.558,73.0]," +
+            "\"parameterSet\":{\"pinocho\":\"19\",\"pepe\":\"-1\"},\"equity\":[[1,1.0,1.0,1.0,1.0],[2,2.0,2.0,2.0,2.0],[3,3.0,3.0,3.0,3.0]]}";
+        private const string _oldValidSerialization = "{\"name\":\"ImABacktestName\",\"id\":\"backtestId\",\"progress\":0.0,\"exitCode\":0," +
             "\"statistics\":[0.374,0.217,0.047,-4.51,2.86,-0.664,52.602,17.800,6300000.00,0.196,1.571,27.0,123.888,77.188,0.63,1.707,1390.49,180.0,0.233,-0.558,73.0]," +
             "\"parameterSet\":{\"pinocho\":\"19\",\"pepe\":\"-1\"},\"equity\":[[1,1.0,1.0,1.0,1.0],[2,2.0,2.0,2.0,2.0],[3,3.0,3.0,3.0,3.0]]}";
-        private const string _oldValidSerialization = "{\"name\":\"ImABacktestName\",\"id\":\"backtestId\",\"progress\":0.0,\"exitCode\":0,"+
+        private const string _oldValid2Serialization = "{\"name\":\"ImABacktestName\",\"id\":\"backtestId\",\"progress\":0.0,\"exitCode\":0,"+
             "\"statistics\":[0.374,0.217,0.047,-4.51,2.86,-0.664,52.602,17.800,6300000.00,0.196,1.571,27.0,123.888,77.188,0.63,1.707,1390.49,180.0,0.233,-0.558,73.0]," +
             "\"parameterSet\":{\"pinocho\":\"19\",\"pepe\":\"-1\"},\"equity\":[[1,1.0],[2,2.0],[3,3.0]]}";
 
@@ -41,8 +45,9 @@ public void SerializationNulls()
             Assert.AreEqual("{}", serialized);
         }
 
-        [Test]
-        public void Serialization()
+        [TestCase(1)]
+        [TestCase(0)]
+        public void Serialization(int version)
         {
             var optimizationBacktest = new OptimizationBacktest(new ParameterSet(18,
                 new Dictionary<string, string>
@@ -80,14 +85,27 @@ public void Serialization()
             {
                 Values = new List<ISeriesPoint> { new Candlestick(1, 1, 1, 1, 1), new Candlestick(2, 2, 2, 2, 2), new Candlestick(3, 3, 3, 3, 3) }
             };
+            if(version > 0)
+            {
+                optimizationBacktest.StartDate = new DateTime(2023, 01, 01);
+                optimizationBacktest.EndDate = new DateTime(2024, 01, 01);
+                optimizationBacktest.OutOfSampleMaxEndDate = new DateTime(2024, 01, 01);
+                optimizationBacktest.OutOfSampleDays = 10;
+            }
 
             var serialized = JsonConvert.SerializeObject(optimizationBacktest);
 
-            Assert.AreEqual(_validSerialization, serialized);
+            var expected = _validSerialization;
+            if (version == 0)
+            {
+                expected = _oldValidSerialization;
+            }
+            Assert.AreEqual(expected, serialized);
         }
 
         [TestCase(_validSerialization)]
         [TestCase(_oldValidSerialization)]
+        [TestCase(_oldValid2Serialization)]
         public void Deserialization(string serialization)
         {
             var deserialized = JsonConvert.DeserializeObject<OptimizationBacktest>(serialization);

Tests/Common/AlgorithmConfigurationTests.cs

@@ -17,8 +17,11 @@
 
 using NUnit.Framework;
 
-using QuantConnect.Brokerages;
+using System;
+using Newtonsoft.Json;
+using QuantConnect.Packets;
 using QuantConnect.Algorithm;
+using QuantConnect.Brokerages;
 
 namespace QuantConnect.Tests.Common
 {
@@ -35,14 +38,34 @@ public void CreatesConfiguration(string currency, BrokerageName brokerageName, A
             algorithm.SetParameters(parameters);
 
 
-            var algorithmConfiguration = AlgorithmConfiguration.Create(algorithm);
+            var algorithmConfiguration = AlgorithmConfiguration.Create(algorithm, null);
 
             Assert.AreEqual(currency, algorithmConfiguration.AccountCurrency);
             Assert.AreEqual(brokerageName, algorithmConfiguration.BrokerageName);
             Assert.AreEqual(accountType, algorithmConfiguration.AccountType);
             CollectionAssert.AreEquivalent(parameters, algorithmConfiguration.Parameters);
         }
 
+        [Test]
+        public void JsonSerialization()
+        {
+            var algorithm = new QCAlgorithm();
+            algorithm.SetAccountCurrency(Currencies.GBP);
+            algorithm.SetBrokerageModel(BrokerageName.GDAX, AccountType.Cash);
+            algorithm.SetParameters(new Dictionary<string, string> { { "a", "A" }, { "b", "B" } });
+
+            var backtestNode = new BacktestNodePacket
+            {
+                OutOfSampleDays = 30,
+                OutOfSampleMaxEndDate = new DateTime(2023, 01, 01)
+            };
+            var algorithmConfiguration = AlgorithmConfiguration.Create(algorithm, backtestNode);
+
+            var serialized = JsonConvert.SerializeObject(algorithmConfiguration);
+
+            Assert.AreEqual("{\"AccountCurrency\":\"GBP\",\"Brokerage\":12,\"AccountType\":1,\"Parameters\":{\"a\":\"A\",\"b\":\"B\"},\"OutOfSampleMaxEndDate\":\"2023-01-01T00:00:00\",\"OutOfSampleDays\":30,\"StartDate\":\"1998-01-01 00:00:00\",\"EndDate\":\"2023-10-26 23:59:59\"}", serialized);
+        }
+
         private static TestCaseData[] AlgorithmConfigurationTestCases => new[]
         {
             new TestCaseData("BTC", BrokerageName.Binance, AccountType.Cash,