From 2502ebc6d07bb05bf84d10d6a33ab88248dde6e5 Mon Sep 17 00:00:00 2001 From: Jhonathan Abreu Date: Tue, 21 Jul 2026 18:50:14 -0400 Subject: [PATCH 01/33] Run a reduced results analysis periodically during backtests Makes the ResultsAnalyzer analysis set overridable and adds an InRunResultsAnalyzer that the BacktestingResultHandler runs on the periodic intermediate result updates, against a thread-safe snapshot of the current backtest state. The in-run set only includes cheap, snapshot-based analyses that detect error conditions early (order response errors, margin calls, stale fills, margin usage, non-positive equity), leaving curve and statistics based analyses to the final analysis. The equity and benchmark curves are skipped for in-run analysis, avoiding benchmark history requests on every update. --- .../Results/Analysis/InRunResultsAnalyzer.cs | 82 +++++++++++++ Engine/Results/Analysis/ResultsAnalyzer.cs | 108 ++++++++++++------ Engine/Results/BacktestingResultHandler.cs | 57 +++++++++ 3 files changed, 209 insertions(+), 38 deletions(-) create mode 100644 Engine/Results/Analysis/InRunResultsAnalyzer.cs diff --git a/Engine/Results/Analysis/InRunResultsAnalyzer.cs b/Engine/Results/Analysis/InRunResultsAnalyzer.cs new file mode 100644 index 000000000000..a8620e8ee269 --- /dev/null +++ b/Engine/Results/Analysis/InRunResultsAnalyzer.cs @@ -0,0 +1,82 @@ +/* + * QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals. + * Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * +*/ +using QuantConnect.Algorithm; +using QuantConnect.Lean.Engine.Results.Analysis.Analyses; +using System.Collections.Generic; + +namespace QuantConnect.Lean.Engine.Results.Analysis +{ + /// + /// Runs a reduced suite of backtest diagnostic tests periodically while the backtest is still running, + /// against a snapshot of the current intermediate results. + /// + public class InRunResultsAnalyzer : ResultsAnalyzer + { + /// + /// Initializes a new instance of the class. + /// + /// A snapshot of the current intermediate backtest result to analyze. + /// The algorithm instance used for history requests and settings. + /// The programming language the algorithm is written in. + /// The list of log lines produced by the backtest so far. + public InRunResultsAnalyzer(Result result, QCAlgorithm algorithm, Language language, IReadOnlyList logs) + : base(result, algorithm, language, logs) + { + } + + /// + /// The equity and benchmark curves are not built for in-run analysis: + /// none of the in-run analyses read them, and building them would issue + /// a benchmark history request on every run. + /// + protected override bool RequiresEquityCurves => false; + + /// + /// Creates the set of diagnostic analyses to run while the backtest is in progress. + /// Only analyses that read the result snapshot (logs, orders, order events, charts) are + /// included: they are cheap, thread-safe, and detect error conditions whose findings + /// don't depend on the backtest being complete. Curve and statistics based analyses are + /// left to the final analysis, since partial-period statistics are noisy and require + /// history requests. + /// + protected override IReadOnlyCollection GetAnalyses() => new BaseResultsAnalysis[] + { + new PortfolioValueIsNotPositiveAnalysis(), + new InsufficientBuyingPowerOrderResponseErrorAnalysis(), + new MarginCallsAnalysis(), + new ExceedsShortableQuantityOrderResponseErrorAnalysis(), + new SecurityPriceZeroOrderResponseErrorAnalysis(), + new OrderQuantityZeroOrderResponseErrorAnalysis(), + new NonTradableSecurityOrderResponseErrorAnalysis(), + new BrokerageModelRefusedToSubmitOrderOrderResponseErrorAnalysis(), + new BrokerageModelRefusedToUpdateOrderOrderResponseErrorAnalysis(), + new TakeProfitAndStopLossOrdersAnalysis(), + new StaleOrderFillsAnalysis(), + new AlgorithmWarmingUpOrderResponseErrorAnalysis(), + new ExchangeNotOpenOrderResponseErrorAnalysis(), + new ForexConversionRateZeroOrderResponseErrorAnalysis(), + new ExceededMaximumOrdersOrderResponseErrorAnalysis(), + new UnsupportedOptionShortPositionExerciseAnalysis(), + new UnsupportedOptionExerciseQuantityAnalysis(), + new EuropeanOptionNotExpiredOnExerciseOrderResponseErrorAnalysis(), + new OptionOrderOnStockSplitOrderResponseErrorAnalysis(), + new MarketOnOpenNotAllowedDuringRegularHoursOrderResponseErrorAnalysis(), + new OrderQuantityLessThanLotSizeOrderResponseErrorAnalysis(), + new InsightsEmittedForDelistedSecuritiesAnalysis(), + new PortfolioMarginUsageAnalysis(), + }; + } +} diff --git a/Engine/Results/Analysis/ResultsAnalyzer.cs b/Engine/Results/Analysis/ResultsAnalyzer.cs index f3eeb0f508fe..a661ae25c75f 100644 --- a/Engine/Results/Analysis/ResultsAnalyzer.cs +++ b/Engine/Results/Analysis/ResultsAnalyzer.cs @@ -62,51 +62,27 @@ public ResultsAnalyzer(Result result, QCAlgorithm algorithm, Language language, /// Up to entries with solutions, ranked by weight. public IReadOnlyList Run(int timeLimitSeconds = 5, int maxFailedAnalyses = 10) { - (_equityCurve, _benchmarkEquityCurve) = ReadEquityCurve(_result, _algorithm); - - var parameters = new ResultsAnalysisRunParameters(_result, _algorithm, _language, _logs, _equityCurve, _benchmarkEquityCurve); + var analyses = GetAnalyses(); + if (analyses.Count == 0) + { + return []; + } - // Instances are sorted by their own Weight — changing a weight automatically reorders execution. - var analyses = new BaseResultsAnalysis[] + _equityCurve = new(); + _benchmarkEquityCurve = new(); + if (RequiresEquityCurves) { - new PortfolioValueIsNotPositiveAnalysis(), - new FlatEquityCurveAnalysis(), - new InsufficientBuyingPowerOrderResponseErrorAnalysis(), - new MarginCallsAnalysis(), - new ExceedsShortableQuantityOrderResponseErrorAnalysis(), - new SecurityPriceZeroOrderResponseErrorAnalysis(), - new OrderQuantityZeroOrderResponseErrorAnalysis(), - new NonTradableSecurityOrderResponseErrorAnalysis(), - new BrokerageModelRefusedToSubmitOrderOrderResponseErrorAnalysis(), - new BrokerageModelRefusedToUpdateOrderOrderResponseErrorAnalysis(), - new TakeProfitAndStopLossOrdersAnalysis(), - new StaleOrderFillsAnalysis(), - new AlgorithmWarmingUpOrderResponseErrorAnalysis(), - new ExchangeNotOpenOrderResponseErrorAnalysis(), - new ForexConversionRateZeroOrderResponseErrorAnalysis(), - new OrderFillsDuringExtendedMarketHoursAnalysis(), - new ExceededMaximumOrdersOrderResponseErrorAnalysis(), - new UnsupportedOptionShortPositionExerciseAnalysis(), - new UnsupportedOptionExerciseQuantityAnalysis(), - new EuropeanOptionNotExpiredOnExerciseOrderResponseErrorAnalysis(), - new OptionOrderOnStockSplitOrderResponseErrorAnalysis(), - new MarketOnOpenNotAllowedDuringRegularHoursOrderResponseErrorAnalysis(), - new OrderQuantityLessThanLotSizeOrderResponseErrorAnalysis(), - new InsightsEmittedForDelistedSecuritiesAnalysis(), - new StatisticalSignificanceOfDailyReturnsAnalysis(), - new PerformanceRelativeToBenchmarkAnalysis(), - new CrisisEventsAnalysis(), - new ExecutionSpeedAnalysis(), - new PortfolioMarginUsageAnalysis(), - new ParameterCountAnalysis(), - new MonteCarloPercentileAnalysis(), - }.OrderByDescending(a => a.Weight); + (_equityCurve, _benchmarkEquityCurve) = ReadEquityCurve(_result, _algorithm); + } + + var parameters = new ResultsAnalysisRunParameters(_result, _algorithm, _language, _logs, _equityCurve, _benchmarkEquityCurve); var responses = new List(); var timer = Stopwatch.StartNew(); var timeLimit = TimeSpan.FromSeconds(timeLimitSeconds); - foreach (var analysis in analyses) + // Instances are sorted by their own Weight — changing a weight automatically reorders execution. + foreach (var analysis in analyses.OrderByDescending(a => a.Weight)) { if (responses.Count >= maxFailedAnalyses) { @@ -131,6 +107,51 @@ public ResultsAnalyzer(Result result, QCAlgorithm algorithm, Language language, return responses; } + /// + /// Whether the equity and benchmark curves should be built before running the analyses. + /// Building them requires a benchmark history request, so analyzers whose analyses + /// don't read the curves can skip it. + /// + protected virtual bool RequiresEquityCurves => true; + + /// + /// Creates the set of diagnostic analyses to run against the backtest. + /// + protected virtual IReadOnlyCollection GetAnalyses() => new BaseResultsAnalysis[] + { + new PortfolioValueIsNotPositiveAnalysis(), + new FlatEquityCurveAnalysis(), + new InsufficientBuyingPowerOrderResponseErrorAnalysis(), + new MarginCallsAnalysis(), + new ExceedsShortableQuantityOrderResponseErrorAnalysis(), + new SecurityPriceZeroOrderResponseErrorAnalysis(), + new OrderQuantityZeroOrderResponseErrorAnalysis(), + new NonTradableSecurityOrderResponseErrorAnalysis(), + new BrokerageModelRefusedToSubmitOrderOrderResponseErrorAnalysis(), + new BrokerageModelRefusedToUpdateOrderOrderResponseErrorAnalysis(), + new TakeProfitAndStopLossOrdersAnalysis(), + new StaleOrderFillsAnalysis(), + new AlgorithmWarmingUpOrderResponseErrorAnalysis(), + new ExchangeNotOpenOrderResponseErrorAnalysis(), + new ForexConversionRateZeroOrderResponseErrorAnalysis(), + new OrderFillsDuringExtendedMarketHoursAnalysis(), + new ExceededMaximumOrdersOrderResponseErrorAnalysis(), + new UnsupportedOptionShortPositionExerciseAnalysis(), + new UnsupportedOptionExerciseQuantityAnalysis(), + new EuropeanOptionNotExpiredOnExerciseOrderResponseErrorAnalysis(), + new OptionOrderOnStockSplitOrderResponseErrorAnalysis(), + new MarketOnOpenNotAllowedDuringRegularHoursOrderResponseErrorAnalysis(), + new OrderQuantityLessThanLotSizeOrderResponseErrorAnalysis(), + new InsightsEmittedForDelistedSecuritiesAnalysis(), + new StatisticalSignificanceOfDailyReturnsAnalysis(), + new PerformanceRelativeToBenchmarkAnalysis(), + new CrisisEventsAnalysis(), + new ExecutionSpeedAnalysis(), + new PortfolioMarginUsageAnalysis(), + new ParameterCountAnalysis(), + new MonteCarloPercentileAnalysis(), + }; + /// /// Reads the backtest's "Strategy Equity" chart and fetches SPY daily history to build /// two time-aligned equity curves: one for the backtest and one for the benchmark. @@ -164,12 +185,23 @@ private static (SortedList BacktestEquity, SortedList(); var historyStart = algorithm.StartDate - TimeSpan.FromDays(3); var historyEnd = algorithm.EndDate + TimeSpan.FromDays(1); + if (historyEnd > algorithm.Time) + { + // When running mid-backtest, requesting past the current algorithm time would get the request + // trimmed by the engine anyway, while also emitting a debug message to the user + historyEnd = algorithm.Time; + } foreach (var bar in algorithm.History(spy, historyStart, historyEnd, Resolution.Daily)) { var time = algorithm.Settings.DailyPreciseEndTime ? bar.EndTime.AddDays(1).Date : bar.EndTime; benchmarkSeries.Add(time.Date.ConvertToUtc(exchangeTimeZone), bar.Close); } + if (benchmarkSeries.Count == 0) + { + return (new(), new()); + } + // ── 3. Resample both to daily data points ──────────────────────────── var sampler = new SeriesSampler(TimeSpan.FromDays(1)); var start = new DateTime( diff --git a/Engine/Results/BacktestingResultHandler.cs b/Engine/Results/BacktestingResultHandler.cs index 8b37736c7b90..a30d95a86ac3 100644 --- a/Engine/Results/BacktestingResultHandler.cs +++ b/Engine/Results/BacktestingResultHandler.cs @@ -228,6 +228,11 @@ private void Update() // we store the last 100 order events, the final packet will contain the full list TransactionHandler.OrderEvents.Reverse().Take(100).ToList(), state: GetAlgorithmState())); + if (RunResultsAnalysis) + { + completeResult.Analysis = RunInRunResultsAnalysis(statisticsResult.TotalPerformance); + } + StoreResult(new BacktestResultPacket(_job, completeResult, Algorithm.EndDate, Algorithm.StartDate, progress)); _nextS3Update = DateTime.UtcNow.AddSeconds(30); @@ -440,6 +445,58 @@ protected void SendFinalResult() } } + /// + /// Runs the in-run results analyzer against a snapshot of the current intermediate backtest state. + /// Invoked periodically while the backtest is still running, unlike the full analysis performed + /// by when the backtest ends. + /// + /// The current total algorithm performance, for analyses that read portfolio statistics + /// The failed analyses with solutions, or null if the analysis could not run + protected virtual IReadOnlyList RunInRunResultsAnalysis(AlgorithmPerformance totalPerformance) + { + try + { + var algorithm = _job.Language == Language.Python ? (Algorithm as AlgorithmPythonWrapper)?.BaseAlgorithm : Algorithm as QCAlgorithm; + if (algorithm == null) + { + return null; + } + + // The analyses read the charts without holding ChartLock, so hand them clones + Dictionary charts; + lock (ChartLock) + { + charts = Charts.ToDictionary(kvp => kvp.Key, kvp => kvp.Value.Clone()); + } + + // Unlike the intermediate result stored to disk, the analyses get the full order + // and order event collections, not just the latest 100 + var snapshot = new BacktestResult(new BacktestResultParameters( + charts, + TransactionHandler.Orders.ToDictionary(), + Algorithm.Transactions.TransactionRecord, + new Dictionary(), + new Dictionary(), + new Dictionary(), + TransactionHandler.OrderEvents.ToList(), + totalPerformance)); + + List logs; + lock (LogStore) + { + logs = LogStore.Select(x => x.Message).ToList(); + } + + // Keep the time budget small: this runs on the result handler thread and delays message processing + return new InRunResultsAnalyzer(snapshot, algorithm, _job.Language, logs).Run(timeLimitSeconds: 1); + } + catch (Exception ex) + { + Log.Error(ex, "Error running in-run backtest analysis"); + return null; + } + } + /// /// Set the Algorithm instance for ths result. /// From 7d7e6c73284bcf6b95c00a4d663a92a5978152c4 Mon Sep 17 00:00:00 2001 From: Jhonathan Abreu Date: Tue, 21 Jul 2026 19:04:21 -0400 Subject: [PATCH 02/33] Analyze only new order events and logs on each in-run analysis run Instead of rescanning the full, monotonically growing order event and log collections on every periodic run, the InRunResultsAnalyzer is now kept alive for the duration of the backtest and tracks the positions already consumed, so each run only receives and scans the new entries. Findings from stream-scanning analyses are accumulated across runs (first sample kept, counts totaled), while state-based analyses (portfolio value, take profit/stop loss orders, margin usage) are recomputed against the full current state and replaced on every run. --- .../Results/Analysis/InRunResultsAnalyzer.cs | 101 +++++++++++++++++- Engine/Results/Analysis/ResultsAnalyzer.cs | 14 ++- Engine/Results/BacktestingResultHandler.cs | 26 +++-- 3 files changed, 126 insertions(+), 15 deletions(-) diff --git a/Engine/Results/Analysis/InRunResultsAnalyzer.cs b/Engine/Results/Analysis/InRunResultsAnalyzer.cs index a8620e8ee269..4d81f6fb4f30 100644 --- a/Engine/Results/Analysis/InRunResultsAnalyzer.cs +++ b/Engine/Results/Analysis/InRunResultsAnalyzer.cs @@ -15,7 +15,9 @@ */ using QuantConnect.Algorithm; using QuantConnect.Lean.Engine.Results.Analysis.Analyses; +using System; using System.Collections.Generic; +using System.Linq; namespace QuantConnect.Lean.Engine.Results.Analysis { @@ -25,16 +27,107 @@ namespace QuantConnect.Lean.Engine.Results.Analysis /// public class InRunResultsAnalyzer : ResultsAnalyzer { + /// + /// Analyses that read the current backtest state (statistics, orders, charts) instead of scanning + /// the append-only order event and log streams. They must run against the full current state on + /// every run, and their previous findings are replaced instead of accumulated. + /// + private static readonly HashSet StateBasedAnalyses = new() + { + nameof(PortfolioValueIsNotPositiveAnalysis), + nameof(TakeProfitAndStopLossOrdersAnalysis), + nameof(PortfolioMarginUsageAnalysis), + }; + + private readonly Dictionary _findings = new(); + + /// + /// The number of order events already consumed by previous runs. The order events + /// in the result passed to + /// are expected to start at this position. + /// + public int OrderEventsPosition { get; private set; } + + /// + /// The number of log entries already consumed by previous runs. The logs passed to + /// are expected to start + /// at this position. + /// + public int LogsPosition { get; private set; } + /// /// Initializes a new instance of the class. + /// The instance is expected to be kept alive for the duration of the backtest, + /// receiving fresh data on each call. /// - /// A snapshot of the current intermediate backtest result to analyze. /// The algorithm instance used for history requests and settings. /// The programming language the algorithm is written in. - /// The list of log lines produced by the backtest so far. - public InRunResultsAnalyzer(Result result, QCAlgorithm algorithm, Language language, IReadOnlyList logs) - : base(result, algorithm, language, logs) + public InRunResultsAnalyzer(QCAlgorithm algorithm, Language language) + : base(null, algorithm, language, null) + { + } + + /// + /// Runs the analyses incrementally: and are + /// expected to contain only the order events and log lines produced since the previous run + /// (per and ), and the returned + /// findings are the merge of this run's findings into the ones accumulated by previous runs. + /// Findings from analyses scanning the order event and log streams are accumulated + /// (first sample kept, counts totaled), while findings from state-based analyses are + /// replaced on every run. + /// + /// A snapshot of the current intermediate backtest result, holding only new order events. + /// The log lines produced since the previous run. + /// Wall-clock seconds allowed for the full chain before early exit. + /// Maximum number of failing analyses to return. + /// The accumulated findings, ranked by analysis weight. + public IReadOnlyList Run(Result result, IReadOnlyList logs, int timeLimitSeconds = 1, int maxFailedAnalyses = 10) + { + SetAnalysisData(result, logs); + var newFindings = Run(timeLimitSeconds, maxFailedAnalyses); + + OrderEventsPosition += result.OrderEvents?.Count ?? 0; + LogsPosition += logs?.Count ?? 0; + + // State-based analyses are recomputed from scratch each run: remove their previous + // findings so they are replaced, or dropped if they no longer fail + foreach (var name in _findings.Keys.Where(IsStateBased).ToList()) + { + _findings.Remove(name); + } + + foreach (var finding in newFindings) + { + if (!IsStateBased(finding.Name) && _findings.TryGetValue(finding.Name, out var previous)) + { + // This run only saw new order events and logs: keep the first sample and total the counts. + // A null count means a single occurrence + finding.Sample = previous.Sample; + finding.Count = (previous.Count ?? 1) + (finding.Count ?? 1); + } + _findings[finding.Name] = finding; + } + + var weights = GetAnalyses().ToDictionary(analysis => analysis.GetType().Name, analysis => analysis.Weight); + return _findings.Values + .OrderByDescending(finding => weights.GetValueOrDefault(BaseAnalysisName(finding.Name))) + .Take(maxFailedAnalyses) + .ToList(); + } + + /// + /// Determines whether the given finding was produced by a state-based analysis. + /// + private static bool IsStateBased(string findingName) => StateBasedAnalyses.Contains(BaseAnalysisName(findingName)); + + /// + /// Gets the analysis class name from a finding name, which aggregated + /// analyses suffix with the sub-analysis name. + /// + private static string BaseAnalysisName(string findingName) { + var separatorIndex = findingName.IndexOf(" / ", StringComparison.Ordinal); + return separatorIndex < 0 ? findingName : findingName[..separatorIndex]; } /// diff --git a/Engine/Results/Analysis/ResultsAnalyzer.cs b/Engine/Results/Analysis/ResultsAnalyzer.cs index a661ae25c75f..f9c56ff66f0a 100644 --- a/Engine/Results/Analysis/ResultsAnalyzer.cs +++ b/Engine/Results/Analysis/ResultsAnalyzer.cs @@ -31,7 +31,7 @@ public class ResultsAnalyzer { private readonly QCAlgorithm _algorithm; private readonly Language _language; - private readonly IReadOnlyList _logs; + private IReadOnlyList _logs; private SortedList _equityCurve; private SortedList _benchmarkEquityCurve; private Result _result; @@ -107,6 +107,18 @@ public ResultsAnalyzer(Result result, QCAlgorithm algorithm, Language language, return responses; } + /// + /// Sets the backtest data to analyze. Used by analyzers that are kept alive + /// and run multiple times against fresh data. + /// + /// The backtest result to analyze. + /// The list of log lines to analyze. + protected void SetAnalysisData(Result result, IReadOnlyList logs) + { + _result = result; + _logs = logs; + } + /// /// Whether the equity and benchmark curves should be built before running the analyses. /// Building them requires a benchmark history request, so analyzers whose analyses diff --git a/Engine/Results/BacktestingResultHandler.cs b/Engine/Results/BacktestingResultHandler.cs index a30d95a86ac3..ba6cfbcca6fd 100644 --- a/Engine/Results/BacktestingResultHandler.cs +++ b/Engine/Results/BacktestingResultHandler.cs @@ -52,6 +52,8 @@ public class BacktestingResultHandler : BaseResultsHandler, IResultHandler private BacktestProgressMonitor _progressMonitor; + private InRunResultsAnalyzer _inRunResultsAnalyzer; + /// /// Calculates the capacity of a strategy per Symbol in real-time /// @@ -469,8 +471,18 @@ protected void SendFinalResult() charts = Charts.ToDictionary(kvp => kvp.Key, kvp => kvp.Value.Clone()); } - // Unlike the intermediate result stored to disk, the analyses get the full order - // and order event collections, not just the latest 100 + _inRunResultsAnalyzer ??= new InRunResultsAnalyzer(algorithm, _job.Language); + + // Only the order events and logs produced since the previous run are analyzed, + // the analyzer accumulates findings across runs + var orderEvents = TransactionHandler.OrderEvents.Skip(_inRunResultsAnalyzer.OrderEventsPosition).ToList(); + + List logs; + lock (LogStore) + { + logs = LogStore.Skip(_inRunResultsAnalyzer.LogsPosition).Select(x => x.Message).ToList(); + } + var snapshot = new BacktestResult(new BacktestResultParameters( charts, TransactionHandler.Orders.ToDictionary(), @@ -478,17 +490,11 @@ protected void SendFinalResult() new Dictionary(), new Dictionary(), new Dictionary(), - TransactionHandler.OrderEvents.ToList(), + orderEvents, totalPerformance)); - List logs; - lock (LogStore) - { - logs = LogStore.Select(x => x.Message).ToList(); - } - // Keep the time budget small: this runs on the result handler thread and delays message processing - return new InRunResultsAnalyzer(snapshot, algorithm, _job.Language, logs).Run(timeLimitSeconds: 1); + return _inRunResultsAnalyzer.Run(snapshot, logs, timeLimitSeconds: 1); } catch (Exception ex) { From 0c3a58a3b2e454c808a720db3162163108936843 Mon Sep 17 00:00:00 2001 From: Jhonathan Abreu Date: Wed, 22 Jul 2026 09:21:33 -0400 Subject: [PATCH 03/33] Send in-run analysis findings to the browser in their own result packet --- Engine/Results/BacktestingResultHandler.cs | 27 ++++++++++++++++++++++ 1 file changed, 27 insertions(+) diff --git a/Engine/Results/BacktestingResultHandler.cs b/Engine/Results/BacktestingResultHandler.cs index ba6cfbcca6fd..6d3c301a8a2f 100644 --- a/Engine/Results/BacktestingResultHandler.cs +++ b/Engine/Results/BacktestingResultHandler.cs @@ -14,6 +14,7 @@ * */ +using Newtonsoft.Json; using QuantConnect.Algorithm; using QuantConnect.AlgorithmFactory.Python.Wrappers; using QuantConnect.Brokerages; @@ -53,6 +54,7 @@ public class BacktestingResultHandler : BaseResultsHandler, IResultHandler private BacktestProgressMonitor _progressMonitor; private InRunResultsAnalyzer _inRunResultsAnalyzer; + private string _lastInRunAnalysisSignature = "[]"; /// /// Calculates the capacity of a strategy per Symbol in real-time @@ -233,6 +235,7 @@ private void Update() if (RunResultsAnalysis) { completeResult.Analysis = RunInRunResultsAnalysis(statisticsResult.TotalPerformance); + SendInRunAnalysis(completeResult.Analysis, progress); } StoreResult(new BacktestResultPacket(_job, completeResult, Algorithm.EndDate, Algorithm.StartDate, progress)); @@ -503,6 +506,30 @@ protected void SendFinalResult() } } + /// + /// Sends the in-run analysis findings to the browser in their own packet, + /// only when they changed since they were last sent. + /// + /// The accumulated in-run analysis findings, or null if the analysis could not run + /// The current backtest progress + private void SendInRunAnalysis(IReadOnlyList findings, decimal progress) + { + if (findings == null) + { + return; + } + + var signature = JsonConvert.SerializeObject(findings); + if (signature == _lastInRunAnalysisSignature) + { + return; + } + _lastInRunAnalysisSignature = signature; + + MessagingHandler.Send(new BacktestResultPacket(_job, new BacktestResult { Analysis = findings }, + Algorithm.EndDate, Algorithm.StartDate, progress)); + } + /// /// Set the Algorithm instance for ths result. /// From b385894e721e37be6b7100ee4a27c2b8254127e1 Mon Sep 17 00:00:00 2001 From: Jhonathan Abreu Date: Wed, 22 Jul 2026 10:07:04 -0400 Subject: [PATCH 04/33] Document the accepted missed-delta behavior on in-run analysis time-limit truncation --- Engine/Results/Analysis/InRunResultsAnalyzer.cs | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/Engine/Results/Analysis/InRunResultsAnalyzer.cs b/Engine/Results/Analysis/InRunResultsAnalyzer.cs index 4d81f6fb4f30..e53a6dbc5925 100644 --- a/Engine/Results/Analysis/InRunResultsAnalyzer.cs +++ b/Engine/Results/Analysis/InRunResultsAnalyzer.cs @@ -86,6 +86,10 @@ public InRunResultsAnalyzer(QCAlgorithm algorithm, Language language) SetAnalysisData(result, logs); var newFindings = Run(timeLimitSeconds, maxFailedAnalyses); + // The positions are advanced even when the time limit truncates a run, so the analyses that + // didn't get to run miss this delta until the final analysis re-scans the complete streams. + // Stress tests show runs complete in a fraction of the time limit, but if its trace message + // starts showing up in logs, revisit this (e.g. track per-analysis positions). OrderEventsPosition += result.OrderEvents?.Count ?? 0; LogsPosition += logs?.Count ?? 0; From 8839190595e60f399fabe570a59c8324eea8317b Mon Sep 17 00:00:00 2001 From: Jhonathan Abreu Date: Wed, 22 Jul 2026 11:13:09 -0400 Subject: [PATCH 05/33] Add unit tests for the results analyzer and the in-run results analyzer --- .../Results/InRunResultsAnalyzerTests.cs | 276 ++++++++++++++++++ Tests/Engine/Results/ResultsAnalyzerTests.cs | 188 ++++++++++++ 2 files changed, 464 insertions(+) create mode 100644 Tests/Engine/Results/InRunResultsAnalyzerTests.cs create mode 100644 Tests/Engine/Results/ResultsAnalyzerTests.cs diff --git a/Tests/Engine/Results/InRunResultsAnalyzerTests.cs b/Tests/Engine/Results/InRunResultsAnalyzerTests.cs new file mode 100644 index 000000000000..f72f346073d7 --- /dev/null +++ b/Tests/Engine/Results/InRunResultsAnalyzerTests.cs @@ -0,0 +1,276 @@ +/* + * QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals. + * Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * +*/ + +using System; +using System.Collections.Generic; +using System.Linq; +using System.Threading; +using NUnit.Framework; +using QuantConnect.Lean.Engine.Results.Analysis; +using QuantConnect.Lean.Engine.Results.Analysis.Analyses; +using QuantConnect.Orders; +using QuantConnect.Packets; + +namespace QuantConnect.Tests.Engine.Results +{ + [TestFixture] + public class InRunResultsAnalyzerTests + { + private static readonly IReadOnlyList SomeSolutions = new[] { "A solution" }; + + [Test] + public void PositionsAdvanceByTheConsumedOrderEventsAndLogs() + { + var analyzer = new TestInRunResultsAnalyzer(new FakeAnalysisA(10)); + + analyzer.Run(MakeResult(3), new[] { "log 1", "log 2" }); + Assert.AreEqual(3, analyzer.OrderEventsPosition); + Assert.AreEqual(2, analyzer.LogsPosition); + + analyzer.Run(MakeResult(5), new[] { "log 3" }); + Assert.AreEqual(8, analyzer.OrderEventsPosition); + Assert.AreEqual(3, analyzer.LogsPosition); + + // Null order events and logs don't move the positions + analyzer.Run(new BacktestResult(), null); + Assert.AreEqual(8, analyzer.OrderEventsPosition); + Assert.AreEqual(3, analyzer.LogsPosition); + } + + [Test] + public void PositionsAdvanceEvenWhenTheTimeLimitTruncatesTheRun() + { + var truncatedRan = false; + // The slow analysis has the higher weight so it runs first and exhausts the time limit + var slow = new FakeAnalysisA(20) { OnRun = () => Thread.Sleep(1100) }; + var truncated = new FakeAnalysisB(10) { OnRun = () => truncatedRan = true }; + var analyzer = new TestInRunResultsAnalyzer(slow, truncated); + + analyzer.Run(MakeResult(4), new[] { "log 1" }, timeLimitSeconds: 1); + + Assert.IsFalse(truncatedRan); + Assert.AreEqual(4, analyzer.OrderEventsPosition); + Assert.AreEqual(1, analyzer.LogsPosition); + } + + [Test] + public void StreamBasedFindingsAccumulateAcrossRuns() + { + var fake = new FakeAnalysisA(10); + var analyzer = new TestInRunResultsAnalyzer(fake); + + fake.Findings = () => MakeFindings(nameof(FakeAnalysisA), "first sample", 3); + analyzer.Run(MakeResult(1), new[] { "log" }); + + fake.Findings = () => MakeFindings(nameof(FakeAnalysisA), "second sample", 2); + var findings = analyzer.Run(MakeResult(1), new[] { "log" }); + + var finding = findings.Single(); + Assert.AreEqual("first sample", finding.Sample); + Assert.AreEqual(5, finding.Count); + } + + [Test] + public void StreamBasedFindingsWithNullCountsCountSingleOccurrences() + { + var fake = new FakeAnalysisA(10) + { + Findings = () => MakeFindings(nameof(FakeAnalysisA), "sample", null) + }; + var analyzer = new TestInRunResultsAnalyzer(fake); + + analyzer.Run(MakeResult(1), new[] { "log" }); + var findings = analyzer.Run(MakeResult(1), new[] { "log" }); + + Assert.AreEqual(2, findings.Single().Count); + } + + [Test] + public void StreamBasedFindingsPersistWhenNotReemitted() + { + var fake = new FakeAnalysisA(10) + { + Findings = () => MakeFindings(nameof(FakeAnalysisA), "sample", 4) + }; + var analyzer = new TestInRunResultsAnalyzer(fake); + analyzer.Run(MakeResult(1), new[] { "log" }); + + // The next delta produces no new occurrences: the accumulated finding is still reported + fake.Findings = () => new List(); + var findings = analyzer.Run(MakeResult(1), new[] { "log" }); + + var finding = findings.Single(); + Assert.AreEqual("sample", finding.Sample); + Assert.AreEqual(4, finding.Count); + } + + [Test] + public void StateBasedFindingsAreReplacedOnEveryRun() + { + var fake = new FakeAnalysisA(10) + { + Findings = () => MakeFindings(nameof(PortfolioValueIsNotPositiveAnalysis), "old sample", 2) + }; + var analyzer = new TestInRunResultsAnalyzer(fake); + analyzer.Run(MakeResult(1), new[] { "log" }); + + fake.Findings = () => MakeFindings(nameof(PortfolioValueIsNotPositiveAnalysis), "new sample", 3); + var findings = analyzer.Run(MakeResult(1), new[] { "log" }); + + // Replaced, not accumulated: latest sample and count win + var finding = findings.Single(); + Assert.AreEqual("new sample", finding.Sample); + Assert.AreEqual(3, finding.Count); + } + + [Test] + public void StateBasedFindingsAreDroppedWhenTheyNoLongerFail() + { + var fake = new FakeAnalysisA(10) + { + Findings = () => MakeFindings(nameof(PortfolioValueIsNotPositiveAnalysis), "sample", 2) + }; + var analyzer = new TestInRunResultsAnalyzer(fake); + Assert.IsNotEmpty(analyzer.Run(MakeResult(1), new[] { "log" })); + + fake.Findings = () => new List(); + var findings = analyzer.Run(MakeResult(1), new[] { "log" }); + + Assert.IsEmpty(findings); + } + + [Test] + public void AggregatedStateBasedFindingsAreReplacedByFullName() + { + // Aggregated analyses emit "AnalysisClass / SubAnalysis" finding names: state-based + // behavior is determined by the base analysis name, replacement is keyed by the full name + var stateBasedName = nameof(PortfolioValueIsNotPositiveAnalysis); + var fake = new FakeAnalysisA(10) + { + Findings = () => MakeFindings($"{stateBasedName} / SubA", "sample a", 1) + .Concat(MakeFindings($"{stateBasedName} / SubB", "sample b", 1)) + .ToList() + }; + var analyzer = new TestInRunResultsAnalyzer(fake); + Assert.AreEqual(2, analyzer.Run(MakeResult(1), new[] { "log" }).Count); + + fake.Findings = () => MakeFindings($"{stateBasedName} / SubA", "new sample a", 2); + var findings = analyzer.Run(MakeResult(1), new[] { "log" }); + + // SubB no longer fails and is dropped; SubA is replaced with the fresh finding + var finding = findings.Single(); + Assert.AreEqual($"{stateBasedName} / SubA", finding.Name); + Assert.AreEqual("new sample a", finding.Sample); + Assert.AreEqual(2, finding.Count); + } + + [Test] + public void FindingsAreRankedByAnalysisWeightAndCapped() + { + var lowWeight = new FakeAnalysisA(10) + { + Findings = () => MakeFindings(nameof(FakeAnalysisA), "sample a", 1) + }; + // Aggregated finding names rank by their base analysis' weight + var midWeight = new FakeAnalysisB(20) + { + Findings = () => MakeFindings($"{nameof(FakeAnalysisB)} / Sub", "sample b", 1) + }; + var highWeight = new FakeAnalysisC(30) + { + Findings = () => MakeFindings(nameof(FakeAnalysisC), "sample c", 1) + }; + var analyzer = new TestInRunResultsAnalyzer(lowWeight, midWeight, highWeight); + + var findings = analyzer.Run(MakeResult(1), new[] { "log" }); + CollectionAssert.AreEqual( + new[] { nameof(FakeAnalysisC), $"{nameof(FakeAnalysisB)} / Sub", nameof(FakeAnalysisA) }, + findings.Select(finding => finding.Name)); + + // The accumulated findings are capped to the top weighted ones + lowWeight.Findings = midWeight.Findings = highWeight.Findings = () => new List(); + findings = analyzer.Run(MakeResult(1), new[] { "log" }, maxFailedAnalyses: 2); + CollectionAssert.AreEqual( + new[] { nameof(FakeAnalysisC), $"{nameof(FakeAnalysisB)} / Sub" }, + findings.Select(finding => finding.Name)); + } + + private static BacktestResult MakeResult(int orderEventsCount) + { + return new BacktestResult + { + OrderEvents = Enumerable.Range(0, orderEventsCount).Select(_ => new OrderEvent()).ToList() + }; + } + + private static List MakeFindings(string name, string sample, int? count) + { + return new List { new(name, "An issue", sample, count, SomeSolutions) }; + } + + private class TestInRunResultsAnalyzer : InRunResultsAnalyzer + { + private readonly IReadOnlyCollection _analyses; + + public TestInRunResultsAnalyzer(params BaseResultsAnalysis[] analyses) + : base(null, Language.CSharp) + { + _analyses = analyses; + } + + protected override IReadOnlyCollection GetAnalyses() => _analyses; + } + + private class FakeAnalysis : BaseResultsAnalysis + { + private readonly int _weight; + + public override string Issue => "A fake issue"; + + public override int Weight => _weight; + + public Func> Findings { get; set; } = () => new List(); + + public Action OnRun { get; set; } + + protected FakeAnalysis(int weight) + { + _weight = weight; + } + + public override IReadOnlyList Run(ResultsAnalysisRunParameters parameters) + { + OnRun?.Invoke(); + return Findings(); + } + } + + private sealed class FakeAnalysisA : FakeAnalysis + { + public FakeAnalysisA(int weight) : base(weight) { } + } + + private sealed class FakeAnalysisB : FakeAnalysis + { + public FakeAnalysisB(int weight) : base(weight) { } + } + + private sealed class FakeAnalysisC : FakeAnalysis + { + public FakeAnalysisC(int weight) : base(weight) { } + } + } +} diff --git a/Tests/Engine/Results/ResultsAnalyzerTests.cs b/Tests/Engine/Results/ResultsAnalyzerTests.cs new file mode 100644 index 000000000000..f0e22c0b50a3 --- /dev/null +++ b/Tests/Engine/Results/ResultsAnalyzerTests.cs @@ -0,0 +1,188 @@ +/* + * QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals. + * Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * +*/ + +using System; +using System.Collections.Generic; +using System.Linq; +using System.Threading; +using NUnit.Framework; +using QuantConnect.Lean.Engine.Results.Analysis; +using QuantConnect.Lean.Engine.Results.Analysis.Analyses; + +namespace QuantConnect.Tests.Engine.Results +{ + [TestFixture] + public class ResultsAnalyzerTests + { + private static readonly IReadOnlyList SomeSolutions = new[] { "A solution" }; + + [Test] + public void RunsTheOverriddenAnalysisSetAndKeepsFindingsWithSolutions() + { + var withSolutions = new FakeAnalysisA(10) + { + Findings = () => new List + { + new(nameof(FakeAnalysisA), "An issue", "sample", null, SomeSolutions) + } + }; + var withoutSolutions = new FakeAnalysisB(20) + { + Findings = () => new List + { + new(nameof(FakeAnalysisB), "An issue", "sample", null, new List()) + } + }; + var analyzer = new TestResultsAnalyzer(false, withSolutions, withoutSolutions); + + var findings = analyzer.Run(); + + // Findings without solutions are not reported + Assert.AreEqual(nameof(FakeAnalysisA), findings.Single().Name); + } + + [Test] + public void EmptyAnalysisSetProducesNoFindings() + { + var analyzer = new TestResultsAnalyzer(false); + Assert.IsEmpty(analyzer.Run()); + } + + [Test] + public void SkipsEquityCurveConstructionWhenNotRequired() + { + ResultsAnalysisRunParameters seenParameters = null; + var fake = new FakeAnalysisA(10) { OnRun = parameters => seenParameters = parameters }; + // Null result and algorithm: building the curves would throw, so a successful + // run proves the equity curves were skipped + var analyzer = new TestResultsAnalyzer(false, fake); + + Assert.DoesNotThrow(() => analyzer.Run()); + + Assert.IsNotNull(seenParameters); + Assert.IsNotNull(seenParameters.EquityCurve); + Assert.IsEmpty(seenParameters.EquityCurve); + Assert.IsNotNull(seenParameters.BenchmarkEquityCurve); + Assert.IsEmpty(seenParameters.BenchmarkEquityCurve); + } + + [Test] + public void AnalysesRunInDescendingWeightOrder() + { + var runOrder = new List(); + var analyzer = new TestResultsAnalyzer(false, + new FakeAnalysisA(10) { OnRun = _ => runOrder.Add(nameof(FakeAnalysisA)) }, + new FakeAnalysisB(30) { OnRun = _ => runOrder.Add(nameof(FakeAnalysisB)) }, + new FakeAnalysisC(20) { OnRun = _ => runOrder.Add(nameof(FakeAnalysisC)) }); + + analyzer.Run(); + + CollectionAssert.AreEqual( + new[] { nameof(FakeAnalysisB), nameof(FakeAnalysisC), nameof(FakeAnalysisA) }, + runOrder); + } + + [Test] + public void TimeLimitStopsTheAnalysisChain() + { + var truncatedRan = false; + // The slow analysis has the higher weight so it runs first and exhausts the time limit + var slow = new FakeAnalysisA(20) { OnRun = _ => Thread.Sleep(1100) }; + var truncated = new FakeAnalysisB(10) { OnRun = _ => truncatedRan = true }; + var analyzer = new TestResultsAnalyzer(false, slow, truncated); + + analyzer.Run(timeLimitSeconds: 1); + + Assert.IsFalse(truncatedRan); + } + + [Test] + public void MaxFailedAnalysesStopsTheAnalysisChain() + { + var skippedRan = false; + var failing = new FakeAnalysisA(20) + { + Findings = () => new List + { + new(nameof(FakeAnalysisA), "An issue", "sample", null, SomeSolutions) + } + }; + var skipped = new FakeAnalysisB(10) { OnRun = _ => skippedRan = true }; + var analyzer = new TestResultsAnalyzer(false, failing, skipped); + + var findings = analyzer.Run(maxFailedAnalyses: 1); + + Assert.IsFalse(skippedRan); + Assert.AreEqual(1, findings.Count); + } + + private class TestResultsAnalyzer : ResultsAnalyzer + { + private readonly bool _requiresEquityCurves; + private readonly IReadOnlyCollection _analyses; + + public TestResultsAnalyzer(bool requiresEquityCurves, params BaseResultsAnalysis[] analyses) + : base(null, null, Language.CSharp, null) + { + _requiresEquityCurves = requiresEquityCurves; + _analyses = analyses; + } + + protected override bool RequiresEquityCurves => _requiresEquityCurves; + + protected override IReadOnlyCollection GetAnalyses() => _analyses; + } + + private class FakeAnalysis : BaseResultsAnalysis + { + private readonly int _weight; + + public override string Issue => "A fake issue"; + + public override int Weight => _weight; + + public Func> Findings { get; set; } = () => new List(); + + public Action OnRun { get; set; } + + protected FakeAnalysis(int weight) + { + _weight = weight; + } + + public override IReadOnlyList Run(ResultsAnalysisRunParameters parameters) + { + OnRun?.Invoke(parameters); + return Findings(); + } + } + + private sealed class FakeAnalysisA : FakeAnalysis + { + public FakeAnalysisA(int weight) : base(weight) { } + } + + private sealed class FakeAnalysisB : FakeAnalysis + { + public FakeAnalysisB(int weight) : base(weight) { } + } + + private sealed class FakeAnalysisC : FakeAnalysis + { + public FakeAnalysisC(int weight) : base(weight) { } + } + } +} From f95d30ba0b75d783268934715e9e76ec5b76dc03 Mon Sep 17 00:00:00 2001 From: Jhonathan Abreu Date: Wed, 22 Jul 2026 12:58:50 -0400 Subject: [PATCH 06/33] Track algorithm speed with an in-run analysis for early stop decisions --- Engine/Engine.cs | 6 +- .../Results/Analysis/AlgorithmSpeedSample.cs | 68 ++++ .../Results/Analysis/AlgorithmSpeedTracker.cs | 204 ++++++++++++ .../Analyses/AlgorithmSpeedAnalysis.cs | 298 ++++++++++++++++++ .../Results/Analysis/InRunResultsAnalyzer.cs | 58 +++- .../Analysis/ResultsAnalysisRunParameters.cs | 10 +- Engine/Results/Analysis/ResultsAnalyzer.cs | 9 +- Engine/Results/BacktestingResultHandler.cs | 17 +- Engine/Results/BaseResultsHandler.cs | 7 + .../ResultHandlerInitializeParameters.cs | 11 +- .../Results/AlgorithmSpeedAnalysisTests.cs | 208 ++++++++++++ .../Results/AlgorithmSpeedTrackerTests.cs | 144 +++++++++ .../Results/InRunResultsAnalyzerTests.cs | 38 ++- 13 files changed, 1060 insertions(+), 18 deletions(-) create mode 100644 Engine/Results/Analysis/AlgorithmSpeedSample.cs create mode 100644 Engine/Results/Analysis/AlgorithmSpeedTracker.cs create mode 100644 Engine/Results/Analysis/Analyses/AlgorithmSpeedAnalysis.cs create mode 100644 Tests/Engine/Results/AlgorithmSpeedAnalysisTests.cs create mode 100644 Tests/Engine/Results/AlgorithmSpeedTrackerTests.cs diff --git a/Engine/Engine.cs b/Engine/Engine.cs index 691d8db248c9..3dafb6aaf9aa 100644 --- a/Engine/Engine.cs +++ b/Engine/Engine.cs @@ -104,12 +104,14 @@ public void Run(AlgorithmNodePacket job, AlgorithmManager manager, string assemb //-> Initialize messaging system SystemHandlers.Notify.SetAuthentication(job); + var performanceTrackingTool = new PerformanceTrackingTool(); + //-> Set the result handler type for this algorithm job, and launch the associated result thread. - AlgorithmHandlers.Results.Initialize(new(job, SystemHandlers.Notify, SystemHandlers.Api, AlgorithmHandlers.Transactions, AlgorithmHandlers.MapFileProvider)); + AlgorithmHandlers.Results.Initialize( + new(job, SystemHandlers.Notify, SystemHandlers.Api, AlgorithmHandlers.Transactions, AlgorithmHandlers.MapFileProvider, performanceTrackingTool)); IBrokerage brokerage = null; DataManager dataManager = null; - var performanceTrackingTool = new PerformanceTrackingTool(); var synchronizer = _liveMode ? new LiveSynchronizer() : new Synchronizer(); try { diff --git a/Engine/Results/Analysis/AlgorithmSpeedSample.cs b/Engine/Results/Analysis/AlgorithmSpeedSample.cs new file mode 100644 index 000000000000..8487e781cb0e --- /dev/null +++ b/Engine/Results/Analysis/AlgorithmSpeedSample.cs @@ -0,0 +1,68 @@ +/* + * QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals. + * Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * +*/ +using System; + +namespace QuantConnect.Lean.Engine.Results.Analysis +{ + /// + /// A point-in-time sample of the engine's cumulative speed counters, fed by the result handler + /// into the on each in-run analysis run. + /// + public readonly struct AlgorithmSpeedSample + { + /// + /// The wall-clock time elapsed since the backtest started. + /// + public TimeSpan Elapsed { get; } + + /// + /// The cumulative data points processed by the main algorithm loop. + /// + public long DataPoints { get; } + + /// + /// The cumulative data points served by the history provider. + /// + public long HistoryDataPoints { get; } + + /// + /// The calendar days the backtest has processed so far. + /// + public int ProcessedDays { get; } + + /// + /// The total calendar days the backtest will run. + /// + public int TotalDays { get; } + + /// + /// Initializes a new instance of the struct. + /// + /// Wall-clock time elapsed since the backtest started. + /// Cumulative data points processed by the main algorithm loop. + /// Cumulative data points served by the history provider. + /// Calendar days the backtest has processed so far. + /// Total calendar days the backtest will run. + public AlgorithmSpeedSample(TimeSpan elapsed, long dataPoints, long historyDataPoints, int processedDays, int totalDays) + { + Elapsed = elapsed; + DataPoints = dataPoints; + HistoryDataPoints = historyDataPoints; + ProcessedDays = processedDays; + TotalDays = totalDays; + } + } +} diff --git a/Engine/Results/Analysis/AlgorithmSpeedTracker.cs b/Engine/Results/Analysis/AlgorithmSpeedTracker.cs new file mode 100644 index 000000000000..69a7c0c6cdae --- /dev/null +++ b/Engine/Results/Analysis/AlgorithmSpeedTracker.cs @@ -0,0 +1,204 @@ +/* + * QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals. + * Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * +*/ +using System; +using System.Collections.Generic; + +namespace QuantConnect.Lean.Engine.Results.Analysis +{ + /// + /// Accumulates periodic samples of the engine's speed counters (data points processed, history data points, + /// backtest days processed) while a backtest runs, and computes the throughput and progress metrics consumed + /// by the in-run . + /// All rates are computed between samples, so setup time before the first sample is excluded. + /// + public class AlgorithmSpeedTracker + { + /// + /// The number of trailing samples that make up the "recent" window used by the windowed rates. + /// At the ~30 second in-run analysis cadence this spans roughly the last two minutes. + /// + public const int RecentWindowSamples = 5; + + private readonly List _samples = new(); + + /// + /// The number of samples recorded so far. + /// + public int SampleCount => _samples.Count; + + /// + /// The total number of calendar days the backtest will run. + /// + public int TotalDays => _samples.Count > 0 ? _samples[^1].TotalDays : 0; + + /// + /// The number of calendar days the backtest has processed as of the latest sample. + /// + public int ProcessedDays => _samples.Count > 0 ? _samples[^1].ProcessedDays : 0; + + /// + /// The backtest progress as of the latest sample, in the [0, 1] range. + /// + public decimal Progress => TotalDays > 0 ? Math.Min((decimal)ProcessedDays / TotalDays, 1m) : 0m; + + /// + /// The wall-clock time elapsed since the backtest started, as of the latest sample. + /// + public TimeSpan Elapsed => _samples.Count > 0 ? _samples[^1].Elapsed : TimeSpan.Zero; + + /// + /// The wall-clock time between the first and the latest sample, that is, + /// the period the rates are measured over. + /// + public TimeSpan SampledSpan => _samples.Count > 1 ? _samples[^1].Elapsed - _samples[0].Elapsed : TimeSpan.Zero; + + /// + /// Whether the main loop data point counter is being fed. When the counter is not wired in + /// (it reads zero), data-point-based rates are not meaningful and should not be used. + /// + public bool HasDataPointCounts => _samples.Count > 0 && _samples[^1].DataPoints > 0; + + /// + /// Records a sample of the cumulative speed counters. Samples with a non-increasing + /// elapsed time are ignored so rates are always computed over positive time deltas. + /// + /// The sample of the cumulative speed counters. + public void AddSample(AlgorithmSpeedSample sample) + { + if (_samples.Count > 0 && sample.Elapsed <= _samples[^1].Elapsed) + { + return; + } + _samples.Add(sample); + } + + /// + /// The average data points processed per second over the whole sampled span, including + /// history data points to match the speed the engine reports on completion. + /// Null when there are not enough samples to measure. + /// + public double? DataPointsPerSecond => RateBetween(0, _samples.Count - 1, TotalDataPoints); + + /// + /// The average data points processed per second over the first samples, + /// used as the early-run baseline for degradation detection. Null when there are not enough samples to measure. + /// + public double? InitialDataPointsPerSecond => RateBetween(0, Math.Min(RecentWindowSamples, _samples.Count) - 1, TotalDataPoints); + + /// + /// The average data points processed per second over the recent window, including history data points. + /// + /// Number of trailing samples to skip, to evaluate the window as of a previous run. + /// The windowed rate, or null when there are not enough samples to measure. + public double? RecentDataPointsPerSecond(int skipLast = 0) + { + var (start, end) = RecentWindow(skipLast); + return RateBetween(start, end, TotalDataPoints); + } + + /// + /// The average backtest calendar days processed per wall-clock second over the recent window. + /// + /// Number of trailing samples to skip, to evaluate the window as of a previous run. + /// The windowed rate, or null when there are not enough samples to measure. + public double? RecentDaysPerSecond(int skipLast = 0) + { + var (start, end) = RecentWindow(skipLast); + return RateBetween(start, end, sample => sample.ProcessedDays); + } + + /// + /// The number of history data points served over the recent window. + /// + /// Number of trailing samples to skip, to evaluate the window as of a previous run. + public long RecentHistoryDataPoints(int skipLast = 0) + { + var (start, end) = RecentWindow(skipLast); + return end > start ? _samples[end].HistoryDataPoints - _samples[start].HistoryDataPoints : 0; + } + + /// + /// The share of the data points processed over the recent window that were served by the history + /// provider, in the [0, 1] range. + /// + /// Number of trailing samples to skip, to evaluate the window as of a previous run. + /// The share, or null when there are not enough samples or no data points were processed in the window. + public double? RecentHistoryDataPointsShare(int skipLast = 0) + { + var (start, end) = RecentWindow(skipLast); + if (end <= start) + { + return null; + } + var totalDelta = TotalDataPoints(_samples[end]) - TotalDataPoints(_samples[start]); + if (totalDelta <= 0) + { + return null; + } + return (_samples[end].HistoryDataPoints - _samples[start].HistoryDataPoints) / totalDelta; + } + + /// + /// The estimated wall-clock time left for the backtest to complete, projecting the recent + /// calendar-days-per-second pace over the remaining backtest days. + /// + /// Number of trailing samples to skip, to evaluate the projection as of a previous run. + /// The estimate, zero when the backtest already reached its end date, or null when the recent pace + /// is zero or there are not enough samples to measure. + public TimeSpan? EstimatedRemainingTime(int skipLast = 0) + { + var end = _samples.Count - 1 - skipLast; + if (end < 0 || TotalDays <= 0) + { + return null; + } + var remainingDays = TotalDays - _samples[end].ProcessedDays; + if (remainingDays <= 0) + { + return TimeSpan.Zero; + } + var daysPerSecond = RecentDaysPerSecond(skipLast); + if (daysPerSecond is null or <= 0) + { + return null; + } + return TimeSpan.FromSeconds(remainingDays / daysPerSecond.Value); + } + + private (int Start, int End) RecentWindow(int skipLast) + { + var end = _samples.Count - 1 - skipLast; + var start = Math.Max(0, end - RecentWindowSamples + 1); + return (start, end); + } + + private double? RateBetween(int startIndex, int endIndex, Func selector) + { + if (startIndex < 0 || endIndex <= startIndex || endIndex >= _samples.Count) + { + return null; + } + var seconds = (_samples[endIndex].Elapsed - _samples[startIndex].Elapsed).TotalSeconds; + if (seconds <= 0) + { + return null; + } + return (selector(_samples[endIndex]) - selector(_samples[startIndex])) / seconds; + } + + private static double TotalDataPoints(AlgorithmSpeedSample sample) => sample.DataPoints + sample.HistoryDataPoints; + } +} diff --git a/Engine/Results/Analysis/Analyses/AlgorithmSpeedAnalysis.cs b/Engine/Results/Analysis/Analyses/AlgorithmSpeedAnalysis.cs new file mode 100644 index 000000000000..487493767357 --- /dev/null +++ b/Engine/Results/Analysis/Analyses/AlgorithmSpeedAnalysis.cs @@ -0,0 +1,298 @@ +/* + * QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals. + * Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * +*/ +using System; +using System.Collections.Generic; +using static QuantConnect.StringExtensions; + +namespace QuantConnect.Lean.Engine.Results.Analysis.Analyses +{ + /// + /// In-run analysis that tracks the algorithm's execution speed so the user can decide to stop + /// a slow backtest early. It reads the throughput and progress metrics accumulated by + /// and reports slow processing speed, a long projected + /// remaining runtime, degrading throughput, and history-request-dominated data loads. + /// Benchmark speeds: https://www.quantconnect.com/performance + /// + public class AlgorithmSpeedAnalysis : BaseResultsAnalysis + { + /// + /// The data points per second under which execution is reported as slow, + /// matching the threshold used by on completed backtests. + /// + public const int SlowDataPointsPerSecond = 40_000; + + /// + /// The recent-to-initial throughput ratio under which throughput is reported as degrading. + /// + public const double DegradationRatio = 0.5; + + /// + /// The share of recently processed data points served by the history provider + /// over which the data load is reported as history-request dominated. + /// + public const double HighHistoryDataPointsShare = 0.5; + + /// + /// The minimum number of history data points in the recent window for the + /// history-request load to be worth reporting. + /// + public const long MinimumRecentHistoryDataPoints = 10_000; + + /// + /// The minimum wall-clock span the metrics must cover before any finding is reported, + /// so early warm-up noise doesn't produce false positives. + /// + public static readonly TimeSpan MinimumSampledSpan = TimeSpan.FromMinutes(1); + + /// + /// The projected remaining runtime over which the backtest is reported as long-running. + /// + public static readonly TimeSpan LongProjectedRemainingTime = TimeSpan.FromHours(1); + + /// + /// The name of the slow execution sub-finding. + /// + public const string SlowExecutionName = "SlowExecution"; + + /// + /// The name of the long projected runtime sub-finding. + /// + public const string LongProjectedRuntimeName = "LongProjectedRuntime"; + + /// + /// The name of the degrading throughput sub-finding. + /// + public const string ThroughputDegradationName = "ThroughputDegradation"; + + /// + /// The name of the history-request load sub-finding. + /// + public const string HistoryRequestLoadName = "HistoryRequestLoad"; + + /// + /// Gets the description of the slow algorithm issue. + /// + public override string Issue { get; } = "The algorithm is running slowly."; + + /// + /// Gets the severity weight for the algorithm speed analysis. + /// + public override int Weight { get; } = 77; + + /// + /// Runs the algorithm speed analysis against the speed metrics tracked for the running backtest. + /// + public override IReadOnlyList Run(ResultsAnalysisRunParameters parameters) => Run(parameters.Speed); + + /// + /// Runs the algorithm speed analysis against the given speed metrics. + /// Each detected condition is reported as its own sub-finding. Every condition must hold for + /// both the current recent window and the window as of the previous run, so a single noisy + /// sample doesn't flag or clear a finding. + /// + /// The speed metrics tracked for the running backtest, or null when not tracked. + /// The failed sub-findings, or empty when speed is not tracked or still within the warm-up span. + public IReadOnlyList Run(AlgorithmSpeedTracker speed) + { + if (speed == null || speed.SampledSpan < MinimumSampledSpan) + { + return []; + } + + var findings = new List(); + AddSlowExecution(speed, findings); + AddLongProjectedRuntime(speed, findings); + AddThroughputDegradation(speed, findings); + AddHistoryRequestLoad(speed, findings); + return CreateAggregatedResponse(findings); + } + + /// + /// Reports slow execution when the recent data points per second are below the platform benchmark. + /// + private static void AddSlowExecution(AlgorithmSpeedTracker speed, List findings) + { + if (!speed.HasDataPointCounts) + { + return; + } + + var recent = speed.RecentDataPointsPerSecond(); + var previous = speed.RecentDataPointsPerSecond(skipLast: 1); + if (recent is null or >= SlowDataPointsPerSecond || previous is null or >= SlowDataPointsPerSecond) + { + return; + } + + var average = speed.DataPointsPerSecond ?? 0; + var remaining = speed.EstimatedRemainingTime(); + var projection = remaining.HasValue + ? Invariant($"about {FormatDuration(remaining.Value)} remaining at the recent pace") + : "the remaining time cannot be estimated yet"; + var sample = Invariant($"Processing {recent.Value / 1000:F1}k data points per second recently ") + + Invariant($"({average / 1000:F1}k average); {speed.Progress * 100:F0}% complete after ") + + Invariant($"{FormatDuration(speed.Elapsed)}, {projection}."); + + findings.Add(new(SlowExecutionName, + Invariant($"The algorithm is running below {SlowDataPointsPerSecond / 1000}k data points per second."), + sample, + null, + [ + "Review the algorithm code for inefficiencies.", + + "If there is a universe, reduce its size.", + + "Reduce the data resolution.", + + "If the algorithm is training a model, reduce the amount of training data or reduce the number of epochs in the training process.", + + "If the projected runtime is not acceptable, stop the backtest, apply the changes above, and run it again.", + ])); + } + + /// + /// Reports a long projected runtime when, at the recent pace, the backtest needs more than + /// to complete, or when it has stopped making + /// backtest-time progress altogether. + /// + private static void AddLongProjectedRuntime(AlgorithmSpeedTracker speed, List findings) + { + if (speed.TotalDays <= 0 || speed.ProcessedDays >= speed.TotalDays) + { + return; + } + + var daysPerSecond = speed.RecentDaysPerSecond(); + var previousDaysPerSecond = speed.RecentDaysPerSecond(skipLast: 1); + + string sample = null; + if (daysPerSecond is 0 && previousDaysPerSecond is 0) + { + sample = Invariant($"The backtest has made no backtest-time progress recently: ") + + Invariant($"still {speed.Progress * 100:F0}% complete after {FormatDuration(speed.Elapsed)}."); + } + else + { + var remaining = speed.EstimatedRemainingTime(); + var previousRemaining = speed.EstimatedRemainingTime(skipLast: 1); + if (remaining > LongProjectedRemainingTime && previousRemaining > LongProjectedRemainingTime) + { + sample = Invariant($"About {FormatDuration(remaining.Value)} of backtest remain at the recent pace ") + + Invariant($"({speed.Progress * 100:F0}% complete after {FormatDuration(speed.Elapsed)})."); + } + } + + if (sample == null) + { + return; + } + + findings.Add(new(LongProjectedRuntimeName, + "The backtest is projected to take a long time to complete.", + sample, + null, + [ + "Reduce the backtest period.", + + "Reduce the data resolution or the universe size.", + + "Review the algorithm code for inefficiencies.", + + "If the projected runtime is not acceptable, stop the backtest, apply the changes above, and run it again.", + ])); + } + + /// + /// Reports degrading throughput when the recent data points per second dropped below + /// of the early-run baseline. Requires enough samples for the + /// baseline and recent windows to not overlap. + /// + private static void AddThroughputDegradation(AlgorithmSpeedTracker speed, List findings) + { + if (!speed.HasDataPointCounts || speed.SampleCount < 2 * AlgorithmSpeedTracker.RecentWindowSamples + 1) + { + return; + } + + var initial = speed.InitialDataPointsPerSecond; + var recent = speed.RecentDataPointsPerSecond(); + var previous = speed.RecentDataPointsPerSecond(skipLast: 1); + if (initial is null or <= 0 || recent == null || previous == null || + recent >= DegradationRatio * initial || previous >= DegradationRatio * initial) + { + return; + } + + findings.Add(new(ThroughputDegradationName, + "The algorithm's processing speed is degrading as the backtest progresses.", + Invariant($"Throughput dropped from {initial.Value / 1000:F1}k data points per second early in the run ") + + Invariant($"to {recent.Value / 1000:F1}k recently."), + null, + [ + "Check for collections that grow unboundedly as the backtest progresses, like lists of past data points; use rolling windows with a fixed size instead.", + + "Check for history requests whose range grows as the backtest progresses, like requests from the algorithm start date to the current time.", + + "If there is a universe, check whether the number of selected securities keeps growing; remove securities that are no longer used.", + + "Check the algorithm's memory usage: sustained growth causes garbage collection pressure that slows the whole run down.", + ])); + } + + /// + /// Reports a history-request-dominated data load when most of the recently processed data + /// points were served by the history provider. + /// + private static void AddHistoryRequestLoad(AlgorithmSpeedTracker speed, List findings) + { + var share = speed.RecentHistoryDataPointsShare(); + var previousShare = speed.RecentHistoryDataPointsShare(skipLast: 1); + if (share is null or <= HighHistoryDataPointsShare || previousShare is null or <= HighHistoryDataPointsShare || + speed.RecentHistoryDataPoints() < MinimumRecentHistoryDataPoints) + { + return; + } + + findings.Add(new(HistoryRequestLoadName, + "Most of the data being processed comes from history requests.", + Invariant($"{share.Value * 100:F0}% of the data points processed recently were served by history requests."), + null, + [ + "Avoid issuing history requests on every data update; maintain the data incrementally with rolling windows or consolidators instead.", + + "Warm up indicators with the automatic indicator warm-up or the algorithm warm-up period instead of history requests.", + + "Reduce the period or resolution of the history requests.", + ])); + } + + /// + /// Formats a duration as a compact human-readable string, like "2h 5m", "12m" or "45s". + /// + private static string FormatDuration(TimeSpan duration) + { + if (duration.TotalHours >= 1) + { + return Invariant($"{(int)duration.TotalHours}h {duration.Minutes}m"); + } + if (duration.TotalMinutes >= 1) + { + return Invariant($"{(int)duration.TotalMinutes}m"); + } + return Invariant($"{(int)duration.TotalSeconds}s"); + } + } +} diff --git a/Engine/Results/Analysis/InRunResultsAnalyzer.cs b/Engine/Results/Analysis/InRunResultsAnalyzer.cs index e53a6dbc5925..05cbad332d9e 100644 --- a/Engine/Results/Analysis/InRunResultsAnalyzer.cs +++ b/Engine/Results/Analysis/InRunResultsAnalyzer.cs @@ -37,10 +37,15 @@ public class InRunResultsAnalyzer : ResultsAnalyzer nameof(PortfolioValueIsNotPositiveAnalysis), nameof(TakeProfitAndStopLossOrdersAnalysis), nameof(PortfolioMarginUsageAnalysis), + nameof(AlgorithmSpeedAnalysis), }; private readonly Dictionary _findings = new(); + private readonly AlgorithmSpeedTracker _speed = new(); + + private readonly QCAlgorithm _algorithm; + /// /// The number of order events already consumed by previous runs. The order events /// in the result passed to @@ -55,6 +60,19 @@ public class InRunResultsAnalyzer : ResultsAnalyzer /// public int LogsPosition { get; private set; } + /// + /// The equity and benchmark curves are not built for in-run analysis: + /// none of the in-run analyses read them, and building them would issue + /// a benchmark history request on every run. + /// + protected override bool RequiresEquityCurves => false; + + /// + /// The in-run analyses read the algorithm speed metrics accumulated from the + /// samples received on each run. + /// + protected override AlgorithmSpeedTracker SpeedTracker => _speed; + /// /// Initializes a new instance of the class. /// The instance is expected to be kept alive for the duration of the backtest, @@ -65,6 +83,7 @@ public class InRunResultsAnalyzer : ResultsAnalyzer public InRunResultsAnalyzer(QCAlgorithm algorithm, Language language) : base(null, algorithm, language, null) { + _algorithm = algorithm; } /// @@ -75,15 +94,30 @@ public InRunResultsAnalyzer(QCAlgorithm algorithm, Language language) /// Findings from analyses scanning the order event and log streams are accumulated /// (first sample kept, counts totaled), while findings from state-based analyses are /// replaced on every run. + /// While the algorithm is warming up, nothing is analyzed or consumed and no findings are reported. /// /// A snapshot of the current intermediate backtest result, holding only new order events. /// The log lines produced since the previous run. + /// A sample of the engine speed counters for the algorithm speed analysis, when available. /// Wall-clock seconds allowed for the full chain before early exit. /// Maximum number of failing analyses to return. /// The accumulated findings, ranked by analysis weight. - public IReadOnlyList Run(Result result, IReadOnlyList logs, int timeLimitSeconds = 1, int maxFailedAnalyses = 10) + public IReadOnlyList Run(Result result, IReadOnlyList logs, AlgorithmSpeedSample? speedSample = null, + int timeLimitSeconds = 1, int maxFailedAnalyses = 10) { + // Nothing is analyzed during the algorithm warm-up period: trading hasn't started, and + // sampling the warm-up pace would skew the speed metrics. The positions don't advance, + // so the order events and logs produced during warm-up are analyzed by the first run after it ends. + if (_algorithm?.IsWarmingUp == true) + { + return []; + } + SetAnalysisData(result, logs); + if (speedSample.HasValue) + { + _speed.AddSample(speedSample.Value); + } var newFindings = Run(timeLimitSeconds, maxFailedAnalyses); // The positions are advanced even when the time limit truncates a run, so the analyses that @@ -112,6 +146,14 @@ public InRunResultsAnalyzer(QCAlgorithm algorithm, Language language) _findings[finding.Name] = finding; } + return RankFindings(maxFailedAnalyses); + } + + /// + /// Ranks the accumulated findings by their analysis weight, capped to the given maximum. + /// + private IReadOnlyList RankFindings(int maxFailedAnalyses) + { var weights = GetAnalyses().ToDictionary(analysis => analysis.GetType().Name, analysis => analysis.Weight); return _findings.Values .OrderByDescending(finding => weights.GetValueOrDefault(BaseAnalysisName(finding.Name))) @@ -134,13 +176,6 @@ private static string BaseAnalysisName(string findingName) return separatorIndex < 0 ? findingName : findingName[..separatorIndex]; } - /// - /// The equity and benchmark curves are not built for in-run analysis: - /// none of the in-run analyses read them, and building them would issue - /// a benchmark history request on every run. - /// - protected override bool RequiresEquityCurves => false; - /// /// Creates the set of diagnostic analyses to run while the backtest is in progress. /// Only analyses that read the result snapshot (logs, orders, order events, charts) are @@ -149,8 +184,8 @@ private static string BaseAnalysisName(string findingName) /// left to the final analysis, since partial-period statistics are noisy and require /// history requests. /// - protected override IReadOnlyCollection GetAnalyses() => new BaseResultsAnalysis[] - { + protected override IReadOnlyCollection GetAnalyses() => + [ new PortfolioValueIsNotPositiveAnalysis(), new InsufficientBuyingPowerOrderResponseErrorAnalysis(), new MarginCallsAnalysis(), @@ -174,6 +209,7 @@ private static string BaseAnalysisName(string findingName) new OrderQuantityLessThanLotSizeOrderResponseErrorAnalysis(), new InsightsEmittedForDelistedSecuritiesAnalysis(), new PortfolioMarginUsageAnalysis(), - }; + new AlgorithmSpeedAnalysis(), + ]; } } diff --git a/Engine/Results/Analysis/ResultsAnalysisRunParameters.cs b/Engine/Results/Analysis/ResultsAnalysisRunParameters.cs index 0f730ccc51cc..2e6ae0d821de 100644 --- a/Engine/Results/Analysis/ResultsAnalysisRunParameters.cs +++ b/Engine/Results/Analysis/ResultsAnalysisRunParameters.cs @@ -55,6 +55,12 @@ public class ResultsAnalysisRunParameters /// public SortedList BenchmarkEquityCurve { get; } + /// + /// The speed metrics tracked for the running backtest. + /// Only available for in-run analysis; null on the final analysis. + /// + public AlgorithmSpeedTracker Speed { get; } + /// /// Initializes a new instance of the class with the specified dependencies. /// @@ -64,7 +70,8 @@ public ResultsAnalysisRunParameters( Language language, IReadOnlyList logs, SortedList equityCurve, - SortedList benchmarkEquityCurve) + SortedList benchmarkEquityCurve, + AlgorithmSpeedTracker speed = null) { Result = result; Algorithm = algorithm; @@ -72,6 +79,7 @@ public ResultsAnalysisRunParameters( Logs = logs; EquityCurve = equityCurve; BenchmarkEquityCurve = benchmarkEquityCurve; + Speed = speed; } } } diff --git a/Engine/Results/Analysis/ResultsAnalyzer.cs b/Engine/Results/Analysis/ResultsAnalyzer.cs index f9c56ff66f0a..8e01905f287b 100644 --- a/Engine/Results/Analysis/ResultsAnalyzer.cs +++ b/Engine/Results/Analysis/ResultsAnalyzer.cs @@ -75,7 +75,7 @@ public ResultsAnalyzer(Result result, QCAlgorithm algorithm, Language language, (_equityCurve, _benchmarkEquityCurve) = ReadEquityCurve(_result, _algorithm); } - var parameters = new ResultsAnalysisRunParameters(_result, _algorithm, _language, _logs, _equityCurve, _benchmarkEquityCurve); + var parameters = new ResultsAnalysisRunParameters(_result, _algorithm, _language, _logs, _equityCurve, _benchmarkEquityCurve, SpeedTracker); var responses = new List(); var timer = Stopwatch.StartNew(); @@ -126,6 +126,13 @@ protected void SetAnalysisData(Result result, IReadOnlyList logs) /// protected virtual bool RequiresEquityCurves => true; + /// + /// The speed metrics tracked for the running backtest, made available to the analyses + /// through . Null unless the analyzer + /// tracks the algorithm speed, like the in-run analyzer does. + /// + protected virtual AlgorithmSpeedTracker SpeedTracker => null; + /// /// Creates the set of diagnostic analyses to run against the backtest. /// diff --git a/Engine/Results/BacktestingResultHandler.cs b/Engine/Results/BacktestingResultHandler.cs index 6d3c301a8a2f..a43a6ef0d3e1 100644 --- a/Engine/Results/BacktestingResultHandler.cs +++ b/Engine/Results/BacktestingResultHandler.cs @@ -461,6 +461,13 @@ protected void SendFinalResult() { try { + // Nothing to analyze until trading starts: skip building the snapshot altogether. + // The analyzer catches up on the warm-up order events and logs on the first run after warm-up ends. + if (Algorithm.IsWarmingUp) + { + return null; + } + var algorithm = _job.Language == Language.Python ? (Algorithm as AlgorithmPythonWrapper)?.BaseAlgorithm : Algorithm as QCAlgorithm; if (algorithm == null) { @@ -476,6 +483,14 @@ protected void SendFinalResult() _inRunResultsAnalyzer ??= new InRunResultsAnalyzer(algorithm, _job.Language); + // Sample the engine speed counters for the algorithm speed analysis + var speedSample = new AlgorithmSpeedSample( + DateTime.UtcNow - StartTime, + PerformanceTrackingTool?.DataPoints ?? 0, + Algorithm.HistoryProvider?.DataPointCount ?? 0, + _progressMonitor?.ProcessedDays ?? 0, + _progressMonitor?.TotalDays ?? 0); + // Only the order events and logs produced since the previous run are analyzed, // the analyzer accumulates findings across runs var orderEvents = TransactionHandler.OrderEvents.Skip(_inRunResultsAnalyzer.OrderEventsPosition).ToList(); @@ -497,7 +512,7 @@ protected void SendFinalResult() totalPerformance)); // Keep the time budget small: this runs on the result handler thread and delays message processing - return _inRunResultsAnalyzer.Run(snapshot, logs, timeLimitSeconds: 1); + return _inRunResultsAnalyzer.Run(snapshot, logs, speedSample, timeLimitSeconds: 1); } catch (Exception ex) { diff --git a/Engine/Results/BaseResultsHandler.cs b/Engine/Results/BaseResultsHandler.cs index 97026fa2e113..ad2e90375c1b 100644 --- a/Engine/Results/BaseResultsHandler.cs +++ b/Engine/Results/BaseResultsHandler.cs @@ -306,6 +306,12 @@ protected Bar CurrentAlgorithmEquity /// protected IMapFileProvider MapFileProvider { get; set; } + /// + /// The tool tracking the engine's performance counters, used by the in-run + /// algorithm speed analysis. May be null when the host doesn't track performance. + /// + protected PerformanceTrackingTool PerformanceTrackingTool { get; set; } + /// /// Creates a new instance /// @@ -498,6 +504,7 @@ public virtual void Initialize(ResultHandlerInitializeParameters parameters) _updateRunner.Start(); State["Hostname"] = _hostName; MapFileProvider = parameters.MapFileProvider; + PerformanceTrackingTool = parameters.PerformanceTrackingTool; SerializerSettings = new() { diff --git a/Engine/Results/ResultHandlerInitializeParameters.cs b/Engine/Results/ResultHandlerInitializeParameters.cs index a30926cb0d17..e198647c42a8 100644 --- a/Engine/Results/ResultHandlerInitializeParameters.cs +++ b/Engine/Results/ResultHandlerInitializeParameters.cs @@ -17,6 +17,7 @@ using QuantConnect.Packets; using QuantConnect.Interfaces; using QuantConnect.Lean.Engine.TransactionHandlers; +using QuantConnect.Util; namespace QuantConnect.Lean.Engine.Results { @@ -50,16 +51,24 @@ public class ResultHandlerInitializeParameters /// public IMapFileProvider MapFileProvider { get; set; } + /// + /// The tool tracking the engine's performance counters, used by the in-run + /// algorithm speed analysis. Optional: may be null when the host doesn't track performance. + /// + public PerformanceTrackingTool PerformanceTrackingTool { get; set; } + /// /// Creates a new instance /// - public ResultHandlerInitializeParameters(AlgorithmNodePacket job, IMessagingHandler messagingHandler, IApi api, ITransactionHandler transactionHandler, IMapFileProvider mapFileProvider) + public ResultHandlerInitializeParameters(AlgorithmNodePacket job, IMessagingHandler messagingHandler, IApi api, ITransactionHandler transactionHandler, + IMapFileProvider mapFileProvider, PerformanceTrackingTool performanceTrackingTool = null) { Job = job; Api = api; MapFileProvider = mapFileProvider; MessagingHandler = messagingHandler; TransactionHandler = transactionHandler; + PerformanceTrackingTool = performanceTrackingTool; } } } diff --git a/Tests/Engine/Results/AlgorithmSpeedAnalysisTests.cs b/Tests/Engine/Results/AlgorithmSpeedAnalysisTests.cs new file mode 100644 index 000000000000..568592270d73 --- /dev/null +++ b/Tests/Engine/Results/AlgorithmSpeedAnalysisTests.cs @@ -0,0 +1,208 @@ +/* + * QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals. + * Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * +*/ + +using System; +using System.Collections.Generic; +using System.Linq; +using NUnit.Framework; +using QuantConnect.Lean.Engine.Results.Analysis; +using QuantConnect.Lean.Engine.Results.Analysis.Analyses; + +namespace QuantConnect.Tests.Engine.Results +{ + [TestFixture] + public class AlgorithmSpeedAnalysisTests + { + [Test] + public void NoFindingsWithoutSpeedMetrics() + { + Assert.IsEmpty(new AlgorithmSpeedAnalysis().Run((AlgorithmSpeedTracker)null)); + + var parameters = new ResultsAnalysisRunParameters(null, null, Language.CSharp, null, null, null); + Assert.IsEmpty(new AlgorithmSpeedAnalysis().Run(parameters)); + } + + [Test] + public void NoFindingsBeforeTheMinimumSampledSpan() + { + // Very slow, but only 30s of samples: still within the warm-up grace period + var tracker = AlgorithmSpeedTrackerTests.BuildUniformTracker(samples: 2, stepSeconds: 30, + dataPointsPerStep: 100, historyDataPointsPerStep: 0, daysPerStep: 0, totalDays: 0); + + Assert.IsEmpty(new AlgorithmSpeedAnalysis().Run(tracker)); + } + + [Test] + public void FlagsSlowExecutionWithProgressAndProjection() + { + // 10k data points per second, 1 backtest day every 30s with only 4 days left: slow, but not long-running + var tracker = AlgorithmSpeedTrackerTests.BuildUniformTracker(samples: 7, stepSeconds: 30, + dataPointsPerStep: 300_000, historyDataPointsPerStep: 0, daysPerStep: 1, totalDays: 10); + + var findings = new AlgorithmSpeedAnalysis().Run(tracker); + + var finding = findings.Single(); + Assert.AreEqual($"{nameof(AlgorithmSpeedAnalysis)} / {AlgorithmSpeedAnalysis.SlowExecutionName}", finding.Name); + var sample = (string)finding.Sample; + StringAssert.Contains("10.0k data points per second", sample); + StringAssert.Contains("60% complete", sample); + StringAssert.Contains("remaining at the recent pace", sample); + Assert.IsNotEmpty(finding.Solutions); + } + + [Test] + public void DoesNotFlagFastExecution() + { + // 100k data points per second and a short remaining runtime + var tracker = AlgorithmSpeedTrackerTests.BuildUniformTracker(samples: 7, stepSeconds: 30, + dataPointsPerStep: 3_000_000, historyDataPointsPerStep: 0, daysPerStep: 1, totalDays: 10); + + Assert.IsEmpty(new AlgorithmSpeedAnalysis().Run(tracker)); + } + + [Test] + public void SlowExecutionRequiresTwoConsecutiveSlowWindows() + { + // A fast run whose very last window stalls: the previous window is still fast, so no flag yet + var tracker = new AlgorithmSpeedTracker(); + for (var i = 0; i < 6; i++) + { + tracker.AddSample(new(TimeSpan.FromSeconds(30 * i), 3_000_000L * i, 0, 0, 0)); + } + tracker.AddSample(new(TimeSpan.FromSeconds(750), 15_000_000, 0, 0, 0)); + + Assert.IsEmpty(new AlgorithmSpeedAnalysis().Run(tracker)); + + // One more stalled window and both recent windows are slow: now it flags + tracker.AddSample(new(TimeSpan.FromSeconds(1350), 15_000_000, 0, 0, 0)); + + var findings = new AlgorithmSpeedAnalysis().Run(tracker); + Assert.IsTrue(findings.Any(finding => finding.Name.EndsWith(AlgorithmSpeedAnalysis.SlowExecutionName, StringComparison.Ordinal))); + } + + [Test] + public void MissingDataPointCountsSuppressDataPointBasedFindings() + { + // The data point counters are not wired in (always zero), but calendar progress is glacial: + // only the projected runtime finding should be reported + var tracker = AlgorithmSpeedTrackerTests.BuildUniformTracker(samples: 12, stepSeconds: 30, + dataPointsPerStep: 0, historyDataPointsPerStep: 0, daysPerStep: 1, totalDays: 100_000); + + var findings = new AlgorithmSpeedAnalysis().Run(tracker); + + var finding = findings.Single(); + Assert.AreEqual($"{nameof(AlgorithmSpeedAnalysis)} / {AlgorithmSpeedAnalysis.LongProjectedRuntimeName}", finding.Name); + } + + [Test] + public void FlagsLongProjectedRuntime() + { + // Fast processing, but ~35 wall-clock days of backtest left at one backtest day per 30s + var tracker = AlgorithmSpeedTrackerTests.BuildUniformTracker(samples: 7, stepSeconds: 30, + dataPointsPerStep: 3_000_000, historyDataPointsPerStep: 0, daysPerStep: 1, totalDays: 100_000); + + var findings = new AlgorithmSpeedAnalysis().Run(tracker); + + var finding = findings.Single(); + Assert.AreEqual($"{nameof(AlgorithmSpeedAnalysis)} / {AlgorithmSpeedAnalysis.LongProjectedRuntimeName}", finding.Name); + StringAssert.Contains("remain at the recent pace", (string)finding.Sample); + Assert.IsNotEmpty(finding.Solutions); + } + + [Test] + public void FlagsStalledCalendarProgressAsLongProjectedRuntime() + { + // Fast processing but the backtest time is not advancing at all + var tracker = new AlgorithmSpeedTracker(); + for (var i = 0; i < 7; i++) + { + tracker.AddSample(new(TimeSpan.FromSeconds(30 * i), 3_000_000L * i, 0, 3, 10)); + } + + var findings = new AlgorithmSpeedAnalysis().Run(tracker); + + var finding = findings.Single(); + Assert.AreEqual($"{nameof(AlgorithmSpeedAnalysis)} / {AlgorithmSpeedAnalysis.LongProjectedRuntimeName}", finding.Name); + StringAssert.Contains("no backtest-time progress", (string)finding.Sample); + } + + [Test] + public void FlagsThroughputDegradation() + { + // 100k data points per second for the first 6 intervals, 10k for the last 5 + var tracker = new AlgorithmSpeedTracker(); + var dataPoints = 0L; + for (var i = 0; i < 12; i++) + { + tracker.AddSample(new(TimeSpan.FromSeconds(30 * i), dataPoints, 0, 0, 0)); + dataPoints += i < 6 ? 3_000_000 : 300_000; + } + + var findings = new AlgorithmSpeedAnalysis().Run(tracker); + + var degradation = findings.Single(finding => + finding.Name.EndsWith(AlgorithmSpeedAnalysis.ThroughputDegradationName, StringComparison.Ordinal)); + StringAssert.Contains("100.0k data points per second early in the run", (string)degradation.Sample); + StringAssert.Contains("10.0k recently", (string)degradation.Sample); + // The recent pace is also below the absolute threshold, so slow execution is reported too + Assert.IsTrue(findings.Any(finding => finding.Name.EndsWith(AlgorithmSpeedAnalysis.SlowExecutionName, StringComparison.Ordinal))); + } + + [Test] + public void NoDegradationFindingBeforeTheBaselineAndRecentWindowsAreDisjoint() + { + var tracker = new AlgorithmSpeedTracker(); + var dataPoints = 0L; + for (var i = 0; i < 9; i++) + { + tracker.AddSample(new(TimeSpan.FromSeconds(30 * i), dataPoints, 0, 0, 0)); + dataPoints += i < 3 ? 3_000_000 : 300_000; + } + + var findings = new AlgorithmSpeedAnalysis().Run(tracker); + + Assert.IsFalse(findings.Any(finding => + finding.Name.EndsWith(AlgorithmSpeedAnalysis.ThroughputDegradationName, StringComparison.Ordinal))); + } + + [Test] + public void FlagsHistoryRequestDominatedLoad() + { + // Fast overall, but 75% of the data points come from history requests + var tracker = AlgorithmSpeedTrackerTests.BuildUniformTracker(samples: 7, stepSeconds: 30, + dataPointsPerStep: 1_000_000, historyDataPointsPerStep: 3_000_000, daysPerStep: 1, totalDays: 10); + + var findings = new AlgorithmSpeedAnalysis().Run(tracker); + + var finding = findings.Single(); + Assert.AreEqual($"{nameof(AlgorithmSpeedAnalysis)} / {AlgorithmSpeedAnalysis.HistoryRequestLoadName}", finding.Name); + StringAssert.Contains("75% of the data points", (string)finding.Sample); + } + + [Test] + public void NoHistoryLoadFindingBelowTheMinimumHistoryDataPointCount() + { + // 60% history share, but only a few hundred history data points in the window + var tracker = AlgorithmSpeedTrackerTests.BuildUniformTracker(samples: 7, stepSeconds: 30, + dataPointsPerStep: 40, historyDataPointsPerStep: 60, daysPerStep: 1, totalDays: 10); + + var findings = new AlgorithmSpeedAnalysis().Run(tracker); + + Assert.IsFalse(findings.Any(finding => + finding.Name.EndsWith(AlgorithmSpeedAnalysis.HistoryRequestLoadName, StringComparison.Ordinal))); + } + } +} diff --git a/Tests/Engine/Results/AlgorithmSpeedTrackerTests.cs b/Tests/Engine/Results/AlgorithmSpeedTrackerTests.cs new file mode 100644 index 000000000000..b0af52d1ef4c --- /dev/null +++ b/Tests/Engine/Results/AlgorithmSpeedTrackerTests.cs @@ -0,0 +1,144 @@ +/* + * QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals. + * Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * +*/ + +using System; +using NUnit.Framework; +using QuantConnect.Lean.Engine.Results.Analysis; + +namespace QuantConnect.Tests.Engine.Results +{ + [TestFixture] + public class AlgorithmSpeedTrackerTests + { + [Test] + public void RatesAreNotMeasurableWithASingleSample() + { + var tracker = new AlgorithmSpeedTracker(); + tracker.AddSample(new(TimeSpan.FromSeconds(30), 1000, 0, 1, 100)); + + Assert.IsNull(tracker.DataPointsPerSecond); + Assert.IsNull(tracker.RecentDataPointsPerSecond()); + Assert.IsNull(tracker.RecentDaysPerSecond()); + Assert.IsNull(tracker.RecentHistoryDataPointsShare()); + Assert.IsNull(tracker.EstimatedRemainingTime()); + Assert.AreEqual(TimeSpan.Zero, tracker.SampledSpan); + } + + [Test] + public void ComputesCumulativeInitialAndRecentRates() + { + // 7 samples 30s apart: 3 fast intervals at 100k dp/s followed by 3 slow ones at 10k dp/s + var tracker = new AlgorithmSpeedTracker(); + var dataPoints = 0L; + for (var i = 0; i < 7; i++) + { + tracker.AddSample(new(TimeSpan.FromSeconds(30 * i), dataPoints, 0, i, 100)); + dataPoints += i < 3 ? 3_000_000 : 300_000; + } + + // (3 * 3M + 3 * 300k) / 180s + Assert.AreEqual(55_000, tracker.DataPointsPerSecond.Value, 1); + // First 5 samples: (3 * 3M + 1 * 300k) / 120s + Assert.AreEqual(77_500, tracker.InitialDataPointsPerSecond.Value, 1); + // Last 5 samples: (1 * 3M + 3 * 300k) / 120s + Assert.AreEqual(32_500, tracker.RecentDataPointsPerSecond().Value, 1); + // Skipping the last sample: (2 * 3M + 2 * 300k) / 120s + Assert.AreEqual(55_000, tracker.RecentDataPointsPerSecond(skipLast: 1).Value, 1); + } + + [Test] + public void IncludesHistoryDataPointsInRatesAndShare() + { + // 100k loop + 200k history data points every 30s + var tracker = BuildUniformTracker(samples: 6, stepSeconds: 30, dataPointsPerStep: 100_000, + historyDataPointsPerStep: 200_000, daysPerStep: 1, totalDays: 100); + + Assert.AreEqual(10_000, tracker.DataPointsPerSecond.Value, 1); + Assert.AreEqual(10_000, tracker.RecentDataPointsPerSecond().Value, 1); + Assert.AreEqual(2.0 / 3.0, tracker.RecentHistoryDataPointsShare().Value, 0.001); + Assert.AreEqual(4 * 200_000, tracker.RecentHistoryDataPoints()); + } + + [Test] + public void EstimatesRemainingTimeFromTheRecentPace() + { + // One backtest day every 30s, 95 days to go after the last sample + var tracker = BuildUniformTracker(samples: 6, stepSeconds: 30, dataPointsPerStep: 100_000, + historyDataPointsPerStep: 0, daysPerStep: 1, totalDays: 100); + + Assert.AreEqual(1.0 / 30, tracker.RecentDaysPerSecond().Value, 0.0001); + Assert.AreEqual(95 * 30, tracker.EstimatedRemainingTime().Value.TotalSeconds, 0.1); + } + + [Test] + public void RemainingTimeIsZeroWhenTheEndDateIsReached() + { + var tracker = BuildUniformTracker(samples: 6, stepSeconds: 30, dataPointsPerStep: 100_000, + historyDataPointsPerStep: 0, daysPerStep: 1, totalDays: 5); + + Assert.AreEqual(TimeSpan.Zero, tracker.EstimatedRemainingTime()); + } + + [Test] + public void RemainingTimeIsNotMeasurableWithoutCalendarProgress() + { + var tracker = BuildUniformTracker(samples: 6, stepSeconds: 30, dataPointsPerStep: 100_000, + historyDataPointsPerStep: 0, daysPerStep: 0, totalDays: 100); + + Assert.AreEqual(0, tracker.RecentDaysPerSecond()); + Assert.IsNull(tracker.EstimatedRemainingTime()); + } + + [Test] + public void IgnoresSamplesWithNonIncreasingElapsedTime() + { + var tracker = new AlgorithmSpeedTracker(); + tracker.AddSample(new(TimeSpan.FromSeconds(30), 1000, 0, 1, 100)); + tracker.AddSample(new(TimeSpan.FromSeconds(30), 2000, 0, 1, 100)); + tracker.AddSample(new(TimeSpan.FromSeconds(20), 3000, 0, 1, 100)); + + Assert.AreEqual(1, tracker.SampleCount); + } + + [Test] + public void TracksProgressAndDataPointCountsAvailability() + { + var tracker = new AlgorithmSpeedTracker(); + Assert.AreEqual(0, tracker.Progress); + Assert.IsFalse(tracker.HasDataPointCounts); + + tracker.AddSample(new(TimeSpan.FromSeconds(30), 0, 1000, 25, 100)); + Assert.AreEqual(0.25m, tracker.Progress); + Assert.IsFalse(tracker.HasDataPointCounts); + + tracker.AddSample(new(TimeSpan.FromSeconds(60), 500, 2000, 50, 100)); + Assert.AreEqual(0.5m, tracker.Progress); + Assert.IsTrue(tracker.HasDataPointCounts); + } + + public static AlgorithmSpeedTracker BuildUniformTracker(int samples, int stepSeconds, long dataPointsPerStep, + long historyDataPointsPerStep, int daysPerStep, int totalDays) + { + var tracker = new AlgorithmSpeedTracker(); + for (var i = 0; i < samples; i++) + { + tracker.AddSample(new(TimeSpan.FromSeconds(stepSeconds * i), dataPointsPerStep * i, + historyDataPointsPerStep * i, daysPerStep * i, totalDays)); + } + return tracker; + } + } +} diff --git a/Tests/Engine/Results/InRunResultsAnalyzerTests.cs b/Tests/Engine/Results/InRunResultsAnalyzerTests.cs index f72f346073d7..a41151d96a05 100644 --- a/Tests/Engine/Results/InRunResultsAnalyzerTests.cs +++ b/Tests/Engine/Results/InRunResultsAnalyzerTests.cs @@ -19,6 +19,7 @@ using System.Linq; using System.Threading; using NUnit.Framework; +using QuantConnect.Algorithm; using QuantConnect.Lean.Engine.Results.Analysis; using QuantConnect.Lean.Engine.Results.Analysis.Analyses; using QuantConnect.Orders; @@ -177,6 +178,36 @@ public void AggregatedStateBasedFindingsAreReplacedByFullName() Assert.AreEqual(2, finding.Count); } + [Test] + public void NothingIsAnalyzedOrConsumedDuringAlgorithmWarmUp() + { + // A fresh algorithm is warming up until the engine flips it + var algorithm = new QCAlgorithm(); + var ran = false; + var fake = new FakeAnalysisA(10) + { + Findings = () => MakeFindings(nameof(FakeAnalysisA), "sample", 1), + OnRun = () => ran = true + }; + var analyzer = new TestInRunResultsAnalyzer(algorithm, fake); + + var findings = analyzer.Run(MakeResult(2), new[] { "log" }); + + Assert.IsFalse(ran); + Assert.IsEmpty(findings); + Assert.AreEqual(0, analyzer.OrderEventsPosition); + Assert.AreEqual(0, analyzer.LogsPosition); + + // Once warm-up finishes, the analysis catches up on the unconsumed order events and logs + algorithm.SetFinishedWarmingUp(); + findings = analyzer.Run(MakeResult(2), new[] { "log" }); + + Assert.IsTrue(ran); + Assert.AreEqual("sample", findings.Single().Sample); + Assert.AreEqual(2, analyzer.OrderEventsPosition); + Assert.AreEqual(1, analyzer.LogsPosition); + } + [Test] public void FindingsAreRankedByAnalysisWeightAndCapped() { @@ -226,7 +257,12 @@ private class TestInRunResultsAnalyzer : InRunResultsAnalyzer private readonly IReadOnlyCollection _analyses; public TestInRunResultsAnalyzer(params BaseResultsAnalysis[] analyses) - : base(null, Language.CSharp) + : this(null, analyses) + { + } + + public TestInRunResultsAnalyzer(QCAlgorithm algorithm, params BaseResultsAnalysis[] analyses) + : base(algorithm, Language.CSharp) { _analyses = analyses; } From 9f97ccf1c3f7316a9714a56be381f3a29c44a8c3 Mon Sep 17 00:00:00 2001 From: Jhonathan Abreu Date: Wed, 22 Jul 2026 15:11:50 -0400 Subject: [PATCH 07/33] Source the speed sample counters from the performance tracking tool and cache the unwrapped algorithm instance --- .../Results/Analysis/AlgorithmSpeedTracker.cs | 26 ++++++++--------- Engine/Results/Analysis/ResultsAnalyzer.cs | 28 +++++++++---------- Engine/Results/BacktestingResultHandler.cs | 14 ++++++---- 3 files changed, 35 insertions(+), 33 deletions(-) diff --git a/Engine/Results/Analysis/AlgorithmSpeedTracker.cs b/Engine/Results/Analysis/AlgorithmSpeedTracker.cs index 69a7c0c6cdae..d4e41df811ee 100644 --- a/Engine/Results/Analysis/AlgorithmSpeedTracker.cs +++ b/Engine/Results/Analysis/AlgorithmSpeedTracker.cs @@ -71,6 +71,19 @@ public class AlgorithmSpeedTracker /// public bool HasDataPointCounts => _samples.Count > 0 && _samples[^1].DataPoints > 0; + /// + /// The average data points processed per second over the whole sampled span, including + /// history data points to match the speed the engine reports on completion. + /// Null when there are not enough samples to measure. + /// + public double? DataPointsPerSecond => RateBetween(0, _samples.Count - 1, TotalDataPoints); + + /// + /// The average data points processed per second over the first samples, + /// used as the early-run baseline for degradation detection. Null when there are not enough samples to measure. + /// + public double? InitialDataPointsPerSecond => RateBetween(0, Math.Min(RecentWindowSamples, _samples.Count) - 1, TotalDataPoints); + /// /// Records a sample of the cumulative speed counters. Samples with a non-increasing /// elapsed time are ignored so rates are always computed over positive time deltas. @@ -85,19 +98,6 @@ public void AddSample(AlgorithmSpeedSample sample) _samples.Add(sample); } - /// - /// The average data points processed per second over the whole sampled span, including - /// history data points to match the speed the engine reports on completion. - /// Null when there are not enough samples to measure. - /// - public double? DataPointsPerSecond => RateBetween(0, _samples.Count - 1, TotalDataPoints); - - /// - /// The average data points processed per second over the first samples, - /// used as the early-run baseline for degradation detection. Null when there are not enough samples to measure. - /// - public double? InitialDataPointsPerSecond => RateBetween(0, Math.Min(RecentWindowSamples, _samples.Count) - 1, TotalDataPoints); - /// /// The average data points processed per second over the recent window, including history data points. /// diff --git a/Engine/Results/Analysis/ResultsAnalyzer.cs b/Engine/Results/Analysis/ResultsAnalyzer.cs index 8e01905f287b..305c71d5af57 100644 --- a/Engine/Results/Analysis/ResultsAnalyzer.cs +++ b/Engine/Results/Analysis/ResultsAnalyzer.cs @@ -36,6 +36,20 @@ public class ResultsAnalyzer private SortedList _benchmarkEquityCurve; private Result _result; + /// + /// Whether the equity and benchmark curves should be built before running the analyses. + /// Building them requires a benchmark history request, so analyzers whose analyses + /// don't read the curves can skip it. + /// + protected virtual bool RequiresEquityCurves => true; + + /// + /// The speed metrics tracked for the running backtest, made available to the analyses + /// through . Null unless the analyzer + /// tracks the algorithm speed, like the in-run analyzer does. + /// + protected virtual AlgorithmSpeedTracker SpeedTracker => null; + /// /// Initializes a new instance of the class. /// @@ -119,20 +133,6 @@ protected void SetAnalysisData(Result result, IReadOnlyList logs) _logs = logs; } - /// - /// Whether the equity and benchmark curves should be built before running the analyses. - /// Building them requires a benchmark history request, so analyzers whose analyses - /// don't read the curves can skip it. - /// - protected virtual bool RequiresEquityCurves => true; - - /// - /// The speed metrics tracked for the running backtest, made available to the analyses - /// through . Null unless the analyzer - /// tracks the algorithm speed, like the in-run analyzer does. - /// - protected virtual AlgorithmSpeedTracker SpeedTracker => null; - /// /// Creates the set of diagnostic analyses to run against the backtest. /// diff --git a/Engine/Results/BacktestingResultHandler.cs b/Engine/Results/BacktestingResultHandler.cs index a43a6ef0d3e1..23a539a7f509 100644 --- a/Engine/Results/BacktestingResultHandler.cs +++ b/Engine/Results/BacktestingResultHandler.cs @@ -66,6 +66,10 @@ public class BacktestingResultHandler : BaseResultsHandler, IResultHandler private string _algorithmId; private int _projectId; + private QCAlgorithm _algorithmInstance; + + private QCAlgorithm AlgorithmInstance => _algorithmInstance ??= _job.Language == Language.Python ? (Algorithm as AlgorithmPythonWrapper)?.BaseAlgorithm : Algorithm as QCAlgorithm; + /// /// Whether or not to run the results analysis at the end of the backtest. /// @@ -418,13 +422,12 @@ protected void SendFinalResult() // Run backtest analyzer if (RunResultsAnalysis) { - var algorithm = _job.Language == Language.Python ? (Algorithm as AlgorithmPythonWrapper)?.BaseAlgorithm : Algorithm as QCAlgorithm; List logs; lock (LogStore) { logs = LogStore.Select(x => x.Message).ToList(); } - var analyzer = new ResultsAnalyzer(result.Results, algorithm, _job.Language, logs); + var analyzer = new ResultsAnalyzer(result.Results, AlgorithmInstance, _job.Language, logs); try { result.Results.Analysis = analyzer.Run(); @@ -468,8 +471,7 @@ protected void SendFinalResult() return null; } - var algorithm = _job.Language == Language.Python ? (Algorithm as AlgorithmPythonWrapper)?.BaseAlgorithm : Algorithm as QCAlgorithm; - if (algorithm == null) + if (AlgorithmInstance == null) { return null; } @@ -481,13 +483,13 @@ protected void SendFinalResult() charts = Charts.ToDictionary(kvp => kvp.Key, kvp => kvp.Value.Clone()); } - _inRunResultsAnalyzer ??= new InRunResultsAnalyzer(algorithm, _job.Language); + _inRunResultsAnalyzer ??= new InRunResultsAnalyzer(AlgorithmInstance, _job.Language); // Sample the engine speed counters for the algorithm speed analysis var speedSample = new AlgorithmSpeedSample( DateTime.UtcNow - StartTime, PerformanceTrackingTool?.DataPoints ?? 0, - Algorithm.HistoryProvider?.DataPointCount ?? 0, + PerformanceTrackingTool?.HistoryDataPoints ?? 0, _progressMonitor?.ProcessedDays ?? 0, _progressMonitor?.TotalDays ?? 0); From 12e082761bf99937f087d00d8b4736fc5b794c49 Mon Sep 17 00:00:00 2001 From: Jhonathan Abreu Date: Wed, 22 Jul 2026 17:13:16 -0400 Subject: [PATCH 08/33] Format sub-1k data point rates as raw counts in speed analysis findings --- .../Analyses/AlgorithmSpeedAnalysis.cs | 19 +++++++++++++++---- .../Results/AlgorithmSpeedAnalysisTests.cs | 13 +++++++++++++ 2 files changed, 28 insertions(+), 4 deletions(-) diff --git a/Engine/Results/Analysis/Analyses/AlgorithmSpeedAnalysis.cs b/Engine/Results/Analysis/Analyses/AlgorithmSpeedAnalysis.cs index 487493767357..b8fb70a5e37d 100644 --- a/Engine/Results/Analysis/Analyses/AlgorithmSpeedAnalysis.cs +++ b/Engine/Results/Analysis/Analyses/AlgorithmSpeedAnalysis.cs @@ -142,8 +142,8 @@ private static void AddSlowExecution(AlgorithmSpeedTracker speed, List + /// Formats a data points per second rate compactly: in thousands like "12.5k" when at least + /// one thousand, as a raw count like "340" below that, so very slow rates don't read as "0.0k". + /// + private static string FormatRate(double dataPointsPerSecond) + { + return dataPointsPerSecond >= 1000 + ? Invariant($"{dataPointsPerSecond / 1000:F1}k") + : Invariant($"{dataPointsPerSecond:F0}"); + } + /// /// Formats a duration as a compact human-readable string, like "2h 5m", "12m" or "45s". /// diff --git a/Tests/Engine/Results/AlgorithmSpeedAnalysisTests.cs b/Tests/Engine/Results/AlgorithmSpeedAnalysisTests.cs index 568592270d73..fd5dc8b0e368 100644 --- a/Tests/Engine/Results/AlgorithmSpeedAnalysisTests.cs +++ b/Tests/Engine/Results/AlgorithmSpeedAnalysisTests.cs @@ -63,6 +63,19 @@ public void FlagsSlowExecutionWithProgressAndProjection() Assert.IsNotEmpty(finding.Solutions); } + [Test] + public void FormatsRatesBelowOneThousandAsRawCounts() + { + // 100 data points per second: formatted as a raw count instead of "0.1k" + var tracker = AlgorithmSpeedTrackerTests.BuildUniformTracker(samples: 7, stepSeconds: 30, + dataPointsPerStep: 3_000, historyDataPointsPerStep: 0, daysPerStep: 1, totalDays: 10); + + var findings = new AlgorithmSpeedAnalysis().Run(tracker); + + var finding = findings.Single(x => x.Name.EndsWith(AlgorithmSpeedAnalysis.SlowExecutionName, StringComparison.Ordinal)); + StringAssert.Contains("Processing 100 data points per second recently (100 average)", (string)finding.Sample); + } + [Test] public void DoesNotFlagFastExecution() { From cf30151c9b47658970503fad02b689e4c7251197 Mon Sep 17 00:00:00 2001 From: Jhonathan Abreu Date: Wed, 22 Jul 2026 17:56:04 -0400 Subject: [PATCH 09/33] Raise the in-run algorithm speed analysis priority above the order-response analyses --- Engine/Results/Analysis/Analyses/AlgorithmSpeedAnalysis.cs | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/Engine/Results/Analysis/Analyses/AlgorithmSpeedAnalysis.cs b/Engine/Results/Analysis/Analyses/AlgorithmSpeedAnalysis.cs index b8fb70a5e37d..7327dae1e1b6 100644 --- a/Engine/Results/Analysis/Analyses/AlgorithmSpeedAnalysis.cs +++ b/Engine/Results/Analysis/Analyses/AlgorithmSpeedAnalysis.cs @@ -88,9 +88,12 @@ public class AlgorithmSpeedAnalysis : BaseResultsAnalysis public override string Issue { get; } = "The algorithm is running slowly."; /// - /// Gets the severity weight for the algorithm speed analysis. + /// Gets the severity weight for the algorithm speed analysis. High enough to run before the + /// order-response error analyses in the in-run chain: this analysis drives the user's decision + /// to stop a slow backtest, and it is one of the cheapest in the set, so it should not be the + /// one skipped when the time limit or the failed-analyses cap truncates a run. /// - public override int Weight { get; } = 77; + public override int Weight { get; } = 96; /// /// Runs the algorithm speed analysis against the speed metrics tracked for the running backtest. From 46430ec3fcf95f3e796fedf4ac0ce1f28a19463a Mon Sep 17 00:00:00 2001 From: Jhonathan Abreu Date: Wed, 22 Jul 2026 17:56:05 -0400 Subject: [PATCH 10/33] Run the in-run analyses during algorithm warm-up, skipping only the speed sampling --- .../Results/Analysis/InRunResultsAnalyzer.cs | 21 ++------- Engine/Results/BacktestingResultHandler.cs | 26 +++++------ .../Results/InRunResultsAnalyzerTests.cs | 46 +++++++------------ 3 files changed, 33 insertions(+), 60 deletions(-) diff --git a/Engine/Results/Analysis/InRunResultsAnalyzer.cs b/Engine/Results/Analysis/InRunResultsAnalyzer.cs index 05cbad332d9e..83c1b2e5148e 100644 --- a/Engine/Results/Analysis/InRunResultsAnalyzer.cs +++ b/Engine/Results/Analysis/InRunResultsAnalyzer.cs @@ -44,18 +44,16 @@ public class InRunResultsAnalyzer : ResultsAnalyzer private readonly AlgorithmSpeedTracker _speed = new(); - private readonly QCAlgorithm _algorithm; - /// /// The number of order events already consumed by previous runs. The order events - /// in the result passed to + /// in the result passed to /// are expected to start at this position. /// public int OrderEventsPosition { get; private set; } /// /// The number of log entries already consumed by previous runs. The logs passed to - /// are expected to start + /// are expected to start /// at this position. /// public int LogsPosition { get; private set; } @@ -76,14 +74,13 @@ public class InRunResultsAnalyzer : ResultsAnalyzer /// /// Initializes a new instance of the class. /// The instance is expected to be kept alive for the duration of the backtest, - /// receiving fresh data on each call. + /// receiving fresh data on each call. /// /// The algorithm instance used for history requests and settings. /// The programming language the algorithm is written in. public InRunResultsAnalyzer(QCAlgorithm algorithm, Language language) : base(null, algorithm, language, null) { - _algorithm = algorithm; } /// @@ -94,25 +91,17 @@ public InRunResultsAnalyzer(QCAlgorithm algorithm, Language language) /// Findings from analyses scanning the order event and log streams are accumulated /// (first sample kept, counts totaled), while findings from state-based analyses are /// replaced on every run. - /// While the algorithm is warming up, nothing is analyzed or consumed and no findings are reported. /// /// A snapshot of the current intermediate backtest result, holding only new order events. /// The log lines produced since the previous run. - /// A sample of the engine speed counters for the algorithm speed analysis, when available. + /// A sample of the engine speed counters for the algorithm speed analysis. + /// Null when the counters should not be sampled, like while the algorithm warms up. /// Wall-clock seconds allowed for the full chain before early exit. /// Maximum number of failing analyses to return. /// The accumulated findings, ranked by analysis weight. public IReadOnlyList Run(Result result, IReadOnlyList logs, AlgorithmSpeedSample? speedSample = null, int timeLimitSeconds = 1, int maxFailedAnalyses = 10) { - // Nothing is analyzed during the algorithm warm-up period: trading hasn't started, and - // sampling the warm-up pace would skew the speed metrics. The positions don't advance, - // so the order events and logs produced during warm-up are analyzed by the first run after it ends. - if (_algorithm?.IsWarmingUp == true) - { - return []; - } - SetAnalysisData(result, logs); if (speedSample.HasValue) { diff --git a/Engine/Results/BacktestingResultHandler.cs b/Engine/Results/BacktestingResultHandler.cs index 23a539a7f509..1ebfcde9ed08 100644 --- a/Engine/Results/BacktestingResultHandler.cs +++ b/Engine/Results/BacktestingResultHandler.cs @@ -464,13 +464,6 @@ protected void SendFinalResult() { try { - // Nothing to analyze until trading starts: skip building the snapshot altogether. - // The analyzer catches up on the warm-up order events and logs on the first run after warm-up ends. - if (Algorithm.IsWarmingUp) - { - return null; - } - if (AlgorithmInstance == null) { return null; @@ -485,13 +478,18 @@ protected void SendFinalResult() _inRunResultsAnalyzer ??= new InRunResultsAnalyzer(AlgorithmInstance, _job.Language); - // Sample the engine speed counters for the algorithm speed analysis - var speedSample = new AlgorithmSpeedSample( - DateTime.UtcNow - StartTime, - PerformanceTrackingTool?.DataPoints ?? 0, - PerformanceTrackingTool?.HistoryDataPoints ?? 0, - _progressMonitor?.ProcessedDays ?? 0, - _progressMonitor?.TotalDays ?? 0); + // Sample the engine speed counters for the algorithm speed analysis, but not while the + // algorithm is warming up: the warm-up pace would skew the speed metrics. The analyses + // themselves do run during warm-up, so conditions like orders submitted while warming up + // surface without waiting for warm-up to end + AlgorithmSpeedSample? speedSample = Algorithm.IsWarmingUp + ? null + : new AlgorithmSpeedSample( + DateTime.UtcNow - StartTime, + PerformanceTrackingTool?.DataPoints ?? 0, + PerformanceTrackingTool?.HistoryDataPoints ?? 0, + _progressMonitor?.ProcessedDays ?? 0, + _progressMonitor?.TotalDays ?? 0); // Only the order events and logs produced since the previous run are analyzed, // the analyzer accumulates findings across runs diff --git a/Tests/Engine/Results/InRunResultsAnalyzerTests.cs b/Tests/Engine/Results/InRunResultsAnalyzerTests.cs index a41151d96a05..b27e302562a3 100644 --- a/Tests/Engine/Results/InRunResultsAnalyzerTests.cs +++ b/Tests/Engine/Results/InRunResultsAnalyzerTests.cs @@ -19,7 +19,6 @@ using System.Linq; using System.Threading; using NUnit.Framework; -using QuantConnect.Algorithm; using QuantConnect.Lean.Engine.Results.Analysis; using QuantConnect.Lean.Engine.Results.Analysis.Analyses; using QuantConnect.Orders; @@ -179,33 +178,22 @@ public void AggregatedStateBasedFindingsAreReplacedByFullName() } [Test] - public void NothingIsAnalyzedOrConsumedDuringAlgorithmWarmUp() + public void SpeedSamplesAreTrackedOnlyWhenProvided() { - // A fresh algorithm is warming up until the engine flips it - var algorithm = new QCAlgorithm(); - var ran = false; - var fake = new FakeAnalysisA(10) - { - Findings = () => MakeFindings(nameof(FakeAnalysisA), "sample", 1), - OnRun = () => ran = true - }; - var analyzer = new TestInRunResultsAnalyzer(algorithm, fake); - - var findings = analyzer.Run(MakeResult(2), new[] { "log" }); + AlgorithmSpeedTracker speed = null; + var fake = new FakeAnalysisA(10) { OnParameters = parameters => speed = parameters.Speed }; + var analyzer = new TestInRunResultsAnalyzer(fake); - Assert.IsFalse(ran); - Assert.IsEmpty(findings); - Assert.AreEqual(0, analyzer.OrderEventsPosition); - Assert.AreEqual(0, analyzer.LogsPosition); + analyzer.Run(MakeResult(1), new[] { "log" }); + Assert.IsNotNull(speed); + Assert.AreEqual(0, speed.SampleCount); - // Once warm-up finishes, the analysis catches up on the unconsumed order events and logs - algorithm.SetFinishedWarmingUp(); - findings = analyzer.Run(MakeResult(2), new[] { "log" }); + analyzer.Run(MakeResult(1), new[] { "log" }, new AlgorithmSpeedSample(TimeSpan.FromSeconds(30), 100, 0, 1, 10)); + Assert.AreEqual(1, speed.SampleCount); - Assert.IsTrue(ran); - Assert.AreEqual("sample", findings.Single().Sample); - Assert.AreEqual(2, analyzer.OrderEventsPosition); - Assert.AreEqual(1, analyzer.LogsPosition); + // No sample provided (e.g. while the algorithm warms up): the tracker is left untouched + analyzer.Run(MakeResult(1), new[] { "log" }); + Assert.AreEqual(1, speed.SampleCount); } [Test] @@ -257,12 +245,7 @@ private class TestInRunResultsAnalyzer : InRunResultsAnalyzer private readonly IReadOnlyCollection _analyses; public TestInRunResultsAnalyzer(params BaseResultsAnalysis[] analyses) - : this(null, analyses) - { - } - - public TestInRunResultsAnalyzer(QCAlgorithm algorithm, params BaseResultsAnalysis[] analyses) - : base(algorithm, Language.CSharp) + : base(null, Language.CSharp) { _analyses = analyses; } @@ -282,6 +265,8 @@ private class FakeAnalysis : BaseResultsAnalysis public Action OnRun { get; set; } + public Action OnParameters { get; set; } + protected FakeAnalysis(int weight) { _weight = weight; @@ -290,6 +275,7 @@ protected FakeAnalysis(int weight) public override IReadOnlyList Run(ResultsAnalysisRunParameters parameters) { OnRun?.Invoke(); + OnParameters?.Invoke(parameters); return Findings(); } } From 557f3a5fb3289d95a527c5874d609ac922d0765b Mon Sep 17 00:00:00 2001 From: Jhonathan Abreu Date: Thu, 23 Jul 2026 10:28:10 -0400 Subject: [PATCH 11/33] Cache the analysis instances across runs and declare the charts the in-run analyses read The analyses are stateless, so the base analyzer now creates them once and reuses them; the in-run findings ranking reads the same cached set instead of re-instantiating the analyses to look up their weights. InRunResultsAnalyzer.RequiredCharts lists the charts its analyses read so the result handler only needs to clone those into the analyzed snapshot. Also reverts the benchmark history end trim to the current algorithm time: the in-run analyzer never builds the equity curves, so the trim only ran in the final analysis, where it could drop the last daily benchmark bar. And documents that state-based findings drop when a time-limit truncated run skips their analysis. --- .../Results/Analysis/InRunResultsAnalyzer.cs | 39 ++++++++++++------- Engine/Results/Analysis/ResultsAnalyzer.cs | 21 +++++----- .../Results/InRunResultsAnalyzerTests.cs | 31 ++++++++++++++- Tests/Engine/Results/ResultsAnalyzerTests.cs | 19 ++++++++- 4 files changed, 83 insertions(+), 27 deletions(-) diff --git a/Engine/Results/Analysis/InRunResultsAnalyzer.cs b/Engine/Results/Analysis/InRunResultsAnalyzer.cs index 83c1b2e5148e..dd8edcc6610b 100644 --- a/Engine/Results/Analysis/InRunResultsAnalyzer.cs +++ b/Engine/Results/Analysis/InRunResultsAnalyzer.cs @@ -44,6 +44,26 @@ public class InRunResultsAnalyzer : ResultsAnalyzer private readonly AlgorithmSpeedTracker _speed = new(); + /// + /// The equity and benchmark curves are not built for in-run analysis: + /// none of the in-run analyses read them, and building them would issue + /// a benchmark history request on every run. + /// + protected override bool RequiresEquityCurves => false; + + /// + /// The in-run analyses read the algorithm speed metrics accumulated from the + /// samples received on each run. + /// + protected override AlgorithmSpeedTracker SpeedTracker => _speed; + + /// + /// The names of the charts the in-run analyses read. Only these need to be + /// cloned into the result snapshot passed to + /// . + /// + public static IReadOnlyList RequiredCharts { get; } = [BaseResultsHandler.PortfolioMarginKey]; + /// /// The number of order events already consumed by previous runs. The order events /// in the result passed to @@ -58,19 +78,6 @@ public class InRunResultsAnalyzer : ResultsAnalyzer /// public int LogsPosition { get; private set; } - /// - /// The equity and benchmark curves are not built for in-run analysis: - /// none of the in-run analyses read them, and building them would issue - /// a benchmark history request on every run. - /// - protected override bool RequiresEquityCurves => false; - - /// - /// The in-run analyses read the algorithm speed metrics accumulated from the - /// samples received on each run. - /// - protected override AlgorithmSpeedTracker SpeedTracker => _speed; - /// /// Initializes a new instance of the class. /// The instance is expected to be kept alive for the duration of the backtest, @@ -117,7 +124,9 @@ public InRunResultsAnalyzer(QCAlgorithm algorithm, Language language) LogsPosition += logs?.Count ?? 0; // State-based analyses are recomputed from scratch each run: remove their previous - // findings so they are replaced, or dropped if they no longer fail + // findings so they are replaced, or dropped if they no longer fail. If a time-limit + // truncated run skipped one of them, its finding drops until a run reaches it again — + // same trade-off as the positions advancement above foreach (var name in _findings.Keys.Where(IsStateBased).ToList()) { _findings.Remove(name); @@ -143,7 +152,7 @@ public InRunResultsAnalyzer(QCAlgorithm algorithm, Language language) /// private IReadOnlyList RankFindings(int maxFailedAnalyses) { - var weights = GetAnalyses().ToDictionary(analysis => analysis.GetType().Name, analysis => analysis.Weight); + var weights = Analyses.ToDictionary(analysis => analysis.GetType().Name, analysis => analysis.Weight); return _findings.Values .OrderByDescending(finding => weights.GetValueOrDefault(BaseAnalysisName(finding.Name))) .Take(maxFailedAnalyses) diff --git a/Engine/Results/Analysis/ResultsAnalyzer.cs b/Engine/Results/Analysis/ResultsAnalyzer.cs index 305c71d5af57..bbe5695b5c19 100644 --- a/Engine/Results/Analysis/ResultsAnalyzer.cs +++ b/Engine/Results/Analysis/ResultsAnalyzer.cs @@ -35,6 +35,13 @@ public class ResultsAnalyzer private SortedList _equityCurve; private SortedList _benchmarkEquityCurve; private Result _result; + private IReadOnlyCollection _analyses; + + /// + /// The diagnostic analyses to run. Created once and reused across runs, + /// since the analyses are stateless. + /// + protected IReadOnlyCollection Analyses => _analyses ??= GetAnalyses(); /// /// Whether the equity and benchmark curves should be built before running the analyses. @@ -76,7 +83,7 @@ public ResultsAnalyzer(Result result, QCAlgorithm algorithm, Language language, /// Up to entries with solutions, ranked by weight. public IReadOnlyList Run(int timeLimitSeconds = 5, int maxFailedAnalyses = 10) { - var analyses = GetAnalyses(); + var analyses = Analyses; if (analyses.Count == 0) { return []; @@ -136,8 +143,8 @@ protected void SetAnalysisData(Result result, IReadOnlyList logs) /// /// Creates the set of diagnostic analyses to run against the backtest. /// - protected virtual IReadOnlyCollection GetAnalyses() => new BaseResultsAnalysis[] - { + protected virtual IReadOnlyCollection GetAnalyses() => + [ new PortfolioValueIsNotPositiveAnalysis(), new FlatEquityCurveAnalysis(), new InsufficientBuyingPowerOrderResponseErrorAnalysis(), @@ -169,7 +176,7 @@ protected void SetAnalysisData(Result result, IReadOnlyList logs) new PortfolioMarginUsageAnalysis(), new ParameterCountAnalysis(), new MonteCarloPercentileAnalysis(), - }; + ]; /// /// Reads the backtest's "Strategy Equity" chart and fetches SPY daily history to build @@ -204,12 +211,6 @@ private static (SortedList BacktestEquity, SortedList(); var historyStart = algorithm.StartDate - TimeSpan.FromDays(3); var historyEnd = algorithm.EndDate + TimeSpan.FromDays(1); - if (historyEnd > algorithm.Time) - { - // When running mid-backtest, requesting past the current algorithm time would get the request - // trimmed by the engine anyway, while also emitting a debug message to the user - historyEnd = algorithm.Time; - } foreach (var bar in algorithm.History(spy, historyStart, historyEnd, Resolution.Daily)) { var time = algorithm.Settings.DailyPreciseEndTime ? bar.EndTime.AddDays(1).Date : bar.EndTime; diff --git a/Tests/Engine/Results/InRunResultsAnalyzerTests.cs b/Tests/Engine/Results/InRunResultsAnalyzerTests.cs index b27e302562a3..2cf807611039 100644 --- a/Tests/Engine/Results/InRunResultsAnalyzerTests.cs +++ b/Tests/Engine/Results/InRunResultsAnalyzerTests.cs @@ -19,6 +19,7 @@ using System.Linq; using System.Threading; using NUnit.Framework; +using QuantConnect.Lean.Engine.Results; using QuantConnect.Lean.Engine.Results.Analysis; using QuantConnect.Lean.Engine.Results.Analysis.Analyses; using QuantConnect.Orders; @@ -227,6 +228,28 @@ public void FindingsAreRankedByAnalysisWeightAndCapped() findings.Select(finding => finding.Name)); } + [Test] + public void RequiredChartsAreTheChartsReadByTheInRunAnalyses() + { + // The result handler only clones these charts into the analyzed snapshot, + // so this must stay in sync with the charts the in-run analyses read + CollectionAssert.AreEquivalent( + new[] { BaseResultsHandler.PortfolioMarginKey }, + InRunResultsAnalyzer.RequiredCharts); + } + + [Test] + public void AnalysesAreCreatedOnceAndReusedAcrossRuns() + { + var analyzer = new TestInRunResultsAnalyzer(new FakeAnalysisA(10)); + + analyzer.Run(MakeResult(1), new[] { "log" }); + analyzer.Run(MakeResult(1), new[] { "log" }); + + // Both the analysis chain and the findings ranking read the cached set + Assert.AreEqual(1, analyzer.GetAnalysesCallCount); + } + private static BacktestResult MakeResult(int orderEventsCount) { return new BacktestResult @@ -250,7 +273,13 @@ public TestInRunResultsAnalyzer(params BaseResultsAnalysis[] analyses) _analyses = analyses; } - protected override IReadOnlyCollection GetAnalyses() => _analyses; + public int GetAnalysesCallCount { get; private set; } + + protected override IReadOnlyCollection GetAnalyses() + { + GetAnalysesCallCount++; + return _analyses; + } } private class FakeAnalysis : BaseResultsAnalysis diff --git a/Tests/Engine/Results/ResultsAnalyzerTests.cs b/Tests/Engine/Results/ResultsAnalyzerTests.cs index f0e22c0b50a3..ebdce70aae2a 100644 --- a/Tests/Engine/Results/ResultsAnalyzerTests.cs +++ b/Tests/Engine/Results/ResultsAnalyzerTests.cs @@ -129,6 +129,17 @@ public void MaxFailedAnalysesStopsTheAnalysisChain() Assert.AreEqual(1, findings.Count); } + [Test] + public void AnalysesAreCreatedOnceAndReusedAcrossRuns() + { + var analyzer = new TestResultsAnalyzer(false, new FakeAnalysisA(10)); + + analyzer.Run(); + analyzer.Run(); + + Assert.AreEqual(1, analyzer.GetAnalysesCallCount); + } + private class TestResultsAnalyzer : ResultsAnalyzer { private readonly bool _requiresEquityCurves; @@ -143,7 +154,13 @@ public TestResultsAnalyzer(bool requiresEquityCurves, params BaseResultsAnalysis protected override bool RequiresEquityCurves => _requiresEquityCurves; - protected override IReadOnlyCollection GetAnalyses() => _analyses; + public int GetAnalysesCallCount { get; private set; } + + protected override IReadOnlyCollection GetAnalyses() + { + GetAnalysesCallCount++; + return _analyses; + } } private class FakeAnalysis : BaseResultsAnalysis From 6227ddfa981f2ec4ae1621aea92ef62762db5b1e Mon Sep 17 00:00:00 2001 From: Jhonathan Abreu Date: Thu, 23 Jul 2026 10:28:26 -0400 Subject: [PATCH 12/33] Withhold the statistics from the in-run analysis until the first equity sample Equity is not sampled while the algorithm warms up, so the intermediate statistics are all-zero defaults that flagged a false non-positive portfolio value finding for the whole warm-up period (and on the first cycle before any sample). The handler now withholds them until the algorithm is done warming up and the equity series has samples, and the portfolio value analysis skips when they are withheld. The handler also now clones only the charts the in-run analyses read instead of deep-cloning every chart on each cycle. --- .../PortfolioValueIsNotPositiveAnalysis.cs | 7 ++ Engine/Results/BacktestingResultHandler.cs | 26 ++++++- ...ortfolioValueIsNotPositiveAnalysisTests.cs | 68 +++++++++++++++++++ 3 files changed, 98 insertions(+), 3 deletions(-) create mode 100644 Tests/Engine/Results/PortfolioValueIsNotPositiveAnalysisTests.cs diff --git a/Engine/Results/Analysis/Analyses/PortfolioValueIsNotPositiveAnalysis.cs b/Engine/Results/Analysis/Analyses/PortfolioValueIsNotPositiveAnalysis.cs index d4b90c210071..d2f1bb96f1ce 100644 --- a/Engine/Results/Analysis/Analyses/PortfolioValueIsNotPositiveAnalysis.cs +++ b/Engine/Results/Analysis/Analyses/PortfolioValueIsNotPositiveAnalysis.cs @@ -44,6 +44,13 @@ public class PortfolioValueIsNotPositiveAnalysis : BaseResultsAnalysis /// Analysis results flagging the issue when ending equity is zero or negative. public IReadOnlyList Run(Result result) { + if (result.TotalPerformance == null) + { + // The statistics are withheld when they are not meaningful yet, + // like while the algorithm warms up + return []; + } + var hasEquity = result.TotalPerformance.PortfolioStatistics.EndEquity > 0; var potentialSolutions = hasEquity ? [] : Solutions(); return SingleResponse(!hasEquity, potentialSolutions); diff --git a/Engine/Results/BacktestingResultHandler.cs b/Engine/Results/BacktestingResultHandler.cs index 1ebfcde9ed08..3ce5eba6e166 100644 --- a/Engine/Results/BacktestingResultHandler.cs +++ b/Engine/Results/BacktestingResultHandler.cs @@ -469,11 +469,31 @@ protected void SendFinalResult() return null; } - // The analyses read the charts without holding ChartLock, so hand them clones - Dictionary charts; + // The analyses read the charts without holding ChartLock, so hand them clones, + // but only of the charts they read + var charts = new Dictionary(); + bool hasEquitySamples; lock (ChartLock) { - charts = Charts.ToDictionary(kvp => kvp.Key, kvp => kvp.Value.Clone()); + foreach (var chartName in InRunResultsAnalyzer.RequiredCharts) + { + if (Charts.TryGetValue(chartName, out var chart)) + { + charts[chartName] = chart.Clone(); + } + } + + hasEquitySamples = Charts.TryGetValue(StrategyEquityKey, out var equityChart) && + equityChart.Series.TryGetValue(EquityKey, out var equitySeries) && + equitySeries.Values.Count > 0; + } + + // Equity is not sampled while the algorithm warms up, so until the first sample exists + // the generated statistics are all-zero defaults that would flag a false non-positive + // portfolio value finding. Withhold them so the analyses reading them skip instead + if (Algorithm.IsWarmingUp || !hasEquitySamples) + { + totalPerformance = null; } _inRunResultsAnalyzer ??= new InRunResultsAnalyzer(AlgorithmInstance, _job.Language); diff --git a/Tests/Engine/Results/PortfolioValueIsNotPositiveAnalysisTests.cs b/Tests/Engine/Results/PortfolioValueIsNotPositiveAnalysisTests.cs new file mode 100644 index 000000000000..45df59e1e01e --- /dev/null +++ b/Tests/Engine/Results/PortfolioValueIsNotPositiveAnalysisTests.cs @@ -0,0 +1,68 @@ +/* + * QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals. + * Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * +*/ + +using System.Linq; +using NUnit.Framework; +using QuantConnect.Lean.Engine.Results.Analysis.Analyses; +using QuantConnect.Packets; +using QuantConnect.Statistics; + +namespace QuantConnect.Tests.Engine.Results +{ + [TestFixture] + public class PortfolioValueIsNotPositiveAnalysisTests + { + [Test] + public void ReturnsNoFindingsWhenStatisticsAreWithheld() + { + // The result handler withholds the statistics when they are not meaningful yet, + // like while the algorithm warms up and no equity has been sampled + var findings = new PortfolioValueIsNotPositiveAnalysis().Run(new BacktestResult()); + + Assert.IsEmpty(findings); + } + + [TestCase(0)] + [TestCase(-100000)] + public void FlagsNonPositiveEndingEquity(int endEquity) + { + var findings = new PortfolioValueIsNotPositiveAnalysis().Run(MakeResult(endEquity)); + + var finding = findings.Single(); + Assert.AreEqual(nameof(PortfolioValueIsNotPositiveAnalysis), finding.Name); + Assert.IsNotEmpty(finding.Solutions); + } + + [Test] + public void ReturnsNoActionableFindingWhenEndingEquityIsPositive() + { + var findings = new PortfolioValueIsNotPositiveAnalysis().Run(MakeResult(100000)); + + Assert.IsEmpty(findings.Single().Solutions); + } + + private static BacktestResult MakeResult(decimal endEquity) + { + return new BacktestResult + { + TotalPerformance = new AlgorithmPerformance + { + PortfolioStatistics = new PortfolioStatistics { EndEquity = endEquity } + } + }; + } + } +} From 93216b29ed1127341a946f49a04ae57625cf4d09 Mon Sep 17 00:00:00 2001 From: Jhonathan Abreu Date: Wed, 29 Jul 2026 10:29:48 -0400 Subject: [PATCH 13/33] Add results analyses detecting the isolator time limit runtime errors --- ...imumRuntimeExceededRuntimeErrorAnalysis.cs | 63 ++++++++ .../OperationCanceledRuntimeErrorAnalysis.cs | 60 +++++++ .../RuntimeErrorAnalysis.cs | 91 +++++++++++ ...ngleTimeLoopTimeoutRuntimeErrorAnalysis.cs | 78 +++++++++ Engine/Results/Analysis/ResultsAnalyzer.cs | 3 + .../Results/RuntimeErrorAnalysesTests.cs | 150 ++++++++++++++++++ 6 files changed, 445 insertions(+) create mode 100644 Engine/Results/Analysis/Analyses/RuntimeErrorAnalyses/MaximumRuntimeExceededRuntimeErrorAnalysis.cs create mode 100644 Engine/Results/Analysis/Analyses/RuntimeErrorAnalyses/OperationCanceledRuntimeErrorAnalysis.cs create mode 100644 Engine/Results/Analysis/Analyses/RuntimeErrorAnalyses/RuntimeErrorAnalysis.cs create mode 100644 Engine/Results/Analysis/Analyses/RuntimeErrorAnalyses/SingleTimeLoopTimeoutRuntimeErrorAnalysis.cs create mode 100644 Tests/Engine/Results/RuntimeErrorAnalysesTests.cs diff --git a/Engine/Results/Analysis/Analyses/RuntimeErrorAnalyses/MaximumRuntimeExceededRuntimeErrorAnalysis.cs b/Engine/Results/Analysis/Analyses/RuntimeErrorAnalyses/MaximumRuntimeExceededRuntimeErrorAnalysis.cs new file mode 100644 index 000000000000..f1bfb6ef2e9d --- /dev/null +++ b/Engine/Results/Analysis/Analyses/RuntimeErrorAnalyses/MaximumRuntimeExceededRuntimeErrorAnalysis.cs @@ -0,0 +1,63 @@ +/* + * QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals. + * Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * +*/ +using System.Collections.Generic; + +namespace QuantConnect.Lean.Engine.Results.Analysis.Analyses +{ + /// + /// Detects algorithms terminated because the whole run outlived the maximum allowed wall-clock + /// time, emitted by the ("Execution Security Error: Operation timed out - + /// N minutes max") or by the engine when the isolator reports an incomplete run ("Failed to + /// complete algorithm within N seconds"). + /// + public class MaximumRuntimeExceededRuntimeErrorAnalysis : RuntimeErrorAnalysis + { + /// + /// Gets the description of the maximum runtime exceeded issue. + /// + public override string Issue { get; } = "The algorithm exceeded the maximum allowed total runtime and was terminated."; + + /// + /// Gets the severity weight for this analysis. The timeout is a fatal error that terminated + /// the run, so it ranks above every non-fatal finding. + /// + public override int Weight { get; } = 100; + + /// + /// Gets the patterns identifying the maximum runtime exceeded error message. + /// + protected override string[][] ErrorMessagePatterns { get; } = + [ + ["Operation timed out", "minutes max"], + ["Failed to complete algorithm within"], + ]; + + /// + /// Gets the suggested solutions to complete the run within the maximum allowed runtime. + /// + protected override List Solutions(Language language) => + [ + "Reduce the backtest period so the run completes within the allowed runtime.", + + "Reduce the universe size and the data resolution to lower the total amount of data processed.", + + "Review the algorithm code for inefficiencies: avoid recomputing values from scratch on every data update, " + + "and avoid history requests whose range grows as the backtest progresses.", + + "Check for infinite or recursive loops that keep the algorithm running without making progress.", + ]; + } +} diff --git a/Engine/Results/Analysis/Analyses/RuntimeErrorAnalyses/OperationCanceledRuntimeErrorAnalysis.cs b/Engine/Results/Analysis/Analyses/RuntimeErrorAnalyses/OperationCanceledRuntimeErrorAnalysis.cs new file mode 100644 index 000000000000..c3ec92ca0acc --- /dev/null +++ b/Engine/Results/Analysis/Analyses/RuntimeErrorAnalyses/OperationCanceledRuntimeErrorAnalysis.cs @@ -0,0 +1,60 @@ +/* + * QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals. + * Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * +*/ +using System.Collections.Generic; + +namespace QuantConnect.Lean.Engine.Results.Analysis.Analyses +{ + /// + /// Detects algorithms that were forcefully canceled, emitted by the when + /// the algorithm is asked to stop but its code is still running once the shutdown grace period + /// expires ("Operation was canceled"). + /// + public class OperationCanceledRuntimeErrorAnalysis : RuntimeErrorAnalysis + { + /// + /// Gets the description of the forced cancellation issue. + /// + public override string Issue { get; } = "The algorithm did not shut down within the grace period after a stop request and was forcefully canceled."; + + /// + /// Gets the severity weight for this analysis. The cancellation is a fatal error that + /// terminated the run, so it ranks above every non-fatal finding. + /// + public override int Weight { get; } = 100; + + /// + /// Gets the patterns identifying the forced cancellation error message. + /// + protected override string[][] ErrorMessagePatterns { get; } = + [ + ["Operation was canceled"], + ]; + + /// + /// Gets the suggested solutions to let the algorithm respond to stop requests promptly. + /// + protected override List Solutions(Language language) => + [ + "This error is raised when the algorithm is asked to stop, typically because the user stopped or deleted it, " + + "but its code keeps running past the shutdown grace period. If you stopped it on purpose, the error is expected and can be ignored.", + + "If you did not stop the algorithm, reduce the work done per event handler so it can respond to stop requests promptly: " + + "break up big loops, avoid long blocking operations, and avoid large history requests or model training on every data update.", + + "Reduce the universe size and the data resolution so each time loop processes less data and control returns to the engine sooner.", + ]; + } +} diff --git a/Engine/Results/Analysis/Analyses/RuntimeErrorAnalyses/RuntimeErrorAnalysis.cs b/Engine/Results/Analysis/Analyses/RuntimeErrorAnalyses/RuntimeErrorAnalysis.cs new file mode 100644 index 000000000000..8996d8376be8 --- /dev/null +++ b/Engine/Results/Analysis/Analyses/RuntimeErrorAnalyses/RuntimeErrorAnalysis.cs @@ -0,0 +1,91 @@ +/* + * QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals. + * Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * +*/ +using System; +using System.Collections.Generic; +using System.Linq; + +namespace QuantConnect.Lean.Engine.Results.Analysis.Analyses +{ + /// + /// Abstract base class for analyses that detect a specific runtime error that terminated the + /// algorithm by inspecting the error message for known text fragments. The runtime error is + /// read from the result state, falling back to the "Runtime Error:" log line for results + /// that carry no state. + /// + public abstract class RuntimeErrorAnalysis : BaseResultsAnalysis + { + /// + /// Gets the patterns identifying the runtime error. Each pattern is a set of text fragments + /// that must all be present in the error message (case-insensitive); the error matches + /// when any pattern does. + /// + protected abstract string[][] ErrorMessagePatterns { get; } + + /// + /// Gets the suggested solutions for the detected runtime error. + /// + protected abstract List Solutions(Language language); + + /// + /// Runs the runtime error analysis against the provided backtest parameters. + /// + public override IReadOnlyList Run(ResultsAnalysisRunParameters parameters) + => Run(parameters.Result?.State, parameters.Logs, parameters.Language); + + /// + /// Runs the runtime error analysis against the algorithm state and logs. + /// + /// The algorithm state of the result, holding the runtime error message if any. + /// The full list of log lines produced by the backtest. + /// The programming language the algorithm is written in. + /// A single response with the matched error message and solutions, or without them when the error is not found. + public IReadOnlyList Run(IDictionary state, IReadOnlyList logs, Language language) + { + var sample = GetRuntimeErrorMessages(state, logs).FirstOrDefault(Matches); + return SingleResponse(sample, sample != null ? Solutions(language) : []); + } + + /// + /// Determines whether the given runtime error message matches any of the + /// . + /// + private bool Matches(string message) + { + return ErrorMessagePatterns.Any( + fragments => fragments.All(fragment => message.Contains(fragment, StringComparison.InvariantCultureIgnoreCase))); + } + + /// + /// Gets the candidate runtime error messages: the result state's runtime error entry when + /// present, plus any "Runtime Error:" lines from the logs. + /// + private static IEnumerable GetRuntimeErrorMessages(IDictionary state, IReadOnlyList logs) + { + if (state != null && state.TryGetValue("RuntimeError", out var error) && !string.IsNullOrEmpty(error)) + { + yield return error; + } + + foreach (var log in logs ?? []) + { + if (log.Contains("Runtime Error", StringComparison.InvariantCultureIgnoreCase)) + { + yield return log; + } + } + } + } +} diff --git a/Engine/Results/Analysis/Analyses/RuntimeErrorAnalyses/SingleTimeLoopTimeoutRuntimeErrorAnalysis.cs b/Engine/Results/Analysis/Analyses/RuntimeErrorAnalyses/SingleTimeLoopTimeoutRuntimeErrorAnalysis.cs new file mode 100644 index 000000000000..e58c0f76ec28 --- /dev/null +++ b/Engine/Results/Analysis/Analyses/RuntimeErrorAnalyses/SingleTimeLoopTimeoutRuntimeErrorAnalysis.cs @@ -0,0 +1,78 @@ +/* + * QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals. + * Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * +*/ +using QuantConnect.Algorithm; +using System.Collections.Generic; + +namespace QuantConnect.Lean.Engine.Results.Analysis.Analyses +{ + /// + /// Detects algorithms terminated because a single time loop exceeded the per-loop time limit, + /// emitted by the when the algorithm manager's time loop tracker trips + /// ("Algorithm took longer than N minutes on a single time loop"). + /// + public class SingleTimeLoopTimeoutRuntimeErrorAnalysis : RuntimeErrorAnalysis + { + /// + /// Gets the description of the single time loop timeout issue. + /// + public override string Issue { get; } = "A single time loop took longer than the maximum allowed, so the algorithm was terminated."; + + /// + /// Gets the severity weight for this analysis. A timeout is a fatal error that terminated + /// the run, so it ranks above every non-fatal finding. + /// + public override int Weight { get; } = 100; + + /// + /// Gets the patterns identifying the single time loop timeout error message. + /// + protected override string[][] ErrorMessagePatterns { get; } = + [ + ["took longer than", "single time loop"], + ]; + + /// + /// Gets the suggested solutions to keep each time loop within the time limit. + /// + protected override List Solutions(Language language) + { + var solutions = new List + { + $"Look for heavy work inside a single event handler (`{FormatCode(nameof(QCAlgorithm.OnData), language)}`, scheduled events, universe selection functions): " + + "large or nested loops over all securities, or values recomputed from scratch on every data update. " + + "Cache the values and update them incrementally with rolling windows, consolidators or indicators instead.", + + "If there is a universe, reduce its size and filter it: return from the selection functions only the securities the strategy actually trades, " + + $"and narrow option and future chain universes with the `{FormatCode("SetFilter", language)}` contract filters (strikes and expirations).", + + "Reduce the number of subscribed securities or the data resolution to lower the amount of data processed on each time loop.", + + "Avoid issuing large history requests inside event handlers; request only the period the strategy needs, " + + "or maintain the data incrementally instead of re-requesting it on every data update.", + + $"If the algorithm trains a machine learning model, run the training through the `{FormatCode(nameof(QCAlgorithm.Train), language)}` method, " + + "which allocates additional time for it to complete.", + + "Check for loops whose exit condition may never be met: an infinite loop in an event handler surfaces as this error.", + }; + if (language == Language.Python) + { + solutions.Add("Vectorize heavy numeric Python loops with numpy or pandas operations, which run orders of magnitude faster."); + } + return solutions; + } + } +} diff --git a/Engine/Results/Analysis/ResultsAnalyzer.cs b/Engine/Results/Analysis/ResultsAnalyzer.cs index bbe5695b5c19..e5ae3605e5a8 100644 --- a/Engine/Results/Analysis/ResultsAnalyzer.cs +++ b/Engine/Results/Analysis/ResultsAnalyzer.cs @@ -145,6 +145,9 @@ protected void SetAnalysisData(Result result, IReadOnlyList logs) /// protected virtual IReadOnlyCollection GetAnalyses() => [ + new SingleTimeLoopTimeoutRuntimeErrorAnalysis(), + new OperationCanceledRuntimeErrorAnalysis(), + new MaximumRuntimeExceededRuntimeErrorAnalysis(), new PortfolioValueIsNotPositiveAnalysis(), new FlatEquityCurveAnalysis(), new InsufficientBuyingPowerOrderResponseErrorAnalysis(), diff --git a/Tests/Engine/Results/RuntimeErrorAnalysesTests.cs b/Tests/Engine/Results/RuntimeErrorAnalysesTests.cs new file mode 100644 index 000000000000..f914d31922a4 --- /dev/null +++ b/Tests/Engine/Results/RuntimeErrorAnalysesTests.cs @@ -0,0 +1,150 @@ +/* + * QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals. + * Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * +*/ + +using System; +using System.Collections.Generic; +using System.Linq; +using NUnit.Framework; +using QuantConnect.Lean.Engine.Results.Analysis.Analyses; + +namespace QuantConnect.Tests.Engine.Results +{ + [TestFixture] + public class RuntimeErrorAnalysesTests + { + private const string SingleTimeLoopError = + "20240101 15:30:00.000 Runtime Error: Algorithm took longer than 10 minutes on a single time loop. CurrentTimeStepElapsed: 10.0 minutes"; + + private const string SingleTimeLoopWithAdditionalTimeError = + "Runtime Error: Algorithm took longer than 10 minutes on a single time loop. " + + "An additional 30 minutes were also allocated and consumed. CurrentTimeStepElapsed: 40.1 minutes"; + + private const string OperationCanceledError = "20240101 15:30:00.000 Runtime Error: Operation was canceled"; + + private const string MaximumRuntimeError = + "Runtime Error: Execution Security Error: Operation timed out - 1440 minutes max. Check for recursive loops."; + + private const string MaximumRuntimeEngineError = + "Runtime Error: Failed to complete algorithm within 86400 seconds. Please make it run faster."; + + private static RuntimeErrorAnalysis[] CreateAnalyses() => + [ + new SingleTimeLoopTimeoutRuntimeErrorAnalysis(), + new OperationCanceledRuntimeErrorAnalysis(), + new MaximumRuntimeExceededRuntimeErrorAnalysis(), + ]; + + [TestCase(SingleTimeLoopError, typeof(SingleTimeLoopTimeoutRuntimeErrorAnalysis))] + [TestCase(SingleTimeLoopWithAdditionalTimeError, typeof(SingleTimeLoopTimeoutRuntimeErrorAnalysis))] + [TestCase(OperationCanceledError, typeof(OperationCanceledRuntimeErrorAnalysis))] + [TestCase(MaximumRuntimeError, typeof(MaximumRuntimeExceededRuntimeErrorAnalysis))] + [TestCase(MaximumRuntimeEngineError, typeof(MaximumRuntimeExceededRuntimeErrorAnalysis))] + public void DetectsItsOwnRuntimeErrorFromState(string runtimeError, Type expectedAnalysisType) + { + var state = new Dictionary { ["RuntimeError"] = runtimeError }; + + foreach (var analysis in CreateAnalyses()) + { + var finding = analysis.Run(state, [], Language.CSharp).Single(); + + if (analysis.GetType() == expectedAnalysisType) + { + Assert.AreEqual(expectedAnalysisType.Name, finding.Name); + Assert.AreEqual(runtimeError, finding.Sample); + Assert.IsNotEmpty(finding.Solutions); + } + else + { + Assert.IsNull(finding.Sample, $"{analysis.GetType().Name} should not flag \"{runtimeError}\""); + Assert.IsEmpty(finding.Solutions); + } + } + } + + [Test] + public void DetectsRuntimeErrorFromLogsWhenStateIsMissing() + { + var logs = new List + { + "20240101 15:29:00.000 Launching analysis for the algorithm", + SingleTimeLoopError, + }; + + var finding = new SingleTimeLoopTimeoutRuntimeErrorAnalysis().Run(null, logs, Language.CSharp).Single(); + + Assert.AreEqual(SingleTimeLoopError, finding.Sample); + Assert.IsNotEmpty(finding.Solutions); + } + + [Test] + public void IgnoresLogLinesThatAreNotRuntimeErrors() + { + // These mention cancellation and timeouts but are not the algorithm's runtime error + var logs = new List + { + "20240101 15:29:00.000 Isolator.ExecuteWithTimeLimit(): Operation was canceled", + "20240101 15:29:00.000 The download operation timed out after 5 minutes max wait", + }; + + foreach (var analysis in CreateAnalyses()) + { + var finding = analysis.Run(new Dictionary(), logs, Language.CSharp).Single(); + + Assert.IsNull(finding.Sample); + Assert.IsEmpty(finding.Solutions); + } + } + + [TestCase("Runtime Error: System.DivideByZeroException: Attempted to divide by zero.")] + [TestCase("")] + public void DoesNotFlagOtherRuntimeErrorsOrCleanRuns(string runtimeError) + { + var state = new Dictionary { ["RuntimeError"] = runtimeError }; + + foreach (var analysis in CreateAnalyses()) + { + var finding = analysis.Run(state, [], Language.CSharp).Single(); + + Assert.IsNull(finding.Sample); + Assert.IsEmpty(finding.Solutions); + } + } + + [TestCase(Language.CSharp, "Train", "OnData")] + [TestCase(Language.Python, "train", "on_data")] + public void FormatsCodeReferencesForTheAlgorithmLanguage(Language language, string trainMethod, string onDataMethod) + { + var state = new Dictionary { ["RuntimeError"] = SingleTimeLoopError }; + + var solutions = new SingleTimeLoopTimeoutRuntimeErrorAnalysis().Run(state, [], language).Single().Solutions; + + Assert.IsTrue(solutions.Any(solution => solution.Contains($"`{trainMethod}`"))); + Assert.IsTrue(solutions.Any(solution => solution.Contains($"`{onDataMethod}`"))); + } + + [Test] + public void PythonSolutionsIncludeVectorizationAdvice() + { + var state = new Dictionary { ["RuntimeError"] = SingleTimeLoopError }; + + var csharpSolutions = new SingleTimeLoopTimeoutRuntimeErrorAnalysis().Run(state, [], Language.CSharp).Single().Solutions; + var pythonSolutions = new SingleTimeLoopTimeoutRuntimeErrorAnalysis().Run(state, [], Language.Python).Single().Solutions; + + Assert.IsFalse(csharpSolutions.Any(solution => solution.Contains("numpy"))); + Assert.IsTrue(pythonSolutions.Any(solution => solution.Contains("numpy"))); + } + } +} From ee6daf1c1718b6210d8b66294703d057829800d4 Mon Sep 17 00:00:00 2001 From: Jhonathan Abreu Date: Thu, 30 Jul 2026 17:54:34 -0400 Subject: [PATCH 14/33] Run the algorithm speed analysis in the final results analysis too --- .../Analyses/AlgorithmSpeedAnalysis.cs | 28 ++++++++---- .../Results/Analysis/InRunResultsAnalyzer.cs | 12 +---- Engine/Results/Analysis/ResultsAnalyzer.cs | 13 ++++-- Engine/Results/BacktestingResultHandler.cs | 45 +++++++++++++------ .../Results/AlgorithmSpeedAnalysisTests.cs | 17 +++++++ Tests/Engine/Results/ResultsAnalyzerTests.cs | 38 +++++++++++++++- 6 files changed, 117 insertions(+), 36 deletions(-) diff --git a/Engine/Results/Analysis/Analyses/AlgorithmSpeedAnalysis.cs b/Engine/Results/Analysis/Analyses/AlgorithmSpeedAnalysis.cs index 7327dae1e1b6..8c4f06aebcf2 100644 --- a/Engine/Results/Analysis/Analyses/AlgorithmSpeedAnalysis.cs +++ b/Engine/Results/Analysis/Analyses/AlgorithmSpeedAnalysis.cs @@ -20,10 +20,11 @@ namespace QuantConnect.Lean.Engine.Results.Analysis.Analyses { /// - /// In-run analysis that tracks the algorithm's execution speed so the user can decide to stop - /// a slow backtest early. It reads the throughput and progress metrics accumulated by - /// and reports slow processing speed, a long projected + /// Tracks the algorithm's execution speed from the throughput and progress metrics accumulated + /// by , reporting slow processing speed, a long projected /// remaining runtime, degrading throughput, and history-request-dominated data loads. + /// It runs periodically while the backtest is in progress, so the user can decide to stop a + /// slow backtest early, and again on the final analysis against the whole run's metrics. /// Benchmark speeds: https://www.quantconnect.com/performance /// public class AlgorithmSpeedAnalysis : BaseResultsAnalysis @@ -141,13 +142,24 @@ private static void AddSlowExecution(AlgorithmSpeedTracker speed, List TimeSpan.Zero) + { + sample += Invariant($", about {FormatDuration(remaining.Value)} remaining at the recent pace."); + } + else + { + sample += "."; + } findings.Add(new(SlowExecutionName, Invariant($"The algorithm is running below {SlowDataPointsPerSecond / 1000}k data points per second."), diff --git a/Engine/Results/Analysis/InRunResultsAnalyzer.cs b/Engine/Results/Analysis/InRunResultsAnalyzer.cs index dd8edcc6610b..515db26948d7 100644 --- a/Engine/Results/Analysis/InRunResultsAnalyzer.cs +++ b/Engine/Results/Analysis/InRunResultsAnalyzer.cs @@ -42,8 +42,6 @@ public class InRunResultsAnalyzer : ResultsAnalyzer private readonly Dictionary _findings = new(); - private readonly AlgorithmSpeedTracker _speed = new(); - /// /// The equity and benchmark curves are not built for in-run analysis: /// none of the in-run analyses read them, and building them would issue @@ -51,12 +49,6 @@ public class InRunResultsAnalyzer : ResultsAnalyzer /// protected override bool RequiresEquityCurves => false; - /// - /// The in-run analyses read the algorithm speed metrics accumulated from the - /// samples received on each run. - /// - protected override AlgorithmSpeedTracker SpeedTracker => _speed; - /// /// The names of the charts the in-run analyses read. Only these need to be /// cloned into the result snapshot passed to @@ -86,7 +78,7 @@ public class InRunResultsAnalyzer : ResultsAnalyzer /// The algorithm instance used for history requests and settings. /// The programming language the algorithm is written in. public InRunResultsAnalyzer(QCAlgorithm algorithm, Language language) - : base(null, algorithm, language, null) + : base(null, algorithm, language, null, new AlgorithmSpeedTracker()) { } @@ -112,7 +104,7 @@ public InRunResultsAnalyzer(QCAlgorithm algorithm, Language language) SetAnalysisData(result, logs); if (speedSample.HasValue) { - _speed.AddSample(speedSample.Value); + SpeedTracker.AddSample(speedSample.Value); } var newFindings = Run(timeLimitSeconds, maxFailedAnalyses); diff --git a/Engine/Results/Analysis/ResultsAnalyzer.cs b/Engine/Results/Analysis/ResultsAnalyzer.cs index e5ae3605e5a8..5c2d9c198c19 100644 --- a/Engine/Results/Analysis/ResultsAnalyzer.cs +++ b/Engine/Results/Analysis/ResultsAnalyzer.cs @@ -52,10 +52,11 @@ public class ResultsAnalyzer /// /// The speed metrics tracked for the running backtest, made available to the analyses - /// through . Null unless the analyzer - /// tracks the algorithm speed, like the in-run analyzer does. + /// through . The in-run analyzer feeds + /// it a sample on each run; the final analyzer receives the same tracker so the speed + /// analysis also runs against the full-run metrics. Null when speed is not tracked. /// - protected virtual AlgorithmSpeedTracker SpeedTracker => null; + public AlgorithmSpeedTracker SpeedTracker { get; } /// /// Initializes a new instance of the class. @@ -64,12 +65,15 @@ public class ResultsAnalyzer /// The algorithm instance used for history requests and settings. /// The programming language the algorithm is written in. /// The full list of log lines produced by the backtest. - public ResultsAnalyzer(Result result, QCAlgorithm algorithm, Language language, IReadOnlyList logs) + /// The speed metrics tracked for the backtest, or null when speed is not tracked. + public ResultsAnalyzer(Result result, QCAlgorithm algorithm, Language language, IReadOnlyList logs, + AlgorithmSpeedTracker speedTracker = null) { _result = result; _algorithm = algorithm; _language = language; _logs = logs; + SpeedTracker = speedTracker; } // ── Test chain ──────────────────────────────────────────────────────────── @@ -176,6 +180,7 @@ protected virtual IReadOnlyCollection GetAnalyses() => new PerformanceRelativeToBenchmarkAnalysis(), new CrisisEventsAnalysis(), new ExecutionSpeedAnalysis(), + new AlgorithmSpeedAnalysis(), new PortfolioMarginUsageAnalysis(), new ParameterCountAnalysis(), new MonteCarloPercentileAnalysis(), diff --git a/Engine/Results/BacktestingResultHandler.cs b/Engine/Results/BacktestingResultHandler.cs index 3ce5eba6e166..e0bae8a58c7e 100644 --- a/Engine/Results/BacktestingResultHandler.cs +++ b/Engine/Results/BacktestingResultHandler.cs @@ -427,7 +427,16 @@ protected void SendFinalResult() { logs = LogStore.Select(x => x.Message).ToList(); } - var analyzer = new ResultsAnalyzer(result.Results, AlgorithmInstance, _job.Language, logs); + // The final analysis reuses the speed metrics accumulated by the in-run analyzer, + // adding one last sample so they cover the backtest through its end + var speedTracker = _inRunResultsAnalyzer?.SpeedTracker; + var speedSample = TakeAlgorithmSpeedSample(); + if (speedTracker != null && speedSample.HasValue) + { + speedTracker.AddSample(speedSample.Value); + } + + var analyzer = new ResultsAnalyzer(result.Results, AlgorithmInstance, _job.Language, logs, speedTracker); try { result.Results.Analysis = analyzer.Run(); @@ -498,18 +507,9 @@ protected void SendFinalResult() _inRunResultsAnalyzer ??= new InRunResultsAnalyzer(AlgorithmInstance, _job.Language); - // Sample the engine speed counters for the algorithm speed analysis, but not while the - // algorithm is warming up: the warm-up pace would skew the speed metrics. The analyses - // themselves do run during warm-up, so conditions like orders submitted while warming up - // surface without waiting for warm-up to end - AlgorithmSpeedSample? speedSample = Algorithm.IsWarmingUp - ? null - : new AlgorithmSpeedSample( - DateTime.UtcNow - StartTime, - PerformanceTrackingTool?.DataPoints ?? 0, - PerformanceTrackingTool?.HistoryDataPoints ?? 0, - _progressMonitor?.ProcessedDays ?? 0, - _progressMonitor?.TotalDays ?? 0); + // The analyses themselves do run during warm-up (the speed sample is null then), so + // conditions like orders submitted while warming up surface without waiting for warm-up to end + var speedSample = TakeAlgorithmSpeedSample(); // Only the order events and logs produced since the previous run are analyzed, // the analyzer accumulates findings across runs @@ -541,6 +541,25 @@ protected void SendFinalResult() } } + /// + /// Takes a sample of the engine speed counters for the algorithm speed analysis. + /// Null while the algorithm warms up, since the warm-up pace would skew the speed metrics. + /// + private AlgorithmSpeedSample? TakeAlgorithmSpeedSample() + { + if (Algorithm == null || Algorithm.IsWarmingUp) + { + return null; + } + + return new AlgorithmSpeedSample( + DateTime.UtcNow - StartTime, + PerformanceTrackingTool?.DataPoints ?? 0, + PerformanceTrackingTool?.HistoryDataPoints ?? 0, + _progressMonitor?.ProcessedDays ?? 0, + _progressMonitor?.TotalDays ?? 0); + } + /// /// Sends the in-run analysis findings to the browser in their own packet, /// only when they changed since they were last sent. diff --git a/Tests/Engine/Results/AlgorithmSpeedAnalysisTests.cs b/Tests/Engine/Results/AlgorithmSpeedAnalysisTests.cs index fd5dc8b0e368..48baaeb66066 100644 --- a/Tests/Engine/Results/AlgorithmSpeedAnalysisTests.cs +++ b/Tests/Engine/Results/AlgorithmSpeedAnalysisTests.cs @@ -63,6 +63,23 @@ public void FlagsSlowExecutionWithProgressAndProjection() Assert.IsNotEmpty(finding.Solutions); } + [Test] + public void SlowExecutionOmitsTheProjectionWhenTheBacktestReachedItsEndDate() + { + // Slow throughout, but all the backtest days are processed, like on the final analysis + // of a completed backtest: no remaining time is projected and no long-runtime finding fails + var tracker = AlgorithmSpeedTrackerTests.BuildUniformTracker(samples: 7, stepSeconds: 30, + dataPointsPerStep: 300_000, historyDataPointsPerStep: 0, daysPerStep: 1, totalDays: 6); + + var findings = new AlgorithmSpeedAnalysis().Run(tracker); + + var finding = findings.Single(); + Assert.AreEqual($"{nameof(AlgorithmSpeedAnalysis)} / {AlgorithmSpeedAnalysis.SlowExecutionName}", finding.Name); + var sample = (string)finding.Sample; + StringAssert.Contains("100% complete", sample); + StringAssert.DoesNotContain("remaining", sample); + } + [Test] public void FormatsRatesBelowOneThousandAsRawCounts() { diff --git a/Tests/Engine/Results/ResultsAnalyzerTests.cs b/Tests/Engine/Results/ResultsAnalyzerTests.cs index ebdce70aae2a..87ad04a1922c 100644 --- a/Tests/Engine/Results/ResultsAnalyzerTests.cs +++ b/Tests/Engine/Results/ResultsAnalyzerTests.cs @@ -140,13 +140,49 @@ public void AnalysesAreCreatedOnceAndReusedAcrossRuns() Assert.AreEqual(1, analyzer.GetAnalysesCallCount); } + [Test] + public void SpeedTrackerIsPassedToTheAnalyses() + { + var tracker = new AlgorithmSpeedTracker(); + ResultsAnalysisRunParameters seenParameters = null; + var fake = new FakeAnalysisA(10) { OnRun = parameters => seenParameters = parameters }; + var analyzer = new TestResultsAnalyzer(false, tracker, fake); + + analyzer.Run(); + + Assert.AreSame(tracker, seenParameters.Speed); + } + + [Test] + public void DefaultAnalysisSetIncludesTheAlgorithmSpeedAnalysis() + { + var analyses = new DefaultSetResultsAnalyzer().DefaultAnalyses; + + Assert.IsTrue(analyses.Any(analysis => analysis is AlgorithmSpeedAnalysis)); + } + + private sealed class DefaultSetResultsAnalyzer : ResultsAnalyzer + { + public DefaultSetResultsAnalyzer() + : base(null, null, Language.CSharp, null) + { + } + + public IReadOnlyCollection DefaultAnalyses => GetAnalyses(); + } + private class TestResultsAnalyzer : ResultsAnalyzer { private readonly bool _requiresEquityCurves; private readonly IReadOnlyCollection _analyses; public TestResultsAnalyzer(bool requiresEquityCurves, params BaseResultsAnalysis[] analyses) - : base(null, null, Language.CSharp, null) + : this(requiresEquityCurves, null, analyses) + { + } + + public TestResultsAnalyzer(bool requiresEquityCurves, AlgorithmSpeedTracker speedTracker, params BaseResultsAnalysis[] analyses) + : base(null, null, Language.CSharp, null, speedTracker) { _requiresEquityCurves = requiresEquityCurves; _analyses = analyses; From b6d8e7527b888ba6d677b46124b12debc98d5453 Mon Sep 17 00:00:00 2001 From: Jhonathan Abreu Date: Thu, 30 Jul 2026 17:54:42 -0400 Subject: [PATCH 15/33] Add contract filter advice to the runtime timeout solutions and make them shorter --- ...imumRuntimeExceededRuntimeErrorAnalysis.cs | 6 +++--- ...ngleTimeLoopTimeoutRuntimeErrorAnalysis.cs | 21 ++++++++----------- 2 files changed, 12 insertions(+), 15 deletions(-) diff --git a/Engine/Results/Analysis/Analyses/RuntimeErrorAnalyses/MaximumRuntimeExceededRuntimeErrorAnalysis.cs b/Engine/Results/Analysis/Analyses/RuntimeErrorAnalyses/MaximumRuntimeExceededRuntimeErrorAnalysis.cs index f1bfb6ef2e9d..7fb2a29055c9 100644 --- a/Engine/Results/Analysis/Analyses/RuntimeErrorAnalyses/MaximumRuntimeExceededRuntimeErrorAnalysis.cs +++ b/Engine/Results/Analysis/Analyses/RuntimeErrorAnalyses/MaximumRuntimeExceededRuntimeErrorAnalysis.cs @@ -50,9 +50,9 @@ public class MaximumRuntimeExceededRuntimeErrorAnalysis : RuntimeErrorAnalysis /// protected override List Solutions(Language language) => [ - "Reduce the backtest period so the run completes within the allowed runtime.", - - "Reduce the universe size and the data resolution to lower the total amount of data processed.", + "Reduce the universe size and the data resolution to lower the total amount of data processed. " + + "For options and futures, tighten the contract filters: narrower strike and expiration ranges, " + + "and fewer contracts per underlying.", "Review the algorithm code for inefficiencies: avoid recomputing values from scratch on every data update, " + "and avoid history requests whose range grows as the backtest progresses.", diff --git a/Engine/Results/Analysis/Analyses/RuntimeErrorAnalyses/SingleTimeLoopTimeoutRuntimeErrorAnalysis.cs b/Engine/Results/Analysis/Analyses/RuntimeErrorAnalyses/SingleTimeLoopTimeoutRuntimeErrorAnalysis.cs index e58c0f76ec28..ff365502f663 100644 --- a/Engine/Results/Analysis/Analyses/RuntimeErrorAnalyses/SingleTimeLoopTimeoutRuntimeErrorAnalysis.cs +++ b/Engine/Results/Analysis/Analyses/RuntimeErrorAnalyses/SingleTimeLoopTimeoutRuntimeErrorAnalysis.cs @@ -51,26 +51,23 @@ protected override List Solutions(Language language) { var solutions = new List { - $"Look for heavy work inside a single event handler (`{FormatCode(nameof(QCAlgorithm.OnData), language)}`, scheduled events, universe selection functions): " + - "large or nested loops over all securities, or values recomputed from scratch on every data update. " + - "Cache the values and update them incrementally with rolling windows, consolidators or indicators instead.", + $"Avoid heavy work in event handlers (`{FormatCode(nameof(QCAlgorithm.OnData), language)}`, scheduled events, universe selection): " + + "instead of recomputing values on every update, update them incrementally with rolling windows, consolidators or indicators.", - "If there is a universe, reduce its size and filter it: return from the selection functions only the securities the strategy actually trades, " + - $"and narrow option and future chain universes with the `{FormatCode("SetFilter", language)}` contract filters (strikes and expirations).", + "Reduce the universe size: select only the securities the strategy trades, " + + $"and narrow option and future chains with the `{FormatCode("SetFilter", language)}` strike and expiration filters.", - "Reduce the number of subscribed securities or the data resolution to lower the amount of data processed on each time loop.", + "Reduce the number of subscribed securities or the data resolution.", - "Avoid issuing large history requests inside event handlers; request only the period the strategy needs, " + - "or maintain the data incrementally instead of re-requesting it on every data update.", + "Avoid large history requests inside event handlers; request only the period needed or maintain the data incrementally.", - $"If the algorithm trains a machine learning model, run the training through the `{FormatCode(nameof(QCAlgorithm.Train), language)}` method, " + - "which allocates additional time for it to complete.", + $"Run machine learning training through the `{FormatCode(nameof(QCAlgorithm.Train), language)}` method, which allocates extra time for it.", - "Check for loops whose exit condition may never be met: an infinite loop in an event handler surfaces as this error.", + "Check for loops that may never exit: an infinite loop in an event handler surfaces as this error.", }; if (language == Language.Python) { - solutions.Add("Vectorize heavy numeric Python loops with numpy or pandas operations, which run orders of magnitude faster."); + solutions.Add("Vectorize heavy numeric loops with numpy or pandas."); } return solutions; } From d66d090d66a65330a0ad2753f06129e1820d41f4 Mon Sep 17 00:00:00 2001 From: Jhonathan Abreu Date: Thu, 30 Jul 2026 18:19:20 -0400 Subject: [PATCH 16/33] Merge the forced cancellation analysis into the single time loop timeout analysis --- .../OperationCanceledRuntimeErrorAnalysis.cs | 60 ------------------- ...ngleTimeLoopTimeoutRuntimeErrorAnalysis.cs | 19 ++++-- Engine/Results/Analysis/ResultsAnalyzer.cs | 1 - .../Results/RuntimeErrorAnalysesTests.cs | 3 +- 4 files changed, 14 insertions(+), 69 deletions(-) delete mode 100644 Engine/Results/Analysis/Analyses/RuntimeErrorAnalyses/OperationCanceledRuntimeErrorAnalysis.cs diff --git a/Engine/Results/Analysis/Analyses/RuntimeErrorAnalyses/OperationCanceledRuntimeErrorAnalysis.cs b/Engine/Results/Analysis/Analyses/RuntimeErrorAnalyses/OperationCanceledRuntimeErrorAnalysis.cs deleted file mode 100644 index c3ec92ca0acc..000000000000 --- a/Engine/Results/Analysis/Analyses/RuntimeErrorAnalyses/OperationCanceledRuntimeErrorAnalysis.cs +++ /dev/null @@ -1,60 +0,0 @@ -/* - * QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals. - * Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * -*/ -using System.Collections.Generic; - -namespace QuantConnect.Lean.Engine.Results.Analysis.Analyses -{ - /// - /// Detects algorithms that were forcefully canceled, emitted by the when - /// the algorithm is asked to stop but its code is still running once the shutdown grace period - /// expires ("Operation was canceled"). - /// - public class OperationCanceledRuntimeErrorAnalysis : RuntimeErrorAnalysis - { - /// - /// Gets the description of the forced cancellation issue. - /// - public override string Issue { get; } = "The algorithm did not shut down within the grace period after a stop request and was forcefully canceled."; - - /// - /// Gets the severity weight for this analysis. The cancellation is a fatal error that - /// terminated the run, so it ranks above every non-fatal finding. - /// - public override int Weight { get; } = 100; - - /// - /// Gets the patterns identifying the forced cancellation error message. - /// - protected override string[][] ErrorMessagePatterns { get; } = - [ - ["Operation was canceled"], - ]; - - /// - /// Gets the suggested solutions to let the algorithm respond to stop requests promptly. - /// - protected override List Solutions(Language language) => - [ - "This error is raised when the algorithm is asked to stop, typically because the user stopped or deleted it, " + - "but its code keeps running past the shutdown grace period. If you stopped it on purpose, the error is expected and can be ignored.", - - "If you did not stop the algorithm, reduce the work done per event handler so it can respond to stop requests promptly: " + - "break up big loops, avoid long blocking operations, and avoid large history requests or model training on every data update.", - - "Reduce the universe size and the data resolution so each time loop processes less data and control returns to the engine sooner.", - ]; - } -} diff --git a/Engine/Results/Analysis/Analyses/RuntimeErrorAnalyses/SingleTimeLoopTimeoutRuntimeErrorAnalysis.cs b/Engine/Results/Analysis/Analyses/RuntimeErrorAnalyses/SingleTimeLoopTimeoutRuntimeErrorAnalysis.cs index ff365502f663..ee08aa2c49c4 100644 --- a/Engine/Results/Analysis/Analyses/RuntimeErrorAnalyses/SingleTimeLoopTimeoutRuntimeErrorAnalysis.cs +++ b/Engine/Results/Analysis/Analyses/RuntimeErrorAnalyses/SingleTimeLoopTimeoutRuntimeErrorAnalysis.cs @@ -19,16 +19,19 @@ namespace QuantConnect.Lean.Engine.Results.Analysis.Analyses { /// - /// Detects algorithms terminated because a single time loop exceeded the per-loop time limit, - /// emitted by the when the algorithm manager's time loop tracker trips - /// ("Algorithm took longer than N minutes on a single time loop"). + /// Detects algorithms terminated because their code kept running past an engine time limit, + /// emitted by the when a single time loop exceeds the per-loop limit + /// ("Algorithm took longer than N minutes on a single time loop") or when the algorithm is + /// asked to stop but its code is still running once the shutdown grace period expires + /// ("Operation was canceled"). /// public class SingleTimeLoopTimeoutRuntimeErrorAnalysis : RuntimeErrorAnalysis { /// - /// Gets the description of the single time loop timeout issue. + /// Gets the description of the time loop timeout issue. /// - public override string Issue { get; } = "A single time loop took longer than the maximum allowed, so the algorithm was terminated."; + public override string Issue { get; } = "The algorithm was terminated: a single time loop took longer than the maximum allowed, " + + "or its code kept running after a stop request."; /// /// Gets the severity weight for this analysis. A timeout is a fatal error that terminated @@ -37,11 +40,12 @@ public class SingleTimeLoopTimeoutRuntimeErrorAnalysis : RuntimeErrorAnalysis public override int Weight { get; } = 100; /// - /// Gets the patterns identifying the single time loop timeout error message. + /// Gets the patterns identifying the time loop timeout and forced cancellation error messages. /// protected override string[][] ErrorMessagePatterns { get; } = [ ["took longer than", "single time loop"], + ["Operation was canceled"], ]; /// @@ -64,6 +68,9 @@ protected override List Solutions(Language language) $"Run machine learning training through the `{FormatCode(nameof(QCAlgorithm.Train), language)}` method, which allocates extra time for it.", "Check for loops that may never exit: an infinite loop in an event handler surfaces as this error.", + + "If the algorithm was stopped or deleted on purpose, an \"Operation was canceled\" error only means " + + "its code was still running when the shutdown grace period expired, and can be ignored.", }; if (language == Language.Python) { diff --git a/Engine/Results/Analysis/ResultsAnalyzer.cs b/Engine/Results/Analysis/ResultsAnalyzer.cs index 5c2d9c198c19..4ed374972f18 100644 --- a/Engine/Results/Analysis/ResultsAnalyzer.cs +++ b/Engine/Results/Analysis/ResultsAnalyzer.cs @@ -150,7 +150,6 @@ protected void SetAnalysisData(Result result, IReadOnlyList logs) protected virtual IReadOnlyCollection GetAnalyses() => [ new SingleTimeLoopTimeoutRuntimeErrorAnalysis(), - new OperationCanceledRuntimeErrorAnalysis(), new MaximumRuntimeExceededRuntimeErrorAnalysis(), new PortfolioValueIsNotPositiveAnalysis(), new FlatEquityCurveAnalysis(), diff --git a/Tests/Engine/Results/RuntimeErrorAnalysesTests.cs b/Tests/Engine/Results/RuntimeErrorAnalysesTests.cs index f914d31922a4..6398537eb05f 100644 --- a/Tests/Engine/Results/RuntimeErrorAnalysesTests.cs +++ b/Tests/Engine/Results/RuntimeErrorAnalysesTests.cs @@ -43,13 +43,12 @@ public class RuntimeErrorAnalysesTests private static RuntimeErrorAnalysis[] CreateAnalyses() => [ new SingleTimeLoopTimeoutRuntimeErrorAnalysis(), - new OperationCanceledRuntimeErrorAnalysis(), new MaximumRuntimeExceededRuntimeErrorAnalysis(), ]; [TestCase(SingleTimeLoopError, typeof(SingleTimeLoopTimeoutRuntimeErrorAnalysis))] [TestCase(SingleTimeLoopWithAdditionalTimeError, typeof(SingleTimeLoopTimeoutRuntimeErrorAnalysis))] - [TestCase(OperationCanceledError, typeof(OperationCanceledRuntimeErrorAnalysis))] + [TestCase(OperationCanceledError, typeof(SingleTimeLoopTimeoutRuntimeErrorAnalysis))] [TestCase(MaximumRuntimeError, typeof(MaximumRuntimeExceededRuntimeErrorAnalysis))] [TestCase(MaximumRuntimeEngineError, typeof(MaximumRuntimeExceededRuntimeErrorAnalysis))] public void DetectsItsOwnRuntimeErrorFromState(string runtimeError, Type expectedAnalysisType) From 4c1e943c4a63101d8ff824a039173f183e8deb55 Mon Sep 17 00:00:00 2001 From: Jhonathan Abreu Date: Thu, 30 Jul 2026 18:19:27 -0400 Subject: [PATCH 17/33] Centralize the slow execution data points per second threshold --- .../Analysis/Analyses/AlgorithmSpeedAnalysis.cs | 11 +++-------- .../Analysis/Analyses/ExecutionSpeedAnalysis.cs | 13 ++++++++++--- 2 files changed, 13 insertions(+), 11 deletions(-) diff --git a/Engine/Results/Analysis/Analyses/AlgorithmSpeedAnalysis.cs b/Engine/Results/Analysis/Analyses/AlgorithmSpeedAnalysis.cs index 8c4f06aebcf2..6d8c276052b7 100644 --- a/Engine/Results/Analysis/Analyses/AlgorithmSpeedAnalysis.cs +++ b/Engine/Results/Analysis/Analyses/AlgorithmSpeedAnalysis.cs @@ -29,12 +29,6 @@ namespace QuantConnect.Lean.Engine.Results.Analysis.Analyses /// public class AlgorithmSpeedAnalysis : BaseResultsAnalysis { - /// - /// The data points per second under which execution is reported as slow, - /// matching the threshold used by on completed backtests. - /// - public const int SlowDataPointsPerSecond = 40_000; - /// /// The recent-to-initial throughput ratio under which throughput is reported as degrading. /// @@ -136,7 +130,8 @@ private static void AddSlowExecution(AlgorithmSpeedTracker speed, List= SlowDataPointsPerSecond || previous is null or >= SlowDataPointsPerSecond) + if (recent is null or >= ExecutionSpeedAnalysis.SlowDataPointsPerSecond || + previous is null or >= ExecutionSpeedAnalysis.SlowDataPointsPerSecond) { return; } @@ -162,7 +157,7 @@ private static void AddSlowExecution(AlgorithmSpeedTracker speed, List public class ExecutionSpeedAnalysis : BaseResultsAnalysis { + /// + /// The data points per second under which execution is reported as slow, from the platform + /// benchmarks. Also used by while the backtest runs. + /// + public const int SlowDataPointsPerSecond = 40_000; + /// /// Gets the description of the slow execution issue. /// - public override string Issue { get; } = "The algorithm ran below 40k data points per second."; + public override string Issue { get; } = $"The algorithm ran below {SlowDataPointsPerSecond / 1000}k data points per second."; /// /// Gets the severity weight for the execution speed analysis. @@ -48,10 +54,11 @@ public class ExecutionSpeedAnalysis : BaseResultsAnalysis /// Parses the backtest logs to determine execution speed and flags backtests that ran slowly. /// /// The full list of log lines produced by the backtest. - /// Analysis results flagging slow execution when below 40k data points per second and runtime is at least 10 seconds. + /// Analysis results flagging slow execution when below and runtime is at least 10 seconds. public IReadOnlyList Run(IReadOnlyList logs) { - var result = TryGetDataPointsPerSecond(logs, out var timeInSeconds, out var dataPointsPerSecond) && timeInSeconds >= 10 && dataPointsPerSecond < 40 + var result = TryGetDataPointsPerSecond(logs, out var timeInSeconds, out var dataPointsPerSecond) && + timeInSeconds >= 10 && dataPointsPerSecond < SlowDataPointsPerSecond / 1000 ? $"The algorithm is slowly executing at only {dataPointsPerSecond}k data points per second" : null; From 1c03c0acfd820c94a337fb8120d3614b286b30ae Mon Sep 17 00:00:00 2001 From: Jhonathan Abreu Date: Mon, 3 Aug 2026 16:11:18 -0400 Subject: [PATCH 18/33] Merge the maximum runtime exceeded analysis into the single time loop timeout analysis --- ...imumRuntimeExceededRuntimeErrorAnalysis.cs | 63 -------- .../RuntimeErrorAnalysis.cs | 91 ------------ ...ngleTimeLoopTimeoutRuntimeErrorAnalysis.cs | 82 ---------- ...ngleTimeLoopTimeoutRuntimeErrorAnalysis.cs | 140 ++++++++++++++++++ Engine/Results/Analysis/ResultsAnalyzer.cs | 1 - ...meLoopTimeoutRuntimeErrorAnalysisTests.cs} | 60 +++----- 6 files changed, 158 insertions(+), 279 deletions(-) delete mode 100644 Engine/Results/Analysis/Analyses/RuntimeErrorAnalyses/MaximumRuntimeExceededRuntimeErrorAnalysis.cs delete mode 100644 Engine/Results/Analysis/Analyses/RuntimeErrorAnalyses/RuntimeErrorAnalysis.cs delete mode 100644 Engine/Results/Analysis/Analyses/RuntimeErrorAnalyses/SingleTimeLoopTimeoutRuntimeErrorAnalysis.cs create mode 100644 Engine/Results/Analysis/Analyses/SingleTimeLoopTimeoutRuntimeErrorAnalysis.cs rename Tests/Engine/Results/{RuntimeErrorAnalysesTests.cs => SingleTimeLoopTimeoutRuntimeErrorAnalysisTests.cs} (68%) diff --git a/Engine/Results/Analysis/Analyses/RuntimeErrorAnalyses/MaximumRuntimeExceededRuntimeErrorAnalysis.cs b/Engine/Results/Analysis/Analyses/RuntimeErrorAnalyses/MaximumRuntimeExceededRuntimeErrorAnalysis.cs deleted file mode 100644 index 7fb2a29055c9..000000000000 --- a/Engine/Results/Analysis/Analyses/RuntimeErrorAnalyses/MaximumRuntimeExceededRuntimeErrorAnalysis.cs +++ /dev/null @@ -1,63 +0,0 @@ -/* - * QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals. - * Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * -*/ -using System.Collections.Generic; - -namespace QuantConnect.Lean.Engine.Results.Analysis.Analyses -{ - /// - /// Detects algorithms terminated because the whole run outlived the maximum allowed wall-clock - /// time, emitted by the ("Execution Security Error: Operation timed out - - /// N minutes max") or by the engine when the isolator reports an incomplete run ("Failed to - /// complete algorithm within N seconds"). - /// - public class MaximumRuntimeExceededRuntimeErrorAnalysis : RuntimeErrorAnalysis - { - /// - /// Gets the description of the maximum runtime exceeded issue. - /// - public override string Issue { get; } = "The algorithm exceeded the maximum allowed total runtime and was terminated."; - - /// - /// Gets the severity weight for this analysis. The timeout is a fatal error that terminated - /// the run, so it ranks above every non-fatal finding. - /// - public override int Weight { get; } = 100; - - /// - /// Gets the patterns identifying the maximum runtime exceeded error message. - /// - protected override string[][] ErrorMessagePatterns { get; } = - [ - ["Operation timed out", "minutes max"], - ["Failed to complete algorithm within"], - ]; - - /// - /// Gets the suggested solutions to complete the run within the maximum allowed runtime. - /// - protected override List Solutions(Language language) => - [ - "Reduce the universe size and the data resolution to lower the total amount of data processed. " + - "For options and futures, tighten the contract filters: narrower strike and expiration ranges, " + - "and fewer contracts per underlying.", - - "Review the algorithm code for inefficiencies: avoid recomputing values from scratch on every data update, " + - "and avoid history requests whose range grows as the backtest progresses.", - - "Check for infinite or recursive loops that keep the algorithm running without making progress.", - ]; - } -} diff --git a/Engine/Results/Analysis/Analyses/RuntimeErrorAnalyses/RuntimeErrorAnalysis.cs b/Engine/Results/Analysis/Analyses/RuntimeErrorAnalyses/RuntimeErrorAnalysis.cs deleted file mode 100644 index 8996d8376be8..000000000000 --- a/Engine/Results/Analysis/Analyses/RuntimeErrorAnalyses/RuntimeErrorAnalysis.cs +++ /dev/null @@ -1,91 +0,0 @@ -/* - * QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals. - * Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * -*/ -using System; -using System.Collections.Generic; -using System.Linq; - -namespace QuantConnect.Lean.Engine.Results.Analysis.Analyses -{ - /// - /// Abstract base class for analyses that detect a specific runtime error that terminated the - /// algorithm by inspecting the error message for known text fragments. The runtime error is - /// read from the result state, falling back to the "Runtime Error:" log line for results - /// that carry no state. - /// - public abstract class RuntimeErrorAnalysis : BaseResultsAnalysis - { - /// - /// Gets the patterns identifying the runtime error. Each pattern is a set of text fragments - /// that must all be present in the error message (case-insensitive); the error matches - /// when any pattern does. - /// - protected abstract string[][] ErrorMessagePatterns { get; } - - /// - /// Gets the suggested solutions for the detected runtime error. - /// - protected abstract List Solutions(Language language); - - /// - /// Runs the runtime error analysis against the provided backtest parameters. - /// - public override IReadOnlyList Run(ResultsAnalysisRunParameters parameters) - => Run(parameters.Result?.State, parameters.Logs, parameters.Language); - - /// - /// Runs the runtime error analysis against the algorithm state and logs. - /// - /// The algorithm state of the result, holding the runtime error message if any. - /// The full list of log lines produced by the backtest. - /// The programming language the algorithm is written in. - /// A single response with the matched error message and solutions, or without them when the error is not found. - public IReadOnlyList Run(IDictionary state, IReadOnlyList logs, Language language) - { - var sample = GetRuntimeErrorMessages(state, logs).FirstOrDefault(Matches); - return SingleResponse(sample, sample != null ? Solutions(language) : []); - } - - /// - /// Determines whether the given runtime error message matches any of the - /// . - /// - private bool Matches(string message) - { - return ErrorMessagePatterns.Any( - fragments => fragments.All(fragment => message.Contains(fragment, StringComparison.InvariantCultureIgnoreCase))); - } - - /// - /// Gets the candidate runtime error messages: the result state's runtime error entry when - /// present, plus any "Runtime Error:" lines from the logs. - /// - private static IEnumerable GetRuntimeErrorMessages(IDictionary state, IReadOnlyList logs) - { - if (state != null && state.TryGetValue("RuntimeError", out var error) && !string.IsNullOrEmpty(error)) - { - yield return error; - } - - foreach (var log in logs ?? []) - { - if (log.Contains("Runtime Error", StringComparison.InvariantCultureIgnoreCase)) - { - yield return log; - } - } - } - } -} diff --git a/Engine/Results/Analysis/Analyses/RuntimeErrorAnalyses/SingleTimeLoopTimeoutRuntimeErrorAnalysis.cs b/Engine/Results/Analysis/Analyses/RuntimeErrorAnalyses/SingleTimeLoopTimeoutRuntimeErrorAnalysis.cs deleted file mode 100644 index ee08aa2c49c4..000000000000 --- a/Engine/Results/Analysis/Analyses/RuntimeErrorAnalyses/SingleTimeLoopTimeoutRuntimeErrorAnalysis.cs +++ /dev/null @@ -1,82 +0,0 @@ -/* - * QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals. - * Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * -*/ -using QuantConnect.Algorithm; -using System.Collections.Generic; - -namespace QuantConnect.Lean.Engine.Results.Analysis.Analyses -{ - /// - /// Detects algorithms terminated because their code kept running past an engine time limit, - /// emitted by the when a single time loop exceeds the per-loop limit - /// ("Algorithm took longer than N minutes on a single time loop") or when the algorithm is - /// asked to stop but its code is still running once the shutdown grace period expires - /// ("Operation was canceled"). - /// - public class SingleTimeLoopTimeoutRuntimeErrorAnalysis : RuntimeErrorAnalysis - { - /// - /// Gets the description of the time loop timeout issue. - /// - public override string Issue { get; } = "The algorithm was terminated: a single time loop took longer than the maximum allowed, " + - "or its code kept running after a stop request."; - - /// - /// Gets the severity weight for this analysis. A timeout is a fatal error that terminated - /// the run, so it ranks above every non-fatal finding. - /// - public override int Weight { get; } = 100; - - /// - /// Gets the patterns identifying the time loop timeout and forced cancellation error messages. - /// - protected override string[][] ErrorMessagePatterns { get; } = - [ - ["took longer than", "single time loop"], - ["Operation was canceled"], - ]; - - /// - /// Gets the suggested solutions to keep each time loop within the time limit. - /// - protected override List Solutions(Language language) - { - var solutions = new List - { - $"Avoid heavy work in event handlers (`{FormatCode(nameof(QCAlgorithm.OnData), language)}`, scheduled events, universe selection): " + - "instead of recomputing values on every update, update them incrementally with rolling windows, consolidators or indicators.", - - "Reduce the universe size: select only the securities the strategy trades, " + - $"and narrow option and future chains with the `{FormatCode("SetFilter", language)}` strike and expiration filters.", - - "Reduce the number of subscribed securities or the data resolution.", - - "Avoid large history requests inside event handlers; request only the period needed or maintain the data incrementally.", - - $"Run machine learning training through the `{FormatCode(nameof(QCAlgorithm.Train), language)}` method, which allocates extra time for it.", - - "Check for loops that may never exit: an infinite loop in an event handler surfaces as this error.", - - "If the algorithm was stopped or deleted on purpose, an \"Operation was canceled\" error only means " + - "its code was still running when the shutdown grace period expired, and can be ignored.", - }; - if (language == Language.Python) - { - solutions.Add("Vectorize heavy numeric loops with numpy or pandas."); - } - return solutions; - } - } -} diff --git a/Engine/Results/Analysis/Analyses/SingleTimeLoopTimeoutRuntimeErrorAnalysis.cs b/Engine/Results/Analysis/Analyses/SingleTimeLoopTimeoutRuntimeErrorAnalysis.cs new file mode 100644 index 000000000000..c8ed7c564c84 --- /dev/null +++ b/Engine/Results/Analysis/Analyses/SingleTimeLoopTimeoutRuntimeErrorAnalysis.cs @@ -0,0 +1,140 @@ +/* + * QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals. + * Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * +*/ +using QuantConnect.Algorithm; +using System; +using System.Collections.Generic; +using System.Linq; + +namespace QuantConnect.Lean.Engine.Results.Analysis.Analyses +{ + /// + /// Detects algorithms terminated by an time limit, inspecting the error + /// message for known text fragments. The runtime error is read from the result state, falling + /// back to the "Runtime Error:" log line for results that carry no state. It covers a single + /// time loop exceeding the per-loop limit ("Algorithm took longer than N minutes on a single + /// time loop"), the whole run outliving the maximum allowed wall-clock time ("Execution + /// Security Error: Operation timed out - N minutes max", "Failed to complete algorithm within + /// N seconds"), and code still running once the shutdown grace period expires after a stop + /// request ("Operation was canceled"). + /// + public class SingleTimeLoopTimeoutRuntimeErrorAnalysis : BaseResultsAnalysis + { + /// + /// The patterns identifying the timeout error messages. Each pattern is a set of text + /// fragments that must all be present in the error message (case-insensitive); the error + /// matches when any pattern does. + /// + private static readonly string[][] ErrorMessagePatterns = + [ + ["took longer than", "single time loop"], + ["Operation timed out", "minutes max"], + ["Failed to complete algorithm within"], + ["Operation was canceled"], + ]; + + /// + /// Gets the description of the timeout issue. + /// + public override string Issue { get; } = "The algorithm was terminated: a single time loop took longer than the maximum allowed, " + + "the whole run exceeded the maximum runtime, or its code kept running after a stop request."; + + /// + /// Gets the severity weight for this analysis. A timeout is a fatal error that terminated + /// the run, so it ranks above every non-fatal finding. + /// + public override int Weight { get; } = 100; + + /// + /// Runs the runtime error analysis against the provided backtest parameters. + /// + public override IReadOnlyList Run(ResultsAnalysisRunParameters parameters) + => Run(parameters.Result?.State, parameters.Logs, parameters.Language); + + /// + /// Runs the runtime error analysis against the algorithm state and logs. + /// + /// The algorithm state of the result, holding the runtime error message if any. + /// The full list of log lines produced by the backtest. + /// The programming language the algorithm is written in. + /// A single response with the matched error message and solutions, or without them when the error is not found. + public IReadOnlyList Run(IDictionary state, IReadOnlyList logs, Language language) + { + var sample = GetRuntimeErrorMessages(state, logs).FirstOrDefault(Matches); + return SingleResponse(sample, sample != null ? Solutions(language) : []); + } + + /// + /// Determines whether the given runtime error message matches any of the + /// . + /// + private static bool Matches(string message) + { + return ErrorMessagePatterns.Any( + fragments => fragments.All(fragment => message.Contains(fragment, StringComparison.InvariantCultureIgnoreCase))); + } + + /// + /// Gets the candidate runtime error messages: the result state's runtime error entry when + /// present, plus any "Runtime Error:" lines from the logs. + /// + private static IEnumerable GetRuntimeErrorMessages(IDictionary state, IReadOnlyList logs) + { + if (state != null && state.TryGetValue("RuntimeError", out var error) && !string.IsNullOrEmpty(error)) + { + yield return error; + } + + foreach (var log in logs ?? []) + { + if (log.Contains("Runtime Error", StringComparison.InvariantCultureIgnoreCase)) + { + yield return log; + } + } + } + + /// + /// Gets the suggested solutions to keep the algorithm within the time limits. + /// + private static List Solutions(Language language) + { + var solutions = new List + { + $"Avoid heavy work in event handlers (`{FormatCode(nameof(QCAlgorithm.OnData), language)}`, scheduled events, universe selection): " + + "instead of recomputing values on every update, update them incrementally with rolling windows, consolidators or indicators.", + + "Reduce the universe size: select only the securities the strategy trades, " + + $"and narrow option and future chains with the `{FormatCode("SetFilter", language)}` strike and expiration filters.", + + "Reduce the number of subscribed securities or the data resolution.", + + "Avoid large history requests inside event handlers; request only the period needed or maintain the data incrementally.", + + $"Run machine learning training through the `{FormatCode(nameof(QCAlgorithm.Train), language)}` method, which allocates extra time for it.", + + "Check for infinite or recursive loops that may never exit: they keep the algorithm running and surface as this error.", + + "If the algorithm was stopped or deleted on purpose, an \"Operation was canceled\" error only means " + + "its code was still running when the shutdown grace period expired, and can be ignored.", + }; + if (language == Language.Python) + { + solutions.Add("Vectorize heavy numeric loops with numpy or pandas."); + } + return solutions; + } + } +} diff --git a/Engine/Results/Analysis/ResultsAnalyzer.cs b/Engine/Results/Analysis/ResultsAnalyzer.cs index 4ed374972f18..a20874c48947 100644 --- a/Engine/Results/Analysis/ResultsAnalyzer.cs +++ b/Engine/Results/Analysis/ResultsAnalyzer.cs @@ -150,7 +150,6 @@ protected void SetAnalysisData(Result result, IReadOnlyList logs) protected virtual IReadOnlyCollection GetAnalyses() => [ new SingleTimeLoopTimeoutRuntimeErrorAnalysis(), - new MaximumRuntimeExceededRuntimeErrorAnalysis(), new PortfolioValueIsNotPositiveAnalysis(), new FlatEquityCurveAnalysis(), new InsufficientBuyingPowerOrderResponseErrorAnalysis(), diff --git a/Tests/Engine/Results/RuntimeErrorAnalysesTests.cs b/Tests/Engine/Results/SingleTimeLoopTimeoutRuntimeErrorAnalysisTests.cs similarity index 68% rename from Tests/Engine/Results/RuntimeErrorAnalysesTests.cs rename to Tests/Engine/Results/SingleTimeLoopTimeoutRuntimeErrorAnalysisTests.cs index 6398537eb05f..fec999eb67fb 100644 --- a/Tests/Engine/Results/RuntimeErrorAnalysesTests.cs +++ b/Tests/Engine/Results/SingleTimeLoopTimeoutRuntimeErrorAnalysisTests.cs @@ -14,7 +14,6 @@ * */ -using System; using System.Collections.Generic; using System.Linq; using NUnit.Framework; @@ -23,7 +22,7 @@ namespace QuantConnect.Tests.Engine.Results { [TestFixture] - public class RuntimeErrorAnalysesTests + public class SingleTimeLoopTimeoutRuntimeErrorAnalysisTests { private const string SingleTimeLoopError = "20240101 15:30:00.000 Runtime Error: Algorithm took longer than 10 minutes on a single time loop. CurrentTimeStepElapsed: 10.0 minutes"; @@ -40,37 +39,20 @@ public class RuntimeErrorAnalysesTests private const string MaximumRuntimeEngineError = "Runtime Error: Failed to complete algorithm within 86400 seconds. Please make it run faster."; - private static RuntimeErrorAnalysis[] CreateAnalyses() => - [ - new SingleTimeLoopTimeoutRuntimeErrorAnalysis(), - new MaximumRuntimeExceededRuntimeErrorAnalysis(), - ]; - - [TestCase(SingleTimeLoopError, typeof(SingleTimeLoopTimeoutRuntimeErrorAnalysis))] - [TestCase(SingleTimeLoopWithAdditionalTimeError, typeof(SingleTimeLoopTimeoutRuntimeErrorAnalysis))] - [TestCase(OperationCanceledError, typeof(SingleTimeLoopTimeoutRuntimeErrorAnalysis))] - [TestCase(MaximumRuntimeError, typeof(MaximumRuntimeExceededRuntimeErrorAnalysis))] - [TestCase(MaximumRuntimeEngineError, typeof(MaximumRuntimeExceededRuntimeErrorAnalysis))] - public void DetectsItsOwnRuntimeErrorFromState(string runtimeError, Type expectedAnalysisType) + [TestCase(SingleTimeLoopError)] + [TestCase(SingleTimeLoopWithAdditionalTimeError)] + [TestCase(OperationCanceledError)] + [TestCase(MaximumRuntimeError)] + [TestCase(MaximumRuntimeEngineError)] + public void DetectsEachTimeoutRuntimeErrorFromState(string runtimeError) { var state = new Dictionary { ["RuntimeError"] = runtimeError }; - foreach (var analysis in CreateAnalyses()) - { - var finding = analysis.Run(state, [], Language.CSharp).Single(); - - if (analysis.GetType() == expectedAnalysisType) - { - Assert.AreEqual(expectedAnalysisType.Name, finding.Name); - Assert.AreEqual(runtimeError, finding.Sample); - Assert.IsNotEmpty(finding.Solutions); - } - else - { - Assert.IsNull(finding.Sample, $"{analysis.GetType().Name} should not flag \"{runtimeError}\""); - Assert.IsEmpty(finding.Solutions); - } - } + var finding = new SingleTimeLoopTimeoutRuntimeErrorAnalysis().Run(state, [], Language.CSharp).Single(); + + Assert.AreEqual(nameof(SingleTimeLoopTimeoutRuntimeErrorAnalysis), finding.Name); + Assert.AreEqual(runtimeError, finding.Sample); + Assert.IsNotEmpty(finding.Solutions); } [Test] @@ -98,13 +80,10 @@ public void IgnoresLogLinesThatAreNotRuntimeErrors() "20240101 15:29:00.000 The download operation timed out after 5 minutes max wait", }; - foreach (var analysis in CreateAnalyses()) - { - var finding = analysis.Run(new Dictionary(), logs, Language.CSharp).Single(); + var finding = new SingleTimeLoopTimeoutRuntimeErrorAnalysis().Run(new Dictionary(), logs, Language.CSharp).Single(); - Assert.IsNull(finding.Sample); - Assert.IsEmpty(finding.Solutions); - } + Assert.IsNull(finding.Sample); + Assert.IsEmpty(finding.Solutions); } [TestCase("Runtime Error: System.DivideByZeroException: Attempted to divide by zero.")] @@ -113,13 +92,10 @@ public void DoesNotFlagOtherRuntimeErrorsOrCleanRuns(string runtimeError) { var state = new Dictionary { ["RuntimeError"] = runtimeError }; - foreach (var analysis in CreateAnalyses()) - { - var finding = analysis.Run(state, [], Language.CSharp).Single(); + var finding = new SingleTimeLoopTimeoutRuntimeErrorAnalysis().Run(state, [], Language.CSharp).Single(); - Assert.IsNull(finding.Sample); - Assert.IsEmpty(finding.Solutions); - } + Assert.IsNull(finding.Sample); + Assert.IsEmpty(finding.Solutions); } [TestCase(Language.CSharp, "Train", "OnData")] From b631c98c681a3d2cd8f7d0eda1caf6a1cb7bf997 Mon Sep 17 00:00:00 2001 From: Jhonathan Abreu Date: Mon, 3 Aug 2026 16:29:16 -0400 Subject: [PATCH 19/33] Test that the final analysis set includes the speed and runtime error analyses --- Tests/Engine/Results/ResultsAnalyzerTests.cs | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/Tests/Engine/Results/ResultsAnalyzerTests.cs b/Tests/Engine/Results/ResultsAnalyzerTests.cs index 87ad04a1922c..2ed61ded6afd 100644 --- a/Tests/Engine/Results/ResultsAnalyzerTests.cs +++ b/Tests/Engine/Results/ResultsAnalyzerTests.cs @@ -154,11 +154,12 @@ public void SpeedTrackerIsPassedToTheAnalyses() } [Test] - public void DefaultAnalysisSetIncludesTheAlgorithmSpeedAnalysis() + public void DefaultAnalysisSetIncludesTheAlgorithmSpeedAndRuntimeErrorAnalyses() { var analyses = new DefaultSetResultsAnalyzer().DefaultAnalyses; Assert.IsTrue(analyses.Any(analysis => analysis is AlgorithmSpeedAnalysis)); + Assert.IsTrue(analyses.Any(analysis => analysis is SingleTimeLoopTimeoutRuntimeErrorAnalysis)); } private sealed class DefaultSetResultsAnalyzer : ResultsAnalyzer From 0532d449d1c27259c9790beb4484dd918dcf194b Mon Sep 17 00:00:00 2001 From: Jhonathan Abreu Date: Mon, 3 Aug 2026 17:08:52 -0400 Subject: [PATCH 20/33] Decouple the in-run results analyzer from the backtesting result handler The analyzer now pulls the backtest data it needs through the new IInRunAnalysisDataProvider interface, implemented by the result handler, keeping its incremental stream consumption, snapshot assembly, and statistics-withholding rules private. --- .../Analysis/IInRunAnalysisDataProvider.cs | 60 +++++ .../Results/Analysis/InRunResultsAnalyzer.cs | 122 +++++++--- Engine/Results/Analysis/ResultsAnalyzer.cs | 2 +- Engine/Results/BacktestingResultHandler.cs | 128 +++++------ .../Results/InRunResultsAnalyzerTests.cs | 213 ++++++++++++++---- 5 files changed, 382 insertions(+), 143 deletions(-) create mode 100644 Engine/Results/Analysis/IInRunAnalysisDataProvider.cs diff --git a/Engine/Results/Analysis/IInRunAnalysisDataProvider.cs b/Engine/Results/Analysis/IInRunAnalysisDataProvider.cs new file mode 100644 index 000000000000..0e2189bf0150 --- /dev/null +++ b/Engine/Results/Analysis/IInRunAnalysisDataProvider.cs @@ -0,0 +1,60 @@ +/* + * QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals. + * Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * +*/ +using QuantConnect.Orders; +using System.Collections.Generic; + +namespace QuantConnect.Lean.Engine.Results.Analysis +{ + /// + /// Provides the access to the data of the running backtest. + /// Implemented by the result handler, which owns the data and its synchronization, while the + /// analyzer decides what to read and how much, keeping its incremental consumption state private. + /// + public interface IInRunAnalysisDataProvider + { + /// + /// Gets the orders placed so far. + /// + IDictionary GetOrders(); + + /// + /// Gets the order events produced from the given position in the order event stream. + /// + List GetOrderEvents(int fromPosition); + + /// + /// Gets the log lines produced from the given position in the log stream. + /// + IReadOnlyList GetLogs(int fromPosition); + + /// + /// Gets clones of the requested charts, safe to read without further synchronization. + /// + IDictionary GetChartSnapshots(IReadOnlyList chartNames); + + /// + /// Whether the strategy equity chart has samples yet. Until the first sample exists, + /// the generated statistics are all-zero defaults. + /// + bool HasEquitySamples(); + + /// + /// Takes a sample of the engine speed counters, or null when the counters should + /// not be sampled, like while the algorithm warms up. + /// + AlgorithmSpeedSample? TakeSpeedSample(); + } +} diff --git a/Engine/Results/Analysis/InRunResultsAnalyzer.cs b/Engine/Results/Analysis/InRunResultsAnalyzer.cs index 515db26948d7..a8ebd7d17cbd 100644 --- a/Engine/Results/Analysis/InRunResultsAnalyzer.cs +++ b/Engine/Results/Analysis/InRunResultsAnalyzer.cs @@ -15,6 +15,8 @@ */ using QuantConnect.Algorithm; using QuantConnect.Lean.Engine.Results.Analysis.Analyses; +using QuantConnect.Packets; +using QuantConnect.Statistics; using System; using System.Collections.Generic; using System.Linq; @@ -23,7 +25,8 @@ namespace QuantConnect.Lean.Engine.Results.Analysis { /// /// Runs a reduced suite of backtest diagnostic tests periodically while the backtest is still running, - /// against a snapshot of the current intermediate results. + /// against a snapshot of the current intermediate results pulled from the + /// . /// public class InRunResultsAnalyzer : ResultsAnalyzer { @@ -41,55 +44,116 @@ public class InRunResultsAnalyzer : ResultsAnalyzer }; private readonly Dictionary _findings = new(); + private readonly QCAlgorithm _algorithm; + private readonly IInRunAnalysisDataProvider _dataProvider; /// - /// The equity and benchmark curves are not built for in-run analysis: - /// none of the in-run analyses read them, and building them would issue - /// a benchmark history request on every run. + /// The number of order events already consumed by previous runs, from which the next run + /// resumes reading the order event stream. /// - protected override bool RequiresEquityCurves => false; + private int _orderEventsPosition; /// - /// The names of the charts the in-run analyses read. Only these need to be - /// cloned into the result snapshot passed to - /// . + /// The number of log entries already consumed by previous runs, from which the next run + /// resumes reading the log stream. /// - public static IReadOnlyList RequiredCharts { get; } = [BaseResultsHandler.PortfolioMarginKey]; + private int _logsPosition; /// - /// The number of order events already consumed by previous runs. The order events - /// in the result passed to - /// are expected to start at this position. + /// The equity and benchmark curves are not built for in-run analysis: + /// none of the in-run analyses read them, and building them would issue + /// a benchmark history request on every run. /// - public int OrderEventsPosition { get; private set; } + protected override bool RequiresEquityCurves => false; /// - /// The number of log entries already consumed by previous runs. The logs passed to - /// are expected to start - /// at this position. + /// The names of the charts the in-run analyses read. Only these are requested + /// from the data provider on each run. /// - public int LogsPosition { get; private set; } + public static IReadOnlyList RequiredCharts { get; } = [BaseResultsHandler.PortfolioMarginKey]; /// /// Initializes a new instance of the class. /// The instance is expected to be kept alive for the duration of the backtest, - /// receiving fresh data on each call. + /// pulling fresh data from on each + /// call. /// /// The algorithm instance used for history requests and settings. /// The programming language the algorithm is written in. - public InRunResultsAnalyzer(QCAlgorithm algorithm, Language language) + /// Provides access to the data of the running backtest. + public InRunResultsAnalyzer(QCAlgorithm algorithm, Language language, IInRunAnalysisDataProvider dataProvider) : base(null, algorithm, language, null, new AlgorithmSpeedTracker()) { + _algorithm = algorithm; + _dataProvider = dataProvider; + } + + /// + /// Runs the analyses against the current backtest state pulled from the data provider: + /// a snapshot of the orders and required charts, plus only the order events and log lines + /// produced since the previous run. The returned findings are the merge of this run's + /// findings into the ones accumulated by previous runs: findings from analyses scanning + /// the order event and log streams are accumulated (first sample kept, counts totaled), + /// while findings from state-based analyses are replaced on every run. + /// + /// The current total algorithm performance, for analyses that read + /// portfolio statistics. Withheld from the analyses until the first equity sample exists, since + /// the statistics are all-zero defaults before that. + /// Wall-clock seconds allowed for the full chain before early exit. + /// The default is small because the analysis runs on the result handler thread, delaying message + /// processing while it runs. + /// Maximum number of failing analyses to return. + /// The accumulated findings, ranked by analysis weight. + public IReadOnlyList Run(AlgorithmPerformance totalPerformance, int timeLimitSeconds = 1, + int maxFailedAnalyses = 10) + { + // The analyses read the charts without synchronizing with the result handler, so they get clones + var charts = _dataProvider.GetChartSnapshots(RequiredCharts); + + // Equity is not sampled while the algorithm warms up, so until the first sample exists + // the generated statistics are all-zero defaults that would flag a false non-positive + // portfolio value finding. Withhold them so the analyses reading them skip instead + if (_algorithm?.IsWarmingUp == true || !_dataProvider.HasEquitySamples()) + { + totalPerformance = null; + } + + var snapshot = new BacktestResult(new BacktestResultParameters( + charts, + _dataProvider.GetOrders(), + _algorithm?.Transactions.TransactionRecord ?? new Dictionary(), + new Dictionary(), + new Dictionary(), + new Dictionary(), + _dataProvider.GetOrderEvents(_orderEventsPosition), + totalPerformance)); + var logs = _dataProvider.GetLogs(_logsPosition); + + // The analyses run during warm-up too (the speed sample is null then), so conditions like + // orders submitted while the algorithm warms up surface without waiting for warm-up to end + return Run(snapshot, logs, _dataProvider.TakeSpeedSample(), timeLimitSeconds, maxFailedAnalyses); + } + + /// + /// Completes the speed metrics with one final sample so they cover the backtest through + /// its end, and returns the tracker for the final analysis to reuse. The tracker is left + /// untouched when no sample can be taken, like when the algorithm never left warm-up. + /// + public AlgorithmSpeedTracker CompleteSpeedTracking() + { + var speedSample = _dataProvider.TakeSpeedSample(); + if (speedSample.HasValue) + { + SpeedTracker.AddSample(speedSample.Value); + } + return SpeedTracker; } /// /// Runs the analyses incrementally: and are - /// expected to contain only the order events and log lines produced since the previous run - /// (per and ), and the returned - /// findings are the merge of this run's findings into the ones accumulated by previous runs. - /// Findings from analyses scanning the order event and log streams are accumulated - /// (first sample kept, counts totaled), while findings from state-based analyses are - /// replaced on every run. + /// expected to contain only the order events and log lines produced since the previous run, + /// and the returned findings are the merge of this run's findings into the ones accumulated + /// by previous runs. /// /// A snapshot of the current intermediate backtest result, holding only new order events. /// The log lines produced since the previous run. @@ -98,8 +162,8 @@ public InRunResultsAnalyzer(QCAlgorithm algorithm, Language language) /// Wall-clock seconds allowed for the full chain before early exit. /// Maximum number of failing analyses to return. /// The accumulated findings, ranked by analysis weight. - public IReadOnlyList Run(Result result, IReadOnlyList logs, AlgorithmSpeedSample? speedSample = null, - int timeLimitSeconds = 1, int maxFailedAnalyses = 10) + private IReadOnlyList Run(Result result, IReadOnlyList logs, AlgorithmSpeedSample? speedSample, + int timeLimitSeconds, int maxFailedAnalyses) { SetAnalysisData(result, logs); if (speedSample.HasValue) @@ -112,8 +176,8 @@ public InRunResultsAnalyzer(QCAlgorithm algorithm, Language language) // didn't get to run miss this delta until the final analysis re-scans the complete streams. // Stress tests show runs complete in a fraction of the time limit, but if its trace message // starts showing up in logs, revisit this (e.g. track per-analysis positions). - OrderEventsPosition += result.OrderEvents?.Count ?? 0; - LogsPosition += logs?.Count ?? 0; + _orderEventsPosition += result.OrderEvents?.Count ?? 0; + _logsPosition += logs?.Count ?? 0; // State-based analyses are recomputed from scratch each run: remove their previous // findings so they are replaced, or dropped if they no longer fail. If a time-limit diff --git a/Engine/Results/Analysis/ResultsAnalyzer.cs b/Engine/Results/Analysis/ResultsAnalyzer.cs index a20874c48947..aa7e377631a1 100644 --- a/Engine/Results/Analysis/ResultsAnalyzer.cs +++ b/Engine/Results/Analysis/ResultsAnalyzer.cs @@ -56,7 +56,7 @@ public class ResultsAnalyzer /// it a sample on each run; the final analyzer receives the same tracker so the speed /// analysis also runs against the full-run metrics. Null when speed is not tracked. /// - public AlgorithmSpeedTracker SpeedTracker { get; } + protected AlgorithmSpeedTracker SpeedTracker { get; } /// /// Initializes a new instance of the class. diff --git a/Engine/Results/BacktestingResultHandler.cs b/Engine/Results/BacktestingResultHandler.cs index e0bae8a58c7e..7dfe45d44368 100644 --- a/Engine/Results/BacktestingResultHandler.cs +++ b/Engine/Results/BacktestingResultHandler.cs @@ -37,7 +37,7 @@ namespace QuantConnect.Lean.Engine.Results /// /// Backtesting result handler passes messages back from the Lean to the User. /// - public class BacktestingResultHandler : BaseResultsHandler, IResultHandler + public class BacktestingResultHandler : BaseResultsHandler, IResultHandler, IInRunAnalysisDataProvider { private const double Samples = 4000; private const double MinimumSamplePeriod = 4; @@ -428,14 +428,8 @@ protected void SendFinalResult() logs = LogStore.Select(x => x.Message).ToList(); } // The final analysis reuses the speed metrics accumulated by the in-run analyzer, - // adding one last sample so they cover the backtest through its end - var speedTracker = _inRunResultsAnalyzer?.SpeedTracker; - var speedSample = TakeAlgorithmSpeedSample(); - if (speedTracker != null && speedSample.HasValue) - { - speedTracker.AddSample(speedSample.Value); - } - + // completed with one last sample so they cover the backtest through its end + var speedTracker = _inRunResultsAnalyzer?.CompleteSpeedTracking(); var analyzer = new ResultsAnalyzer(result.Results, AlgorithmInstance, _job.Language, logs, speedTracker); try { @@ -463,9 +457,10 @@ protected void SendFinalResult() } /// - /// Runs the in-run results analyzer against a snapshot of the current intermediate backtest state. - /// Invoked periodically while the backtest is still running, unlike the full analysis performed - /// by when the backtest ends. + /// Runs the in-run results analyzer against the current intermediate backtest state, + /// accessed through the implementation. + /// Invoked periodically while the backtest is still running, unlike the full analysis + /// performed by when the backtest ends. /// /// The current total algorithm performance, for analyses that read portfolio statistics /// The failed analyses with solutions, or null if the analysis could not run @@ -478,66 +473,69 @@ protected void SendFinalResult() return null; } - // The analyses read the charts without holding ChartLock, so hand them clones, - // but only of the charts they read - var charts = new Dictionary(); - bool hasEquitySamples; - lock (ChartLock) - { - foreach (var chartName in InRunResultsAnalyzer.RequiredCharts) - { - if (Charts.TryGetValue(chartName, out var chart)) - { - charts[chartName] = chart.Clone(); - } - } - - hasEquitySamples = Charts.TryGetValue(StrategyEquityKey, out var equityChart) && - equityChart.Series.TryGetValue(EquityKey, out var equitySeries) && - equitySeries.Values.Count > 0; - } + _inRunResultsAnalyzer ??= new InRunResultsAnalyzer(AlgorithmInstance, _job.Language, this); + return _inRunResultsAnalyzer.Run(totalPerformance); + } + catch (Exception ex) + { + Log.Error(ex, "Error running in-run backtest analysis"); + return null; + } + } - // Equity is not sampled while the algorithm warms up, so until the first sample exists - // the generated statistics are all-zero defaults that would flag a false non-positive - // portfolio value finding. Withhold them so the analyses reading them skip instead - if (Algorithm.IsWarmingUp || !hasEquitySamples) - { - totalPerformance = null; - } + #region IInRunAnalysisDataProvider implementation - _inRunResultsAnalyzer ??= new InRunResultsAnalyzer(AlgorithmInstance, _job.Language); + /// + /// Gets the orders placed so far. + /// + IDictionary IInRunAnalysisDataProvider.GetOrders() => TransactionHandler.Orders.ToDictionary(); - // The analyses themselves do run during warm-up (the speed sample is null then), so - // conditions like orders submitted while warming up surface without waiting for warm-up to end - var speedSample = TakeAlgorithmSpeedSample(); + /// + /// Gets the order events produced from the given position in the order event stream. + /// + List IInRunAnalysisDataProvider.GetOrderEvents(int fromPosition) + => TransactionHandler.OrderEvents.Skip(fromPosition).ToList(); - // Only the order events and logs produced since the previous run are analyzed, - // the analyzer accumulates findings across runs - var orderEvents = TransactionHandler.OrderEvents.Skip(_inRunResultsAnalyzer.OrderEventsPosition).ToList(); + /// + /// Gets the log lines produced from the given position in the log stream. + /// + IReadOnlyList IInRunAnalysisDataProvider.GetLogs(int fromPosition) + { + lock (LogStore) + { + return LogStore.Skip(fromPosition).Select(x => x.Message).ToList(); + } + } - List logs; - lock (LogStore) + /// + /// Gets clones of the requested charts, safe for the analyses to read without holding the chart lock. + /// + IDictionary IInRunAnalysisDataProvider.GetChartSnapshots(IReadOnlyList chartNames) + { + var charts = new Dictionary(); + lock (ChartLock) + { + foreach (var chartName in chartNames) { - logs = LogStore.Skip(_inRunResultsAnalyzer.LogsPosition).Select(x => x.Message).ToList(); + if (Charts.TryGetValue(chartName, out var chart)) + { + charts[chartName] = chart.Clone(); + } } - - var snapshot = new BacktestResult(new BacktestResultParameters( - charts, - TransactionHandler.Orders.ToDictionary(), - Algorithm.Transactions.TransactionRecord, - new Dictionary(), - new Dictionary(), - new Dictionary(), - orderEvents, - totalPerformance)); - - // Keep the time budget small: this runs on the result handler thread and delays message processing - return _inRunResultsAnalyzer.Run(snapshot, logs, speedSample, timeLimitSeconds: 1); } - catch (Exception ex) + return charts; + } + + /// + /// Whether the strategy equity chart has samples yet. + /// + bool IInRunAnalysisDataProvider.HasEquitySamples() + { + lock (ChartLock) { - Log.Error(ex, "Error running in-run backtest analysis"); - return null; + return Charts.TryGetValue(StrategyEquityKey, out var equityChart) && + equityChart.Series.TryGetValue(EquityKey, out var equitySeries) && + equitySeries.Values.Count > 0; } } @@ -545,7 +543,7 @@ protected void SendFinalResult() /// Takes a sample of the engine speed counters for the algorithm speed analysis. /// Null while the algorithm warms up, since the warm-up pace would skew the speed metrics. /// - private AlgorithmSpeedSample? TakeAlgorithmSpeedSample() + AlgorithmSpeedSample? IInRunAnalysisDataProvider.TakeSpeedSample() { if (Algorithm == null || Algorithm.IsWarmingUp) { @@ -560,6 +558,8 @@ protected void SendFinalResult() _progressMonitor?.TotalDays ?? 0); } + #endregion + /// /// Sends the in-run analysis findings to the browser in their own packet, /// only when they changed since they were last sent. diff --git a/Tests/Engine/Results/InRunResultsAnalyzerTests.cs b/Tests/Engine/Results/InRunResultsAnalyzerTests.cs index 2cf807611039..7d290dc609d4 100644 --- a/Tests/Engine/Results/InRunResultsAnalyzerTests.cs +++ b/Tests/Engine/Results/InRunResultsAnalyzerTests.cs @@ -24,6 +24,7 @@ using QuantConnect.Lean.Engine.Results.Analysis.Analyses; using QuantConnect.Orders; using QuantConnect.Packets; +using QuantConnect.Statistics; namespace QuantConnect.Tests.Engine.Results { @@ -33,26 +34,22 @@ public class InRunResultsAnalyzerTests private static readonly IReadOnlyList SomeSolutions = new[] { "A solution" }; [Test] - public void PositionsAdvanceByTheConsumedOrderEventsAndLogs() + public void OrderEventAndLogStreamsAreConsumedIncrementally() { var analyzer = new TestInRunResultsAnalyzer(new FakeAnalysisA(10)); - analyzer.Run(MakeResult(3), new[] { "log 1", "log 2" }); - Assert.AreEqual(3, analyzer.OrderEventsPosition); - Assert.AreEqual(2, analyzer.LogsPosition); + analyzer.Run(3, new[] { "log 1", "log 2" }); + analyzer.Run(5, new[] { "log 3" }); + // Runs without new order events or logs don't move the read positions + analyzer.Run(0, null); + analyzer.Run(0, null); - analyzer.Run(MakeResult(5), new[] { "log 3" }); - Assert.AreEqual(8, analyzer.OrderEventsPosition); - Assert.AreEqual(3, analyzer.LogsPosition); - - // Null order events and logs don't move the positions - analyzer.Run(new BacktestResult(), null); - Assert.AreEqual(8, analyzer.OrderEventsPosition); - Assert.AreEqual(3, analyzer.LogsPosition); + CollectionAssert.AreEqual(new[] { 0, 3, 8, 8 }, analyzer.Provider.RequestedOrderEventsPositions); + CollectionAssert.AreEqual(new[] { 0, 2, 3, 3 }, analyzer.Provider.RequestedLogsPositions); } [Test] - public void PositionsAdvanceEvenWhenTheTimeLimitTruncatesTheRun() + public void StreamPositionsAdvanceEvenWhenTheTimeLimitTruncatesTheRun() { var truncatedRan = false; // The slow analysis has the higher weight so it runs first and exhausts the time limit @@ -60,11 +57,14 @@ public void PositionsAdvanceEvenWhenTheTimeLimitTruncatesTheRun() var truncated = new FakeAnalysisB(10) { OnRun = () => truncatedRan = true }; var analyzer = new TestInRunResultsAnalyzer(slow, truncated); - analyzer.Run(MakeResult(4), new[] { "log 1" }, timeLimitSeconds: 1); - + analyzer.Run(4, new[] { "log 1" }, timeLimitSeconds: 1); Assert.IsFalse(truncatedRan); - Assert.AreEqual(4, analyzer.OrderEventsPosition); - Assert.AreEqual(1, analyzer.LogsPosition); + + // The next run still resumes after the consumed order events and logs + slow.OnRun = null; + analyzer.Run(0, null); + CollectionAssert.AreEqual(new[] { 0, 4 }, analyzer.Provider.RequestedOrderEventsPositions); + CollectionAssert.AreEqual(new[] { 0, 1 }, analyzer.Provider.RequestedLogsPositions); } [Test] @@ -74,10 +74,10 @@ public void StreamBasedFindingsAccumulateAcrossRuns() var analyzer = new TestInRunResultsAnalyzer(fake); fake.Findings = () => MakeFindings(nameof(FakeAnalysisA), "first sample", 3); - analyzer.Run(MakeResult(1), new[] { "log" }); + analyzer.Run(1, new[] { "log" }); fake.Findings = () => MakeFindings(nameof(FakeAnalysisA), "second sample", 2); - var findings = analyzer.Run(MakeResult(1), new[] { "log" }); + var findings = analyzer.Run(1, new[] { "log" }); var finding = findings.Single(); Assert.AreEqual("first sample", finding.Sample); @@ -93,8 +93,8 @@ public void StreamBasedFindingsWithNullCountsCountSingleOccurrences() }; var analyzer = new TestInRunResultsAnalyzer(fake); - analyzer.Run(MakeResult(1), new[] { "log" }); - var findings = analyzer.Run(MakeResult(1), new[] { "log" }); + analyzer.Run(1, new[] { "log" }); + var findings = analyzer.Run(1, new[] { "log" }); Assert.AreEqual(2, findings.Single().Count); } @@ -107,11 +107,11 @@ public void StreamBasedFindingsPersistWhenNotReemitted() Findings = () => MakeFindings(nameof(FakeAnalysisA), "sample", 4) }; var analyzer = new TestInRunResultsAnalyzer(fake); - analyzer.Run(MakeResult(1), new[] { "log" }); + analyzer.Run(1, new[] { "log" }); // The next delta produces no new occurrences: the accumulated finding is still reported fake.Findings = () => new List(); - var findings = analyzer.Run(MakeResult(1), new[] { "log" }); + var findings = analyzer.Run(1, new[] { "log" }); var finding = findings.Single(); Assert.AreEqual("sample", finding.Sample); @@ -126,10 +126,10 @@ public void StateBasedFindingsAreReplacedOnEveryRun() Findings = () => MakeFindings(nameof(PortfolioValueIsNotPositiveAnalysis), "old sample", 2) }; var analyzer = new TestInRunResultsAnalyzer(fake); - analyzer.Run(MakeResult(1), new[] { "log" }); + analyzer.Run(1, new[] { "log" }); fake.Findings = () => MakeFindings(nameof(PortfolioValueIsNotPositiveAnalysis), "new sample", 3); - var findings = analyzer.Run(MakeResult(1), new[] { "log" }); + var findings = analyzer.Run(1, new[] { "log" }); // Replaced, not accumulated: latest sample and count win var finding = findings.Single(); @@ -145,10 +145,10 @@ public void StateBasedFindingsAreDroppedWhenTheyNoLongerFail() Findings = () => MakeFindings(nameof(PortfolioValueIsNotPositiveAnalysis), "sample", 2) }; var analyzer = new TestInRunResultsAnalyzer(fake); - Assert.IsNotEmpty(analyzer.Run(MakeResult(1), new[] { "log" })); + Assert.IsNotEmpty(analyzer.Run(1, new[] { "log" })); fake.Findings = () => new List(); - var findings = analyzer.Run(MakeResult(1), new[] { "log" }); + var findings = analyzer.Run(1, new[] { "log" }); Assert.IsEmpty(findings); } @@ -166,10 +166,10 @@ public void AggregatedStateBasedFindingsAreReplacedByFullName() .ToList() }; var analyzer = new TestInRunResultsAnalyzer(fake); - Assert.AreEqual(2, analyzer.Run(MakeResult(1), new[] { "log" }).Count); + Assert.AreEqual(2, analyzer.Run(1, new[] { "log" }).Count); fake.Findings = () => MakeFindings($"{stateBasedName} / SubA", "new sample a", 2); - var findings = analyzer.Run(MakeResult(1), new[] { "log" }); + var findings = analyzer.Run(1, new[] { "log" }); // SubB no longer fails and is dropped; SubA is replaced with the fresh finding var finding = findings.Single(); @@ -179,24 +179,81 @@ public void AggregatedStateBasedFindingsAreReplacedByFullName() } [Test] - public void SpeedSamplesAreTrackedOnlyWhenProvided() + public void SpeedSamplesAreTrackedOnlyWhenTheProviderTakesThem() { AlgorithmSpeedTracker speed = null; var fake = new FakeAnalysisA(10) { OnParameters = parameters => speed = parameters.Speed }; var analyzer = new TestInRunResultsAnalyzer(fake); - analyzer.Run(MakeResult(1), new[] { "log" }); + analyzer.Run(1, new[] { "log" }); Assert.IsNotNull(speed); Assert.AreEqual(0, speed.SampleCount); - analyzer.Run(MakeResult(1), new[] { "log" }, new AlgorithmSpeedSample(TimeSpan.FromSeconds(30), 100, 0, 1, 10)); + analyzer.Run(1, new[] { "log" }, new AlgorithmSpeedSample(TimeSpan.FromSeconds(30), 100, 0, 1, 10)); Assert.AreEqual(1, speed.SampleCount); - // No sample provided (e.g. while the algorithm warms up): the tracker is left untouched - analyzer.Run(MakeResult(1), new[] { "log" }); + // No sample taken (e.g. while the algorithm warms up): the tracker is left untouched + analyzer.Run(1, new[] { "log" }); Assert.AreEqual(1, speed.SampleCount); } + [Test] + public void CompletedSpeedTrackingAddsAFinalSampleAndReturnsTheTracker() + { + AlgorithmSpeedTracker speed = null; + var fake = new FakeAnalysisA(10) { OnParameters = parameters => speed = parameters.Speed }; + var analyzer = new TestInRunResultsAnalyzer(fake); + analyzer.Run(1, new[] { "log" }, new AlgorithmSpeedSample(TimeSpan.FromSeconds(30), 100, 0, 1, 10)); + + analyzer.Provider.NextSpeedSample = new AlgorithmSpeedSample(TimeSpan.FromSeconds(60), 200, 0, 2, 10); + var tracker = analyzer.CompleteSpeedTracking(); + + // The final analysis receives the same tracker the in-run analyses saw, with the final sample added + Assert.AreSame(speed, tracker); + Assert.AreEqual(2, tracker.SampleCount); + + // Without a final sample (e.g. the algorithm never left warm-up), the tracker is left untouched + analyzer.Provider.NextSpeedSample = null; + Assert.AreEqual(2, analyzer.CompleteSpeedTracking().SampleCount); + } + + [Test] + public void SnapshotIsBuiltFromTheDataProvider() + { + ResultsAnalysisRunParameters seenParameters = null; + var fake = new FakeAnalysisA(10) { OnParameters = parameters => seenParameters = parameters }; + var analyzer = new TestInRunResultsAnalyzer(fake); + analyzer.Provider.Orders[1] = new MarketOrder(); + analyzer.Provider.Charts["a chart"] = new Chart("a chart"); + + analyzer.Run(2, new[] { "log" }); + + // Only the charts the in-run analyses read are requested from the provider + CollectionAssert.AreEqual(InRunResultsAnalyzer.RequiredCharts, analyzer.Provider.RequestedChartNames); + Assert.IsTrue(seenParameters.Result.Charts.ContainsKey("a chart")); + Assert.AreEqual(1, seenParameters.Result.Orders.Count); + Assert.AreEqual(2, seenParameters.Result.OrderEvents.Count); + CollectionAssert.AreEqual(new[] { "log" }, seenParameters.Logs); + } + + [Test] + public void StatisticsAreWithheldUntilEquityHasSamples() + { + BacktestResult seenResult = null; + var fake = new FakeAnalysisA(10) { OnParameters = parameters => seenResult = (BacktestResult)parameters.Result }; + var analyzer = new TestInRunResultsAnalyzer(fake); + var performance = new AlgorithmPerformance(); + + // Equity has no samples yet: the all-zero default statistics are withheld + analyzer.Provider.EquityHasSamples = false; + analyzer.Run(performance); + Assert.IsNull(seenResult.TotalPerformance); + + analyzer.Provider.EquityHasSamples = true; + analyzer.Run(performance); + Assert.AreSame(performance, seenResult.TotalPerformance); + } + [Test] public void FindingsAreRankedByAnalysisWeightAndCapped() { @@ -215,14 +272,14 @@ public void FindingsAreRankedByAnalysisWeightAndCapped() }; var analyzer = new TestInRunResultsAnalyzer(lowWeight, midWeight, highWeight); - var findings = analyzer.Run(MakeResult(1), new[] { "log" }); + var findings = analyzer.Run(1, new[] { "log" }); CollectionAssert.AreEqual( new[] { nameof(FakeAnalysisC), $"{nameof(FakeAnalysisB)} / Sub", nameof(FakeAnalysisA) }, findings.Select(finding => finding.Name)); // The accumulated findings are capped to the top weighted ones lowWeight.Findings = midWeight.Findings = highWeight.Findings = () => new List(); - findings = analyzer.Run(MakeResult(1), new[] { "log" }, maxFailedAnalyses: 2); + findings = analyzer.Run(1, new[] { "log" }, maxFailedAnalyses: 2); CollectionAssert.AreEqual( new[] { nameof(FakeAnalysisC), $"{nameof(FakeAnalysisB)} / Sub" }, findings.Select(finding => finding.Name)); @@ -231,7 +288,7 @@ public void FindingsAreRankedByAnalysisWeightAndCapped() [Test] public void RequiredChartsAreTheChartsReadByTheInRunAnalyses() { - // The result handler only clones these charts into the analyzed snapshot, + // The data provider only clones these charts into the analyzed snapshot, // so this must stay in sync with the charts the in-run analyses read CollectionAssert.AreEquivalent( new[] { BaseResultsHandler.PortfolioMarginKey }, @@ -243,21 +300,13 @@ public void AnalysesAreCreatedOnceAndReusedAcrossRuns() { var analyzer = new TestInRunResultsAnalyzer(new FakeAnalysisA(10)); - analyzer.Run(MakeResult(1), new[] { "log" }); - analyzer.Run(MakeResult(1), new[] { "log" }); + analyzer.Run(1, new[] { "log" }); + analyzer.Run(1, new[] { "log" }); // Both the analysis chain and the findings ranking read the cached set Assert.AreEqual(1, analyzer.GetAnalysesCallCount); } - private static BacktestResult MakeResult(int orderEventsCount) - { - return new BacktestResult - { - OrderEvents = Enumerable.Range(0, orderEventsCount).Select(_ => new OrderEvent()).ToList() - }; - } - private static List MakeFindings(string name, string sample, int? count) { return new List { new(name, "An issue", sample, count, SomeSolutions) }; @@ -267,13 +316,34 @@ private class TestInRunResultsAnalyzer : InRunResultsAnalyzer { private readonly IReadOnlyCollection _analyses; + public FakeDataProvider Provider { get; } + + public int GetAnalysesCallCount { get; private set; } + public TestInRunResultsAnalyzer(params BaseResultsAnalysis[] analyses) - : base(null, Language.CSharp) + : this(new FakeDataProvider(), analyses) + { + } + + private TestInRunResultsAnalyzer(FakeDataProvider provider, BaseResultsAnalysis[] analyses) + : base(null, Language.CSharp, provider) { + Provider = provider; _analyses = analyses; } - public int GetAnalysesCallCount { get; private set; } + /// + /// Appends the new order events and logs to the provider's streams and runs the analyzer, + /// mirroring the incremental stream growth the analyzer sees in a running backtest. + /// + public IReadOnlyList Run(int newOrderEventsCount, string[] newLogs, + AlgorithmSpeedSample? speedSample = null, int timeLimitSeconds = 1, int maxFailedAnalyses = 10) + { + Provider.OrderEvents.AddRange(Enumerable.Range(0, newOrderEventsCount).Select(_ => new OrderEvent())); + Provider.Logs.AddRange(newLogs ?? Array.Empty()); + Provider.NextSpeedSample = speedSample; + return Run(totalPerformance: null, timeLimitSeconds, maxFailedAnalyses); + } protected override IReadOnlyCollection GetAnalyses() { @@ -282,6 +352,51 @@ protected override IReadOnlyCollection GetAnalyses() } } + private sealed class FakeDataProvider : IInRunAnalysisDataProvider + { + public Dictionary Orders { get; } = new(); + + public List OrderEvents { get; } = new(); + + public List Logs { get; } = new(); + + public Dictionary Charts { get; } = new(); + + public bool EquityHasSamples { get; set; } = true; + + public AlgorithmSpeedSample? NextSpeedSample { get; set; } + + public List RequestedOrderEventsPositions { get; } = new(); + + public List RequestedLogsPositions { get; } = new(); + + public IReadOnlyList RequestedChartNames { get; private set; } + + public IDictionary GetOrders() => Orders; + + public List GetOrderEvents(int fromPosition) + { + RequestedOrderEventsPositions.Add(fromPosition); + return OrderEvents.Skip(fromPosition).ToList(); + } + + public IReadOnlyList GetLogs(int fromPosition) + { + RequestedLogsPositions.Add(fromPosition); + return Logs.Skip(fromPosition).ToList(); + } + + public IDictionary GetChartSnapshots(IReadOnlyList chartNames) + { + RequestedChartNames = chartNames; + return Charts; + } + + public bool HasEquitySamples() => EquityHasSamples; + + public AlgorithmSpeedSample? TakeSpeedSample() => NextSpeedSample; + } + private class FakeAnalysis : BaseResultsAnalysis { private readonly int _weight; From 76194979958457a60a891ea11b6720d29eafff69 Mon Sep 17 00:00:00 2001 From: Jhonathan Abreu Date: Mon, 3 Aug 2026 17:43:45 -0400 Subject: [PATCH 21/33] Move the in-run and state-based analysis classification onto the analyses Each analysis now declares whether it can run while the backtest is in progress (RunsInRun) and whether it is state-based (IsStateBased), so the in-run analyzer derives its set by filtering the final analysis set instead of maintaining a duplicated list and a name-based state-based set. --- .../Analyses/AlgorithmSpeedAnalysis.cs | 11 +++ .../Analysis/Analyses/BaseResultsAnalysis.cs | 16 +++++ ...htsEmittedForDelistedSecuritiesAnalysis.cs | 5 ++ .../Analysis/Analyses/MarginCallsAnalysis.cs | 5 ++ ...ithmWarmingUpOrderResponseErrorAnalysis.cs | 5 ++ ...ToSubmitOrderOrderResponseErrorAnalysis.cs | 5 ++ ...ToUpdateOrderOrderResponseErrorAnalysis.cs | 5 ++ ...redOnExerciseOrderResponseErrorAnalysis.cs | 5 ++ ...MaximumOrdersOrderResponseErrorAnalysis.cs | 5 ++ ...tableQuantityOrderResponseErrorAnalysis.cs | 5 ++ ...changeNotOpenOrderResponseErrorAnalysis.cs | 5 ++ ...rsionRateZeroOrderResponseErrorAnalysis.cs | 5 ++ ...ntBuyingPowerOrderResponseErrorAnalysis.cs | 5 ++ ...gRegularHoursOrderResponseErrorAnalysis.cs | 5 ++ ...dableSecurityOrderResponseErrorAnalysis.cs | 5 ++ ...rOnStockSplitOrderResponseErrorAnalysis.cs | 5 ++ ...ssThanLotSizeOrderResponseErrorAnalysis.cs | 5 ++ ...rQuantityZeroOrderResponseErrorAnalysis.cs | 5 ++ ...rityPriceZeroOrderResponseErrorAnalysis.cs | 5 ++ ...supportedOptionExerciseQuantityAnalysis.cs | 5 ++ ...rtedOptionShortPositionExerciseAnalysis.cs | 5 ++ .../Analyses/PortfolioMarginUsageAnalysis.cs | 11 +++ .../PortfolioValueIsNotPositiveAnalysis.cs | 11 +++ .../Analyses/StaleOrderFillsAnalysis.cs | 5 ++ .../TakeProfitAndStopLossOrdersAnalysis.cs | 11 +++ .../Results/Analysis/InRunResultsAnalyzer.cs | 67 ++++++------------- Engine/Results/Analysis/ResultsAnalyzer.cs | 4 +- .../Results/InRunResultsAnalyzerTests.cs | 39 +++++++++-- Tests/Engine/Results/ResultsAnalyzerTests.cs | 39 +++++++++++ 29 files changed, 257 insertions(+), 52 deletions(-) diff --git a/Engine/Results/Analysis/Analyses/AlgorithmSpeedAnalysis.cs b/Engine/Results/Analysis/Analyses/AlgorithmSpeedAnalysis.cs index 6d8c276052b7..dae23e2d5c3f 100644 --- a/Engine/Results/Analysis/Analyses/AlgorithmSpeedAnalysis.cs +++ b/Engine/Results/Analysis/Analyses/AlgorithmSpeedAnalysis.cs @@ -77,6 +77,17 @@ public class AlgorithmSpeedAnalysis : BaseResultsAnalysis /// public const string HistoryRequestLoadName = "HistoryRequestLoad"; + /// + /// This analysis reads only the tracked speed metrics, so it also runs while the backtest is in progress. + /// + public override bool RunsInRun { get; } = true; + + /// + /// This analysis reads the current speed metrics instead of scanning the order event + /// and log streams, so its in-run findings are replaced on every run. + /// + public override bool IsStateBased { get; } = true; + /// /// Gets the description of the slow algorithm issue. /// diff --git a/Engine/Results/Analysis/Analyses/BaseResultsAnalysis.cs b/Engine/Results/Analysis/Analyses/BaseResultsAnalysis.cs index ba27fbb39e9d..d90cf01245cb 100644 --- a/Engine/Results/Analysis/Analyses/BaseResultsAnalysis.cs +++ b/Engine/Results/Analysis/Analyses/BaseResultsAnalysis.cs @@ -34,6 +34,22 @@ public abstract class BaseResultsAnalysis /// public abstract int Weight { get; } + /// + /// Whether this analysis can also run while the backtest is still in progress, against a + /// snapshot of the intermediate results. Analyses that need the completed run (runtime + /// errors, equity curves, final statistics, completion logs) or that read algorithm state + /// that is not safe to access while it runs are left to the final analysis only. + /// + public virtual bool RunsInRun { get; } + + /// + /// Whether this analysis reads the current backtest state (statistics, orders, charts) + /// instead of scanning the append-only order event and log streams. When run in-run, it + /// runs against the full current state on every run and its findings replace the + /// previous ones instead of accumulating. + /// + public virtual bool IsStateBased { get; } + /// /// Runs the analysis against all backtest data provided in . /// diff --git a/Engine/Results/Analysis/Analyses/InsightsEmittedForDelistedSecuritiesAnalysis.cs b/Engine/Results/Analysis/Analyses/InsightsEmittedForDelistedSecuritiesAnalysis.cs index 839b245fdbb4..95cc64ba5149 100644 --- a/Engine/Results/Analysis/Analyses/InsightsEmittedForDelistedSecuritiesAnalysis.cs +++ b/Engine/Results/Analysis/Analyses/InsightsEmittedForDelistedSecuritiesAnalysis.cs @@ -23,6 +23,11 @@ namespace QuantConnect.Lean.Engine.Results.Analysis.Analyses /// public class InsightsEmittedForDelistedSecuritiesAnalysis : MessageAnalysis { + /// + /// This analysis reads only the result snapshot, so it also runs while the backtest is in progress. + /// + public override bool RunsInRun { get; } = true; + /// /// Description of the delisted-security insight emission issue detected by this analysis. /// diff --git a/Engine/Results/Analysis/Analyses/MarginCallsAnalysis.cs b/Engine/Results/Analysis/Analyses/MarginCallsAnalysis.cs index 554ad652220e..aa2ceb64d8e2 100644 --- a/Engine/Results/Analysis/Analyses/MarginCallsAnalysis.cs +++ b/Engine/Results/Analysis/Analyses/MarginCallsAnalysis.cs @@ -24,6 +24,11 @@ namespace QuantConnect.Lean.Engine.Results.Analysis.Analyses /// public class MarginCallsAnalysis : MessageAnalysis { + /// + /// This analysis reads only the result snapshot, so it also runs while the backtest is in progress. + /// + public override bool RunsInRun { get; } = true; + /// /// Description of the margin-call issue detected by this analysis. /// diff --git a/Engine/Results/Analysis/Analyses/OrderResponseErrorsAnalyses/AlgorithmWarmingUpOrderResponseErrorAnalysis.cs b/Engine/Results/Analysis/Analyses/OrderResponseErrorsAnalyses/AlgorithmWarmingUpOrderResponseErrorAnalysis.cs index ef0b487f99a4..d0dc5b825884 100644 --- a/Engine/Results/Analysis/Analyses/OrderResponseErrorsAnalyses/AlgorithmWarmingUpOrderResponseErrorAnalysis.cs +++ b/Engine/Results/Analysis/Analyses/OrderResponseErrorsAnalyses/AlgorithmWarmingUpOrderResponseErrorAnalysis.cs @@ -25,6 +25,11 @@ namespace QuantConnect.Lean.Engine.Results.Analysis.Analyses /// public class AlgorithmWarmingUpOrderResponseErrorAnalysis : MessageAnalysis { + /// + /// This analysis reads only the result snapshot, so it also runs while the backtest is in progress. + /// + public override bool RunsInRun { get; } = true; + /// /// Gets a description of the warm-up period ordering violation. /// diff --git a/Engine/Results/Analysis/Analyses/OrderResponseErrorsAnalyses/BrokerageModelRefusedToSubmitOrderOrderResponseErrorAnalysis.cs b/Engine/Results/Analysis/Analyses/OrderResponseErrorsAnalyses/BrokerageModelRefusedToSubmitOrderOrderResponseErrorAnalysis.cs index d07b9e22faf1..e39c9b2d957b 100644 --- a/Engine/Results/Analysis/Analyses/OrderResponseErrorsAnalyses/BrokerageModelRefusedToSubmitOrderOrderResponseErrorAnalysis.cs +++ b/Engine/Results/Analysis/Analyses/OrderResponseErrorsAnalyses/BrokerageModelRefusedToSubmitOrderOrderResponseErrorAnalysis.cs @@ -36,6 +36,11 @@ namespace QuantConnect.Lean.Engine.Results.Analysis.Analyses /// public class BrokerageModelRefusedToSubmitOrderOrderResponseErrorAnalysis : OrderResponseErrorAnalysis { + /// + /// This analysis reads only the result snapshot, so it also runs while the backtest is in progress. + /// + public override bool RunsInRun { get; } = true; + /// /// Gets a description of the brokerage-refused-to-submit-order issue. /// diff --git a/Engine/Results/Analysis/Analyses/OrderResponseErrorsAnalyses/BrokerageModelRefusedToUpdateOrderOrderResponseErrorAnalysis.cs b/Engine/Results/Analysis/Analyses/OrderResponseErrorsAnalyses/BrokerageModelRefusedToUpdateOrderOrderResponseErrorAnalysis.cs index 13918fc38f3b..4731a79cc85a 100644 --- a/Engine/Results/Analysis/Analyses/OrderResponseErrorsAnalyses/BrokerageModelRefusedToUpdateOrderOrderResponseErrorAnalysis.cs +++ b/Engine/Results/Analysis/Analyses/OrderResponseErrorsAnalyses/BrokerageModelRefusedToUpdateOrderOrderResponseErrorAnalysis.cs @@ -28,6 +28,11 @@ namespace QuantConnect.Lean.Engine.Results.Analysis.Analyses /// public class BrokerageModelRefusedToUpdateOrderOrderResponseErrorAnalysis : OrderResponseErrorAnalysis { + /// + /// This analysis reads only the result snapshot, so it also runs while the backtest is in progress. + /// + public override bool RunsInRun { get; } = true; + /// /// Gets a description of the brokerage-refused-to-update-order issue. /// diff --git a/Engine/Results/Analysis/Analyses/OrderResponseErrorsAnalyses/EuropeanOptionNotExpiredOnExerciseOrderResponseErrorAnalysis.cs b/Engine/Results/Analysis/Analyses/OrderResponseErrorsAnalyses/EuropeanOptionNotExpiredOnExerciseOrderResponseErrorAnalysis.cs index 23920fb83c9d..aabba0a28dc0 100644 --- a/Engine/Results/Analysis/Analyses/OrderResponseErrorsAnalyses/EuropeanOptionNotExpiredOnExerciseOrderResponseErrorAnalysis.cs +++ b/Engine/Results/Analysis/Analyses/OrderResponseErrorsAnalyses/EuropeanOptionNotExpiredOnExerciseOrderResponseErrorAnalysis.cs @@ -24,6 +24,11 @@ namespace QuantConnect.Lean.Engine.Results.Analysis.Analyses /// public class EuropeanOptionNotExpiredOnExerciseOrderResponseErrorAnalysis : MessageAnalysis { + /// + /// This analysis reads only the result snapshot, so it also runs while the backtest is in progress. + /// + public override bool RunsInRun { get; } = true; + /// /// Gets a description of the premature European option exercise issue. /// diff --git a/Engine/Results/Analysis/Analyses/OrderResponseErrorsAnalyses/ExceededMaximumOrdersOrderResponseErrorAnalysis.cs b/Engine/Results/Analysis/Analyses/OrderResponseErrorsAnalyses/ExceededMaximumOrdersOrderResponseErrorAnalysis.cs index dc2ab53a445c..c19d6ad45c96 100644 --- a/Engine/Results/Analysis/Analyses/OrderResponseErrorsAnalyses/ExceededMaximumOrdersOrderResponseErrorAnalysis.cs +++ b/Engine/Results/Analysis/Analyses/OrderResponseErrorsAnalyses/ExceededMaximumOrdersOrderResponseErrorAnalysis.cs @@ -23,6 +23,11 @@ namespace QuantConnect.Lean.Engine.Results.Analysis.Analyses /// public class ExceededMaximumOrdersOrderResponseErrorAnalysis : MessageAnalysis { + /// + /// This analysis reads only the result snapshot, so it also runs while the backtest is in progress. + /// + public override bool RunsInRun { get; } = true; + /// /// Gets a description of the exceeded maximum orders issue. /// diff --git a/Engine/Results/Analysis/Analyses/OrderResponseErrorsAnalyses/ExceedsShortableQuantityOrderResponseErrorAnalysis.cs b/Engine/Results/Analysis/Analyses/OrderResponseErrorsAnalyses/ExceedsShortableQuantityOrderResponseErrorAnalysis.cs index 3a58ac6ea5a7..8c70fd4cc938 100644 --- a/Engine/Results/Analysis/Analyses/OrderResponseErrorsAnalyses/ExceedsShortableQuantityOrderResponseErrorAnalysis.cs +++ b/Engine/Results/Analysis/Analyses/OrderResponseErrorsAnalyses/ExceedsShortableQuantityOrderResponseErrorAnalysis.cs @@ -25,6 +25,11 @@ namespace QuantConnect.Lean.Engine.Results.Analysis.Analyses /// public class ExceedsShortableQuantityOrderResponseErrorAnalysis : BaseResultsAnalysis { + /// + /// This analysis reads only the result snapshot, so it also runs while the backtest is in progress. + /// + public override bool RunsInRun { get; } = true; + /// /// Gets a description of the exceeded shortable quantity issue. /// diff --git a/Engine/Results/Analysis/Analyses/OrderResponseErrorsAnalyses/ExchangeNotOpenOrderResponseErrorAnalysis.cs b/Engine/Results/Analysis/Analyses/OrderResponseErrorsAnalyses/ExchangeNotOpenOrderResponseErrorAnalysis.cs index 93e63ab3bef2..50a5d72ef248 100644 --- a/Engine/Results/Analysis/Analyses/OrderResponseErrorsAnalyses/ExchangeNotOpenOrderResponseErrorAnalysis.cs +++ b/Engine/Results/Analysis/Analyses/OrderResponseErrorsAnalyses/ExchangeNotOpenOrderResponseErrorAnalysis.cs @@ -25,6 +25,11 @@ namespace QuantConnect.Lean.Engine.Results.Analysis.Analyses /// public class ExchangeNotOpenOrderResponseErrorAnalysis : BaseResultsAnalysis { + /// + /// This analysis reads only the result snapshot, so it also runs while the backtest is in progress. + /// + public override bool RunsInRun { get; } = true; + /// /// Gets a description of the exchange-not-open ordering issue. /// diff --git a/Engine/Results/Analysis/Analyses/OrderResponseErrorsAnalyses/ForexConversionRateZeroOrderResponseErrorAnalysis.cs b/Engine/Results/Analysis/Analyses/OrderResponseErrorsAnalyses/ForexConversionRateZeroOrderResponseErrorAnalysis.cs index 8ec708316c8f..b2b5d2a6775c 100644 --- a/Engine/Results/Analysis/Analyses/OrderResponseErrorsAnalyses/ForexConversionRateZeroOrderResponseErrorAnalysis.cs +++ b/Engine/Results/Analysis/Analyses/OrderResponseErrorsAnalyses/ForexConversionRateZeroOrderResponseErrorAnalysis.cs @@ -23,6 +23,11 @@ namespace QuantConnect.Lean.Engine.Results.Analysis.Analyses /// public class ForexConversionRateZeroOrderResponseErrorAnalysis : MessageAnalysis { + /// + /// This analysis reads only the result snapshot, so it also runs while the backtest is in progress. + /// + public override bool RunsInRun { get; } = true; + /// /// Gets a description of the zero Forex conversion rate issue. /// diff --git a/Engine/Results/Analysis/Analyses/OrderResponseErrorsAnalyses/InsufficientBuyingPowerOrderResponseErrorAnalysis.cs b/Engine/Results/Analysis/Analyses/OrderResponseErrorsAnalyses/InsufficientBuyingPowerOrderResponseErrorAnalysis.cs index 2ce5f7789b25..253ab6d7cd94 100644 --- a/Engine/Results/Analysis/Analyses/OrderResponseErrorsAnalyses/InsufficientBuyingPowerOrderResponseErrorAnalysis.cs +++ b/Engine/Results/Analysis/Analyses/OrderResponseErrorsAnalyses/InsufficientBuyingPowerOrderResponseErrorAnalysis.cs @@ -23,6 +23,11 @@ namespace QuantConnect.Lean.Engine.Results.Analysis.Analyses /// public class InsufficientBuyingPowerOrderResponseErrorAnalysis : OrderResponseErrorAnalysis { + /// + /// This analysis reads only the result snapshot, so it also runs while the backtest is in progress. + /// + public override bool RunsInRun { get; } = true; + /// /// Gets a description of the insufficient buying power issue. /// diff --git a/Engine/Results/Analysis/Analyses/OrderResponseErrorsAnalyses/MarketOnOpenNotAllowedDuringRegularHoursOrderResponseErrorAnalysis.cs b/Engine/Results/Analysis/Analyses/OrderResponseErrorsAnalyses/MarketOnOpenNotAllowedDuringRegularHoursOrderResponseErrorAnalysis.cs index a5b963a91da6..4c7f7186e1ab 100644 --- a/Engine/Results/Analysis/Analyses/OrderResponseErrorsAnalyses/MarketOnOpenNotAllowedDuringRegularHoursOrderResponseErrorAnalysis.cs +++ b/Engine/Results/Analysis/Analyses/OrderResponseErrorsAnalyses/MarketOnOpenNotAllowedDuringRegularHoursOrderResponseErrorAnalysis.cs @@ -23,6 +23,11 @@ namespace QuantConnect.Lean.Engine.Results.Analysis.Analyses /// public class MarketOnOpenNotAllowedDuringRegularHoursOrderResponseErrorAnalysis : MessageAnalysis { + /// + /// This analysis reads only the result snapshot, so it also runs while the backtest is in progress. + /// + public override bool RunsInRun { get; } = true; + /// /// Gets a description of the market-on-open during regular hours issue. /// diff --git a/Engine/Results/Analysis/Analyses/OrderResponseErrorsAnalyses/NonTradableSecurityOrderResponseErrorAnalysis.cs b/Engine/Results/Analysis/Analyses/OrderResponseErrorsAnalyses/NonTradableSecurityOrderResponseErrorAnalysis.cs index 35752b5613a4..f0b904a826bf 100644 --- a/Engine/Results/Analysis/Analyses/OrderResponseErrorsAnalyses/NonTradableSecurityOrderResponseErrorAnalysis.cs +++ b/Engine/Results/Analysis/Analyses/OrderResponseErrorsAnalyses/NonTradableSecurityOrderResponseErrorAnalysis.cs @@ -23,6 +23,11 @@ namespace QuantConnect.Lean.Engine.Results.Analysis.Analyses /// public class NonTradableSecurityOrderResponseErrorAnalysis : MessageAnalysis { + /// + /// This analysis reads only the result snapshot, so it also runs while the backtest is in progress. + /// + public override bool RunsInRun { get; } = true; + /// /// Gets a description of the non-tradable security ordering issue. /// diff --git a/Engine/Results/Analysis/Analyses/OrderResponseErrorsAnalyses/OptionOrderOnStockSplitOrderResponseErrorAnalysis.cs b/Engine/Results/Analysis/Analyses/OrderResponseErrorsAnalyses/OptionOrderOnStockSplitOrderResponseErrorAnalysis.cs index 402c469a16c6..dc201b049877 100644 --- a/Engine/Results/Analysis/Analyses/OrderResponseErrorsAnalyses/OptionOrderOnStockSplitOrderResponseErrorAnalysis.cs +++ b/Engine/Results/Analysis/Analyses/OrderResponseErrorsAnalyses/OptionOrderOnStockSplitOrderResponseErrorAnalysis.cs @@ -23,6 +23,11 @@ namespace QuantConnect.Lean.Engine.Results.Analysis.Analyses /// public class OptionOrderOnStockSplitOrderResponseErrorAnalysis : MessageAnalysis { + /// + /// This analysis reads only the result snapshot, so it also runs while the backtest is in progress. + /// + public override bool RunsInRun { get; } = true; + /// /// Gets a description of the option order during stock split issue. /// diff --git a/Engine/Results/Analysis/Analyses/OrderResponseErrorsAnalyses/OrderQuantityLessThanLotSizeOrderResponseErrorAnalysis.cs b/Engine/Results/Analysis/Analyses/OrderResponseErrorsAnalyses/OrderQuantityLessThanLotSizeOrderResponseErrorAnalysis.cs index e23063f49ebb..5081eba46d76 100644 --- a/Engine/Results/Analysis/Analyses/OrderResponseErrorsAnalyses/OrderQuantityLessThanLotSizeOrderResponseErrorAnalysis.cs +++ b/Engine/Results/Analysis/Analyses/OrderResponseErrorsAnalyses/OrderQuantityLessThanLotSizeOrderResponseErrorAnalysis.cs @@ -23,6 +23,11 @@ namespace QuantConnect.Lean.Engine.Results.Analysis.Analyses /// public class OrderQuantityLessThanLotSizeOrderResponseErrorAnalysis : MessageAnalysis { + /// + /// This analysis reads only the result snapshot, so it also runs while the backtest is in progress. + /// + public override bool RunsInRun { get; } = true; + /// /// Gets a description of the order quantity below lot size issue. /// diff --git a/Engine/Results/Analysis/Analyses/OrderResponseErrorsAnalyses/OrderQuantityZeroOrderResponseErrorAnalysis.cs b/Engine/Results/Analysis/Analyses/OrderResponseErrorsAnalyses/OrderQuantityZeroOrderResponseErrorAnalysis.cs index df4ea6831644..18028437c0e9 100644 --- a/Engine/Results/Analysis/Analyses/OrderResponseErrorsAnalyses/OrderQuantityZeroOrderResponseErrorAnalysis.cs +++ b/Engine/Results/Analysis/Analyses/OrderResponseErrorsAnalyses/OrderQuantityZeroOrderResponseErrorAnalysis.cs @@ -24,6 +24,11 @@ namespace QuantConnect.Lean.Engine.Results.Analysis.Analyses /// public class OrderQuantityZeroOrderResponseErrorAnalysis : MessageAnalysis { + /// + /// This analysis reads only the result snapshot, so it also runs while the backtest is in progress. + /// + public override bool RunsInRun { get; } = true; + /// /// Gets a description of the zero order quantity issue. /// diff --git a/Engine/Results/Analysis/Analyses/OrderResponseErrorsAnalyses/SecurityPriceZeroOrderResponseErrorAnalysis.cs b/Engine/Results/Analysis/Analyses/OrderResponseErrorsAnalyses/SecurityPriceZeroOrderResponseErrorAnalysis.cs index 00413b747be6..6365760b37eb 100644 --- a/Engine/Results/Analysis/Analyses/OrderResponseErrorsAnalyses/SecurityPriceZeroOrderResponseErrorAnalysis.cs +++ b/Engine/Results/Analysis/Analyses/OrderResponseErrorsAnalyses/SecurityPriceZeroOrderResponseErrorAnalysis.cs @@ -23,6 +23,11 @@ namespace QuantConnect.Lean.Engine.Results.Analysis.Analyses /// public class SecurityPriceZeroOrderResponseErrorAnalysis : MessageAnalysis { + /// + /// This analysis reads only the result snapshot, so it also runs while the backtest is in progress. + /// + public override bool RunsInRun { get; } = true; + /// /// Gets a description of the zero security price ordering issue. /// diff --git a/Engine/Results/Analysis/Analyses/OrderResponseErrorsAnalyses/UnsupportedOptionExerciseQuantityAnalysis.cs b/Engine/Results/Analysis/Analyses/OrderResponseErrorsAnalyses/UnsupportedOptionExerciseQuantityAnalysis.cs index f05b7ce75175..40fe302745a2 100644 --- a/Engine/Results/Analysis/Analyses/OrderResponseErrorsAnalyses/UnsupportedOptionExerciseQuantityAnalysis.cs +++ b/Engine/Results/Analysis/Analyses/OrderResponseErrorsAnalyses/UnsupportedOptionExerciseQuantityAnalysis.cs @@ -23,6 +23,11 @@ namespace QuantConnect.Lean.Engine.Results.Analysis.Analyses /// public class UnsupportedOptionExerciseQuantityAnalysis : MessageAnalysis { + /// + /// This analysis reads only the result snapshot, so it also runs while the backtest is in progress. + /// + public override bool RunsInRun { get; } = true; + /// /// Gets a description of the excess-quantity option exercise issue. /// diff --git a/Engine/Results/Analysis/Analyses/OrderResponseErrorsAnalyses/UnsupportedOptionShortPositionExerciseAnalysis.cs b/Engine/Results/Analysis/Analyses/OrderResponseErrorsAnalyses/UnsupportedOptionShortPositionExerciseAnalysis.cs index d29d4d98fdfe..23e2f5a4a94d 100644 --- a/Engine/Results/Analysis/Analyses/OrderResponseErrorsAnalyses/UnsupportedOptionShortPositionExerciseAnalysis.cs +++ b/Engine/Results/Analysis/Analyses/OrderResponseErrorsAnalyses/UnsupportedOptionShortPositionExerciseAnalysis.cs @@ -23,6 +23,11 @@ namespace QuantConnect.Lean.Engine.Results.Analysis.Analyses /// public class UnsupportedOptionShortPositionExerciseAnalysis : MessageAnalysis { + /// + /// This analysis reads only the result snapshot, so it also runs while the backtest is in progress. + /// + public override bool RunsInRun { get; } = true; + /// /// Gets a description of the short-position option exercise issue. /// diff --git a/Engine/Results/Analysis/Analyses/PortfolioMarginUsageAnalysis.cs b/Engine/Results/Analysis/Analyses/PortfolioMarginUsageAnalysis.cs index 2aaecbb7f576..8f42b5994b2a 100644 --- a/Engine/Results/Analysis/Analyses/PortfolioMarginUsageAnalysis.cs +++ b/Engine/Results/Analysis/Analyses/PortfolioMarginUsageAnalysis.cs @@ -25,6 +25,17 @@ namespace QuantConnect.Lean.Engine.Results.Analysis.Analyses /// public class PortfolioMarginUsageAnalysis : BaseResultsAnalysis { + /// + /// This analysis reads only the result snapshot, so it also runs while the backtest is in progress. + /// + public override bool RunsInRun { get; } = true; + + /// + /// This analysis reads the current margin chart instead of scanning the order event + /// and log streams, so its in-run findings are replaced on every run. + /// + public override bool IsStateBased { get; } = true; + /// /// Gets the description of the detected margin under-utilisation issue. /// diff --git a/Engine/Results/Analysis/Analyses/PortfolioValueIsNotPositiveAnalysis.cs b/Engine/Results/Analysis/Analyses/PortfolioValueIsNotPositiveAnalysis.cs index d2f1bb96f1ce..147de93e49f3 100644 --- a/Engine/Results/Analysis/Analyses/PortfolioValueIsNotPositiveAnalysis.cs +++ b/Engine/Results/Analysis/Analyses/PortfolioValueIsNotPositiveAnalysis.cs @@ -22,6 +22,17 @@ namespace QuantConnect.Lean.Engine.Results.Analysis.Analyses /// public class PortfolioValueIsNotPositiveAnalysis : BaseResultsAnalysis { + /// + /// This analysis reads only the result snapshot, so it also runs while the backtest is in progress. + /// + public override bool RunsInRun { get; } = true; + + /// + /// This analysis reads the current portfolio statistics instead of scanning the order event + /// and log streams, so its in-run findings are replaced on every run. + /// + public override bool IsStateBased { get; } = true; + /// /// Gets the description of the non-positive portfolio equity issue. /// diff --git a/Engine/Results/Analysis/Analyses/StaleOrderFillsAnalysis.cs b/Engine/Results/Analysis/Analyses/StaleOrderFillsAnalysis.cs index a0ba22fc516c..05c6a60729a3 100644 --- a/Engine/Results/Analysis/Analyses/StaleOrderFillsAnalysis.cs +++ b/Engine/Results/Analysis/Analyses/StaleOrderFillsAnalysis.cs @@ -25,6 +25,11 @@ namespace QuantConnect.Lean.Engine.Results.Analysis.Analyses /// public class StaleOrderFillsAnalysis : BaseResultsAnalysis { + /// + /// This analysis reads only the result snapshot, so it also runs while the backtest is in progress. + /// + public override bool RunsInRun { get; } = true; + /// /// Gets the description of the stale order fill issue. /// diff --git a/Engine/Results/Analysis/Analyses/TakeProfitAndStopLossOrdersAnalysis.cs b/Engine/Results/Analysis/Analyses/TakeProfitAndStopLossOrdersAnalysis.cs index 6c7c3770f805..2ef3dd5a14a8 100644 --- a/Engine/Results/Analysis/Analyses/TakeProfitAndStopLossOrdersAnalysis.cs +++ b/Engine/Results/Analysis/Analyses/TakeProfitAndStopLossOrdersAnalysis.cs @@ -26,6 +26,17 @@ namespace QuantConnect.Lean.Engine.Results.Analysis.Analyses /// public class TakeProfitAndStopLossOrdersAnalysis : BaseResultsAnalysis { + /// + /// This analysis reads only the result snapshot, so it also runs while the backtest is in progress. + /// + public override bool RunsInRun { get; } = true; + + /// + /// This analysis reads the current orders collection instead of scanning the order event + /// and log streams, so its in-run findings are replaced on every run. + /// + public override bool IsStateBased { get; } = true; + /// /// Gets the description of the TP/SL order handling issue. /// diff --git a/Engine/Results/Analysis/InRunResultsAnalyzer.cs b/Engine/Results/Analysis/InRunResultsAnalyzer.cs index a8ebd7d17cbd..2fd7aefc01a7 100644 --- a/Engine/Results/Analysis/InRunResultsAnalyzer.cs +++ b/Engine/Results/Analysis/InRunResultsAnalyzer.cs @@ -30,23 +30,17 @@ namespace QuantConnect.Lean.Engine.Results.Analysis /// public class InRunResultsAnalyzer : ResultsAnalyzer { - /// - /// Analyses that read the current backtest state (statistics, orders, charts) instead of scanning - /// the append-only order event and log streams. They must run against the full current state on - /// every run, and their previous findings are replaced instead of accumulated. - /// - private static readonly HashSet StateBasedAnalyses = new() - { - nameof(PortfolioValueIsNotPositiveAnalysis), - nameof(TakeProfitAndStopLossOrdersAnalysis), - nameof(PortfolioMarginUsageAnalysis), - nameof(AlgorithmSpeedAnalysis), - }; - private readonly Dictionary _findings = new(); private readonly QCAlgorithm _algorithm; private readonly IInRunAnalysisDataProvider _dataProvider; + /// + /// The names of the analyses in the in-run set that declare themselves state-based + /// (see ), whose findings are replaced + /// on every run instead of accumulated. + /// + private HashSet _stateBasedAnalyses; + /// /// The number of order events already consumed by previous runs, from which the next run /// resumes reading the order event stream. @@ -218,7 +212,14 @@ public AlgorithmSpeedTracker CompleteSpeedTracking() /// /// Determines whether the given finding was produced by a state-based analysis. /// - private static bool IsStateBased(string findingName) => StateBasedAnalyses.Contains(BaseAnalysisName(findingName)); + private bool IsStateBased(string findingName) + { + _stateBasedAnalyses ??= Analyses + .Where(analysis => analysis.IsStateBased) + .Select(analysis => analysis.GetType().Name) + .ToHashSet(); + return _stateBasedAnalyses.Contains(BaseAnalysisName(findingName)); + } /// /// Gets the analysis class name from a finding name, which aggregated @@ -231,39 +232,11 @@ private static string BaseAnalysisName(string findingName) } /// - /// Creates the set of diagnostic analyses to run while the backtest is in progress. - /// Only analyses that read the result snapshot (logs, orders, order events, charts) are - /// included: they are cheap, thread-safe, and detect error conditions whose findings - /// don't depend on the backtest being complete. Curve and statistics based analyses are - /// left to the final analysis, since partial-period statistics are noisy and require - /// history requests. + /// Creates the set of diagnostic analyses to run while the backtest is in progress: + /// the final analysis set filtered to the analyses declaring they can run in-run + /// (see ). /// - protected override IReadOnlyCollection GetAnalyses() => - [ - new PortfolioValueIsNotPositiveAnalysis(), - new InsufficientBuyingPowerOrderResponseErrorAnalysis(), - new MarginCallsAnalysis(), - new ExceedsShortableQuantityOrderResponseErrorAnalysis(), - new SecurityPriceZeroOrderResponseErrorAnalysis(), - new OrderQuantityZeroOrderResponseErrorAnalysis(), - new NonTradableSecurityOrderResponseErrorAnalysis(), - new BrokerageModelRefusedToSubmitOrderOrderResponseErrorAnalysis(), - new BrokerageModelRefusedToUpdateOrderOrderResponseErrorAnalysis(), - new TakeProfitAndStopLossOrdersAnalysis(), - new StaleOrderFillsAnalysis(), - new AlgorithmWarmingUpOrderResponseErrorAnalysis(), - new ExchangeNotOpenOrderResponseErrorAnalysis(), - new ForexConversionRateZeroOrderResponseErrorAnalysis(), - new ExceededMaximumOrdersOrderResponseErrorAnalysis(), - new UnsupportedOptionShortPositionExerciseAnalysis(), - new UnsupportedOptionExerciseQuantityAnalysis(), - new EuropeanOptionNotExpiredOnExerciseOrderResponseErrorAnalysis(), - new OptionOrderOnStockSplitOrderResponseErrorAnalysis(), - new MarketOnOpenNotAllowedDuringRegularHoursOrderResponseErrorAnalysis(), - new OrderQuantityLessThanLotSizeOrderResponseErrorAnalysis(), - new InsightsEmittedForDelistedSecuritiesAnalysis(), - new PortfolioMarginUsageAnalysis(), - new AlgorithmSpeedAnalysis(), - ]; + protected override IReadOnlyCollection GetAnalyses() + => base.GetAnalyses().Where(analysis => analysis.RunsInRun).ToList(); } } diff --git a/Engine/Results/Analysis/ResultsAnalyzer.cs b/Engine/Results/Analysis/ResultsAnalyzer.cs index aa7e377631a1..f064ff22c6f6 100644 --- a/Engine/Results/Analysis/ResultsAnalyzer.cs +++ b/Engine/Results/Analysis/ResultsAnalyzer.cs @@ -145,7 +145,9 @@ protected void SetAnalysisData(Result result, IReadOnlyList logs) } /// - /// Creates the set of diagnostic analyses to run against the backtest. + /// Creates the full set of diagnostic analyses to run against the backtest. + /// Each analysis declares through whether it can + /// also run while the backtest is in progress, which the in-run analyzer filters this set by. /// protected virtual IReadOnlyCollection GetAnalyses() => [ diff --git a/Tests/Engine/Results/InRunResultsAnalyzerTests.cs b/Tests/Engine/Results/InRunResultsAnalyzerTests.cs index 7d290dc609d4..999bf8554d9c 100644 --- a/Tests/Engine/Results/InRunResultsAnalyzerTests.cs +++ b/Tests/Engine/Results/InRunResultsAnalyzerTests.cs @@ -123,12 +123,13 @@ public void StateBasedFindingsAreReplacedOnEveryRun() { var fake = new FakeAnalysisA(10) { - Findings = () => MakeFindings(nameof(PortfolioValueIsNotPositiveAnalysis), "old sample", 2) + StateBased = true, + Findings = () => MakeFindings(nameof(FakeAnalysisA), "old sample", 2) }; var analyzer = new TestInRunResultsAnalyzer(fake); analyzer.Run(1, new[] { "log" }); - fake.Findings = () => MakeFindings(nameof(PortfolioValueIsNotPositiveAnalysis), "new sample", 3); + fake.Findings = () => MakeFindings(nameof(FakeAnalysisA), "new sample", 3); var findings = analyzer.Run(1, new[] { "log" }); // Replaced, not accumulated: latest sample and count win @@ -142,7 +143,8 @@ public void StateBasedFindingsAreDroppedWhenTheyNoLongerFail() { var fake = new FakeAnalysisA(10) { - Findings = () => MakeFindings(nameof(PortfolioValueIsNotPositiveAnalysis), "sample", 2) + StateBased = true, + Findings = () => MakeFindings(nameof(FakeAnalysisA), "sample", 2) }; var analyzer = new TestInRunResultsAnalyzer(fake); Assert.IsNotEmpty(analyzer.Run(1, new[] { "log" })); @@ -158,9 +160,10 @@ public void AggregatedStateBasedFindingsAreReplacedByFullName() { // Aggregated analyses emit "AnalysisClass / SubAnalysis" finding names: state-based // behavior is determined by the base analysis name, replacement is keyed by the full name - var stateBasedName = nameof(PortfolioValueIsNotPositiveAnalysis); + var stateBasedName = nameof(FakeAnalysisA); var fake = new FakeAnalysisA(10) { + StateBased = true, Findings = () => MakeFindings($"{stateBasedName} / SubA", "sample a", 1) .Concat(MakeFindings($"{stateBasedName} / SubB", "sample b", 1)) .ToList() @@ -295,6 +298,20 @@ public void RequiredChartsAreTheChartsReadByTheInRunAnalyses() InRunResultsAnalyzer.RequiredCharts); } + [Test] + public void DefaultAnalysisSetIsTheInRunCapableSubsetOfTheFinalSet() + { + var analyses = new DefaultSetInRunResultsAnalyzer().DefaultAnalyses; + + Assert.IsNotEmpty(analyses); + Assert.IsTrue(analyses.All(analysis => analysis.RunsInRun)); + // Representative membership checks: state-based and stream-based in-run analyses + // are included, final-only ones are not + Assert.IsTrue(analyses.Any(analysis => analysis is AlgorithmSpeedAnalysis)); + Assert.IsTrue(analyses.Any(analysis => analysis is MarginCallsAnalysis)); + Assert.IsFalse(analyses.Any(analysis => analysis is ExecutionSpeedAnalysis)); + } + [Test] public void AnalysesAreCreatedOnceAndReusedAcrossRuns() { @@ -352,6 +369,16 @@ protected override IReadOnlyCollection GetAnalyses() } } + private sealed class DefaultSetInRunResultsAnalyzer : InRunResultsAnalyzer + { + public DefaultSetInRunResultsAnalyzer() + : base(null, Language.CSharp, new FakeDataProvider()) + { + } + + public IReadOnlyCollection DefaultAnalyses => Analyses; + } + private sealed class FakeDataProvider : IInRunAnalysisDataProvider { public Dictionary Orders { get; } = new(); @@ -405,6 +432,10 @@ private class FakeAnalysis : BaseResultsAnalysis public override int Weight => _weight; + public override bool IsStateBased => StateBased; + + public bool StateBased { get; set; } + public Func> Findings { get; set; } = () => new List(); public Action OnRun { get; set; } diff --git a/Tests/Engine/Results/ResultsAnalyzerTests.cs b/Tests/Engine/Results/ResultsAnalyzerTests.cs index 2ed61ded6afd..d2639eb88602 100644 --- a/Tests/Engine/Results/ResultsAnalyzerTests.cs +++ b/Tests/Engine/Results/ResultsAnalyzerTests.cs @@ -162,6 +162,45 @@ public void DefaultAnalysisSetIncludesTheAlgorithmSpeedAndRuntimeErrorAnalyses() Assert.IsTrue(analyses.Any(analysis => analysis is SingleTimeLoopTimeoutRuntimeErrorAnalysis)); } + [Test] + public void DefaultAnalysisSetDeclaresTheFinalOnlyAnalyses() + { + var finalOnly = new DefaultSetResultsAnalyzer().DefaultAnalyses + .Where(analysis => !analysis.RunsInRun) + .Select(analysis => analysis.GetType().Name); + + // These need the completed run (runtime errors, equity curves, final statistics, + // completion logs) or read algorithm state that is not safe to access while it runs + CollectionAssert.AreEquivalent(new[] + { + nameof(SingleTimeLoopTimeoutRuntimeErrorAnalysis), + nameof(FlatEquityCurveAnalysis), + nameof(OrderFillsDuringExtendedMarketHoursAnalysis), + nameof(StatisticalSignificanceOfDailyReturnsAnalysis), + nameof(PerformanceRelativeToBenchmarkAnalysis), + nameof(CrisisEventsAnalysis), + nameof(ExecutionSpeedAnalysis), + nameof(ParameterCountAnalysis), + nameof(MonteCarloPercentileAnalysis), + }, finalOnly); + } + + [Test] + public void DefaultAnalysisSetDeclaresTheStateBasedAnalyses() + { + var stateBased = new DefaultSetResultsAnalyzer().DefaultAnalyses + .Where(analysis => analysis.IsStateBased) + .Select(analysis => analysis.GetType().Name); + + CollectionAssert.AreEquivalent(new[] + { + nameof(PortfolioValueIsNotPositiveAnalysis), + nameof(TakeProfitAndStopLossOrdersAnalysis), + nameof(PortfolioMarginUsageAnalysis), + nameof(AlgorithmSpeedAnalysis), + }, stateBased); + } + private sealed class DefaultSetResultsAnalyzer : ResultsAnalyzer { public DefaultSetResultsAnalyzer() From e48ec49eb2b1d61172e8b51980f6d315872d2b85 Mon Sep 17 00:00:00 2001 From: Jhonathan Abreu Date: Mon, 3 Aug 2026 18:24:03 -0400 Subject: [PATCH 22/33] Merge the in-run results analyzer into the ResultsAnalyzer class A single ResultsAnalyzer now serves both analysis modes through named factory methods: CreateForFinalAnalysis builds a one-shot instance for the completed backtest, and CreateForInRunAnalysis builds the long-lived instance that runs the in-run capable analyses incrementally against the data pulled from its provider. --- .../Analysis/IInRunAnalysisDataProvider.cs | 7 +- .../Results/Analysis/InRunResultsAnalyzer.cs | 242 --------------- Engine/Results/Analysis/ResultsAnalyzer.cs | 288 +++++++++++++++++- Engine/Results/BacktestingResultHandler.cs | 6 +- ...rTests.cs => InRunResultsAnalysisTests.cs} | 13 +- Tests/Engine/Results/ResultsAnalyzerTests.cs | 10 + 6 files changed, 299 insertions(+), 267 deletions(-) delete mode 100644 Engine/Results/Analysis/InRunResultsAnalyzer.cs rename Tests/Engine/Results/{InRunResultsAnalyzerTests.cs => InRunResultsAnalysisTests.cs} (97%) diff --git a/Engine/Results/Analysis/IInRunAnalysisDataProvider.cs b/Engine/Results/Analysis/IInRunAnalysisDataProvider.cs index 0e2189bf0150..f8eb1d2a1db0 100644 --- a/Engine/Results/Analysis/IInRunAnalysisDataProvider.cs +++ b/Engine/Results/Analysis/IInRunAnalysisDataProvider.cs @@ -19,9 +19,10 @@ namespace QuantConnect.Lean.Engine.Results.Analysis { /// - /// Provides the access to the data of the running backtest. - /// Implemented by the result handler, which owns the data and its synchronization, while the - /// analyzer decides what to read and how much, keeping its incremental consumption state private. + /// Provides an in-run instance access to the data of the running + /// backtest. Implemented by the result handler, which owns the data and its synchronization, + /// while the analyzer decides what to read and how much, keeping its incremental consumption + /// state private. /// public interface IInRunAnalysisDataProvider { diff --git a/Engine/Results/Analysis/InRunResultsAnalyzer.cs b/Engine/Results/Analysis/InRunResultsAnalyzer.cs deleted file mode 100644 index 2fd7aefc01a7..000000000000 --- a/Engine/Results/Analysis/InRunResultsAnalyzer.cs +++ /dev/null @@ -1,242 +0,0 @@ -/* - * QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals. - * Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * -*/ -using QuantConnect.Algorithm; -using QuantConnect.Lean.Engine.Results.Analysis.Analyses; -using QuantConnect.Packets; -using QuantConnect.Statistics; -using System; -using System.Collections.Generic; -using System.Linq; - -namespace QuantConnect.Lean.Engine.Results.Analysis -{ - /// - /// Runs a reduced suite of backtest diagnostic tests periodically while the backtest is still running, - /// against a snapshot of the current intermediate results pulled from the - /// . - /// - public class InRunResultsAnalyzer : ResultsAnalyzer - { - private readonly Dictionary _findings = new(); - private readonly QCAlgorithm _algorithm; - private readonly IInRunAnalysisDataProvider _dataProvider; - - /// - /// The names of the analyses in the in-run set that declare themselves state-based - /// (see ), whose findings are replaced - /// on every run instead of accumulated. - /// - private HashSet _stateBasedAnalyses; - - /// - /// The number of order events already consumed by previous runs, from which the next run - /// resumes reading the order event stream. - /// - private int _orderEventsPosition; - - /// - /// The number of log entries already consumed by previous runs, from which the next run - /// resumes reading the log stream. - /// - private int _logsPosition; - - /// - /// The equity and benchmark curves are not built for in-run analysis: - /// none of the in-run analyses read them, and building them would issue - /// a benchmark history request on every run. - /// - protected override bool RequiresEquityCurves => false; - - /// - /// The names of the charts the in-run analyses read. Only these are requested - /// from the data provider on each run. - /// - public static IReadOnlyList RequiredCharts { get; } = [BaseResultsHandler.PortfolioMarginKey]; - - /// - /// Initializes a new instance of the class. - /// The instance is expected to be kept alive for the duration of the backtest, - /// pulling fresh data from on each - /// call. - /// - /// The algorithm instance used for history requests and settings. - /// The programming language the algorithm is written in. - /// Provides access to the data of the running backtest. - public InRunResultsAnalyzer(QCAlgorithm algorithm, Language language, IInRunAnalysisDataProvider dataProvider) - : base(null, algorithm, language, null, new AlgorithmSpeedTracker()) - { - _algorithm = algorithm; - _dataProvider = dataProvider; - } - - /// - /// Runs the analyses against the current backtest state pulled from the data provider: - /// a snapshot of the orders and required charts, plus only the order events and log lines - /// produced since the previous run. The returned findings are the merge of this run's - /// findings into the ones accumulated by previous runs: findings from analyses scanning - /// the order event and log streams are accumulated (first sample kept, counts totaled), - /// while findings from state-based analyses are replaced on every run. - /// - /// The current total algorithm performance, for analyses that read - /// portfolio statistics. Withheld from the analyses until the first equity sample exists, since - /// the statistics are all-zero defaults before that. - /// Wall-clock seconds allowed for the full chain before early exit. - /// The default is small because the analysis runs on the result handler thread, delaying message - /// processing while it runs. - /// Maximum number of failing analyses to return. - /// The accumulated findings, ranked by analysis weight. - public IReadOnlyList Run(AlgorithmPerformance totalPerformance, int timeLimitSeconds = 1, - int maxFailedAnalyses = 10) - { - // The analyses read the charts without synchronizing with the result handler, so they get clones - var charts = _dataProvider.GetChartSnapshots(RequiredCharts); - - // Equity is not sampled while the algorithm warms up, so until the first sample exists - // the generated statistics are all-zero defaults that would flag a false non-positive - // portfolio value finding. Withhold them so the analyses reading them skip instead - if (_algorithm?.IsWarmingUp == true || !_dataProvider.HasEquitySamples()) - { - totalPerformance = null; - } - - var snapshot = new BacktestResult(new BacktestResultParameters( - charts, - _dataProvider.GetOrders(), - _algorithm?.Transactions.TransactionRecord ?? new Dictionary(), - new Dictionary(), - new Dictionary(), - new Dictionary(), - _dataProvider.GetOrderEvents(_orderEventsPosition), - totalPerformance)); - var logs = _dataProvider.GetLogs(_logsPosition); - - // The analyses run during warm-up too (the speed sample is null then), so conditions like - // orders submitted while the algorithm warms up surface without waiting for warm-up to end - return Run(snapshot, logs, _dataProvider.TakeSpeedSample(), timeLimitSeconds, maxFailedAnalyses); - } - - /// - /// Completes the speed metrics with one final sample so they cover the backtest through - /// its end, and returns the tracker for the final analysis to reuse. The tracker is left - /// untouched when no sample can be taken, like when the algorithm never left warm-up. - /// - public AlgorithmSpeedTracker CompleteSpeedTracking() - { - var speedSample = _dataProvider.TakeSpeedSample(); - if (speedSample.HasValue) - { - SpeedTracker.AddSample(speedSample.Value); - } - return SpeedTracker; - } - - /// - /// Runs the analyses incrementally: and are - /// expected to contain only the order events and log lines produced since the previous run, - /// and the returned findings are the merge of this run's findings into the ones accumulated - /// by previous runs. - /// - /// A snapshot of the current intermediate backtest result, holding only new order events. - /// The log lines produced since the previous run. - /// A sample of the engine speed counters for the algorithm speed analysis. - /// Null when the counters should not be sampled, like while the algorithm warms up. - /// Wall-clock seconds allowed for the full chain before early exit. - /// Maximum number of failing analyses to return. - /// The accumulated findings, ranked by analysis weight. - private IReadOnlyList Run(Result result, IReadOnlyList logs, AlgorithmSpeedSample? speedSample, - int timeLimitSeconds, int maxFailedAnalyses) - { - SetAnalysisData(result, logs); - if (speedSample.HasValue) - { - SpeedTracker.AddSample(speedSample.Value); - } - var newFindings = Run(timeLimitSeconds, maxFailedAnalyses); - - // The positions are advanced even when the time limit truncates a run, so the analyses that - // didn't get to run miss this delta until the final analysis re-scans the complete streams. - // Stress tests show runs complete in a fraction of the time limit, but if its trace message - // starts showing up in logs, revisit this (e.g. track per-analysis positions). - _orderEventsPosition += result.OrderEvents?.Count ?? 0; - _logsPosition += logs?.Count ?? 0; - - // State-based analyses are recomputed from scratch each run: remove their previous - // findings so they are replaced, or dropped if they no longer fail. If a time-limit - // truncated run skipped one of them, its finding drops until a run reaches it again — - // same trade-off as the positions advancement above - foreach (var name in _findings.Keys.Where(IsStateBased).ToList()) - { - _findings.Remove(name); - } - - foreach (var finding in newFindings) - { - if (!IsStateBased(finding.Name) && _findings.TryGetValue(finding.Name, out var previous)) - { - // This run only saw new order events and logs: keep the first sample and total the counts. - // A null count means a single occurrence - finding.Sample = previous.Sample; - finding.Count = (previous.Count ?? 1) + (finding.Count ?? 1); - } - _findings[finding.Name] = finding; - } - - return RankFindings(maxFailedAnalyses); - } - - /// - /// Ranks the accumulated findings by their analysis weight, capped to the given maximum. - /// - private IReadOnlyList RankFindings(int maxFailedAnalyses) - { - var weights = Analyses.ToDictionary(analysis => analysis.GetType().Name, analysis => analysis.Weight); - return _findings.Values - .OrderByDescending(finding => weights.GetValueOrDefault(BaseAnalysisName(finding.Name))) - .Take(maxFailedAnalyses) - .ToList(); - } - - /// - /// Determines whether the given finding was produced by a state-based analysis. - /// - private bool IsStateBased(string findingName) - { - _stateBasedAnalyses ??= Analyses - .Where(analysis => analysis.IsStateBased) - .Select(analysis => analysis.GetType().Name) - .ToHashSet(); - return _stateBasedAnalyses.Contains(BaseAnalysisName(findingName)); - } - - /// - /// Gets the analysis class name from a finding name, which aggregated - /// analyses suffix with the sub-analysis name. - /// - private static string BaseAnalysisName(string findingName) - { - var separatorIndex = findingName.IndexOf(" / ", StringComparison.Ordinal); - return separatorIndex < 0 ? findingName : findingName[..separatorIndex]; - } - - /// - /// Creates the set of diagnostic analyses to run while the backtest is in progress: - /// the final analysis set filtered to the analyses declaring they can run in-run - /// (see ). - /// - protected override IReadOnlyCollection GetAnalyses() - => base.GetAnalyses().Where(analysis => analysis.RunsInRun).ToList(); - } -} diff --git a/Engine/Results/Analysis/ResultsAnalyzer.cs b/Engine/Results/Analysis/ResultsAnalyzer.cs index f064ff22c6f6..bd04c5ee1fa4 100644 --- a/Engine/Results/Analysis/ResultsAnalyzer.cs +++ b/Engine/Results/Analysis/ResultsAnalyzer.cs @@ -16,7 +16,9 @@ using QuantConnect.Algorithm; using QuantConnect.Lean.Engine.Results.Analysis.Analyses; using QuantConnect.Logging; +using QuantConnect.Packets; using QuantConnect.Securities; +using QuantConnect.Statistics; using System; using System.Collections.Generic; using System.Diagnostics; @@ -25,12 +27,20 @@ namespace QuantConnect.Lean.Engine.Results.Analysis { /// - /// Runs the full suite of backtest diagnostic tests against a single backtest. + /// Runs the suite of backtest diagnostic tests against a single backtest, in one of two modes + /// depending on how the instance is created: + /// a final analysis instance is created with the completed result and logs, and runs the + /// full analysis set once; an in-run analysis instance is created with an + /// , is kept alive for the duration of the backtest, + /// and periodically runs the in-run capable analyses incrementally against snapshots of the + /// intermediate results pulled from the provider. /// public class ResultsAnalyzer { private readonly QCAlgorithm _algorithm; private readonly Language _language; + private readonly IInRunAnalysisDataProvider _dataProvider; + private readonly Dictionary _findings = new(); private IReadOnlyList _logs; private SortedList _equityCurve; private SortedList _benchmarkEquityCurve; @@ -38,35 +48,74 @@ public class ResultsAnalyzer private IReadOnlyCollection _analyses; /// - /// The diagnostic analyses to run. Created once and reused across runs, - /// since the analyses are stateless. + /// The names of the analyses in the in-run set that declare themselves state-based + /// (see ), whose findings are replaced + /// on every run instead of accumulated. /// - protected IReadOnlyCollection Analyses => _analyses ??= GetAnalyses(); + private HashSet _stateBasedAnalyses; + + /// + /// The number of order events already consumed by previous in-run runs, from which the + /// next run resumes reading the order event stream. + /// + private int _orderEventsPosition; + + /// + /// The number of log entries already consumed by previous in-run runs, from which the + /// next run resumes reading the log stream. + /// + private int _logsPosition; + + /// + /// Whether this instance was created for in-run analysis of a backtest still in progress + /// (see ), as opposed to the final analysis of a + /// completed backtest. + /// + protected bool IsInRun => _dataProvider != null; + + /// + /// The diagnostic analyses to run. Created once and reused across runs, since the analyses + /// are stateless. In-run instances filter the set to the analyses that declare they can run + /// while the backtest is in progress (see ). + /// + protected IReadOnlyCollection Analyses => _analyses ??= IsInRun + ? GetAnalyses().Where(analysis => analysis.RunsInRun).ToList() + : GetAnalyses(); /// /// Whether the equity and benchmark curves should be built before running the analyses. - /// Building them requires a benchmark history request, so analyzers whose analyses - /// don't read the curves can skip it. + /// Building them requires a benchmark history request, so in-run instances skip it: + /// none of the in-run analyses read the curves, and building them would issue the + /// history request on every run. /// - protected virtual bool RequiresEquityCurves => true; + protected virtual bool RequiresEquityCurves => !IsInRun; /// /// The speed metrics tracked for the running backtest, made available to the analyses - /// through . The in-run analyzer feeds - /// it a sample on each run; the final analyzer receives the same tracker so the speed - /// analysis also runs against the full-run metrics. Null when speed is not tracked. + /// through . An in-run instance owns its + /// tracker and feeds it a sample on each run; the final analysis instance receives the same + /// tracker so the speed analysis also runs against the full-run metrics. Null when speed is + /// not tracked. /// protected AlgorithmSpeedTracker SpeedTracker { get; } /// - /// Initializes a new instance of the class. + /// The names of the charts the in-run analyses read. Only these are requested + /// from the data provider on each in-run run. + /// + public static IReadOnlyList RequiredCharts { get; } = [BaseResultsHandler.PortfolioMarginKey]; + + /// + /// Initializes a new instance of the class for the final + /// analysis of a completed backtest. Use or + /// to create instances. /// /// The backtest result to analyze. /// The algorithm instance used for history requests and settings. /// The programming language the algorithm is written in. /// The full list of log lines produced by the backtest. /// The speed metrics tracked for the backtest, or null when speed is not tracked. - public ResultsAnalyzer(Result result, QCAlgorithm algorithm, Language language, IReadOnlyList logs, + protected ResultsAnalyzer(Result result, QCAlgorithm algorithm, Language language, IReadOnlyList logs, AlgorithmSpeedTracker speedTracker = null) { _result = result; @@ -76,6 +125,51 @@ public ResultsAnalyzer(Result result, QCAlgorithm algorithm, Language language, SpeedTracker = speedTracker; } + /// + /// Initializes a new instance of the class for in-run analysis + /// of a backtest still in progress. Use to create instances. + /// + /// The algorithm instance used for history requests and settings. + /// The programming language the algorithm is written in. + /// Provides access to the data of the running backtest. + protected ResultsAnalyzer(QCAlgorithm algorithm, Language language, IInRunAnalysisDataProvider dataProvider) + : this(null, algorithm, language, null, new AlgorithmSpeedTracker()) + { + _dataProvider = dataProvider; + } + + /// + /// Creates an analyzer for the final analysis of a completed backtest, running the full + /// analysis set once through . + /// + /// The backtest result to analyze. + /// The algorithm instance used for history requests and settings. + /// The programming language the algorithm is written in. + /// The full list of log lines produced by the backtest. + /// The speed metrics tracked for the backtest, typically completed by the + /// in-run analyzer through , or null when speed is not tracked. + /// The final analysis instance. + public static ResultsAnalyzer CreateForFinalAnalysis(Result result, QCAlgorithm algorithm, Language language, + IReadOnlyList logs, AlgorithmSpeedTracker speedTracker = null) + { + return new ResultsAnalyzer(result, algorithm, language, logs, speedTracker); + } + + /// + /// Creates an analyzer for in-run analysis of a backtest still in progress. The instance is + /// expected to be kept alive for the duration of the backtest, pulling fresh data from + /// on each call. + /// + /// The algorithm instance used for history requests and settings. + /// The programming language the algorithm is written in. + /// Provides access to the data of the running backtest. + /// The in-run analysis instance. + public static ResultsAnalyzer CreateForInRunAnalysis(QCAlgorithm algorithm, Language language, + IInRunAnalysisDataProvider dataProvider) + { + return new ResultsAnalyzer(algorithm, language, dataProvider); + } + // ── Test chain ──────────────────────────────────────────────────────────── /// @@ -133,7 +227,72 @@ public ResultsAnalyzer(Result result, QCAlgorithm algorithm, Language language, } /// - /// Sets the backtest data to analyze. Used by analyzers that are kept alive + /// Runs the in-run analyses against the current backtest state pulled from the data provider: + /// a snapshot of the orders and required charts, plus only the order events and log lines + /// produced since the previous run. The returned findings are the merge of this run's + /// findings into the ones accumulated by previous runs: findings from analyses scanning + /// the order event and log streams are accumulated (first sample kept, counts totaled), + /// while findings from state-based analyses are replaced on every run. + /// + /// The current total algorithm performance, for analyses that read + /// portfolio statistics. Withheld from the analyses until the first equity sample exists, since + /// the statistics are all-zero defaults before that. + /// Wall-clock seconds allowed for the full chain before early exit. + /// The default is small because the analysis runs on the result handler thread, delaying message + /// processing while it runs. + /// Maximum number of failing analyses to return. + /// The accumulated findings, ranked by analysis weight. + public IReadOnlyList Run(AlgorithmPerformance totalPerformance, int timeLimitSeconds = 1, + int maxFailedAnalyses = 10) + { + ThrowIfNotInRunInstance(); + + // The analyses read the charts without synchronizing with the result handler, so they get clones + var charts = _dataProvider.GetChartSnapshots(RequiredCharts); + + // Equity is not sampled while the algorithm warms up, so until the first sample exists + // the generated statistics are all-zero defaults that would flag a false non-positive + // portfolio value finding. Withhold them so the analyses reading them skip instead + if (_algorithm?.IsWarmingUp == true || !_dataProvider.HasEquitySamples()) + { + totalPerformance = null; + } + + var snapshot = new BacktestResult(new BacktestResultParameters( + charts, + _dataProvider.GetOrders(), + _algorithm?.Transactions.TransactionRecord ?? new Dictionary(), + new Dictionary(), + new Dictionary(), + new Dictionary(), + _dataProvider.GetOrderEvents(_orderEventsPosition), + totalPerformance)); + var logs = _dataProvider.GetLogs(_logsPosition); + + // The analyses run during warm-up too (the speed sample is null then), so conditions like + // orders submitted while the algorithm warms up surface without waiting for warm-up to end + return Run(snapshot, logs, _dataProvider.TakeSpeedSample(), timeLimitSeconds, maxFailedAnalyses); + } + + /// + /// Completes the speed metrics with one final sample so they cover the backtest through + /// its end, and returns the tracker for the final analysis to reuse. The tracker is left + /// untouched when no sample can be taken, like when the algorithm never left warm-up. + /// + public AlgorithmSpeedTracker CompleteSpeedTracking() + { + ThrowIfNotInRunInstance(); + + var speedSample = _dataProvider.TakeSpeedSample(); + if (speedSample.HasValue) + { + SpeedTracker.AddSample(speedSample.Value); + } + return SpeedTracker; + } + + /// + /// Sets the backtest data to analyze. Used by in-run instances, which are kept alive /// and run multiple times against fresh data. /// /// The backtest result to analyze. @@ -147,7 +306,7 @@ protected void SetAnalysisData(Result result, IReadOnlyList logs) /// /// Creates the full set of diagnostic analyses to run against the backtest. /// Each analysis declares through whether it can - /// also run while the backtest is in progress, which the in-run analyzer filters this set by. + /// also run while the backtest is in progress, which in-run instances filter this set by. /// protected virtual IReadOnlyCollection GetAnalyses() => [ @@ -186,6 +345,107 @@ protected virtual IReadOnlyCollection GetAnalyses() => new MonteCarloPercentileAnalysis(), ]; + /// + /// Runs the in-run analyses incrementally: and + /// are expected to contain only the order events and log lines produced since the previous run, + /// and the returned findings are the merge of this run's findings into the ones accumulated + /// by previous runs. + /// + /// A snapshot of the current intermediate backtest result, holding only new order events. + /// The log lines produced since the previous run. + /// A sample of the engine speed counters for the algorithm speed analysis. + /// Null when the counters should not be sampled, like while the algorithm warms up. + /// Wall-clock seconds allowed for the full chain before early exit. + /// Maximum number of failing analyses to return. + /// The accumulated findings, ranked by analysis weight. + private IReadOnlyList Run(Result result, IReadOnlyList logs, AlgorithmSpeedSample? speedSample, + int timeLimitSeconds, int maxFailedAnalyses) + { + SetAnalysisData(result, logs); + if (speedSample.HasValue) + { + SpeedTracker.AddSample(speedSample.Value); + } + var newFindings = Run(timeLimitSeconds, maxFailedAnalyses); + + // The positions are advanced even when the time limit truncates a run, so the analyses that + // didn't get to run miss this delta until the final analysis re-scans the complete streams. + // Stress tests show runs complete in a fraction of the time limit, but if its trace message + // starts showing up in logs, revisit this (e.g. track per-analysis positions). + _orderEventsPosition += result.OrderEvents?.Count ?? 0; + _logsPosition += logs?.Count ?? 0; + + // State-based analyses are recomputed from scratch each run: remove their previous + // findings so they are replaced, or dropped if they no longer fail. If a time-limit + // truncated run skipped one of them, its finding drops until a run reaches it again — + // same trade-off as the positions advancement above + foreach (var name in _findings.Keys.Where(IsStateBased).ToList()) + { + _findings.Remove(name); + } + + foreach (var finding in newFindings) + { + if (!IsStateBased(finding.Name) && _findings.TryGetValue(finding.Name, out var previous)) + { + // This run only saw new order events and logs: keep the first sample and total the counts. + // A null count means a single occurrence + finding.Sample = previous.Sample; + finding.Count = (previous.Count ?? 1) + (finding.Count ?? 1); + } + _findings[finding.Name] = finding; + } + + return RankFindings(maxFailedAnalyses); + } + + /// + /// Ranks the accumulated findings by their analysis weight, capped to the given maximum. + /// + private IReadOnlyList RankFindings(int maxFailedAnalyses) + { + var weights = Analyses.ToDictionary(analysis => analysis.GetType().Name, analysis => analysis.Weight); + return _findings.Values + .OrderByDescending(finding => weights.GetValueOrDefault(BaseAnalysisName(finding.Name))) + .Take(maxFailedAnalyses) + .ToList(); + } + + /// + /// Determines whether the given finding was produced by a state-based analysis. + /// + private bool IsStateBased(string findingName) + { + _stateBasedAnalyses ??= Analyses + .Where(analysis => analysis.IsStateBased) + .Select(analysis => analysis.GetType().Name) + .ToHashSet(); + return _stateBasedAnalyses.Contains(BaseAnalysisName(findingName)); + } + + /// + /// Gets the analysis class name from a finding name, which aggregated + /// analyses suffix with the sub-analysis name. + /// + private static string BaseAnalysisName(string findingName) + { + var separatorIndex = findingName.IndexOf(" / ", StringComparison.Ordinal); + return separatorIndex < 0 ? findingName : findingName[..separatorIndex]; + } + + /// + /// Throws when this instance was not created for in-run analysis, guarding the members + /// that read the data provider. + /// + private void ThrowIfNotInRunInstance() + { + if (!IsInRun) + { + throw new InvalidOperationException( + "This operation requires an instance created for in-run analysis, with a data provider."); + } + } + /// /// Reads the backtest's "Strategy Equity" chart and fetches SPY daily history to build /// two time-aligned equity curves: one for the backtest and one for the benchmark. diff --git a/Engine/Results/BacktestingResultHandler.cs b/Engine/Results/BacktestingResultHandler.cs index 7dfe45d44368..912a84adcbd2 100644 --- a/Engine/Results/BacktestingResultHandler.cs +++ b/Engine/Results/BacktestingResultHandler.cs @@ -53,7 +53,7 @@ public class BacktestingResultHandler : BaseResultsHandler, IResultHandler, IInR private BacktestProgressMonitor _progressMonitor; - private InRunResultsAnalyzer _inRunResultsAnalyzer; + private ResultsAnalyzer _inRunResultsAnalyzer; private string _lastInRunAnalysisSignature = "[]"; /// @@ -430,7 +430,7 @@ protected void SendFinalResult() // The final analysis reuses the speed metrics accumulated by the in-run analyzer, // completed with one last sample so they cover the backtest through its end var speedTracker = _inRunResultsAnalyzer?.CompleteSpeedTracking(); - var analyzer = new ResultsAnalyzer(result.Results, AlgorithmInstance, _job.Language, logs, speedTracker); + var analyzer = ResultsAnalyzer.CreateForFinalAnalysis(result.Results, AlgorithmInstance, _job.Language, logs, speedTracker); try { result.Results.Analysis = analyzer.Run(); @@ -473,7 +473,7 @@ protected void SendFinalResult() return null; } - _inRunResultsAnalyzer ??= new InRunResultsAnalyzer(AlgorithmInstance, _job.Language, this); + _inRunResultsAnalyzer ??= ResultsAnalyzer.CreateForInRunAnalysis(AlgorithmInstance, _job.Language, this); return _inRunResultsAnalyzer.Run(totalPerformance); } catch (Exception ex) diff --git a/Tests/Engine/Results/InRunResultsAnalyzerTests.cs b/Tests/Engine/Results/InRunResultsAnalysisTests.cs similarity index 97% rename from Tests/Engine/Results/InRunResultsAnalyzerTests.cs rename to Tests/Engine/Results/InRunResultsAnalysisTests.cs index 999bf8554d9c..44f026becbd2 100644 --- a/Tests/Engine/Results/InRunResultsAnalyzerTests.cs +++ b/Tests/Engine/Results/InRunResultsAnalysisTests.cs @@ -29,7 +29,7 @@ namespace QuantConnect.Tests.Engine.Results { [TestFixture] - public class InRunResultsAnalyzerTests + public class InRunResultsAnalysisTests { private static readonly IReadOnlyList SomeSolutions = new[] { "A solution" }; @@ -232,7 +232,7 @@ public void SnapshotIsBuiltFromTheDataProvider() analyzer.Run(2, new[] { "log" }); // Only the charts the in-run analyses read are requested from the provider - CollectionAssert.AreEqual(InRunResultsAnalyzer.RequiredCharts, analyzer.Provider.RequestedChartNames); + CollectionAssert.AreEqual(ResultsAnalyzer.RequiredCharts, analyzer.Provider.RequestedChartNames); Assert.IsTrue(seenParameters.Result.Charts.ContainsKey("a chart")); Assert.AreEqual(1, seenParameters.Result.Orders.Count); Assert.AreEqual(2, seenParameters.Result.OrderEvents.Count); @@ -295,7 +295,7 @@ public void RequiredChartsAreTheChartsReadByTheInRunAnalyses() // so this must stay in sync with the charts the in-run analyses read CollectionAssert.AreEquivalent( new[] { BaseResultsHandler.PortfolioMarginKey }, - InRunResultsAnalyzer.RequiredCharts); + ResultsAnalyzer.RequiredCharts); } [Test] @@ -329,7 +329,7 @@ public void AnalysesAreCreatedOnceAndReusedAcrossRuns() return new List { new(name, "An issue", sample, count, SomeSolutions) }; } - private class TestInRunResultsAnalyzer : InRunResultsAnalyzer + private class TestInRunResultsAnalyzer : ResultsAnalyzer { private readonly IReadOnlyCollection _analyses; @@ -369,7 +369,7 @@ protected override IReadOnlyCollection GetAnalyses() } } - private sealed class DefaultSetInRunResultsAnalyzer : InRunResultsAnalyzer + private sealed class DefaultSetInRunResultsAnalyzer : ResultsAnalyzer { public DefaultSetInRunResultsAnalyzer() : base(null, Language.CSharp, new FakeDataProvider()) @@ -432,6 +432,9 @@ private class FakeAnalysis : BaseResultsAnalysis public override int Weight => _weight; + // The in-run instances only run the analyses declaring RunsInRun + public override bool RunsInRun { get; } = true; + public override bool IsStateBased => StateBased; public bool StateBased { get; set; } diff --git a/Tests/Engine/Results/ResultsAnalyzerTests.cs b/Tests/Engine/Results/ResultsAnalyzerTests.cs index d2639eb88602..72b95820b46b 100644 --- a/Tests/Engine/Results/ResultsAnalyzerTests.cs +++ b/Tests/Engine/Results/ResultsAnalyzerTests.cs @@ -201,6 +201,16 @@ public void DefaultAnalysisSetDeclaresTheStateBasedAnalyses() }, stateBased); } + [Test] + public void InRunOperationsRequireAnInstanceCreatedWithADataProvider() + { + // This is a final-analysis instance: the in-run entry points need the data provider + var analyzer = new TestResultsAnalyzer(false, new FakeAnalysisA(10)); + + Assert.Throws(() => analyzer.Run(totalPerformance: null)); + Assert.Throws(() => analyzer.CompleteSpeedTracking()); + } + private sealed class DefaultSetResultsAnalyzer : ResultsAnalyzer { public DefaultSetResultsAnalyzer() From 3042cb2ad1c653f1e887be82bd28730aeb042058 Mon Sep 17 00:00:00 2001 From: Jhonathan Abreu Date: Tue, 4 Aug 2026 09:30:39 -0400 Subject: [PATCH 23/33] Default RunsInRun to true, overridden only by the final-only analyses --- .../Results/Analysis/Analyses/AlgorithmSpeedAnalysis.cs | 5 ----- Engine/Results/Analysis/Analyses/BaseResultsAnalysis.cs | 9 +++++---- Engine/Results/Analysis/Analyses/CrisisEventsAnalysis.cs | 5 +++++ .../Results/Analysis/Analyses/ExecutionSpeedAnalysis.cs | 6 ++++++ .../Results/Analysis/Analyses/FlatEquityCurveAnalysis.cs | 5 +++++ .../InsightsEmittedForDelistedSecuritiesAnalysis.cs | 5 ----- Engine/Results/Analysis/Analyses/MarginCallsAnalysis.cs | 5 ----- .../Analysis/Analyses/MonteCarloPercentileAnalysis.cs | 5 +++++ .../OrderFillsDuringExtendedMarketHoursAnalysis.cs | 5 +++++ .../AlgorithmWarmingUpOrderResponseErrorAnalysis.cs | 5 ----- ...odelRefusedToSubmitOrderOrderResponseErrorAnalysis.cs | 5 ----- ...odelRefusedToUpdateOrderOrderResponseErrorAnalysis.cs | 5 ----- ...tionNotExpiredOnExerciseOrderResponseErrorAnalysis.cs | 5 ----- .../ExceededMaximumOrdersOrderResponseErrorAnalysis.cs | 5 ----- ...ExceedsShortableQuantityOrderResponseErrorAnalysis.cs | 5 ----- .../ExchangeNotOpenOrderResponseErrorAnalysis.cs | 5 ----- .../ForexConversionRateZeroOrderResponseErrorAnalysis.cs | 5 ----- .../InsufficientBuyingPowerOrderResponseErrorAnalysis.cs | 5 ----- ...llowedDuringRegularHoursOrderResponseErrorAnalysis.cs | 5 ----- .../NonTradableSecurityOrderResponseErrorAnalysis.cs | 5 ----- .../OptionOrderOnStockSplitOrderResponseErrorAnalysis.cs | 5 ----- ...rQuantityLessThanLotSizeOrderResponseErrorAnalysis.cs | 5 ----- .../OrderQuantityZeroOrderResponseErrorAnalysis.cs | 5 ----- .../SecurityPriceZeroOrderResponseErrorAnalysis.cs | 5 ----- .../UnsupportedOptionExerciseQuantityAnalysis.cs | 5 ----- .../UnsupportedOptionShortPositionExerciseAnalysis.cs | 5 ----- .../Results/Analysis/Analyses/ParameterCountAnalysis.cs | 6 ++++++ .../Analyses/PerformanceRelativeToBenchmarkAnalysis.cs | 5 +++++ .../Analysis/Analyses/PortfolioMarginUsageAnalysis.cs | 5 ----- .../Analyses/PortfolioValueIsNotPositiveAnalysis.cs | 5 ----- .../SingleTimeLoopTimeoutRuntimeErrorAnalysis.cs | 5 +++++ .../Results/Analysis/Analyses/StaleOrderFillsAnalysis.cs | 5 ----- .../StatisticalSignificanceOfDailyReturnsAnalysis.cs | 5 +++++ .../Analyses/TakeProfitAndStopLossOrdersAnalysis.cs | 5 ----- Tests/Engine/Results/InRunResultsAnalysisTests.cs | 3 --- 35 files changed, 52 insertions(+), 127 deletions(-) diff --git a/Engine/Results/Analysis/Analyses/AlgorithmSpeedAnalysis.cs b/Engine/Results/Analysis/Analyses/AlgorithmSpeedAnalysis.cs index dae23e2d5c3f..d270fa90f77d 100644 --- a/Engine/Results/Analysis/Analyses/AlgorithmSpeedAnalysis.cs +++ b/Engine/Results/Analysis/Analyses/AlgorithmSpeedAnalysis.cs @@ -77,11 +77,6 @@ public class AlgorithmSpeedAnalysis : BaseResultsAnalysis /// public const string HistoryRequestLoadName = "HistoryRequestLoad"; - /// - /// This analysis reads only the tracked speed metrics, so it also runs while the backtest is in progress. - /// - public override bool RunsInRun { get; } = true; - /// /// This analysis reads the current speed metrics instead of scanning the order event /// and log streams, so its in-run findings are replaced on every run. diff --git a/Engine/Results/Analysis/Analyses/BaseResultsAnalysis.cs b/Engine/Results/Analysis/Analyses/BaseResultsAnalysis.cs index d90cf01245cb..e6d39a4bd520 100644 --- a/Engine/Results/Analysis/Analyses/BaseResultsAnalysis.cs +++ b/Engine/Results/Analysis/Analyses/BaseResultsAnalysis.cs @@ -36,11 +36,12 @@ public abstract class BaseResultsAnalysis /// /// Whether this analysis can also run while the backtest is still in progress, against a - /// snapshot of the intermediate results. Analyses that need the completed run (runtime - /// errors, equity curves, final statistics, completion logs) or that read algorithm state - /// that is not safe to access while it runs are left to the final analysis only. + /// snapshot of the intermediate results. Most analyses read only the result snapshot, so + /// this defaults to true. Analyses that need the completed run (runtime errors, equity + /// curves, final statistics, completion logs) or that read algorithm state that is not + /// safe to access while it runs override this to leave them to the final analysis only. /// - public virtual bool RunsInRun { get; } + public virtual bool RunsInRun { get; } = true; /// /// Whether this analysis reads the current backtest state (statistics, orders, charts) diff --git a/Engine/Results/Analysis/Analyses/CrisisEventsAnalysis.cs b/Engine/Results/Analysis/Analyses/CrisisEventsAnalysis.cs index a24c32988882..343a9e97668e 100644 --- a/Engine/Results/Analysis/Analyses/CrisisEventsAnalysis.cs +++ b/Engine/Results/Analysis/Analyses/CrisisEventsAnalysis.cs @@ -30,6 +30,11 @@ namespace QuantConnect.Lean.Engine.Results.Analysis.Analyses /// public class CrisisEventsAnalysis : BaseResultsAnalysis { + /// + /// This analysis compares the equity and benchmark curves, which are only built for the final analysis. + /// + public override bool RunsInRun { get; } = false; + /// /// Gets the description indicating that the strategy underperformed the benchmark during crisis events. /// diff --git a/Engine/Results/Analysis/Analyses/ExecutionSpeedAnalysis.cs b/Engine/Results/Analysis/Analyses/ExecutionSpeedAnalysis.cs index 051e7432a1e6..4a18ab43427c 100644 --- a/Engine/Results/Analysis/Analyses/ExecutionSpeedAnalysis.cs +++ b/Engine/Results/Analysis/Analyses/ExecutionSpeedAnalysis.cs @@ -31,6 +31,12 @@ public class ExecutionSpeedAnalysis : BaseResultsAnalysis /// public const int SlowDataPointsPerSecond = 40_000; + /// + /// This analysis reads the engine's completion logs, which only exist once the backtest ends. + /// While it runs, tracks the algorithm's speed instead. + /// + public override bool RunsInRun { get; } = false; + /// /// Gets the description of the slow execution issue. /// diff --git a/Engine/Results/Analysis/Analyses/FlatEquityCurveAnalysis.cs b/Engine/Results/Analysis/Analyses/FlatEquityCurveAnalysis.cs index 78864a90f2d5..b03916ab3292 100644 --- a/Engine/Results/Analysis/Analyses/FlatEquityCurveAnalysis.cs +++ b/Engine/Results/Analysis/Analyses/FlatEquityCurveAnalysis.cs @@ -24,6 +24,11 @@ namespace QuantConnect.Lean.Engine.Results.Analysis.Analyses /// public class FlatEquityCurveAnalysis : BaseResultsAnalysis { + /// + /// This analysis scans the equity curve, which is only built for the final analysis. + /// + public override bool RunsInRun { get; } = false; + /// /// Gets the description of the flat equity curve issue. /// diff --git a/Engine/Results/Analysis/Analyses/InsightsEmittedForDelistedSecuritiesAnalysis.cs b/Engine/Results/Analysis/Analyses/InsightsEmittedForDelistedSecuritiesAnalysis.cs index 95cc64ba5149..839b245fdbb4 100644 --- a/Engine/Results/Analysis/Analyses/InsightsEmittedForDelistedSecuritiesAnalysis.cs +++ b/Engine/Results/Analysis/Analyses/InsightsEmittedForDelistedSecuritiesAnalysis.cs @@ -23,11 +23,6 @@ namespace QuantConnect.Lean.Engine.Results.Analysis.Analyses /// public class InsightsEmittedForDelistedSecuritiesAnalysis : MessageAnalysis { - /// - /// This analysis reads only the result snapshot, so it also runs while the backtest is in progress. - /// - public override bool RunsInRun { get; } = true; - /// /// Description of the delisted-security insight emission issue detected by this analysis. /// diff --git a/Engine/Results/Analysis/Analyses/MarginCallsAnalysis.cs b/Engine/Results/Analysis/Analyses/MarginCallsAnalysis.cs index aa2ceb64d8e2..554ad652220e 100644 --- a/Engine/Results/Analysis/Analyses/MarginCallsAnalysis.cs +++ b/Engine/Results/Analysis/Analyses/MarginCallsAnalysis.cs @@ -24,11 +24,6 @@ namespace QuantConnect.Lean.Engine.Results.Analysis.Analyses /// public class MarginCallsAnalysis : MessageAnalysis { - /// - /// This analysis reads only the result snapshot, so it also runs while the backtest is in progress. - /// - public override bool RunsInRun { get; } = true; - /// /// Description of the margin-call issue detected by this analysis. /// diff --git a/Engine/Results/Analysis/Analyses/MonteCarloPercentileAnalysis.cs b/Engine/Results/Analysis/Analyses/MonteCarloPercentileAnalysis.cs index 8690657758c3..c2801c0aaaf8 100644 --- a/Engine/Results/Analysis/Analyses/MonteCarloPercentileAnalysis.cs +++ b/Engine/Results/Analysis/Analyses/MonteCarloPercentileAnalysis.cs @@ -25,6 +25,11 @@ namespace QuantConnect.Lean.Engine.Results.Analysis.Analyses /// public class MonteCarloPercentileAnalysis : BaseResultsAnalysis { + /// + /// This analysis runs simulations over the equity curve, which is only built for the final analysis. + /// + public override bool RunsInRun { get; } = false; + /// /// Gets the description of the overly optimistic equity curve issue. /// diff --git a/Engine/Results/Analysis/Analyses/OrderFillsDuringExtendedMarketHoursAnalysis.cs b/Engine/Results/Analysis/Analyses/OrderFillsDuringExtendedMarketHoursAnalysis.cs index e835704cf3b5..9112c65f5607 100644 --- a/Engine/Results/Analysis/Analyses/OrderFillsDuringExtendedMarketHoursAnalysis.cs +++ b/Engine/Results/Analysis/Analyses/OrderFillsDuringExtendedMarketHoursAnalysis.cs @@ -26,6 +26,11 @@ namespace QuantConnect.Lean.Engine.Results.Analysis.Analyses /// public class OrderFillsDuringExtendedMarketHoursAnalysis : BaseResultsAnalysis { + /// + /// This analysis reads the algorithm's securities, which are not safe to access while it runs. + /// + public override bool RunsInRun { get; } = false; + /// /// Gets the description of the extended market hours fill issue. /// diff --git a/Engine/Results/Analysis/Analyses/OrderResponseErrorsAnalyses/AlgorithmWarmingUpOrderResponseErrorAnalysis.cs b/Engine/Results/Analysis/Analyses/OrderResponseErrorsAnalyses/AlgorithmWarmingUpOrderResponseErrorAnalysis.cs index d0dc5b825884..ef0b487f99a4 100644 --- a/Engine/Results/Analysis/Analyses/OrderResponseErrorsAnalyses/AlgorithmWarmingUpOrderResponseErrorAnalysis.cs +++ b/Engine/Results/Analysis/Analyses/OrderResponseErrorsAnalyses/AlgorithmWarmingUpOrderResponseErrorAnalysis.cs @@ -25,11 +25,6 @@ namespace QuantConnect.Lean.Engine.Results.Analysis.Analyses /// public class AlgorithmWarmingUpOrderResponseErrorAnalysis : MessageAnalysis { - /// - /// This analysis reads only the result snapshot, so it also runs while the backtest is in progress. - /// - public override bool RunsInRun { get; } = true; - /// /// Gets a description of the warm-up period ordering violation. /// diff --git a/Engine/Results/Analysis/Analyses/OrderResponseErrorsAnalyses/BrokerageModelRefusedToSubmitOrderOrderResponseErrorAnalysis.cs b/Engine/Results/Analysis/Analyses/OrderResponseErrorsAnalyses/BrokerageModelRefusedToSubmitOrderOrderResponseErrorAnalysis.cs index e39c9b2d957b..d07b9e22faf1 100644 --- a/Engine/Results/Analysis/Analyses/OrderResponseErrorsAnalyses/BrokerageModelRefusedToSubmitOrderOrderResponseErrorAnalysis.cs +++ b/Engine/Results/Analysis/Analyses/OrderResponseErrorsAnalyses/BrokerageModelRefusedToSubmitOrderOrderResponseErrorAnalysis.cs @@ -36,11 +36,6 @@ namespace QuantConnect.Lean.Engine.Results.Analysis.Analyses /// public class BrokerageModelRefusedToSubmitOrderOrderResponseErrorAnalysis : OrderResponseErrorAnalysis { - /// - /// This analysis reads only the result snapshot, so it also runs while the backtest is in progress. - /// - public override bool RunsInRun { get; } = true; - /// /// Gets a description of the brokerage-refused-to-submit-order issue. /// diff --git a/Engine/Results/Analysis/Analyses/OrderResponseErrorsAnalyses/BrokerageModelRefusedToUpdateOrderOrderResponseErrorAnalysis.cs b/Engine/Results/Analysis/Analyses/OrderResponseErrorsAnalyses/BrokerageModelRefusedToUpdateOrderOrderResponseErrorAnalysis.cs index 4731a79cc85a..13918fc38f3b 100644 --- a/Engine/Results/Analysis/Analyses/OrderResponseErrorsAnalyses/BrokerageModelRefusedToUpdateOrderOrderResponseErrorAnalysis.cs +++ b/Engine/Results/Analysis/Analyses/OrderResponseErrorsAnalyses/BrokerageModelRefusedToUpdateOrderOrderResponseErrorAnalysis.cs @@ -28,11 +28,6 @@ namespace QuantConnect.Lean.Engine.Results.Analysis.Analyses /// public class BrokerageModelRefusedToUpdateOrderOrderResponseErrorAnalysis : OrderResponseErrorAnalysis { - /// - /// This analysis reads only the result snapshot, so it also runs while the backtest is in progress. - /// - public override bool RunsInRun { get; } = true; - /// /// Gets a description of the brokerage-refused-to-update-order issue. /// diff --git a/Engine/Results/Analysis/Analyses/OrderResponseErrorsAnalyses/EuropeanOptionNotExpiredOnExerciseOrderResponseErrorAnalysis.cs b/Engine/Results/Analysis/Analyses/OrderResponseErrorsAnalyses/EuropeanOptionNotExpiredOnExerciseOrderResponseErrorAnalysis.cs index aabba0a28dc0..23920fb83c9d 100644 --- a/Engine/Results/Analysis/Analyses/OrderResponseErrorsAnalyses/EuropeanOptionNotExpiredOnExerciseOrderResponseErrorAnalysis.cs +++ b/Engine/Results/Analysis/Analyses/OrderResponseErrorsAnalyses/EuropeanOptionNotExpiredOnExerciseOrderResponseErrorAnalysis.cs @@ -24,11 +24,6 @@ namespace QuantConnect.Lean.Engine.Results.Analysis.Analyses /// public class EuropeanOptionNotExpiredOnExerciseOrderResponseErrorAnalysis : MessageAnalysis { - /// - /// This analysis reads only the result snapshot, so it also runs while the backtest is in progress. - /// - public override bool RunsInRun { get; } = true; - /// /// Gets a description of the premature European option exercise issue. /// diff --git a/Engine/Results/Analysis/Analyses/OrderResponseErrorsAnalyses/ExceededMaximumOrdersOrderResponseErrorAnalysis.cs b/Engine/Results/Analysis/Analyses/OrderResponseErrorsAnalyses/ExceededMaximumOrdersOrderResponseErrorAnalysis.cs index c19d6ad45c96..dc2ab53a445c 100644 --- a/Engine/Results/Analysis/Analyses/OrderResponseErrorsAnalyses/ExceededMaximumOrdersOrderResponseErrorAnalysis.cs +++ b/Engine/Results/Analysis/Analyses/OrderResponseErrorsAnalyses/ExceededMaximumOrdersOrderResponseErrorAnalysis.cs @@ -23,11 +23,6 @@ namespace QuantConnect.Lean.Engine.Results.Analysis.Analyses /// public class ExceededMaximumOrdersOrderResponseErrorAnalysis : MessageAnalysis { - /// - /// This analysis reads only the result snapshot, so it also runs while the backtest is in progress. - /// - public override bool RunsInRun { get; } = true; - /// /// Gets a description of the exceeded maximum orders issue. /// diff --git a/Engine/Results/Analysis/Analyses/OrderResponseErrorsAnalyses/ExceedsShortableQuantityOrderResponseErrorAnalysis.cs b/Engine/Results/Analysis/Analyses/OrderResponseErrorsAnalyses/ExceedsShortableQuantityOrderResponseErrorAnalysis.cs index 8c70fd4cc938..3a58ac6ea5a7 100644 --- a/Engine/Results/Analysis/Analyses/OrderResponseErrorsAnalyses/ExceedsShortableQuantityOrderResponseErrorAnalysis.cs +++ b/Engine/Results/Analysis/Analyses/OrderResponseErrorsAnalyses/ExceedsShortableQuantityOrderResponseErrorAnalysis.cs @@ -25,11 +25,6 @@ namespace QuantConnect.Lean.Engine.Results.Analysis.Analyses /// public class ExceedsShortableQuantityOrderResponseErrorAnalysis : BaseResultsAnalysis { - /// - /// This analysis reads only the result snapshot, so it also runs while the backtest is in progress. - /// - public override bool RunsInRun { get; } = true; - /// /// Gets a description of the exceeded shortable quantity issue. /// diff --git a/Engine/Results/Analysis/Analyses/OrderResponseErrorsAnalyses/ExchangeNotOpenOrderResponseErrorAnalysis.cs b/Engine/Results/Analysis/Analyses/OrderResponseErrorsAnalyses/ExchangeNotOpenOrderResponseErrorAnalysis.cs index 50a5d72ef248..93e63ab3bef2 100644 --- a/Engine/Results/Analysis/Analyses/OrderResponseErrorsAnalyses/ExchangeNotOpenOrderResponseErrorAnalysis.cs +++ b/Engine/Results/Analysis/Analyses/OrderResponseErrorsAnalyses/ExchangeNotOpenOrderResponseErrorAnalysis.cs @@ -25,11 +25,6 @@ namespace QuantConnect.Lean.Engine.Results.Analysis.Analyses /// public class ExchangeNotOpenOrderResponseErrorAnalysis : BaseResultsAnalysis { - /// - /// This analysis reads only the result snapshot, so it also runs while the backtest is in progress. - /// - public override bool RunsInRun { get; } = true; - /// /// Gets a description of the exchange-not-open ordering issue. /// diff --git a/Engine/Results/Analysis/Analyses/OrderResponseErrorsAnalyses/ForexConversionRateZeroOrderResponseErrorAnalysis.cs b/Engine/Results/Analysis/Analyses/OrderResponseErrorsAnalyses/ForexConversionRateZeroOrderResponseErrorAnalysis.cs index b2b5d2a6775c..8ec708316c8f 100644 --- a/Engine/Results/Analysis/Analyses/OrderResponseErrorsAnalyses/ForexConversionRateZeroOrderResponseErrorAnalysis.cs +++ b/Engine/Results/Analysis/Analyses/OrderResponseErrorsAnalyses/ForexConversionRateZeroOrderResponseErrorAnalysis.cs @@ -23,11 +23,6 @@ namespace QuantConnect.Lean.Engine.Results.Analysis.Analyses /// public class ForexConversionRateZeroOrderResponseErrorAnalysis : MessageAnalysis { - /// - /// This analysis reads only the result snapshot, so it also runs while the backtest is in progress. - /// - public override bool RunsInRun { get; } = true; - /// /// Gets a description of the zero Forex conversion rate issue. /// diff --git a/Engine/Results/Analysis/Analyses/OrderResponseErrorsAnalyses/InsufficientBuyingPowerOrderResponseErrorAnalysis.cs b/Engine/Results/Analysis/Analyses/OrderResponseErrorsAnalyses/InsufficientBuyingPowerOrderResponseErrorAnalysis.cs index 253ab6d7cd94..2ce5f7789b25 100644 --- a/Engine/Results/Analysis/Analyses/OrderResponseErrorsAnalyses/InsufficientBuyingPowerOrderResponseErrorAnalysis.cs +++ b/Engine/Results/Analysis/Analyses/OrderResponseErrorsAnalyses/InsufficientBuyingPowerOrderResponseErrorAnalysis.cs @@ -23,11 +23,6 @@ namespace QuantConnect.Lean.Engine.Results.Analysis.Analyses /// public class InsufficientBuyingPowerOrderResponseErrorAnalysis : OrderResponseErrorAnalysis { - /// - /// This analysis reads only the result snapshot, so it also runs while the backtest is in progress. - /// - public override bool RunsInRun { get; } = true; - /// /// Gets a description of the insufficient buying power issue. /// diff --git a/Engine/Results/Analysis/Analyses/OrderResponseErrorsAnalyses/MarketOnOpenNotAllowedDuringRegularHoursOrderResponseErrorAnalysis.cs b/Engine/Results/Analysis/Analyses/OrderResponseErrorsAnalyses/MarketOnOpenNotAllowedDuringRegularHoursOrderResponseErrorAnalysis.cs index 4c7f7186e1ab..a5b963a91da6 100644 --- a/Engine/Results/Analysis/Analyses/OrderResponseErrorsAnalyses/MarketOnOpenNotAllowedDuringRegularHoursOrderResponseErrorAnalysis.cs +++ b/Engine/Results/Analysis/Analyses/OrderResponseErrorsAnalyses/MarketOnOpenNotAllowedDuringRegularHoursOrderResponseErrorAnalysis.cs @@ -23,11 +23,6 @@ namespace QuantConnect.Lean.Engine.Results.Analysis.Analyses /// public class MarketOnOpenNotAllowedDuringRegularHoursOrderResponseErrorAnalysis : MessageAnalysis { - /// - /// This analysis reads only the result snapshot, so it also runs while the backtest is in progress. - /// - public override bool RunsInRun { get; } = true; - /// /// Gets a description of the market-on-open during regular hours issue. /// diff --git a/Engine/Results/Analysis/Analyses/OrderResponseErrorsAnalyses/NonTradableSecurityOrderResponseErrorAnalysis.cs b/Engine/Results/Analysis/Analyses/OrderResponseErrorsAnalyses/NonTradableSecurityOrderResponseErrorAnalysis.cs index f0b904a826bf..35752b5613a4 100644 --- a/Engine/Results/Analysis/Analyses/OrderResponseErrorsAnalyses/NonTradableSecurityOrderResponseErrorAnalysis.cs +++ b/Engine/Results/Analysis/Analyses/OrderResponseErrorsAnalyses/NonTradableSecurityOrderResponseErrorAnalysis.cs @@ -23,11 +23,6 @@ namespace QuantConnect.Lean.Engine.Results.Analysis.Analyses /// public class NonTradableSecurityOrderResponseErrorAnalysis : MessageAnalysis { - /// - /// This analysis reads only the result snapshot, so it also runs while the backtest is in progress. - /// - public override bool RunsInRun { get; } = true; - /// /// Gets a description of the non-tradable security ordering issue. /// diff --git a/Engine/Results/Analysis/Analyses/OrderResponseErrorsAnalyses/OptionOrderOnStockSplitOrderResponseErrorAnalysis.cs b/Engine/Results/Analysis/Analyses/OrderResponseErrorsAnalyses/OptionOrderOnStockSplitOrderResponseErrorAnalysis.cs index dc201b049877..402c469a16c6 100644 --- a/Engine/Results/Analysis/Analyses/OrderResponseErrorsAnalyses/OptionOrderOnStockSplitOrderResponseErrorAnalysis.cs +++ b/Engine/Results/Analysis/Analyses/OrderResponseErrorsAnalyses/OptionOrderOnStockSplitOrderResponseErrorAnalysis.cs @@ -23,11 +23,6 @@ namespace QuantConnect.Lean.Engine.Results.Analysis.Analyses /// public class OptionOrderOnStockSplitOrderResponseErrorAnalysis : MessageAnalysis { - /// - /// This analysis reads only the result snapshot, so it also runs while the backtest is in progress. - /// - public override bool RunsInRun { get; } = true; - /// /// Gets a description of the option order during stock split issue. /// diff --git a/Engine/Results/Analysis/Analyses/OrderResponseErrorsAnalyses/OrderQuantityLessThanLotSizeOrderResponseErrorAnalysis.cs b/Engine/Results/Analysis/Analyses/OrderResponseErrorsAnalyses/OrderQuantityLessThanLotSizeOrderResponseErrorAnalysis.cs index 5081eba46d76..e23063f49ebb 100644 --- a/Engine/Results/Analysis/Analyses/OrderResponseErrorsAnalyses/OrderQuantityLessThanLotSizeOrderResponseErrorAnalysis.cs +++ b/Engine/Results/Analysis/Analyses/OrderResponseErrorsAnalyses/OrderQuantityLessThanLotSizeOrderResponseErrorAnalysis.cs @@ -23,11 +23,6 @@ namespace QuantConnect.Lean.Engine.Results.Analysis.Analyses /// public class OrderQuantityLessThanLotSizeOrderResponseErrorAnalysis : MessageAnalysis { - /// - /// This analysis reads only the result snapshot, so it also runs while the backtest is in progress. - /// - public override bool RunsInRun { get; } = true; - /// /// Gets a description of the order quantity below lot size issue. /// diff --git a/Engine/Results/Analysis/Analyses/OrderResponseErrorsAnalyses/OrderQuantityZeroOrderResponseErrorAnalysis.cs b/Engine/Results/Analysis/Analyses/OrderResponseErrorsAnalyses/OrderQuantityZeroOrderResponseErrorAnalysis.cs index 18028437c0e9..df4ea6831644 100644 --- a/Engine/Results/Analysis/Analyses/OrderResponseErrorsAnalyses/OrderQuantityZeroOrderResponseErrorAnalysis.cs +++ b/Engine/Results/Analysis/Analyses/OrderResponseErrorsAnalyses/OrderQuantityZeroOrderResponseErrorAnalysis.cs @@ -24,11 +24,6 @@ namespace QuantConnect.Lean.Engine.Results.Analysis.Analyses /// public class OrderQuantityZeroOrderResponseErrorAnalysis : MessageAnalysis { - /// - /// This analysis reads only the result snapshot, so it also runs while the backtest is in progress. - /// - public override bool RunsInRun { get; } = true; - /// /// Gets a description of the zero order quantity issue. /// diff --git a/Engine/Results/Analysis/Analyses/OrderResponseErrorsAnalyses/SecurityPriceZeroOrderResponseErrorAnalysis.cs b/Engine/Results/Analysis/Analyses/OrderResponseErrorsAnalyses/SecurityPriceZeroOrderResponseErrorAnalysis.cs index 6365760b37eb..00413b747be6 100644 --- a/Engine/Results/Analysis/Analyses/OrderResponseErrorsAnalyses/SecurityPriceZeroOrderResponseErrorAnalysis.cs +++ b/Engine/Results/Analysis/Analyses/OrderResponseErrorsAnalyses/SecurityPriceZeroOrderResponseErrorAnalysis.cs @@ -23,11 +23,6 @@ namespace QuantConnect.Lean.Engine.Results.Analysis.Analyses /// public class SecurityPriceZeroOrderResponseErrorAnalysis : MessageAnalysis { - /// - /// This analysis reads only the result snapshot, so it also runs while the backtest is in progress. - /// - public override bool RunsInRun { get; } = true; - /// /// Gets a description of the zero security price ordering issue. /// diff --git a/Engine/Results/Analysis/Analyses/OrderResponseErrorsAnalyses/UnsupportedOptionExerciseQuantityAnalysis.cs b/Engine/Results/Analysis/Analyses/OrderResponseErrorsAnalyses/UnsupportedOptionExerciseQuantityAnalysis.cs index 40fe302745a2..f05b7ce75175 100644 --- a/Engine/Results/Analysis/Analyses/OrderResponseErrorsAnalyses/UnsupportedOptionExerciseQuantityAnalysis.cs +++ b/Engine/Results/Analysis/Analyses/OrderResponseErrorsAnalyses/UnsupportedOptionExerciseQuantityAnalysis.cs @@ -23,11 +23,6 @@ namespace QuantConnect.Lean.Engine.Results.Analysis.Analyses /// public class UnsupportedOptionExerciseQuantityAnalysis : MessageAnalysis { - /// - /// This analysis reads only the result snapshot, so it also runs while the backtest is in progress. - /// - public override bool RunsInRun { get; } = true; - /// /// Gets a description of the excess-quantity option exercise issue. /// diff --git a/Engine/Results/Analysis/Analyses/OrderResponseErrorsAnalyses/UnsupportedOptionShortPositionExerciseAnalysis.cs b/Engine/Results/Analysis/Analyses/OrderResponseErrorsAnalyses/UnsupportedOptionShortPositionExerciseAnalysis.cs index 23e2f5a4a94d..d29d4d98fdfe 100644 --- a/Engine/Results/Analysis/Analyses/OrderResponseErrorsAnalyses/UnsupportedOptionShortPositionExerciseAnalysis.cs +++ b/Engine/Results/Analysis/Analyses/OrderResponseErrorsAnalyses/UnsupportedOptionShortPositionExerciseAnalysis.cs @@ -23,11 +23,6 @@ namespace QuantConnect.Lean.Engine.Results.Analysis.Analyses /// public class UnsupportedOptionShortPositionExerciseAnalysis : MessageAnalysis { - /// - /// This analysis reads only the result snapshot, so it also runs while the backtest is in progress. - /// - public override bool RunsInRun { get; } = true; - /// /// Gets a description of the short-position option exercise issue. /// diff --git a/Engine/Results/Analysis/Analyses/ParameterCountAnalysis.cs b/Engine/Results/Analysis/Analyses/ParameterCountAnalysis.cs index b5e78cab6363..b63cbd67b087 100644 --- a/Engine/Results/Analysis/Analyses/ParameterCountAnalysis.cs +++ b/Engine/Results/Analysis/Analyses/ParameterCountAnalysis.cs @@ -23,6 +23,12 @@ namespace QuantConnect.Lean.Engine.Results.Analysis.Analyses /// public class ParameterCountAnalysis : BaseResultsAnalysis { + /// + /// This analysis reads the algorithm's parameters, and its overfitting-risk warning is only + /// meaningful for the completed run. + /// + public override bool RunsInRun { get; } = false; + /// /// Gets the description of the excessive parameter count issue. /// diff --git a/Engine/Results/Analysis/Analyses/PerformanceRelativeToBenchmarkAnalysis.cs b/Engine/Results/Analysis/Analyses/PerformanceRelativeToBenchmarkAnalysis.cs index 232e334e8af5..5fc52cf6af19 100644 --- a/Engine/Results/Analysis/Analyses/PerformanceRelativeToBenchmarkAnalysis.cs +++ b/Engine/Results/Analysis/Analyses/PerformanceRelativeToBenchmarkAnalysis.cs @@ -24,6 +24,11 @@ namespace QuantConnect.Lean.Engine.Results.Analysis.Analyses /// public class PerformanceRelativeToBenchmarkAnalysis : BaseResultsAnalysis { + /// + /// This analysis compares the equity and benchmark curves, which are only built for the final analysis. + /// + public override bool RunsInRun { get; } = false; + /// /// Gets the description of the underperformance relative to benchmark issue. /// diff --git a/Engine/Results/Analysis/Analyses/PortfolioMarginUsageAnalysis.cs b/Engine/Results/Analysis/Analyses/PortfolioMarginUsageAnalysis.cs index 8f42b5994b2a..30c213ff90da 100644 --- a/Engine/Results/Analysis/Analyses/PortfolioMarginUsageAnalysis.cs +++ b/Engine/Results/Analysis/Analyses/PortfolioMarginUsageAnalysis.cs @@ -25,11 +25,6 @@ namespace QuantConnect.Lean.Engine.Results.Analysis.Analyses /// public class PortfolioMarginUsageAnalysis : BaseResultsAnalysis { - /// - /// This analysis reads only the result snapshot, so it also runs while the backtest is in progress. - /// - public override bool RunsInRun { get; } = true; - /// /// This analysis reads the current margin chart instead of scanning the order event /// and log streams, so its in-run findings are replaced on every run. diff --git a/Engine/Results/Analysis/Analyses/PortfolioValueIsNotPositiveAnalysis.cs b/Engine/Results/Analysis/Analyses/PortfolioValueIsNotPositiveAnalysis.cs index 147de93e49f3..b21b4668e961 100644 --- a/Engine/Results/Analysis/Analyses/PortfolioValueIsNotPositiveAnalysis.cs +++ b/Engine/Results/Analysis/Analyses/PortfolioValueIsNotPositiveAnalysis.cs @@ -22,11 +22,6 @@ namespace QuantConnect.Lean.Engine.Results.Analysis.Analyses /// public class PortfolioValueIsNotPositiveAnalysis : BaseResultsAnalysis { - /// - /// This analysis reads only the result snapshot, so it also runs while the backtest is in progress. - /// - public override bool RunsInRun { get; } = true; - /// /// This analysis reads the current portfolio statistics instead of scanning the order event /// and log streams, so its in-run findings are replaced on every run. diff --git a/Engine/Results/Analysis/Analyses/SingleTimeLoopTimeoutRuntimeErrorAnalysis.cs b/Engine/Results/Analysis/Analyses/SingleTimeLoopTimeoutRuntimeErrorAnalysis.cs index c8ed7c564c84..83fae07aa88f 100644 --- a/Engine/Results/Analysis/Analyses/SingleTimeLoopTimeoutRuntimeErrorAnalysis.cs +++ b/Engine/Results/Analysis/Analyses/SingleTimeLoopTimeoutRuntimeErrorAnalysis.cs @@ -45,6 +45,11 @@ public class SingleTimeLoopTimeoutRuntimeErrorAnalysis : BaseResultsAnalysis ["Operation was canceled"], ]; + /// + /// A timeout runtime error terminates the backtest, so there is no in-progress run to analyze. + /// + public override bool RunsInRun { get; } = false; + /// /// Gets the description of the timeout issue. /// diff --git a/Engine/Results/Analysis/Analyses/StaleOrderFillsAnalysis.cs b/Engine/Results/Analysis/Analyses/StaleOrderFillsAnalysis.cs index 05c6a60729a3..a0ba22fc516c 100644 --- a/Engine/Results/Analysis/Analyses/StaleOrderFillsAnalysis.cs +++ b/Engine/Results/Analysis/Analyses/StaleOrderFillsAnalysis.cs @@ -25,11 +25,6 @@ namespace QuantConnect.Lean.Engine.Results.Analysis.Analyses /// public class StaleOrderFillsAnalysis : BaseResultsAnalysis { - /// - /// This analysis reads only the result snapshot, so it also runs while the backtest is in progress. - /// - public override bool RunsInRun { get; } = true; - /// /// Gets the description of the stale order fill issue. /// diff --git a/Engine/Results/Analysis/Analyses/StatisticalSignificanceOfDailyReturnsAnalysis.cs b/Engine/Results/Analysis/Analyses/StatisticalSignificanceOfDailyReturnsAnalysis.cs index 1ec487074155..227485f65c79 100644 --- a/Engine/Results/Analysis/Analyses/StatisticalSignificanceOfDailyReturnsAnalysis.cs +++ b/Engine/Results/Analysis/Analyses/StatisticalSignificanceOfDailyReturnsAnalysis.cs @@ -28,6 +28,11 @@ namespace QuantConnect.Lean.Engine.Results.Analysis.Analyses /// public class StatisticalSignificanceOfDailyReturnsAnalysis : BaseResultsAnalysis { + /// + /// This analysis compares the equity and benchmark curves, which are only built for the final analysis. + /// + public override bool RunsInRun { get; } = false; + /// /// Gets the description of the statistical insignificance issue. /// diff --git a/Engine/Results/Analysis/Analyses/TakeProfitAndStopLossOrdersAnalysis.cs b/Engine/Results/Analysis/Analyses/TakeProfitAndStopLossOrdersAnalysis.cs index 2ef3dd5a14a8..531ae0a294b9 100644 --- a/Engine/Results/Analysis/Analyses/TakeProfitAndStopLossOrdersAnalysis.cs +++ b/Engine/Results/Analysis/Analyses/TakeProfitAndStopLossOrdersAnalysis.cs @@ -26,11 +26,6 @@ namespace QuantConnect.Lean.Engine.Results.Analysis.Analyses /// public class TakeProfitAndStopLossOrdersAnalysis : BaseResultsAnalysis { - /// - /// This analysis reads only the result snapshot, so it also runs while the backtest is in progress. - /// - public override bool RunsInRun { get; } = true; - /// /// This analysis reads the current orders collection instead of scanning the order event /// and log streams, so its in-run findings are replaced on every run. diff --git a/Tests/Engine/Results/InRunResultsAnalysisTests.cs b/Tests/Engine/Results/InRunResultsAnalysisTests.cs index 44f026becbd2..d2c11736a34f 100644 --- a/Tests/Engine/Results/InRunResultsAnalysisTests.cs +++ b/Tests/Engine/Results/InRunResultsAnalysisTests.cs @@ -432,9 +432,6 @@ private class FakeAnalysis : BaseResultsAnalysis public override int Weight => _weight; - // The in-run instances only run the analyses declaring RunsInRun - public override bool RunsInRun { get; } = true; - public override bool IsStateBased => StateBased; public bool StateBased { get; set; } From 1a6f73f07f5f4a58683a81695b6ae37354b4677b Mon Sep 17 00:00:00 2001 From: Jhonathan Abreu Date: Tue, 4 Aug 2026 09:43:13 -0400 Subject: [PATCH 24/33] Rename the in-run analysis tests fixture to ResultsAnalyzerInRunTests --- ...esultsAnalysisTests.cs => ResultsAnalyzerInRunTests.cs} | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) rename Tests/Engine/Results/{InRunResultsAnalysisTests.cs => ResultsAnalyzerInRunTests.cs} (98%) diff --git a/Tests/Engine/Results/InRunResultsAnalysisTests.cs b/Tests/Engine/Results/ResultsAnalyzerInRunTests.cs similarity index 98% rename from Tests/Engine/Results/InRunResultsAnalysisTests.cs rename to Tests/Engine/Results/ResultsAnalyzerInRunTests.cs index d2c11736a34f..78de05c56150 100644 --- a/Tests/Engine/Results/InRunResultsAnalysisTests.cs +++ b/Tests/Engine/Results/ResultsAnalyzerInRunTests.cs @@ -28,8 +28,13 @@ namespace QuantConnect.Tests.Engine.Results { + /// + /// Tests the in-run mode of , driven through a fake + /// . The core, mode-independent behavior + /// is covered by . + /// [TestFixture] - public class InRunResultsAnalysisTests + public class ResultsAnalyzerInRunTests { private static readonly IReadOnlyList SomeSolutions = new[] { "A solution" }; From fad227ed6a21de8502a64beee3864bf59a962a01 Mon Sep 17 00:00:00 2001 From: Jhonathan Abreu Date: Tue, 4 Aug 2026 12:56:22 -0400 Subject: [PATCH 25/33] Merge the execution speed analysis into the algorithm speed analysis as a completion-log fallback --- .../Analyses/AlgorithmSpeedAnalysis.cs | 113 +++++++++++++++--- .../Analyses/ExecutionSpeedAnalysis.cs | 113 ------------------ Engine/Results/Analysis/ResultsAnalyzer.cs | 1 - .../Results/AlgorithmSpeedAnalysisTests.cs | 63 ++++++++++ .../Results/ResultsAnalyzerInRunTests.cs | 2 +- Tests/Engine/Results/ResultsAnalyzerTests.cs | 1 - 6 files changed, 160 insertions(+), 133 deletions(-) delete mode 100644 Engine/Results/Analysis/Analyses/ExecutionSpeedAnalysis.cs diff --git a/Engine/Results/Analysis/Analyses/AlgorithmSpeedAnalysis.cs b/Engine/Results/Analysis/Analyses/AlgorithmSpeedAnalysis.cs index d270fa90f77d..699b69ead71b 100644 --- a/Engine/Results/Analysis/Analyses/AlgorithmSpeedAnalysis.cs +++ b/Engine/Results/Analysis/Analyses/AlgorithmSpeedAnalysis.cs @@ -15,6 +15,8 @@ */ using System; using System.Collections.Generic; +using System.Globalization; +using System.Text.RegularExpressions; using static QuantConnect.StringExtensions; namespace QuantConnect.Lean.Engine.Results.Analysis.Analyses @@ -25,10 +27,33 @@ namespace QuantConnect.Lean.Engine.Results.Analysis.Analyses /// remaining runtime, degrading throughput, and history-request-dominated data loads. /// It runs periodically while the backtest is in progress, so the user can decide to stop a /// slow backtest early, and again on the final analysis against the whole run's metrics. + /// When the tracked metrics cannot measure the processing speed, the engine's completion log + /// line is parsed for the whole-run average rate as a fallback; the line only exists once the + /// backtest ends, so the fallback can only fire on the final analysis. /// Benchmark speeds: https://www.quantconnect.com/performance /// public class AlgorithmSpeedAnalysis : BaseResultsAnalysis { + /// + /// Matches the engine's completion log line, capturing the execution time and the data + /// points per second (in thousands). Example match: "Algorithm Id:(Foo) completed in + /// 25.68 seconds at 85k data points per second." gives seconds=25.68, rate=85. + /// + private static readonly Regex CompletionLogLineRegex = new( + @"Algorithm Id:\([^)]+\) completed in ([\d.]+) seconds at (\d+)k data points per second\. Processing total of [\d,]+ data points\.", + RegexOptions.Compiled); + + /// + /// The data points per second under which execution is reported as slow, from the platform benchmarks. + /// + public const int SlowDataPointsPerSecond = 40_000; + + /// + /// The minimum runtime a completed backtest must have for its whole-run average rate, + /// parsed from the completion log line, to be worth reporting as slow. + /// + public const int MinimumCompletedRuntimeSeconds = 10; + /// /// The recent-to-initial throughput ratio under which throughput is reported as degrading. /// @@ -97,49 +122,63 @@ public class AlgorithmSpeedAnalysis : BaseResultsAnalysis public override int Weight { get; } = 96; /// - /// Runs the algorithm speed analysis against the speed metrics tracked for the running backtest. + /// Runs the algorithm speed analysis against the speed metrics tracked for the backtest, + /// falling back to the completion log line when they cannot measure the speed. /// - public override IReadOnlyList Run(ResultsAnalysisRunParameters parameters) => Run(parameters.Speed); + public override IReadOnlyList Run(ResultsAnalysisRunParameters parameters) => Run(parameters.Speed, parameters.Logs); /// /// Runs the algorithm speed analysis against the given speed metrics. /// Each detected condition is reported as its own sub-finding. Every condition must hold for /// both the current recent window and the window as of the previous run, so a single noisy /// sample doesn't flag or clear a finding. + /// When the metrics cannot measure the processing speed — the tracker isn't wired in, the + /// backtest finished before it got enough samples, or the data point counters aren't fed — + /// the completion log line's whole-run average is used to detect slow execution instead. /// /// The speed metrics tracked for the running backtest, or null when not tracked. - /// The failed sub-findings, or empty when speed is not tracked or still within the warm-up span. - public IReadOnlyList Run(AlgorithmSpeedTracker speed) + /// The log lines to search for the completion line, or null when not available. + /// The failed sub-findings, or empty when no speed condition failed or none could be measured. + public IReadOnlyList Run(AlgorithmSpeedTracker speed, IReadOnlyList logs = null) { - if (speed == null || speed.SampledSpan < MinimumSampledSpan) + var findings = new List(); + var speedMeasured = false; + if (speed != null && speed.SampledSpan >= MinimumSampledSpan) { - return []; + speedMeasured = AddSlowExecution(speed, findings); + AddLongProjectedRuntime(speed, findings); + AddThroughputDegradation(speed, findings); + AddHistoryRequestLoad(speed, findings); + } + + if (!speedMeasured) + { + AddSlowExecutionFromCompletionLog(logs, findings); } - var findings = new List(); - AddSlowExecution(speed, findings); - AddLongProjectedRuntime(speed, findings); - AddThroughputDegradation(speed, findings); - AddHistoryRequestLoad(speed, findings); return CreateAggregatedResponse(findings); } /// /// Reports slow execution when the recent data points per second are below the platform benchmark. /// - private static void AddSlowExecution(AlgorithmSpeedTracker speed, List findings) + /// Whether the speed could be measured, regardless of it being slow or not. + private static bool AddSlowExecution(AlgorithmSpeedTracker speed, List findings) { if (!speed.HasDataPointCounts) { - return; + return false; } var recent = speed.RecentDataPointsPerSecond(); var previous = speed.RecentDataPointsPerSecond(skipLast: 1); - if (recent is null or >= ExecutionSpeedAnalysis.SlowDataPointsPerSecond || - previous is null or >= ExecutionSpeedAnalysis.SlowDataPointsPerSecond) + if (recent == null || previous == null) { - return; + return false; + } + if (recent >= SlowDataPointsPerSecond || previous >= SlowDataPointsPerSecond) + { + return true; } var average = speed.DataPointsPerSecond ?? 0; @@ -163,7 +202,7 @@ private static void AddSlowExecution(AlgorithmSpeedTracker speed, List + /// Fallback slow-execution detection for when the tracked metrics cannot measure the speed: + /// parses the engine's completion log line for the whole-run average rate. The line is only + /// logged once the backtest ends, so in-run log deltas never match and the fallback can + /// only fire on the final analysis. + /// + private static void AddSlowExecutionFromCompletionLog(IReadOnlyList logs, List findings) + { + for (var i = (logs?.Count ?? 0) - 1; i >= 0; i--) + { + var match = CompletionLogLineRegex.Match(logs[i]); + if (!match.Success) + { + continue; + } + + var timeInSeconds = double.Parse(match.Groups[1].Value, NumberFormatInfo.InvariantInfo); + var dataPointsPerSecond = int.Parse(match.Groups[2].Value, NumberFormatInfo.InvariantInfo); + if (timeInSeconds >= MinimumCompletedRuntimeSeconds && dataPointsPerSecond < SlowDataPointsPerSecond / 1000) + { + findings.Add(new(SlowExecutionName, + Invariant($"The algorithm is running below {SlowDataPointsPerSecond / 1000}k data points per second."), + Invariant($"The algorithm executed at only {dataPointsPerSecond}k data points per second ") + + Invariant($"over the whole {FormatDuration(TimeSpan.FromSeconds(timeInSeconds))} run."), + null, + [ + "Review the algorithm code for inefficiencies.", + + "If there is a universe, reduce its size.", + + "Reduce the data resolution.", + + "If the algorithm is training a model, reduce the amount of training data or reduce the number of epochs in the training process.", + ])); + } + return; + } } /// diff --git a/Engine/Results/Analysis/Analyses/ExecutionSpeedAnalysis.cs b/Engine/Results/Analysis/Analyses/ExecutionSpeedAnalysis.cs deleted file mode 100644 index 4a18ab43427c..000000000000 --- a/Engine/Results/Analysis/Analyses/ExecutionSpeedAnalysis.cs +++ /dev/null @@ -1,113 +0,0 @@ -/* - * QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals. - * Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * -*/ -using System.Collections.Generic; -using System.Globalization; -using System.Text.RegularExpressions; - -namespace QuantConnect.Lean.Engine.Results.Analysis.Analyses -{ - /// - /// Detects slow execution by parsing the last log line. - /// Benchmark speeds: https://www.quantconnect.com/performance - /// - public class ExecutionSpeedAnalysis : BaseResultsAnalysis - { - /// - /// The data points per second under which execution is reported as slow, from the platform - /// benchmarks. Also used by while the backtest runs. - /// - public const int SlowDataPointsPerSecond = 40_000; - - /// - /// This analysis reads the engine's completion logs, which only exist once the backtest ends. - /// While it runs, tracks the algorithm's speed instead. - /// - public override bool RunsInRun { get; } = false; - - /// - /// Gets the description of the slow execution issue. - /// - public override string Issue { get; } = $"The algorithm ran below {SlowDataPointsPerSecond / 1000}k data points per second."; - - /// - /// Gets the severity weight for the execution speed analysis. - /// - public override int Weight { get; } = 77; - - /// - /// Runs the execution speed analysis against the provided backtest parameters. - /// - public override IReadOnlyList Run(ResultsAnalysisRunParameters parameters) => Run(parameters.Logs); - - private static readonly Regex DataPointsPerSecondRegex = new( - @"Algorithm Id:\([^)]+\) completed in ([\d.]+) seconds at (\d+)k data points per second\. Processing total of [\d,]+ data points\.", - RegexOptions.Compiled); - - /// - /// Parses the backtest logs to determine execution speed and flags backtests that ran slowly. - /// - /// The full list of log lines produced by the backtest. - /// Analysis results flagging slow execution when below and runtime is at least 10 seconds. - public IReadOnlyList Run(IReadOnlyList logs) - { - var result = TryGetDataPointsPerSecond(logs, out var timeInSeconds, out var dataPointsPerSecond) && - timeInSeconds >= 10 && dataPointsPerSecond < SlowDataPointsPerSecond / 1000 - ? $"The algorithm is slowly executing at only {dataPointsPerSecond}k data points per second" - : null; - - var potentialSolutions = result is not null ? Solutions() : []; - return SingleResponse(result, potentialSolutions); - } - - /// - /// Searches in reverse order for a completion line and extracts - /// the execution time and data points per second (in thousands). - /// Example match: "Algorithm Id:(Foo) completed in 25.68 seconds at 85k data points per second." - /// returns seconds=25.68, dataPointsPerSecond=85. - /// - private static bool TryGetDataPointsPerSecond(IReadOnlyList logs, out double? timeInSeconds, out int? dataPointsPerSecond) - { - for (var i = logs.Count - 1; i >= 0; i--) - { - var match = DataPointsPerSecondRegex.Match(logs[i]); - if (match.Success) - { - timeInSeconds = double.Parse(match.Groups[1].Value, NumberFormatInfo.InvariantInfo); - dataPointsPerSecond = int.Parse(match.Groups[2].Value, NumberFormatInfo.InvariantInfo); - return true; - } - } - - timeInSeconds = null; - dataPointsPerSecond = null; - return false; - } - - /// - /// Returns suggested solutions for improving execution speed. - /// - private static List Solutions() => - [ - "Review the algorithm code for inefficiencies.", - - "If there is a universe, reduce its size.", - - "Reduce the data resolution.", - - "If the algorithm is training a model, reduce the amount of training data or reduce the number of epochs in the training process.", - ]; - } -} diff --git a/Engine/Results/Analysis/ResultsAnalyzer.cs b/Engine/Results/Analysis/ResultsAnalyzer.cs index bd04c5ee1fa4..4c5ad75d8357 100644 --- a/Engine/Results/Analysis/ResultsAnalyzer.cs +++ b/Engine/Results/Analysis/ResultsAnalyzer.cs @@ -338,7 +338,6 @@ protected virtual IReadOnlyCollection GetAnalyses() => new StatisticalSignificanceOfDailyReturnsAnalysis(), new PerformanceRelativeToBenchmarkAnalysis(), new CrisisEventsAnalysis(), - new ExecutionSpeedAnalysis(), new AlgorithmSpeedAnalysis(), new PortfolioMarginUsageAnalysis(), new ParameterCountAnalysis(), diff --git a/Tests/Engine/Results/AlgorithmSpeedAnalysisTests.cs b/Tests/Engine/Results/AlgorithmSpeedAnalysisTests.cs index 48baaeb66066..7ebe52df3210 100644 --- a/Tests/Engine/Results/AlgorithmSpeedAnalysisTests.cs +++ b/Tests/Engine/Results/AlgorithmSpeedAnalysisTests.cs @@ -26,6 +26,9 @@ namespace QuantConnect.Tests.Engine.Results [TestFixture] public class AlgorithmSpeedAnalysisTests { + private const string SlowCompletionLogLine = + "Algorithm Id:(BasicTemplateAlgorithm) completed in 120.50 seconds at 10k data points per second. Processing total of 1,205,000 data points."; + [Test] public void NoFindingsWithoutSpeedMetrics() { @@ -123,6 +126,66 @@ public void SlowExecutionRequiresTwoConsecutiveSlowWindows() Assert.IsTrue(findings.Any(finding => finding.Name.EndsWith(AlgorithmSpeedAnalysis.SlowExecutionName, StringComparison.Ordinal))); } + [Test] + public void CompletionLogLineIsTheSlowExecutionFallbackWhenSpeedIsNotTracked() + { + var logs = new[] { "Some log line", SlowCompletionLogLine, "Another log line" }; + + var findings = new AlgorithmSpeedAnalysis().Run(null, logs); + + var finding = findings.Single(); + Assert.AreEqual($"{nameof(AlgorithmSpeedAnalysis)} / {AlgorithmSpeedAnalysis.SlowExecutionName}", finding.Name); + StringAssert.Contains("10k data points per second", (string)finding.Sample); + StringAssert.Contains("whole", (string)finding.Sample); + Assert.IsNotEmpty(finding.Solutions); + } + + [TestCase("Algorithm Id:(BasicTemplateAlgorithm) completed in 120.50 seconds at 85k data points per second. Processing total of 10,242,500 data points.", + TestName = "CompletionLogFallbackDoesNotFlagFastRuns")] + [TestCase("Algorithm Id:(BasicTemplateAlgorithm) completed in 5.20 seconds at 5k data points per second. Processing total of 26,000 data points.", + TestName = "CompletionLogFallbackDoesNotFlagVeryShortRuns")] + [TestCase("A log line without a completion line, like the deltas the in-run analysis cycles see", + TestName = "CompletionLogFallbackNeedsACompletionLine")] + public void CompletionLogFallbackOnlyFlagsSlowCompletedRuns(string logLine) + { + Assert.IsEmpty(new AlgorithmSpeedAnalysis().Run(null, new[] { logLine })); + } + + [Test] + public void TrackedMetricsSupersedeTheCompletionLogFallback() + { + // Fast per the tracked metrics: the slow whole-run average in the log is not reported + var fastTracker = AlgorithmSpeedTrackerTests.BuildUniformTracker(samples: 7, stepSeconds: 30, + dataPointsPerStep: 3_000_000, historyDataPointsPerStep: 0, daysPerStep: 1, totalDays: 10); + + Assert.IsEmpty(new AlgorithmSpeedAnalysis().Run(fastTracker, new[] { SlowCompletionLogLine })); + + // Slow per the tracked metrics: a single metric-based finding, not the log-based one + var slowTracker = AlgorithmSpeedTrackerTests.BuildUniformTracker(samples: 7, stepSeconds: 30, + dataPointsPerStep: 300_000, historyDataPointsPerStep: 0, daysPerStep: 1, totalDays: 10); + + var findings = new AlgorithmSpeedAnalysis().Run(slowTracker, new[] { SlowCompletionLogLine }); + + var finding = findings.Single(); + Assert.AreEqual($"{nameof(AlgorithmSpeedAnalysis)} / {AlgorithmSpeedAnalysis.SlowExecutionName}", finding.Name); + StringAssert.Contains("recently", (string)finding.Sample); + } + + [Test] + public void CompletionLogFallbackFiresWhenTheDataPointCountersAreNotWiredIn() + { + // The tracker samples calendar progress but the data point counters are always zero: + // the metrics cannot measure the speed, so the completion line's average is used + var tracker = AlgorithmSpeedTrackerTests.BuildUniformTracker(samples: 7, stepSeconds: 30, + dataPointsPerStep: 0, historyDataPointsPerStep: 0, daysPerStep: 1, totalDays: 10); + + var findings = new AlgorithmSpeedAnalysis().Run(tracker, new[] { SlowCompletionLogLine }); + + var finding = findings.Single(); + Assert.AreEqual($"{nameof(AlgorithmSpeedAnalysis)} / {AlgorithmSpeedAnalysis.SlowExecutionName}", finding.Name); + StringAssert.Contains("whole", (string)finding.Sample); + } + [Test] public void MissingDataPointCountsSuppressDataPointBasedFindings() { diff --git a/Tests/Engine/Results/ResultsAnalyzerInRunTests.cs b/Tests/Engine/Results/ResultsAnalyzerInRunTests.cs index 78de05c56150..5216c4897812 100644 --- a/Tests/Engine/Results/ResultsAnalyzerInRunTests.cs +++ b/Tests/Engine/Results/ResultsAnalyzerInRunTests.cs @@ -314,7 +314,7 @@ public void DefaultAnalysisSetIsTheInRunCapableSubsetOfTheFinalSet() // are included, final-only ones are not Assert.IsTrue(analyses.Any(analysis => analysis is AlgorithmSpeedAnalysis)); Assert.IsTrue(analyses.Any(analysis => analysis is MarginCallsAnalysis)); - Assert.IsFalse(analyses.Any(analysis => analysis is ExecutionSpeedAnalysis)); + Assert.IsFalse(analyses.Any(analysis => analysis is MonteCarloPercentileAnalysis)); } [Test] diff --git a/Tests/Engine/Results/ResultsAnalyzerTests.cs b/Tests/Engine/Results/ResultsAnalyzerTests.cs index 72b95820b46b..9bbf8b475f8b 100644 --- a/Tests/Engine/Results/ResultsAnalyzerTests.cs +++ b/Tests/Engine/Results/ResultsAnalyzerTests.cs @@ -179,7 +179,6 @@ public void DefaultAnalysisSetDeclaresTheFinalOnlyAnalyses() nameof(StatisticalSignificanceOfDailyReturnsAnalysis), nameof(PerformanceRelativeToBenchmarkAnalysis), nameof(CrisisEventsAnalysis), - nameof(ExecutionSpeedAnalysis), nameof(ParameterCountAnalysis), nameof(MonteCarloPercentileAnalysis), }, finalOnly); From 6f17f6485aadaedacbc13f65fffc3607e7ab0efb Mon Sep 17 00:00:00 2001 From: Jhonathan Abreu Date: Tue, 4 Aug 2026 13:27:37 -0400 Subject: [PATCH 26/33] Sort the analyses by weight once when the analysis set is created --- Engine/Results/Analysis/ResultsAnalyzer.cs | 28 ++++++++++++++-------- 1 file changed, 18 insertions(+), 10 deletions(-) diff --git a/Engine/Results/Analysis/ResultsAnalyzer.cs b/Engine/Results/Analysis/ResultsAnalyzer.cs index 4c5ad75d8357..80fc4202fedf 100644 --- a/Engine/Results/Analysis/ResultsAnalyzer.cs +++ b/Engine/Results/Analysis/ResultsAnalyzer.cs @@ -54,6 +54,11 @@ public class ResultsAnalyzer /// private HashSet _stateBasedAnalyses; + /// + /// The weight of each analysis by name, for ranking the findings. + /// + private Dictionary _analysisWeights; + /// /// The number of order events already consumed by previous in-run runs, from which the /// next run resumes reading the order event stream. @@ -74,13 +79,17 @@ public class ResultsAnalyzer protected bool IsInRun => _dataProvider != null; /// - /// The diagnostic analyses to run. Created once and reused across runs, since the analyses - /// are stateless. In-run instances filter the set to the analyses that declare they can run - /// while the backtest is in progress (see ). + /// The diagnostic analyses to run, in execution order: descending by weight, so changing an + /// analysis weight automatically reorders execution. Created once and reused across runs, + /// since the analyses are stateless and their weights are constant. In-run instances filter + /// the set to the analyses that declare they can run while the backtest is in progress + /// (see ). /// - protected IReadOnlyCollection Analyses => _analyses ??= IsInRun - ? GetAnalyses().Where(analysis => analysis.RunsInRun).ToList() - : GetAnalyses(); + protected IReadOnlyCollection Analyses => _analyses ??= (IsInRun + ? GetAnalyses().Where(analysis => analysis.RunsInRun) + : GetAnalyses()) + .OrderByDescending(analysis => analysis.Weight) + .ToList(); /// /// Whether the equity and benchmark curves should be built before running the analyses. @@ -200,8 +209,7 @@ public static ResultsAnalyzer CreateForInRunAnalysis(QCAlgorithm algorithm, Lang var timer = Stopwatch.StartNew(); var timeLimit = TimeSpan.FromSeconds(timeLimitSeconds); - // Instances are sorted by their own Weight — changing a weight automatically reorders execution. - foreach (var analysis in analyses.OrderByDescending(a => a.Weight)) + foreach (var analysis in analyses) { if (responses.Count >= maxFailedAnalyses) { @@ -403,9 +411,9 @@ protected virtual IReadOnlyCollection GetAnalyses() => /// private IReadOnlyList RankFindings(int maxFailedAnalyses) { - var weights = Analyses.ToDictionary(analysis => analysis.GetType().Name, analysis => analysis.Weight); + _analysisWeights ??= Analyses.ToDictionary(analysis => analysis.GetType().Name, analysis => analysis.Weight); return _findings.Values - .OrderByDescending(finding => weights.GetValueOrDefault(BaseAnalysisName(finding.Name))) + .OrderByDescending(finding => _analysisWeights.GetValueOrDefault(BaseAnalysisName(finding.Name))) .Take(maxFailedAnalyses) .ToList(); } From 5ae1518971842dc00c51553334dea9d6d845b019 Mon Sep 17 00:00:00 2001 From: Jhonathan Abreu Date: Tue, 4 Aug 2026 14:52:18 -0400 Subject: [PATCH 27/33] Analyze the intermediate complete result in the in-run analysis instead of pulling the backtest data through the provider --- .../Analysis/IInRunAnalysisDataProvider.cs | 28 +--- Engine/Results/Analysis/ResultsAnalyzer.cs | 107 ++++++++++---- Engine/Results/BacktestingResultHandler.cs | 61 ++------ .../Results/ResultsAnalyzerInRunTests.cs | 138 ++++++++++-------- Tests/Engine/Results/ResultsAnalyzerTests.cs | 2 +- 5 files changed, 166 insertions(+), 170 deletions(-) diff --git a/Engine/Results/Analysis/IInRunAnalysisDataProvider.cs b/Engine/Results/Analysis/IInRunAnalysisDataProvider.cs index f8eb1d2a1db0..eb6091d1a58d 100644 --- a/Engine/Results/Analysis/IInRunAnalysisDataProvider.cs +++ b/Engine/Results/Analysis/IInRunAnalysisDataProvider.cs @@ -13,45 +13,23 @@ * limitations under the License. * */ -using QuantConnect.Orders; using System.Collections.Generic; namespace QuantConnect.Lean.Engine.Results.Analysis { /// /// Provides an in-run instance access to the data of the running - /// backtest. Implemented by the result handler, which owns the data and its synchronization, - /// while the analyzer decides what to read and how much, keeping its incremental consumption - /// state private. + /// backtest that the intermediate results handed to it don't carry. Implemented by the result + /// handler, which owns the data and its synchronization, while the analyzer decides what to + /// read and how much, keeping its incremental consumption state private. /// public interface IInRunAnalysisDataProvider { - /// - /// Gets the orders placed so far. - /// - IDictionary GetOrders(); - - /// - /// Gets the order events produced from the given position in the order event stream. - /// - List GetOrderEvents(int fromPosition); - /// /// Gets the log lines produced from the given position in the log stream. /// IReadOnlyList GetLogs(int fromPosition); - /// - /// Gets clones of the requested charts, safe to read without further synchronization. - /// - IDictionary GetChartSnapshots(IReadOnlyList chartNames); - - /// - /// Whether the strategy equity chart has samples yet. Until the first sample exists, - /// the generated statistics are all-zero defaults. - /// - bool HasEquitySamples(); - /// /// Takes a sample of the engine speed counters, or null when the counters should /// not be sampled, like while the algorithm warms up. diff --git a/Engine/Results/Analysis/ResultsAnalyzer.cs b/Engine/Results/Analysis/ResultsAnalyzer.cs index 80fc4202fedf..b0876f17a3e1 100644 --- a/Engine/Results/Analysis/ResultsAnalyzer.cs +++ b/Engine/Results/Analysis/ResultsAnalyzer.cs @@ -16,6 +16,7 @@ using QuantConnect.Algorithm; using QuantConnect.Lean.Engine.Results.Analysis.Analyses; using QuantConnect.Logging; +using QuantConnect.Orders; using QuantConnect.Packets; using QuantConnect.Securities; using QuantConnect.Statistics; @@ -32,8 +33,8 @@ namespace QuantConnect.Lean.Engine.Results.Analysis /// a final analysis instance is created with the completed result and logs, and runs the /// full analysis set once; an in-run analysis instance is created with an /// , is kept alive for the duration of the backtest, - /// and periodically runs the in-run capable analyses incrementally against snapshots of the - /// intermediate results pulled from the provider. + /// and periodically runs the in-run capable analyses incrementally against the intermediate + /// results, complemented with data pulled from the provider. /// public class ResultsAnalyzer { @@ -60,10 +61,14 @@ public class ResultsAnalyzer private Dictionary _analysisWeights; /// - /// The number of order events already consumed by previous in-run runs, from which the - /// next run resumes reading the order event stream. + /// The identity of the newest order event analyzed by previous in-run runs: the order id + /// and the per-order event id. The intermediate results carry a truncated, newest-first + /// window of the order events, so each run consumes the window until it finds this + /// watermark. When the watermark is not in the window the whole window is new, and any + /// events already evicted from it are missed until the final analysis re-scans the + /// complete stream. /// - private int _orderEventsPosition; + private (int OrderId, int Id)? _lastAnalyzedOrderEvent; /// /// The number of log entries already consumed by previous in-run runs, from which the @@ -108,12 +113,6 @@ public class ResultsAnalyzer /// protected AlgorithmSpeedTracker SpeedTracker { get; } - /// - /// The names of the charts the in-run analyses read. Only these are requested - /// from the data provider on each in-run run. - /// - public static IReadOnlyList RequiredCharts { get; } = [BaseResultsHandler.PortfolioMarginKey]; - /// /// Initializes a new instance of the class for the final /// analysis of a completed backtest. Use or @@ -235,13 +234,18 @@ public static ResultsAnalyzer CreateForInRunAnalysis(QCAlgorithm algorithm, Lang } /// - /// Runs the in-run analyses against the current backtest state pulled from the data provider: - /// a snapshot of the orders and required charts, plus only the order events and log lines - /// produced since the previous run. The returned findings are the merge of this run's - /// findings into the ones accumulated by previous runs: findings from analyses scanning - /// the order event and log streams are accumulated (first sample kept, counts totaled), - /// while findings from state-based analyses are replaced on every run. + /// Runs the in-run analyses against the given intermediate backtest result, complemented + /// with the log lines produced since the previous run, pulled from the provider. The + /// returned findings are the merge of this run's findings into the ones accumulated by + /// previous runs: findings from analyses scanning the order event and log streams are + /// accumulated (first sample kept, counts totaled), while findings from state-based + /// analyses are replaced on every run. /// + /// The current intermediate backtest result. Its orders and order events + /// are windows truncated to the most recent ones, so the in-run analyses can miss orders and + /// events already evicted from them; the final analysis re-scans the complete data. Its charts + /// are the handler's live ones, read without synchronization: a torn read while the algorithm + /// thread updates them can fail a run, which the handler catches, and the next run retries. /// The current total algorithm performance, for analyses that read /// portfolio statistics. Withheld from the analyses until the first equity sample exists, since /// the statistics are all-zero defaults before that. @@ -250,30 +254,27 @@ public static ResultsAnalyzer CreateForInRunAnalysis(QCAlgorithm algorithm, Lang /// processing while it runs. /// Maximum number of failing analyses to return. /// The accumulated findings, ranked by analysis weight. - public IReadOnlyList Run(AlgorithmPerformance totalPerformance, int timeLimitSeconds = 1, - int maxFailedAnalyses = 10) + public IReadOnlyList Run(BacktestResult result, AlgorithmPerformance totalPerformance, + int timeLimitSeconds = 1, int maxFailedAnalyses = 10) { ThrowIfNotInRunInstance(); - // The analyses read the charts without synchronizing with the result handler, so they get clones - var charts = _dataProvider.GetChartSnapshots(RequiredCharts); - // Equity is not sampled while the algorithm warms up, so until the first sample exists // the generated statistics are all-zero defaults that would flag a false non-positive // portfolio value finding. Withhold them so the analyses reading them skip instead - if (_algorithm?.IsWarmingUp == true || !_dataProvider.HasEquitySamples()) + if (_algorithm?.IsWarmingUp == true || !HasEquitySamples(result)) { totalPerformance = null; } var snapshot = new BacktestResult(new BacktestResultParameters( - charts, - _dataProvider.GetOrders(), - _algorithm?.Transactions.TransactionRecord ?? new Dictionary(), + result?.Charts ?? new Dictionary(), + result?.Orders ?? new Dictionary(), + result?.ProfitLoss ?? new Dictionary(), new Dictionary(), new Dictionary(), new Dictionary(), - _dataProvider.GetOrderEvents(_orderEventsPosition), + ExtractNewOrderEvents(result?.OrderEvents), totalPerformance)); var logs = _dataProvider.GetLogs(_logsPosition); @@ -282,6 +283,48 @@ public static ResultsAnalyzer CreateForInRunAnalysis(QCAlgorithm algorithm, Lang return Run(snapshot, logs, _dataProvider.TakeSpeedSample(), timeLimitSeconds, maxFailedAnalyses); } + /// + /// Whether the strategy equity chart in the given result has samples yet. Until the first + /// sample exists, the generated statistics are all-zero defaults. + /// + private static bool HasEquitySamples(Result result) + { + return result?.Charts != null && + result.Charts.TryGetValue(BaseResultsHandler.StrategyEquityKey, out var equityChart) && + equityChart.Series.TryGetValue(BaseResultsHandler.EquityKey, out var equitySeries) && + equitySeries.Values.Count > 0; + } + + /// + /// Extracts the order events not yet analyzed from the newest-first, truncated order events + /// window the intermediate results carry, returned in chronological order, and advances the + /// watermark to the newest one. The watermark advances even if the time limit later + /// truncates the run, so analyses that didn't get to run miss this delta until the final + /// analysis re-scans the complete stream — same trade-off as the log position advancement. + /// + private List ExtractNewOrderEvents(IReadOnlyList orderEvents) + { + var newOrderEvents = new List(); + for (var i = 0; i < (orderEvents?.Count ?? 0); i++) + { + var orderEvent = orderEvents[i]; + if (orderEvent.OrderId == _lastAnalyzedOrderEvent?.OrderId && orderEvent.Id == _lastAnalyzedOrderEvent?.Id) + { + break; + } + newOrderEvents.Add(orderEvent); + } + + if (newOrderEvents.Count > 0) + { + newOrderEvents.Reverse(); + var newest = newOrderEvents[^1]; + _lastAnalyzedOrderEvent = (newest.OrderId, newest.Id); + } + + return newOrderEvents; + } + /// /// Completes the speed metrics with one final sample so they cover the backtest through /// its end, and returns the tracker for the final analysis to reuse. The tracker is left @@ -375,11 +418,11 @@ protected virtual IReadOnlyCollection GetAnalyses() => } var newFindings = Run(timeLimitSeconds, maxFailedAnalyses); - // The positions are advanced even when the time limit truncates a run, so the analyses that - // didn't get to run miss this delta until the final analysis re-scans the complete streams. - // Stress tests show runs complete in a fraction of the time limit, but if its trace message - // starts showing up in logs, revisit this (e.g. track per-analysis positions). - _orderEventsPosition += result.OrderEvents?.Count ?? 0; + // The log position is advanced (like the order event watermark was on extraction) even + // when the time limit truncates a run, so the analyses that didn't get to run miss this + // delta until the final analysis re-scans the complete streams. Stress tests show runs + // complete in a fraction of the time limit, but if its trace message starts showing up + // in logs, revisit this (e.g. track per-analysis positions). _logsPosition += logs?.Count ?? 0; // State-based analyses are recomputed from scratch each run: remove their previous diff --git a/Engine/Results/BacktestingResultHandler.cs b/Engine/Results/BacktestingResultHandler.cs index 912a84adcbd2..f4b35910f38a 100644 --- a/Engine/Results/BacktestingResultHandler.cs +++ b/Engine/Results/BacktestingResultHandler.cs @@ -238,7 +238,7 @@ private void Update() if (RunResultsAnalysis) { - completeResult.Analysis = RunInRunResultsAnalysis(statisticsResult.TotalPerformance); + completeResult.Analysis = RunInRunResultsAnalysis(completeResult, statisticsResult.TotalPerformance); SendInRunAnalysis(completeResult.Analysis, progress); } @@ -457,14 +457,18 @@ protected void SendFinalResult() } /// - /// Runs the in-run results analyzer against the current intermediate backtest state, - /// accessed through the implementation. - /// Invoked periodically while the backtest is still running, unlike the full analysis - /// performed by when the backtest ends. + /// Runs the in-run results analyzer against the current intermediate backtest result, + /// complemented with data accessed through the + /// implementation. Invoked periodically while the backtest is still running, unlike the + /// full analysis performed by when the backtest ends. /// + /// The current intermediate backtest result. Its orders and order + /// events are truncated to the most recent ones, so the in-run analyses can miss data between + /// runs; the final analysis re-scans the complete streams. /// The current total algorithm performance, for analyses that read portfolio statistics /// The failed analyses with solutions, or null if the analysis could not run - protected virtual IReadOnlyList RunInRunResultsAnalysis(AlgorithmPerformance totalPerformance) + protected virtual IReadOnlyList RunInRunResultsAnalysis(BacktestResult completeResult, + AlgorithmPerformance totalPerformance) { try { @@ -474,7 +478,7 @@ protected void SendFinalResult() } _inRunResultsAnalyzer ??= ResultsAnalyzer.CreateForInRunAnalysis(AlgorithmInstance, _job.Language, this); - return _inRunResultsAnalyzer.Run(totalPerformance); + return _inRunResultsAnalyzer.Run(completeResult, totalPerformance); } catch (Exception ex) { @@ -485,17 +489,6 @@ protected void SendFinalResult() #region IInRunAnalysisDataProvider implementation - /// - /// Gets the orders placed so far. - /// - IDictionary IInRunAnalysisDataProvider.GetOrders() => TransactionHandler.Orders.ToDictionary(); - - /// - /// Gets the order events produced from the given position in the order event stream. - /// - List IInRunAnalysisDataProvider.GetOrderEvents(int fromPosition) - => TransactionHandler.OrderEvents.Skip(fromPosition).ToList(); - /// /// Gets the log lines produced from the given position in the log stream. /// @@ -507,38 +500,6 @@ IReadOnlyList IInRunAnalysisDataProvider.GetLogs(int fromPosition) } } - /// - /// Gets clones of the requested charts, safe for the analyses to read without holding the chart lock. - /// - IDictionary IInRunAnalysisDataProvider.GetChartSnapshots(IReadOnlyList chartNames) - { - var charts = new Dictionary(); - lock (ChartLock) - { - foreach (var chartName in chartNames) - { - if (Charts.TryGetValue(chartName, out var chart)) - { - charts[chartName] = chart.Clone(); - } - } - } - return charts; - } - - /// - /// Whether the strategy equity chart has samples yet. - /// - bool IInRunAnalysisDataProvider.HasEquitySamples() - { - lock (ChartLock) - { - return Charts.TryGetValue(StrategyEquityKey, out var equityChart) && - equityChart.Series.TryGetValue(EquityKey, out var equitySeries) && - equitySeries.Values.Count > 0; - } - } - /// /// Takes a sample of the engine speed counters for the algorithm speed analysis. /// Null while the algorithm warms up, since the warm-up pace would skew the speed metrics. diff --git a/Tests/Engine/Results/ResultsAnalyzerInRunTests.cs b/Tests/Engine/Results/ResultsAnalyzerInRunTests.cs index 5216c4897812..6229d9a30974 100644 --- a/Tests/Engine/Results/ResultsAnalyzerInRunTests.cs +++ b/Tests/Engine/Results/ResultsAnalyzerInRunTests.cs @@ -29,7 +29,8 @@ namespace QuantConnect.Tests.Engine.Results { /// - /// Tests the in-run mode of , driven through a fake + /// Tests the in-run mode of , driven through intermediate + /// results carrying truncated order and order event windows, complemented by a fake /// . The core, mode-independent behavior /// is covered by . /// @@ -41,24 +42,50 @@ public class ResultsAnalyzerInRunTests [Test] public void OrderEventAndLogStreamsAreConsumedIncrementally() { - var analyzer = new TestInRunResultsAnalyzer(new FakeAnalysisA(10)); + var seenOrderEvents = new List(); + var fake = new FakeAnalysisA(10) { OnParameters = parameters => seenOrderEvents.AddRange(parameters.Result.OrderEvents) }; + var analyzer = new TestInRunResultsAnalyzer(fake); + // The order event windows fully overlap (every event fits in the window), but the + // watermark dedupes them: each event is analyzed exactly once, in chronological order analyzer.Run(3, new[] { "log 1", "log 2" }); analyzer.Run(5, new[] { "log 3" }); - // Runs without new order events or logs don't move the read positions + // Runs without new order events or logs produce empty deltas and don't move the log position analyzer.Run(0, null); analyzer.Run(0, null); - CollectionAssert.AreEqual(new[] { 0, 3, 8, 8 }, analyzer.Provider.RequestedOrderEventsPositions); + CollectionAssert.AreEqual(analyzer.OrderEventStream, seenOrderEvents); CollectionAssert.AreEqual(new[] { 0, 2, 3, 3 }, analyzer.Provider.RequestedLogsPositions); } + [Test] + public void OrderEventsEvictedFromTheTruncatedWindowAreMissed() + { + var seenOrderEvents = new List(); + var fake = new FakeAnalysisA(10) { OnParameters = parameters => seenOrderEvents.AddRange(parameters.Result.OrderEvents) }; + var analyzer = new TestInRunResultsAnalyzer(fake); + + // 3 new events, but only the newest 2 fit the window + analyzer.Run(3, null, orderEventsWindowSize: 2); + // 4 more events: the watermark is not in the window, so the whole window is new + // and the evicted events in between are missed + analyzer.Run(4, null, orderEventsWindowSize: 2); + + var stream = analyzer.OrderEventStream; + CollectionAssert.AreEqual(new[] { stream[1], stream[2], stream[5], stream[6] }, seenOrderEvents); + } + [Test] public void StreamPositionsAdvanceEvenWhenTheTimeLimitTruncatesTheRun() { + var seenOrderEventCounts = new List(); var truncatedRan = false; // The slow analysis has the higher weight so it runs first and exhausts the time limit - var slow = new FakeAnalysisA(20) { OnRun = () => Thread.Sleep(1100) }; + var slow = new FakeAnalysisA(20) + { + OnParameters = parameters => seenOrderEventCounts.Add(parameters.Result.OrderEvents.Count), + OnRun = () => Thread.Sleep(1100) + }; var truncated = new FakeAnalysisB(10) { OnRun = () => truncatedRan = true }; var analyzer = new TestInRunResultsAnalyzer(slow, truncated); @@ -68,7 +95,7 @@ public void StreamPositionsAdvanceEvenWhenTheTimeLimitTruncatesTheRun() // The next run still resumes after the consumed order events and logs slow.OnRun = null; analyzer.Run(0, null); - CollectionAssert.AreEqual(new[] { 0, 4 }, analyzer.Provider.RequestedOrderEventsPositions); + CollectionAssert.AreEqual(new[] { 4, 0 }, seenOrderEventCounts); CollectionAssert.AreEqual(new[] { 0, 1 }, analyzer.Provider.RequestedLogsPositions); } @@ -226,20 +253,19 @@ public void CompletedSpeedTrackingAddsAFinalSampleAndReturnsTheTracker() } [Test] - public void SnapshotIsBuiltFromTheDataProvider() + public void SnapshotIsBuiltFromTheIntermediateResult() { ResultsAnalysisRunParameters seenParameters = null; var fake = new FakeAnalysisA(10) { OnParameters = parameters => seenParameters = parameters }; var analyzer = new TestInRunResultsAnalyzer(fake); - analyzer.Provider.Orders[1] = new MarketOrder(); - analyzer.Provider.Charts["a chart"] = new Chart("a chart"); + var charts = new Dictionary { ["a chart"] = new Chart("a chart") }; + var orders = new Dictionary { [1] = new MarketOrder() }; - analyzer.Run(2, new[] { "log" }); + analyzer.Run(2, new[] { "log" }, orders: orders, charts: charts); - // Only the charts the in-run analyses read are requested from the provider - CollectionAssert.AreEqual(ResultsAnalyzer.RequiredCharts, analyzer.Provider.RequestedChartNames); - Assert.IsTrue(seenParameters.Result.Charts.ContainsKey("a chart")); - Assert.AreEqual(1, seenParameters.Result.Orders.Count); + // The charts, orders and order events come from the intermediate result + Assert.AreSame(charts, seenParameters.Result.Charts); + Assert.AreSame(orders, seenParameters.Result.Orders); Assert.AreEqual(2, seenParameters.Result.OrderEvents.Count); CollectionAssert.AreEqual(new[] { "log" }, seenParameters.Logs); } @@ -252,13 +278,20 @@ public void StatisticsAreWithheldUntilEquityHasSamples() var analyzer = new TestInRunResultsAnalyzer(fake); var performance = new AlgorithmPerformance(); - // Equity has no samples yet: the all-zero default statistics are withheld - analyzer.Provider.EquityHasSamples = false; - analyzer.Run(performance); + // The equity chart has no samples yet: the all-zero default statistics are withheld + analyzer.Run(new BacktestResult(), totalPerformance: performance); Assert.IsNull(seenResult.TotalPerformance); - analyzer.Provider.EquityHasSamples = true; - analyzer.Run(performance); + var equitySeries = new Series(BaseResultsHandler.EquityKey); + equitySeries.AddPoint(new DateTime(2024, 01, 02), 100000m); + var equityChart = new Chart(BaseResultsHandler.StrategyEquityKey); + equityChart.AddSeries(equitySeries); + var result = new BacktestResult + { + Charts = new Dictionary { [equityChart.Name] = equityChart } + }; + + analyzer.Run(result, totalPerformance: performance); Assert.AreSame(performance, seenResult.TotalPerformance); } @@ -293,16 +326,6 @@ public void FindingsAreRankedByAnalysisWeightAndCapped() findings.Select(finding => finding.Name)); } - [Test] - public void RequiredChartsAreTheChartsReadByTheInRunAnalyses() - { - // The data provider only clones these charts into the analyzed snapshot, - // so this must stay in sync with the charts the in-run analyses read - CollectionAssert.AreEquivalent( - new[] { BaseResultsHandler.PortfolioMarginKey }, - ResultsAnalyzer.RequiredCharts); - } - [Test] public void DefaultAnalysisSetIsTheInRunCapableSubsetOfTheFinalSet() { @@ -340,6 +363,12 @@ private class TestInRunResultsAnalyzer : ResultsAnalyzer public FakeDataProvider Provider { get; } + /// + /// The full, chronological order event stream of the simulated backtest, from which the + /// intermediate results' truncated windows are built. + /// + public List OrderEventStream { get; } = new(); + public int GetAnalysesCallCount { get; private set; } public TestInRunResultsAnalyzer(params BaseResultsAnalysis[] analyses) @@ -355,16 +384,29 @@ private TestInRunResultsAnalyzer(FakeDataProvider provider, BaseResultsAnalysis[ } /// - /// Appends the new order events and logs to the provider's streams and runs the analyzer, - /// mirroring the incremental stream growth the analyzer sees in a running backtest. + /// Appends the new order events and logs to the backtest's streams and runs the analyzer + /// against an intermediate result carrying the truncated, newest-first order events + /// window the backtesting result handler builds. /// public IReadOnlyList Run(int newOrderEventsCount, string[] newLogs, - AlgorithmSpeedSample? speedSample = null, int timeLimitSeconds = 1, int maxFailedAnalyses = 10) + AlgorithmSpeedSample? speedSample = null, int timeLimitSeconds = 1, int maxFailedAnalyses = 10, + int orderEventsWindowSize = 100, Dictionary orders = null, Dictionary charts = null) { - Provider.OrderEvents.AddRange(Enumerable.Range(0, newOrderEventsCount).Select(_ => new OrderEvent())); + for (var i = 0; i < newOrderEventsCount; i++) + { + // One event per order, so the (order id, per-order event id) pairs stay unique + OrderEventStream.Add(new OrderEvent { OrderId = OrderEventStream.Count + 1, Id = 1 }); + } Provider.Logs.AddRange(newLogs ?? Array.Empty()); Provider.NextSpeedSample = speedSample; - return Run(totalPerformance: null, timeLimitSeconds, maxFailedAnalyses); + + var result = new BacktestResult + { + Charts = charts ?? new Dictionary(), + Orders = orders ?? new Dictionary(), + OrderEvents = Enumerable.Reverse(OrderEventStream).Take(orderEventsWindowSize).ToList() + }; + return Run(result, totalPerformance: null, timeLimitSeconds, maxFailedAnalyses); } protected override IReadOnlyCollection GetAnalyses() @@ -386,46 +428,18 @@ public DefaultSetInRunResultsAnalyzer() private sealed class FakeDataProvider : IInRunAnalysisDataProvider { - public Dictionary Orders { get; } = new(); - - public List OrderEvents { get; } = new(); - public List Logs { get; } = new(); - public Dictionary Charts { get; } = new(); - - public bool EquityHasSamples { get; set; } = true; - public AlgorithmSpeedSample? NextSpeedSample { get; set; } - public List RequestedOrderEventsPositions { get; } = new(); - public List RequestedLogsPositions { get; } = new(); - public IReadOnlyList RequestedChartNames { get; private set; } - - public IDictionary GetOrders() => Orders; - - public List GetOrderEvents(int fromPosition) - { - RequestedOrderEventsPositions.Add(fromPosition); - return OrderEvents.Skip(fromPosition).ToList(); - } - public IReadOnlyList GetLogs(int fromPosition) { RequestedLogsPositions.Add(fromPosition); return Logs.Skip(fromPosition).ToList(); } - public IDictionary GetChartSnapshots(IReadOnlyList chartNames) - { - RequestedChartNames = chartNames; - return Charts; - } - - public bool HasEquitySamples() => EquityHasSamples; - public AlgorithmSpeedSample? TakeSpeedSample() => NextSpeedSample; } diff --git a/Tests/Engine/Results/ResultsAnalyzerTests.cs b/Tests/Engine/Results/ResultsAnalyzerTests.cs index 9bbf8b475f8b..754c815cc067 100644 --- a/Tests/Engine/Results/ResultsAnalyzerTests.cs +++ b/Tests/Engine/Results/ResultsAnalyzerTests.cs @@ -206,7 +206,7 @@ public void InRunOperationsRequireAnInstanceCreatedWithADataProvider() // This is a final-analysis instance: the in-run entry points need the data provider var analyzer = new TestResultsAnalyzer(false, new FakeAnalysisA(10)); - Assert.Throws(() => analyzer.Run(totalPerformance: null)); + Assert.Throws(() => analyzer.Run(result: null, totalPerformance: null)); Assert.Throws(() => analyzer.CompleteSpeedTracking()); } From a5a7efa036cdc59fce901af2ac3f1dc4986d1119 Mon Sep 17 00:00:00 2001 From: Jhonathan Abreu Date: Tue, 4 Aug 2026 15:16:26 -0400 Subject: [PATCH 28/33] Mute accumulated in-run findings once they reach a maximum reported occurrence count --- Engine/Results/Analysis/ResultsAnalyzer.cs | 34 ++++++++++++- .../Results/ResultsAnalyzerInRunTests.cs | 50 +++++++++++++++++++ 2 files changed, 83 insertions(+), 1 deletion(-) diff --git a/Engine/Results/Analysis/ResultsAnalyzer.cs b/Engine/Results/Analysis/ResultsAnalyzer.cs index b0876f17a3e1..94a77d789dcd 100644 --- a/Engine/Results/Analysis/ResultsAnalyzer.cs +++ b/Engine/Results/Analysis/ResultsAnalyzer.cs @@ -40,8 +40,25 @@ public class ResultsAnalyzer { private readonly QCAlgorithm _algorithm; private readonly Language _language; + /// + /// The accumulated occurrence count at which an in-run stream-based finding stops being + /// returned. A finding is returned while its occurrences are below this value, and one last + /// time on the run that reaches it; after that it keeps accumulating occurrences silently. + /// This bounds how often a recurring finding is re-reported, so consumers of the periodic + /// findings (like LLMs) see its first occurrences without the ever-growing count polluting + /// their context on every update. State-based findings are not affected: their counts are + /// recomputed snapshots, not accumulated occurrences. + /// + private const int MaxReportedFindingOccurrences = 5; + private readonly IInRunAnalysisDataProvider _dataProvider; private readonly Dictionary _findings = new(); + + /// + /// The names of the accumulated findings that reached + /// and are no longer included in the returned findings, while still accumulating occurrences. + /// + private readonly HashSet _mutedFindings = new(); private IReadOnlyList _logs; private SortedList _equityCurve; private SortedList _benchmarkEquityCurve; @@ -451,14 +468,29 @@ protected virtual IReadOnlyCollection GetAnalyses() => /// /// Ranks the accumulated findings by their analysis weight, capped to the given maximum. + /// Muted findings are left out, and the returned stream-based findings that reached + /// are muted for the runs that follow. /// private IReadOnlyList RankFindings(int maxFailedAnalyses) { _analysisWeights ??= Analyses.ToDictionary(analysis => analysis.GetType().Name, analysis => analysis.Weight); - return _findings.Values + var findings = _findings.Values + .Where(finding => !_mutedFindings.Contains(finding.Name)) .OrderByDescending(finding => _analysisWeights.GetValueOrDefault(BaseAnalysisName(finding.Name))) .Take(maxFailedAnalyses) .ToList(); + + // Only findings that were actually returned are muted, so a finding a truncated run + // never got to report is not silenced before it is seen at least once + foreach (var finding in findings) + { + if (!IsStateBased(finding.Name) && (finding.Count ?? 1) >= MaxReportedFindingOccurrences) + { + _mutedFindings.Add(finding.Name); + } + } + + return findings; } /// diff --git a/Tests/Engine/Results/ResultsAnalyzerInRunTests.cs b/Tests/Engine/Results/ResultsAnalyzerInRunTests.cs index 6229d9a30974..373bb035642a 100644 --- a/Tests/Engine/Results/ResultsAnalyzerInRunTests.cs +++ b/Tests/Engine/Results/ResultsAnalyzerInRunTests.cs @@ -150,6 +150,56 @@ public void StreamBasedFindingsPersistWhenNotReemitted() Assert.AreEqual(4, finding.Count); } + [Test] + public void AccumulatedFindingsAreMutedOnceTheyReachTheMaxReportedOccurrences() + { + // The analyzer mutes accumulated findings once they reach 5 reported occurrences + var fake = new FakeAnalysisA(10) + { + Findings = () => MakeFindings(nameof(FakeAnalysisA), "sample", 3) + }; + var analyzer = new TestInRunResultsAnalyzer(fake); + + // Below the cap: reported + Assert.AreEqual(3, analyzer.Run(1, new[] { "log" }).Single().Count); + // The run that reaches the cap still reports the finding, with its full count + Assert.AreEqual(6, analyzer.Run(1, new[] { "log" }).Single().Count); + // Beyond the cap the finding keeps accumulating, but is no longer reported + Assert.IsEmpty(analyzer.Run(1, new[] { "log" })); + fake.Findings = () => new List(); + Assert.IsEmpty(analyzer.Run(1, new[] { "log" })); + } + + [Test] + public void FindingsWhoseFirstOccurrencesAlreadyExceedTheCapAreReportedOnce() + { + // A single delta can carry more occurrences than the cap: the finding is still + // reported once before being muted, so it is never silently dropped + var fake = new FakeAnalysisA(10) + { + Findings = () => MakeFindings(nameof(FakeAnalysisA), "sample", 100) + }; + var analyzer = new TestInRunResultsAnalyzer(fake); + + Assert.AreEqual(100, analyzer.Run(1, new[] { "log" }).Single().Count); + Assert.IsEmpty(analyzer.Run(1, new[] { "log" })); + } + + [Test] + public void StateBasedFindingsAreNotMutedByTheReportedOccurrencesCap() + { + // State-based counts are recomputed snapshots, not accumulated occurrences + var fake = new FakeAnalysisA(10) + { + StateBased = true, + Findings = () => MakeFindings(nameof(FakeAnalysisA), "sample", 100) + }; + var analyzer = new TestInRunResultsAnalyzer(fake); + + Assert.AreEqual(100, analyzer.Run(1, new[] { "log" }).Single().Count); + Assert.AreEqual(100, analyzer.Run(1, new[] { "log" }).Single().Count); + } + [Test] public void StateBasedFindingsAreReplacedOnEveryRun() { From bafc5859560240e86231b5ee42a3a4eaa086e63d Mon Sep 17 00:00:00 2001 From: Jhonathan Abreu Date: Wed, 5 Aug 2026 09:55:39 -0400 Subject: [PATCH 29/33] Mute state-based in-run findings once reported in a maximum number of runs --- Engine/Results/Analysis/ResultsAnalyzer.cs | 31 ++++++++++------- .../Results/ResultsAnalyzerInRunTests.cs | 33 ++++++++++++++++--- 2 files changed, 49 insertions(+), 15 deletions(-) diff --git a/Engine/Results/Analysis/ResultsAnalyzer.cs b/Engine/Results/Analysis/ResultsAnalyzer.cs index 94a77d789dcd..9ed525fe110e 100644 --- a/Engine/Results/Analysis/ResultsAnalyzer.cs +++ b/Engine/Results/Analysis/ResultsAnalyzer.cs @@ -41,13 +41,11 @@ public class ResultsAnalyzer private readonly QCAlgorithm _algorithm; private readonly Language _language; /// - /// The accumulated occurrence count at which an in-run stream-based finding stops being - /// returned. A finding is returned while its occurrences are below this value, and one last - /// time on the run that reaches it; after that it keeps accumulating occurrences silently. - /// This bounds how often a recurring finding is re-reported, so consumers of the periodic - /// findings (like LLMs) see its first occurrences without the ever-growing count polluting - /// their context on every update. State-based findings are not affected: their counts are - /// recomputed snapshots, not accumulated occurrences. + /// The occurrence count at which an in-run finding is muted: reported while below this + /// value and one last time on the run that reaches it, then no longer returned, so a + /// recurring finding is not re-sent to consumers (like LLMs) on every update. + /// Stream-based findings count their accumulated occurrences; state-based findings count + /// the runs they were reported in, since their counts are recomputed snapshots. /// private const int MaxReportedFindingOccurrences = 5; @@ -55,10 +53,17 @@ public class ResultsAnalyzer private readonly Dictionary _findings = new(); /// - /// The names of the accumulated findings that reached - /// and are no longer included in the returned findings, while still accumulating occurrences. + /// The names of the findings that reached + /// and are no longer included in the returned findings. Stream-based findings keep + /// accumulating occurrences while muted. /// private readonly HashSet _mutedFindings = new(); + + /// + /// The number of runs each state-based finding has been reported in, used to mute it. + /// Never reset: a finding that clears and later fails again stays muted. + /// + private readonly Dictionary _reportedStateBasedFindingRuns = new(); private IReadOnlyList _logs; private SortedList _equityCurve; private SortedList _benchmarkEquityCurve; @@ -468,7 +473,7 @@ protected virtual IReadOnlyCollection GetAnalyses() => /// /// Ranks the accumulated findings by their analysis weight, capped to the given maximum. - /// Muted findings are left out, and the returned stream-based findings that reached + /// Muted findings are left out, and the returned findings that reached /// are muted for the runs that follow. /// private IReadOnlyList RankFindings(int maxFailedAnalyses) @@ -484,7 +489,11 @@ protected virtual IReadOnlyCollection GetAnalyses() => // never got to report is not silenced before it is seen at least once foreach (var finding in findings) { - if (!IsStateBased(finding.Name) && (finding.Count ?? 1) >= MaxReportedFindingOccurrences) + // State-based counts are recomputed snapshots, so count reported runs instead + var occurrences = IsStateBased(finding.Name) + ? _reportedStateBasedFindingRuns[finding.Name] = _reportedStateBasedFindingRuns.GetValueOrDefault(finding.Name) + 1 + : finding.Count ?? 1; + if (occurrences >= MaxReportedFindingOccurrences) { _mutedFindings.Add(finding.Name); } diff --git a/Tests/Engine/Results/ResultsAnalyzerInRunTests.cs b/Tests/Engine/Results/ResultsAnalyzerInRunTests.cs index 373bb035642a..428500cdb27d 100644 --- a/Tests/Engine/Results/ResultsAnalyzerInRunTests.cs +++ b/Tests/Engine/Results/ResultsAnalyzerInRunTests.cs @@ -186,9 +186,9 @@ public void FindingsWhoseFirstOccurrencesAlreadyExceedTheCapAreReportedOnce() } [Test] - public void StateBasedFindingsAreNotMutedByTheReportedOccurrencesCap() + public void StateBasedFindingsAreMutedOnceReportedInTheMaxOccurrenceRuns() { - // State-based counts are recomputed snapshots, not accumulated occurrences + // For state-based findings the cap counts reported runs, not the snapshot count var fake = new FakeAnalysisA(10) { StateBased = true, @@ -196,8 +196,33 @@ public void StateBasedFindingsAreNotMutedByTheReportedOccurrencesCap() }; var analyzer = new TestInRunResultsAnalyzer(fake); - Assert.AreEqual(100, analyzer.Run(1, new[] { "log" }).Single().Count); - Assert.AreEqual(100, analyzer.Run(1, new[] { "log" }).Single().Count); + for (var run = 0; run < 5; run++) + { + Assert.AreEqual(100, analyzer.Run(1, new[] { "log" }).Single().Count); + } + // Muted from the sixth run on, even though the analysis still fails + Assert.IsEmpty(analyzer.Run(1, new[] { "log" })); + } + + [Test] + public void MutedStateBasedFindingsStayMutedWhenTheyClearAndFailAgain() + { + var fake = new FakeAnalysisA(10) + { + StateBased = true, + Findings = () => MakeFindings(nameof(FakeAnalysisA), "sample", 2) + }; + var analyzer = new TestInRunResultsAnalyzer(fake); + for (var run = 0; run < 5; run++) + { + Assert.IsNotEmpty(analyzer.Run(1, new[] { "log" })); + } + + // Clears, then fails again: the reported runs are not reset + fake.Findings = () => new List(); + Assert.IsEmpty(analyzer.Run(1, new[] { "log" })); + fake.Findings = () => MakeFindings(nameof(FakeAnalysisA), "sample", 2); + Assert.IsEmpty(analyzer.Run(1, new[] { "log" })); } [Test] From 02cdef8379bd6c099aaf86313616b7628ac4a7a4 Mon Sep 17 00:00:00 2001 From: Jhonathan Abreu Date: Wed, 5 Aug 2026 10:40:16 -0400 Subject: [PATCH 30/33] Mute in-run findings once returned a maximum number of times regardless of their kind --- Engine/Results/Analysis/ResultsAnalyzer.cs | 42 ++++++------------- .../Results/ResultsAnalyzerInRunTests.cs | 36 +++++----------- 2 files changed, 23 insertions(+), 55 deletions(-) diff --git a/Engine/Results/Analysis/ResultsAnalyzer.cs b/Engine/Results/Analysis/ResultsAnalyzer.cs index 9ed525fe110e..9b4445b3df14 100644 --- a/Engine/Results/Analysis/ResultsAnalyzer.cs +++ b/Engine/Results/Analysis/ResultsAnalyzer.cs @@ -41,29 +41,20 @@ public class ResultsAnalyzer private readonly QCAlgorithm _algorithm; private readonly Language _language; /// - /// The occurrence count at which an in-run finding is muted: reported while below this - /// value and one last time on the run that reaches it, then no longer returned, so a - /// recurring finding is not re-sent to consumers (like LLMs) on every update. - /// Stream-based findings count their accumulated occurrences; state-based findings count - /// the runs they were reported in, since their counts are recomputed snapshots. + /// The number of times an in-run finding is returned before it is muted and no longer + /// reported, so a recurring finding is not re-sent to consumers (like LLMs) on every update. /// - private const int MaxReportedFindingOccurrences = 5; + private const int MaxFindingReports = 5; private readonly IInRunAnalysisDataProvider _dataProvider; private readonly Dictionary _findings = new(); /// - /// The names of the findings that reached - /// and are no longer included in the returned findings. Stream-based findings keep - /// accumulating occurrences while muted. + /// The number of times each finding has been returned, muting it once it reaches + /// . Never reset: a muted finding that clears and later + /// fails again stays muted. /// - private readonly HashSet _mutedFindings = new(); - - /// - /// The number of runs each state-based finding has been reported in, used to mute it. - /// Never reset: a finding that clears and later fails again stays muted. - /// - private readonly Dictionary _reportedStateBasedFindingRuns = new(); + private readonly Dictionary _findingReportCounts = new(); private IReadOnlyList _logs; private SortedList _equityCurve; private SortedList _benchmarkEquityCurve; @@ -473,30 +464,23 @@ protected virtual IReadOnlyCollection GetAnalyses() => /// /// Ranks the accumulated findings by their analysis weight, capped to the given maximum. - /// Muted findings are left out, and the returned findings that reached - /// are muted for the runs that follow. + /// Findings already returned times are muted and left out, + /// and each returned finding counts one report toward that cap. /// private IReadOnlyList RankFindings(int maxFailedAnalyses) { _analysisWeights ??= Analyses.ToDictionary(analysis => analysis.GetType().Name, analysis => analysis.Weight); var findings = _findings.Values - .Where(finding => !_mutedFindings.Contains(finding.Name)) + .Where(finding => _findingReportCounts.GetValueOrDefault(finding.Name) < MaxFindingReports) .OrderByDescending(finding => _analysisWeights.GetValueOrDefault(BaseAnalysisName(finding.Name))) .Take(maxFailedAnalyses) .ToList(); - // Only findings that were actually returned are muted, so a finding a truncated run - // never got to report is not silenced before it is seen at least once + // Only findings that are actually returned count toward muting, so a finding a + // truncated run never got to report is not silenced before it is seen foreach (var finding in findings) { - // State-based counts are recomputed snapshots, so count reported runs instead - var occurrences = IsStateBased(finding.Name) - ? _reportedStateBasedFindingRuns[finding.Name] = _reportedStateBasedFindingRuns.GetValueOrDefault(finding.Name) + 1 - : finding.Count ?? 1; - if (occurrences >= MaxReportedFindingOccurrences) - { - _mutedFindings.Add(finding.Name); - } + _findingReportCounts[finding.Name] = _findingReportCounts.GetValueOrDefault(finding.Name) + 1; } return findings; diff --git a/Tests/Engine/Results/ResultsAnalyzerInRunTests.cs b/Tests/Engine/Results/ResultsAnalyzerInRunTests.cs index 428500cdb27d..aa9dac2b8ccd 100644 --- a/Tests/Engine/Results/ResultsAnalyzerInRunTests.cs +++ b/Tests/Engine/Results/ResultsAnalyzerInRunTests.cs @@ -151,44 +151,28 @@ public void StreamBasedFindingsPersistWhenNotReemitted() } [Test] - public void AccumulatedFindingsAreMutedOnceTheyReachTheMaxReportedOccurrences() + public void FindingsAreMutedOnceReturnedTheMaxNumberOfTimes() { - // The analyzer mutes accumulated findings once they reach 5 reported occurrences + // A finding is returned 5 times and then muted, regardless of its occurrence counts var fake = new FakeAnalysisA(10) { - Findings = () => MakeFindings(nameof(FakeAnalysisA), "sample", 3) + Findings = () => MakeFindings(nameof(FakeAnalysisA), "sample", 100) }; var analyzer = new TestInRunResultsAnalyzer(fake); - // Below the cap: reported - Assert.AreEqual(3, analyzer.Run(1, new[] { "log" }).Single().Count); - // The run that reaches the cap still reports the finding, with its full count - Assert.AreEqual(6, analyzer.Run(1, new[] { "log" }).Single().Count); - // Beyond the cap the finding keeps accumulating, but is no longer reported + for (var report = 1; report <= 5; report++) + { + Assert.AreEqual(100 * report, analyzer.Run(1, new[] { "log" }).Single().Count); + } + // Muted from the sixth run on, even though the finding keeps accumulating Assert.IsEmpty(analyzer.Run(1, new[] { "log" })); fake.Findings = () => new List(); Assert.IsEmpty(analyzer.Run(1, new[] { "log" })); } [Test] - public void FindingsWhoseFirstOccurrencesAlreadyExceedTheCapAreReportedOnce() + public void StateBasedFindingsAreMutedOnceReturnedTheMaxNumberOfTimes() { - // A single delta can carry more occurrences than the cap: the finding is still - // reported once before being muted, so it is never silently dropped - var fake = new FakeAnalysisA(10) - { - Findings = () => MakeFindings(nameof(FakeAnalysisA), "sample", 100) - }; - var analyzer = new TestInRunResultsAnalyzer(fake); - - Assert.AreEqual(100, analyzer.Run(1, new[] { "log" }).Single().Count); - Assert.IsEmpty(analyzer.Run(1, new[] { "log" })); - } - - [Test] - public void StateBasedFindingsAreMutedOnceReportedInTheMaxOccurrenceRuns() - { - // For state-based findings the cap counts reported runs, not the snapshot count var fake = new FakeAnalysisA(10) { StateBased = true, @@ -196,7 +180,7 @@ public void StateBasedFindingsAreMutedOnceReportedInTheMaxOccurrenceRuns() }; var analyzer = new TestInRunResultsAnalyzer(fake); - for (var run = 0; run < 5; run++) + for (var report = 1; report <= 5; report++) { Assert.AreEqual(100, analyzer.Run(1, new[] { "log" }).Single().Count); } From f6011a2e2e84a24ce00d185cd0be735ed03fb81f Mon Sep 17 00:00:00 2001 From: Jhonathan Abreu Date: Wed, 5 Aug 2026 13:43:33 -0400 Subject: [PATCH 31/33] Remove the in-run analysis data provider, the analyzer samples the speed counters and slices the logs itself --- .../Analysis/IInRunAnalysisDataProvider.cs | 39 --------- Engine/Results/Analysis/ResultsAnalyzer.cs | 86 +++++++++++++------ Engine/Results/BacktestingResultHandler.cs | 57 +++--------- Engine/Results/BaseResultsHandler.cs | 12 ++- .../Results/ResultsAnalyzerInRunTests.cs | 84 +++++++++--------- Tests/Engine/Results/ResultsAnalyzerTests.cs | 6 +- 6 files changed, 125 insertions(+), 159 deletions(-) delete mode 100644 Engine/Results/Analysis/IInRunAnalysisDataProvider.cs diff --git a/Engine/Results/Analysis/IInRunAnalysisDataProvider.cs b/Engine/Results/Analysis/IInRunAnalysisDataProvider.cs deleted file mode 100644 index eb6091d1a58d..000000000000 --- a/Engine/Results/Analysis/IInRunAnalysisDataProvider.cs +++ /dev/null @@ -1,39 +0,0 @@ -/* - * QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals. - * Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * -*/ -using System.Collections.Generic; - -namespace QuantConnect.Lean.Engine.Results.Analysis -{ - /// - /// Provides an in-run instance access to the data of the running - /// backtest that the intermediate results handed to it don't carry. Implemented by the result - /// handler, which owns the data and its synchronization, while the analyzer decides what to - /// read and how much, keeping its incremental consumption state private. - /// - public interface IInRunAnalysisDataProvider - { - /// - /// Gets the log lines produced from the given position in the log stream. - /// - IReadOnlyList GetLogs(int fromPosition); - - /// - /// Takes a sample of the engine speed counters, or null when the counters should - /// not be sampled, like while the algorithm warms up. - /// - AlgorithmSpeedSample? TakeSpeedSample(); - } -} diff --git a/Engine/Results/Analysis/ResultsAnalyzer.cs b/Engine/Results/Analysis/ResultsAnalyzer.cs index 9b4445b3df14..45e1428c0249 100644 --- a/Engine/Results/Analysis/ResultsAnalyzer.cs +++ b/Engine/Results/Analysis/ResultsAnalyzer.cs @@ -20,6 +20,7 @@ using QuantConnect.Packets; using QuantConnect.Securities; using QuantConnect.Statistics; +using QuantConnect.Util; using System; using System.Collections.Generic; using System.Diagnostics; @@ -31,10 +32,9 @@ namespace QuantConnect.Lean.Engine.Results.Analysis /// Runs the suite of backtest diagnostic tests against a single backtest, in one of two modes /// depending on how the instance is created: /// a final analysis instance is created with the completed result and logs, and runs the - /// full analysis set once; an in-run analysis instance is created with an - /// , is kept alive for the duration of the backtest, - /// and periodically runs the in-run capable analyses incrementally against the intermediate - /// results, complemented with data pulled from the provider. + /// full analysis set once; an in-run analysis instance is created with the engine's + /// speed counters, is kept alive for the duration of the backtest, and periodically runs the + /// in-run capable analyses incrementally against the intermediate results and logs. /// public class ResultsAnalyzer { @@ -46,7 +46,10 @@ public class ResultsAnalyzer /// private const int MaxFindingReports = 5; - private readonly IInRunAnalysisDataProvider _dataProvider; + private readonly bool _isInRun; + private readonly DateTime _startTime; + private readonly PerformanceTrackingTool _performanceTrackingTool; + private readonly BacktestProgressMonitor _progressMonitor; private readonly Dictionary _findings = new(); /// @@ -94,7 +97,7 @@ public class ResultsAnalyzer /// (see ), as opposed to the final analysis of a /// completed backtest. /// - protected bool IsInRun => _dataProvider != null; + protected bool IsInRun => _isInRun; /// /// The diagnostic analyses to run, in execution order: descending by weight, so changing an @@ -152,11 +155,17 @@ protected ResultsAnalyzer(Result result, QCAlgorithm algorithm, Language languag /// /// The algorithm instance used for history requests and settings. /// The programming language the algorithm is written in. - /// Provides access to the data of the running backtest. - protected ResultsAnalyzer(QCAlgorithm algorithm, Language language, IInRunAnalysisDataProvider dataProvider) + /// The UTC time the backtest started, for the speed samples' elapsed time. + /// The engine's data point counters, for the speed samples. + /// The backtest day-progress monitor, for the speed samples. + protected ResultsAnalyzer(QCAlgorithm algorithm, Language language, DateTime startTime, + PerformanceTrackingTool performanceTrackingTool, BacktestProgressMonitor progressMonitor) : this(null, algorithm, language, null, new AlgorithmSpeedTracker()) { - _dataProvider = dataProvider; + _isInRun = true; + _startTime = startTime; + _performanceTrackingTool = performanceTrackingTool; + _progressMonitor = progressMonitor; } /// @@ -178,17 +187,19 @@ public static ResultsAnalyzer CreateForFinalAnalysis(Result result, QCAlgorithm /// /// Creates an analyzer for in-run analysis of a backtest still in progress. The instance is - /// expected to be kept alive for the duration of the backtest, pulling fresh data from - /// on each call. + /// expected to be kept alive for the duration of the backtest, sampling the given engine + /// speed counters on each call. /// /// The algorithm instance used for history requests and settings. /// The programming language the algorithm is written in. - /// Provides access to the data of the running backtest. + /// The UTC time the backtest started, for the speed samples' elapsed time. + /// The engine's data point counters, for the speed samples. + /// The backtest day-progress monitor, for the speed samples. /// The in-run analysis instance. public static ResultsAnalyzer CreateForInRunAnalysis(QCAlgorithm algorithm, Language language, - IInRunAnalysisDataProvider dataProvider) + DateTime startTime, PerformanceTrackingTool performanceTrackingTool, BacktestProgressMonitor progressMonitor) { - return new ResultsAnalyzer(algorithm, language, dataProvider); + return new ResultsAnalyzer(algorithm, language, startTime, performanceTrackingTool, progressMonitor); } // ── Test chain ──────────────────────────────────────────────────────────── @@ -247,18 +258,19 @@ public static ResultsAnalyzer CreateForInRunAnalysis(QCAlgorithm algorithm, Lang } /// - /// Runs the in-run analyses against the given intermediate backtest result, complemented - /// with the log lines produced since the previous run, pulled from the provider. The - /// returned findings are the merge of this run's findings into the ones accumulated by - /// previous runs: findings from analyses scanning the order event and log streams are - /// accumulated (first sample kept, counts totaled), while findings from state-based - /// analyses are replaced on every run. + /// Runs the in-run analyses against the given intermediate backtest result and the log + /// lines produced since the previous run. The returned findings are the merge of this + /// run's findings into the ones accumulated by previous runs: findings from analyses + /// scanning the order event and log streams are accumulated (first sample kept, counts + /// totaled), while findings from state-based analyses are replaced on every run. /// /// The current intermediate backtest result. Its orders and order events /// are windows truncated to the most recent ones, so the in-run analyses can miss orders and /// events already evicted from them; the final analysis re-scans the complete data. Its charts /// are the handler's live ones, read without synchronization: a torn read while the algorithm /// thread updates them can fail a run, which the handler catches, and the next run retries. + /// The full list of log lines produced so far; the analyzer analyzes the + /// lines past the ones consumed by previous runs. /// The current total algorithm performance, for analyses that read /// portfolio statistics. Withheld from the analyses until the first equity sample exists, since /// the statistics are all-zero defaults before that. @@ -267,8 +279,8 @@ public static ResultsAnalyzer CreateForInRunAnalysis(QCAlgorithm algorithm, Lang /// processing while it runs. /// Maximum number of failing analyses to return. /// The accumulated findings, ranked by analysis weight. - public IReadOnlyList Run(BacktestResult result, AlgorithmPerformance totalPerformance, - int timeLimitSeconds = 1, int maxFailedAnalyses = 10) + public IReadOnlyList Run(BacktestResult result, IReadOnlyList logs, + AlgorithmPerformance totalPerformance, int timeLimitSeconds = 1, int maxFailedAnalyses = 10) { ThrowIfNotInRunInstance(); @@ -289,11 +301,30 @@ public static ResultsAnalyzer CreateForInRunAnalysis(QCAlgorithm algorithm, Lang new Dictionary(), ExtractNewOrderEvents(result?.OrderEvents), totalPerformance)); - var logs = _dataProvider.GetLogs(_logsPosition); + var newLogs = logs?.Skip(_logsPosition).ToList(); // The analyses run during warm-up too (the speed sample is null then), so conditions like // orders submitted while the algorithm warms up surface without waiting for warm-up to end - return Run(snapshot, logs, _dataProvider.TakeSpeedSample(), timeLimitSeconds, maxFailedAnalyses); + return Run(snapshot, newLogs, TakeSpeedSample(), timeLimitSeconds, maxFailedAnalyses); + } + + /// + /// Takes a sample of the engine speed counters for the algorithm speed analysis. + /// Null while the algorithm warms up, since the warm-up pace would skew the speed metrics. + /// + protected virtual AlgorithmSpeedSample? TakeSpeedSample() + { + if (_algorithm == null || _algorithm.IsWarmingUp) + { + return null; + } + + return new AlgorithmSpeedSample( + DateTime.UtcNow - _startTime, + _performanceTrackingTool?.DataPoints ?? 0, + _performanceTrackingTool?.HistoryDataPoints ?? 0, + _progressMonitor?.ProcessedDays ?? 0, + _progressMonitor?.TotalDays ?? 0); } /// @@ -347,7 +378,7 @@ public AlgorithmSpeedTracker CompleteSpeedTracking() { ThrowIfNotInRunInstance(); - var speedSample = _dataProvider.TakeSpeedSample(); + var speedSample = TakeSpeedSample(); if (speedSample.HasValue) { SpeedTracker.AddSample(speedSample.Value); @@ -510,14 +541,13 @@ private static string BaseAnalysisName(string findingName) /// /// Throws when this instance was not created for in-run analysis, guarding the members - /// that read the data provider. + /// that track incremental state across runs. /// private void ThrowIfNotInRunInstance() { if (!IsInRun) { - throw new InvalidOperationException( - "This operation requires an instance created for in-run analysis, with a data provider."); + throw new InvalidOperationException("This operation requires an instance created for in-run analysis."); } } diff --git a/Engine/Results/BacktestingResultHandler.cs b/Engine/Results/BacktestingResultHandler.cs index f4b35910f38a..cfa14ce79c92 100644 --- a/Engine/Results/BacktestingResultHandler.cs +++ b/Engine/Results/BacktestingResultHandler.cs @@ -37,7 +37,7 @@ namespace QuantConnect.Lean.Engine.Results /// /// Backtesting result handler passes messages back from the Lean to the User. /// - public class BacktestingResultHandler : BaseResultsHandler, IResultHandler, IInRunAnalysisDataProvider + public class BacktestingResultHandler : BaseResultsHandler, IResultHandler { private const double Samples = 4000; private const double MinimumSamplePeriod = 4; @@ -422,11 +422,7 @@ protected void SendFinalResult() // Run backtest analyzer if (RunResultsAnalysis) { - List logs; - lock (LogStore) - { - logs = LogStore.Select(x => x.Message).ToList(); - } + var logs = CloneLogs(); // The final analysis reuses the speed metrics accumulated by the in-run analyzer, // completed with one last sample so they cover the backtest through its end var speedTracker = _inRunResultsAnalyzer?.CompleteSpeedTracking(); @@ -457,10 +453,9 @@ protected void SendFinalResult() } /// - /// Runs the in-run results analyzer against the current intermediate backtest result, - /// complemented with data accessed through the - /// implementation. Invoked periodically while the backtest is still running, unlike the - /// full analysis performed by when the backtest ends. + /// Runs the in-run results analyzer against the current intermediate backtest result and + /// the accumulated logs. Invoked periodically while the backtest is still running, unlike + /// the full analysis performed by when the backtest ends. /// /// The current intermediate backtest result. Its orders and order /// events are truncated to the most recent ones, so the in-run analyses can miss data between @@ -477,8 +472,11 @@ protected void SendFinalResult() return null; } - _inRunResultsAnalyzer ??= ResultsAnalyzer.CreateForInRunAnalysis(AlgorithmInstance, _job.Language, this); - return _inRunResultsAnalyzer.Run(completeResult, totalPerformance); + var logs = CloneLogs(); + + _inRunResultsAnalyzer ??= ResultsAnalyzer.CreateForInRunAnalysis(AlgorithmInstance, _job.Language, + StartTime, PerformanceTrackingTool, _progressMonitor); + return _inRunResultsAnalyzer.Run(completeResult, logs, totalPerformance); } catch (Exception ex) { @@ -487,40 +485,17 @@ protected void SendFinalResult() } } - #region IInRunAnalysisDataProvider implementation - /// - /// Gets the log lines produced from the given position in the log stream. + /// Takes a snapshot of the accumulated log messages under the log store lock. /// - IReadOnlyList IInRunAnalysisDataProvider.GetLogs(int fromPosition) + private List CloneLogs() { lock (LogStore) { - return LogStore.Skip(fromPosition).Select(x => x.Message).ToList(); - } - } - - /// - /// Takes a sample of the engine speed counters for the algorithm speed analysis. - /// Null while the algorithm warms up, since the warm-up pace would skew the speed metrics. - /// - AlgorithmSpeedSample? IInRunAnalysisDataProvider.TakeSpeedSample() - { - if (Algorithm == null || Algorithm.IsWarmingUp) - { - return null; + return LogStore.Select(x => x.Message).ToList(); } - - return new AlgorithmSpeedSample( - DateTime.UtcNow - StartTime, - PerformanceTrackingTool?.DataPoints ?? 0, - PerformanceTrackingTool?.HistoryDataPoints ?? 0, - _progressMonitor?.ProcessedDays ?? 0, - _progressMonitor?.TotalDays ?? 0); } - #endregion - /// /// Sends the in-run analysis findings to the browser in their own packet, /// only when they changed since they were last sent. @@ -772,11 +747,7 @@ public override void Exit() if (!ExitTriggered) { Log.Trace("BacktestingResultHandler.Exit(): starting..."); - List copy; - lock (LogStore) - { - copy = LogStore.ToList(); - } + var copy = CloneLogs(); ProcessSynchronousEvents(true); Log.Trace("BacktestingResultHandler.Exit(): Saving logs..."); var logLocation = SaveLogs(_algorithmId, copy); diff --git a/Engine/Results/BaseResultsHandler.cs b/Engine/Results/BaseResultsHandler.cs index ad2e90375c1b..eda3d16db557 100644 --- a/Engine/Results/BaseResultsHandler.cs +++ b/Engine/Results/BaseResultsHandler.cs @@ -602,10 +602,20 @@ public virtual void OnSecuritiesChanged(SecurityChanges changes) /// The logs to save /// The path to the logs public virtual string SaveLogs(string id, List logs) + { + return SaveLogs(id, logs.Select(x => x.Message)); + } + + /// + /// Returns the location of the logs + /// + /// Id that will be incorporated into the algorithm log name + /// The log lines to save + /// The path to the logs + public virtual string SaveLogs(string id, IEnumerable logLines) { var filename = $"{id}-log.txt"; var path = GetResultsPath(filename); - var logLines = logs.Select(x => x.Message); File.WriteAllLines(path, logLines); return path; } diff --git a/Tests/Engine/Results/ResultsAnalyzerInRunTests.cs b/Tests/Engine/Results/ResultsAnalyzerInRunTests.cs index aa9dac2b8ccd..267de5145a38 100644 --- a/Tests/Engine/Results/ResultsAnalyzerInRunTests.cs +++ b/Tests/Engine/Results/ResultsAnalyzerInRunTests.cs @@ -30,9 +30,8 @@ namespace QuantConnect.Tests.Engine.Results { /// /// Tests the in-run mode of , driven through intermediate - /// results carrying truncated order and order event windows, complemented by a fake - /// . The core, mode-independent behavior - /// is covered by . + /// results carrying truncated order and order event windows plus the accumulated logs. + /// The core, mode-independent behavior is covered by . /// [TestFixture] public class ResultsAnalyzerInRunTests @@ -43,19 +42,28 @@ public class ResultsAnalyzerInRunTests public void OrderEventAndLogStreamsAreConsumedIncrementally() { var seenOrderEvents = new List(); - var fake = new FakeAnalysisA(10) { OnParameters = parameters => seenOrderEvents.AddRange(parameters.Result.OrderEvents) }; + var seenLogs = new List(); + var fake = new FakeAnalysisA(10) + { + OnParameters = parameters => + { + seenOrderEvents.AddRange(parameters.Result.OrderEvents); + seenLogs.AddRange(parameters.Logs); + } + }; var analyzer = new TestInRunResultsAnalyzer(fake); // The order event windows fully overlap (every event fits in the window), but the - // watermark dedupes them: each event is analyzed exactly once, in chronological order + // watermark dedupes them, and the full logs are sliced from the consumed position: + // each event and log line is analyzed exactly once, in chronological order analyzer.Run(3, new[] { "log 1", "log 2" }); analyzer.Run(5, new[] { "log 3" }); - // Runs without new order events or logs produce empty deltas and don't move the log position + // Runs without new order events or logs produce empty deltas analyzer.Run(0, null); analyzer.Run(0, null); CollectionAssert.AreEqual(analyzer.OrderEventStream, seenOrderEvents); - CollectionAssert.AreEqual(new[] { 0, 2, 3, 3 }, analyzer.Provider.RequestedLogsPositions); + CollectionAssert.AreEqual(new[] { "log 1", "log 2", "log 3" }, seenLogs); } [Test] @@ -79,11 +87,16 @@ public void OrderEventsEvictedFromTheTruncatedWindowAreMissed() public void StreamPositionsAdvanceEvenWhenTheTimeLimitTruncatesTheRun() { var seenOrderEventCounts = new List(); + var seenLogCounts = new List(); var truncatedRan = false; // The slow analysis has the higher weight so it runs first and exhausts the time limit var slow = new FakeAnalysisA(20) { - OnParameters = parameters => seenOrderEventCounts.Add(parameters.Result.OrderEvents.Count), + OnParameters = parameters => + { + seenOrderEventCounts.Add(parameters.Result.OrderEvents.Count); + seenLogCounts.Add(parameters.Logs.Count); + }, OnRun = () => Thread.Sleep(1100) }; var truncated = new FakeAnalysisB(10) { OnRun = () => truncatedRan = true }; @@ -96,7 +109,7 @@ public void StreamPositionsAdvanceEvenWhenTheTimeLimitTruncatesTheRun() slow.OnRun = null; analyzer.Run(0, null); CollectionAssert.AreEqual(new[] { 4, 0 }, seenOrderEventCounts); - CollectionAssert.AreEqual(new[] { 0, 1 }, analyzer.Provider.RequestedLogsPositions); + CollectionAssert.AreEqual(new[] { 1, 0 }, seenLogCounts); } [Test] @@ -273,7 +286,7 @@ public void AggregatedStateBasedFindingsAreReplacedByFullName() } [Test] - public void SpeedSamplesAreTrackedOnlyWhenTheProviderTakesThem() + public void SpeedSamplesAreTrackedOnlyWhenTheyCanBeTaken() { AlgorithmSpeedTracker speed = null; var fake = new FakeAnalysisA(10) { OnParameters = parameters => speed = parameters.Speed }; @@ -299,7 +312,7 @@ public void CompletedSpeedTrackingAddsAFinalSampleAndReturnsTheTracker() var analyzer = new TestInRunResultsAnalyzer(fake); analyzer.Run(1, new[] { "log" }, new AlgorithmSpeedSample(TimeSpan.FromSeconds(30), 100, 0, 1, 10)); - analyzer.Provider.NextSpeedSample = new AlgorithmSpeedSample(TimeSpan.FromSeconds(60), 200, 0, 2, 10); + analyzer.NextSpeedSample = new AlgorithmSpeedSample(TimeSpan.FromSeconds(60), 200, 0, 2, 10); var tracker = analyzer.CompleteSpeedTracking(); // The final analysis receives the same tracker the in-run analyses saw, with the final sample added @@ -307,7 +320,7 @@ public void CompletedSpeedTrackingAddsAFinalSampleAndReturnsTheTracker() Assert.AreEqual(2, tracker.SampleCount); // Without a final sample (e.g. the algorithm never left warm-up), the tracker is left untouched - analyzer.Provider.NextSpeedSample = null; + analyzer.NextSpeedSample = null; Assert.AreEqual(2, analyzer.CompleteSpeedTracking().SampleCount); } @@ -338,7 +351,7 @@ public void StatisticsAreWithheldUntilEquityHasSamples() var performance = new AlgorithmPerformance(); // The equity chart has no samples yet: the all-zero default statistics are withheld - analyzer.Run(new BacktestResult(), totalPerformance: performance); + analyzer.Run(new BacktestResult(), logs: null, totalPerformance: performance); Assert.IsNull(seenResult.TotalPerformance); var equitySeries = new Series(BaseResultsHandler.EquityKey); @@ -350,7 +363,7 @@ public void StatisticsAreWithheldUntilEquityHasSamples() Charts = new Dictionary { [equityChart.Name] = equityChart } }; - analyzer.Run(result, totalPerformance: performance); + analyzer.Run(result, logs: null, totalPerformance: performance); Assert.AreSame(performance, seenResult.TotalPerformance); } @@ -420,32 +433,28 @@ private class TestInRunResultsAnalyzer : ResultsAnalyzer { private readonly IReadOnlyCollection _analyses; - public FakeDataProvider Provider { get; } - /// /// The full, chronological order event stream of the simulated backtest, from which the /// intermediate results' truncated windows are built. /// public List OrderEventStream { get; } = new(); + public List Logs { get; } = new(); + + public AlgorithmSpeedSample? NextSpeedSample { get; set; } + public int GetAnalysesCallCount { get; private set; } public TestInRunResultsAnalyzer(params BaseResultsAnalysis[] analyses) - : this(new FakeDataProvider(), analyses) + : base(null, Language.CSharp, default, null, null) { - } - - private TestInRunResultsAnalyzer(FakeDataProvider provider, BaseResultsAnalysis[] analyses) - : base(null, Language.CSharp, provider) - { - Provider = provider; _analyses = analyses; } /// /// Appends the new order events and logs to the backtest's streams and runs the analyzer /// against an intermediate result carrying the truncated, newest-first order events - /// window the backtesting result handler builds. + /// window the backtesting result handler builds, plus the full accumulated logs. /// public IReadOnlyList Run(int newOrderEventsCount, string[] newLogs, AlgorithmSpeedSample? speedSample = null, int timeLimitSeconds = 1, int maxFailedAnalyses = 10, @@ -456,8 +465,8 @@ private TestInRunResultsAnalyzer(FakeDataProvider provider, BaseResultsAnalysis[ // One event per order, so the (order id, per-order event id) pairs stay unique OrderEventStream.Add(new OrderEvent { OrderId = OrderEventStream.Count + 1, Id = 1 }); } - Provider.Logs.AddRange(newLogs ?? Array.Empty()); - Provider.NextSpeedSample = speedSample; + Logs.AddRange(newLogs ?? Array.Empty()); + NextSpeedSample = speedSample; var result = new BacktestResult { @@ -465,9 +474,11 @@ private TestInRunResultsAnalyzer(FakeDataProvider provider, BaseResultsAnalysis[ Orders = orders ?? new Dictionary(), OrderEvents = Enumerable.Reverse(OrderEventStream).Take(orderEventsWindowSize).ToList() }; - return Run(result, totalPerformance: null, timeLimitSeconds, maxFailedAnalyses); + return Run(result, Logs, totalPerformance: null, timeLimitSeconds, maxFailedAnalyses); } + protected override AlgorithmSpeedSample? TakeSpeedSample() => NextSpeedSample; + protected override IReadOnlyCollection GetAnalyses() { GetAnalysesCallCount++; @@ -478,30 +489,13 @@ protected override IReadOnlyCollection GetAnalyses() private sealed class DefaultSetInRunResultsAnalyzer : ResultsAnalyzer { public DefaultSetInRunResultsAnalyzer() - : base(null, Language.CSharp, new FakeDataProvider()) + : base(null, Language.CSharp, default, null, null) { } public IReadOnlyCollection DefaultAnalyses => Analyses; } - private sealed class FakeDataProvider : IInRunAnalysisDataProvider - { - public List Logs { get; } = new(); - - public AlgorithmSpeedSample? NextSpeedSample { get; set; } - - public List RequestedLogsPositions { get; } = new(); - - public IReadOnlyList GetLogs(int fromPosition) - { - RequestedLogsPositions.Add(fromPosition); - return Logs.Skip(fromPosition).ToList(); - } - - public AlgorithmSpeedSample? TakeSpeedSample() => NextSpeedSample; - } - private class FakeAnalysis : BaseResultsAnalysis { private readonly int _weight; diff --git a/Tests/Engine/Results/ResultsAnalyzerTests.cs b/Tests/Engine/Results/ResultsAnalyzerTests.cs index 754c815cc067..3371976388ab 100644 --- a/Tests/Engine/Results/ResultsAnalyzerTests.cs +++ b/Tests/Engine/Results/ResultsAnalyzerTests.cs @@ -201,12 +201,12 @@ public void DefaultAnalysisSetDeclaresTheStateBasedAnalyses() } [Test] - public void InRunOperationsRequireAnInstanceCreatedWithADataProvider() + public void InRunOperationsRequireAnInRunInstance() { - // This is a final-analysis instance: the in-run entry points need the data provider + // This is a final-analysis instance: the in-run entry points must throw var analyzer = new TestResultsAnalyzer(false, new FakeAnalysisA(10)); - Assert.Throws(() => analyzer.Run(result: null, totalPerformance: null)); + Assert.Throws(() => analyzer.Run(result: null, logs: null, totalPerformance: null)); Assert.Throws(() => analyzer.CompleteSpeedTracking()); } From 8a9f9713574dea1e439c65ac285312553eb05e99 Mon Sep 17 00:00:00 2001 From: Jhonathan Abreu Date: Wed, 5 Aug 2026 14:24:23 -0400 Subject: [PATCH 32/33] Reduce the maximum in-run finding reports to 3 --- Engine/Results/Analysis/ResultsAnalyzer.cs | 2 +- Tests/Engine/Results/ResultsAnalyzerInRunTests.cs | 12 ++++++------ 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/Engine/Results/Analysis/ResultsAnalyzer.cs b/Engine/Results/Analysis/ResultsAnalyzer.cs index 45e1428c0249..b276cf88c5cc 100644 --- a/Engine/Results/Analysis/ResultsAnalyzer.cs +++ b/Engine/Results/Analysis/ResultsAnalyzer.cs @@ -44,7 +44,7 @@ public class ResultsAnalyzer /// The number of times an in-run finding is returned before it is muted and no longer /// reported, so a recurring finding is not re-sent to consumers (like LLMs) on every update. /// - private const int MaxFindingReports = 5; + private const int MaxFindingReports = 3; private readonly bool _isInRun; private readonly DateTime _startTime; diff --git a/Tests/Engine/Results/ResultsAnalyzerInRunTests.cs b/Tests/Engine/Results/ResultsAnalyzerInRunTests.cs index 267de5145a38..321ad4831796 100644 --- a/Tests/Engine/Results/ResultsAnalyzerInRunTests.cs +++ b/Tests/Engine/Results/ResultsAnalyzerInRunTests.cs @@ -166,18 +166,18 @@ public void StreamBasedFindingsPersistWhenNotReemitted() [Test] public void FindingsAreMutedOnceReturnedTheMaxNumberOfTimes() { - // A finding is returned 5 times and then muted, regardless of its occurrence counts + // A finding is returned 3 times and then muted, regardless of its occurrence counts var fake = new FakeAnalysisA(10) { Findings = () => MakeFindings(nameof(FakeAnalysisA), "sample", 100) }; var analyzer = new TestInRunResultsAnalyzer(fake); - for (var report = 1; report <= 5; report++) + for (var report = 1; report <= 3; report++) { Assert.AreEqual(100 * report, analyzer.Run(1, new[] { "log" }).Single().Count); } - // Muted from the sixth run on, even though the finding keeps accumulating + // Muted from the fourth run on, even though the finding keeps accumulating Assert.IsEmpty(analyzer.Run(1, new[] { "log" })); fake.Findings = () => new List(); Assert.IsEmpty(analyzer.Run(1, new[] { "log" })); @@ -193,11 +193,11 @@ public void StateBasedFindingsAreMutedOnceReturnedTheMaxNumberOfTimes() }; var analyzer = new TestInRunResultsAnalyzer(fake); - for (var report = 1; report <= 5; report++) + for (var report = 1; report <= 3; report++) { Assert.AreEqual(100, analyzer.Run(1, new[] { "log" }).Single().Count); } - // Muted from the sixth run on, even though the analysis still fails + // Muted from the fourth run on, even though the analysis still fails Assert.IsEmpty(analyzer.Run(1, new[] { "log" })); } @@ -210,7 +210,7 @@ public void MutedStateBasedFindingsStayMutedWhenTheyClearAndFailAgain() Findings = () => MakeFindings(nameof(FakeAnalysisA), "sample", 2) }; var analyzer = new TestInRunResultsAnalyzer(fake); - for (var run = 0; run < 5; run++) + for (var run = 0; run < 3; run++) { Assert.IsNotEmpty(analyzer.Run(1, new[] { "log" })); } From 09d741d47b464fe725840975aa65cb0859078e18 Mon Sep 17 00:00:00 2001 From: Jhonathan Abreu Date: Wed, 5 Aug 2026 14:24:23 -0400 Subject: [PATCH 33/33] Send the in-run analysis findings on every cycle instead of deduplicating by serialized signature --- Engine/Results/BacktestingResultHandler.cs | 12 +----------- 1 file changed, 1 insertion(+), 11 deletions(-) diff --git a/Engine/Results/BacktestingResultHandler.cs b/Engine/Results/BacktestingResultHandler.cs index cfa14ce79c92..71ea8fba5d25 100644 --- a/Engine/Results/BacktestingResultHandler.cs +++ b/Engine/Results/BacktestingResultHandler.cs @@ -14,7 +14,6 @@ * */ -using Newtonsoft.Json; using QuantConnect.Algorithm; using QuantConnect.AlgorithmFactory.Python.Wrappers; using QuantConnect.Brokerages; @@ -54,7 +53,6 @@ public class BacktestingResultHandler : BaseResultsHandler, IResultHandler private BacktestProgressMonitor _progressMonitor; private ResultsAnalyzer _inRunResultsAnalyzer; - private string _lastInRunAnalysisSignature = "[]"; /// /// Calculates the capacity of a strategy per Symbol in real-time @@ -497,8 +495,7 @@ private List CloneLogs() } /// - /// Sends the in-run analysis findings to the browser in their own packet, - /// only when they changed since they were last sent. + /// Sends the in-run analysis findings to the browser in their own packet. /// /// The accumulated in-run analysis findings, or null if the analysis could not run /// The current backtest progress @@ -509,13 +506,6 @@ private void SendInRunAnalysis(IReadOnlyList findings, de return; } - var signature = JsonConvert.SerializeObject(findings); - if (signature == _lastInRunAnalysisSignature) - { - return; - } - _lastInRunAnalysisSignature = signature; - MessagingHandler.Send(new BacktestResultPacket(_job, new BacktestResult { Analysis = findings }, Algorithm.EndDate, Algorithm.StartDate, progress)); }