Smart iteration count based on confidence intervals and error

src/Directory.Build.props

@@ -1,6 +1,6 @@
 <Project>
     <PropertyGroup>
-        <Version>0.2.1</Version>
+        <Version>0.3.0</Version>
         <Authors>Tony Redondo, Grégory Léocadie</Authors>
         <TargetFrameworks>net6.0;net7.0;net8.0;net9.0</TargetFrameworks>
         <ImplicitUsings>enable</ImplicitUsings>

src/TimeItSharp.Common/Configuration/Builder/ConfigBuilder.cs

@@ -202,6 +202,61 @@ public ConfigBuilder WithDebugMode()
         _configuration.DebugMode = true;
         return this;
     }
+
+    /// <summary>
+    /// Sets the acceptable relative width for the confidence interval where timeit will consider the results as valid and stop iterating
+    /// </summary>
+    /// <param name="acceptableRelativeWidth">Acceptable relative width</param>
+    /// <returns>Configuration builder instance</returns>
+    public ConfigBuilder WithAcceptableRelativeWidth(double acceptableRelativeWidth)
+    {
+        _configuration.AcceptableRelativeWidth = acceptableRelativeWidth;
+        return this;
+    }
+
+    /// <summary>
+    /// Sets the confidence level for the confidence interval where timeit will compare the acceptable relative width
+    /// </summary>
+    /// <param name="confidenceLevel">Confidence level</param>
+    /// <returns>Configuration builder instance</returns>
+    public ConfigBuilder WithConfidenceLevel(double confidenceLevel)
+    {
+        _configuration.ConfidenceLevel = confidenceLevel;
+        return this;
+    }
+
+    /// <summary>
+    /// Sets the maximum duration in minutes for all scenarios to run
+    /// </summary>
+    /// <param name="maximumDurationInMinutes">Maximum number of minutes</param>
+    /// <returns>Configuration builder instance</returns>
+    public ConfigBuilder WithMaximumDurationInMinutes(int maximumDurationInMinutes)
+    {
+        _configuration.MaximumDurationInMinutes = maximumDurationInMinutes;
+        return this;
+    }
+
+    /// <summary>
+    /// Sets the interval in which timeit will evaluate the results and decide if there's error reductions.
+    /// </summary>
+    /// <param name="evaluationInterval">Interval in number of iterations</param>
+    /// <returns>Configuration builder instance</returns>
+    public ConfigBuilder WithEvaluationInterval(int evaluationInterval)
+    {
+        _configuration.EvaluationInterval = evaluationInterval;
+        return this;
+    }
+
+    /// <summary>
+    /// Sets the minimum error reduction required for timeit to consider the results as valid and stop iterating
+    /// </summary>
+    /// <param name="minimumErrorReduction">Minimum error reduction required</param>
+    /// <returns>Configuration builder instance</returns>
+    public ConfigBuilder WithMinimumErrorReduction(double minimumErrorReduction)
+    {
+        _configuration.MinimumErrorReduction = minimumErrorReduction;
+        return this;
+    }
 
     #region WithExporter
 

src/TimeItSharp.Common/Configuration/Config.cs

@@ -56,6 +56,22 @@ public class Config : ProcessData
     [JsonPropertyName("debugMode")]
     public bool DebugMode { get; set; }
 
+    [JsonPropertyName("acceptableRelativeWidth")]
+    public double AcceptableRelativeWidth { get; set; }
+
+    [JsonPropertyName("confidenceLevel")]
+    public double ConfidenceLevel { get; set; }
+
+    [JsonPropertyName("maximumDurationInMinutes")]
+    public int MaximumDurationInMinutes { get; set; }
+
+    [JsonPropertyName("evaluationInterval")]
+    public int EvaluationInterval { get; set; }
+
+    [JsonPropertyName("minimumErrorReduction")]
+    public double MinimumErrorReduction { get; set; }
+
+
     public Config()
     {
         FilePath = string.Empty;
@@ -75,6 +91,11 @@ public Config()
         ProcessFailedDataPoints = false;
         ShowStdOutForFirstRun = false;
         DebugMode = false;
+        AcceptableRelativeWidth = 0.01;
+        ConfidenceLevel = 0.95;
+        MaximumDurationInMinutes = 45;
+        EvaluationInterval = 10;
+        MinimumErrorReduction = 0.0005;
     }
 
     public static Config LoadConfiguration(string filePath)
@@ -133,5 +154,10 @@ public static Config LoadConfiguration(string filePath)
         ProcessFailedDataPoints = ProcessFailedDataPoints,
         ShowStdOutForFirstRun = ShowStdOutForFirstRun,
         DebugMode = DebugMode,
+        AcceptableRelativeWidth = AcceptableRelativeWidth,
+        ConfidenceLevel = ConfidenceLevel,
+        MaximumDurationInMinutes = MaximumDurationInMinutes,
+        EvaluationInterval = EvaluationInterval,
+        MinimumErrorReduction = MinimumErrorReduction,
     };
 }

src/TimeItSharp.Common/Exporters/ConsoleExporter.cs

@@ -30,21 +30,22 @@ public void Export(TimeitResult results)
 
         // ****************************************
         // Results table
-        AnsiConsole.MarkupLine("[aqua bold underline]### Results:[/]");
+        AnsiConsole.MarkupLine("[aqua bold underline]### Results (last 10):[/]");
         var resultsTable = new Table()
             .MarkdownBorder();
 
         // Add columns
         resultsTable.AddColumns(results.Scenarios.Select(r => new TableColumn($"[dodgerblue1 bold]{r.Name}[/]").Centered()).ToArray());
 
         // Add rows
-        for (var i = 0; i < _options.Configuration.Count; i++)
+        var minDurationCount = Math.Min(results.Scenarios.Select(r => r.Durations.Count).Min(), 10);
+        for (var i = minDurationCount; i > 0; i--)
         {
             resultsTable.AddRow(results.Scenarios.Select(r =>
             {
                 if (i < r.Durations.Count)
                 {
-                    return Math.Round(Utils.FromNanosecondsToMilliseconds(r.Durations[i]), 3) + "ms";
+                    return Math.Round(Utils.FromNanosecondsToMilliseconds(r.Durations[^i]), 3) + "ms";
                 }
 
                 return "-";
@@ -56,10 +57,10 @@ public void Export(TimeitResult results)
 
         // ****************************************
         // Outliers table
-        var maxOutliersCount = results.Scenarios.Select(r => r.Outliers.Count).Max();
+        var maxOutliersCount = Math.Min(results.Scenarios.Select(r => r.Outliers.Count).Max(), 5);
         if (maxOutliersCount > 0)
         {
-            AnsiConsole.MarkupLine("[aqua bold underline]### Outliers:[/]");
+            AnsiConsole.MarkupLine("[aqua bold underline]### Outliers (last 5):[/]");
             var outliersTable = new Table()
                 .MarkdownBorder();
 
@@ -68,13 +69,13 @@ public void Export(TimeitResult results)
                 .Select(r => new TableColumn($"[dodgerblue1 bold]{r.Name}[/]").Centered()).ToArray());
 
             // Add rows
-            for (var i = 0; i < maxOutliersCount; i++)
+            for (var i = maxOutliersCount; i > 0; i--)
             {
                 outliersTable.AddRow(results.Scenarios.Select(r =>
                 {
                     if (i < r.Outliers.Count)
                     {
-                        return Math.Round(Utils.FromNanosecondsToMilliseconds(r.Outliers[i]), 3) + "ms";
+                        return Math.Round(Utils.FromNanosecondsToMilliseconds(r.Outliers[^i]), 3) + "ms";
                     }
 
                     return "-";

src/TimeItSharp.Common/ScenarioProcessor.cs

@@ -9,6 +9,7 @@
 using CliWrap;
 using CliWrap.Buffered;
 using DatadogTestLogger.Vendors.Datadog.Trace;
+using MathNet.Numerics.Distributions;
 using MathNet.Numerics.Statistics;
 using Spectre.Console;
 using TimeItSharp.Common.Assertors;
@@ -29,6 +30,8 @@ internal sealed class ScenarioProcessor
 
     private static readonly IDictionary EnvironmentVariables = Environment.GetEnvironmentVariables();
 
+    private double _remainingTimeInMinutes;
+
     public ScenarioProcessor(
         Config configuration,
         TemplateVariables templateVariables,
@@ -41,6 +44,7 @@ public ScenarioProcessor(
         _assertors = assertors;
         _services = services;
         _callbacksTriggers = callbacksTriggers;
+        _remainingTimeInMinutes = configuration.MaximumDurationInMinutes;
     }
 
     [UnconditionalSuppressMessage("SingleFile", "IL3000:Avoid accessing Assembly file path when publishing as a single file", Justification = "Case is being handled")]
@@ -212,6 +216,7 @@ public void CleanScenario(Scenario scenario)
             AnsiConsole.Markup("  [gold3_1]Warming up[/]");
             watch.Restart();
             await RunScenarioAsync(_configuration.WarmUpCount, index, scenario, TimeItPhase.WarmUp, false,
+                stopwatch: watch,
                 cancellationToken: cancellationToken).ConfigureAwait(false);
             watch.Stop();
             if (cancellationToken.IsCancellationRequested)
@@ -225,7 +230,9 @@ await RunScenarioAsync(_configuration.WarmUpCount, index, scenario, TimeItPhase.
         AnsiConsole.Markup("  [green3]Run[/]");
         var start = DateTime.UtcNow;
         watch.Restart();
-        var dataPoints = await RunScenarioAsync(_configuration.Count, index, scenario, TimeItPhase.Run, true, cancellationToken: cancellationToken).ConfigureAwait(false);
+        var dataPoints = await RunScenarioAsync(_configuration.Count, index, scenario, TimeItPhase.Run, true,
+            stopwatch: watch,
+            cancellationToken: cancellationToken).ConfigureAwait(false);
         watch.Stop();
         if (cancellationToken.IsCancellationRequested)
         {
@@ -241,6 +248,7 @@ await RunScenarioAsync(_configuration.WarmUpCount, index, scenario, TimeItPhase.
             scenario.ParentService = repeat.ServiceAskingForRepeat;
             watch.Restart();
             await RunScenarioAsync(repeat.Count, index, scenario, TimeItPhase.ExtraRun, false,
+                stopwatch: watch,
                 cancellationToken: cancellationToken).ConfigureAwait(false);
             watch.Stop();
             if (cancellationToken.IsCancellationRequested)
@@ -416,8 +424,17 @@ await RunScenarioAsync(repeat.Count, index, scenario, TimeItPhase.ExtraRun, fals
         return scenarioResult;
     }
 
-    private async Task<List<DataPoint>> RunScenarioAsync(int count, int index, Scenario scenario, TimeItPhase phase, bool checkShouldContinue, CancellationToken cancellationToken)
+    private async Task<List<DataPoint>> RunScenarioAsync(int count, int index, Scenario scenario, TimeItPhase phase, bool checkShouldContinue, Stopwatch stopwatch, CancellationToken cancellationToken)
     {
+        var minIterations = count / 3;
+        minIterations = minIterations < 10 ? 10 : minIterations;
+        var confidenceLevel = _configuration.ConfidenceLevel;
+        if (confidenceLevel is <= 0 or >= 1)
+        {
+            confidenceLevel = 0.95;
+        }
+        var previousRelativeWidth = double.MaxValue;
+
         var dataPoints = new List<DataPoint>();
         AnsiConsole.Markup(" ");
         for (var i = 0; i < count; i++)
@@ -440,10 +457,119 @@ private async Task<List<DataPoint>> RunScenarioAsync(int count, int index, Scena
             {
                 break;
             }
+
+            try
+            {
+                // If we are in a run phase, let's do the automatic checks
+                if (phase == TimeItPhase.Run)
+                {
+                    static double GetDuration(DataPoint point)
+                    {
+#if NET7_0_OR_GREATER
+                        return point.Duration.TotalNanoseconds;
+#else
+                        return Utils.FromTimeSpanToNanoseconds(point.Duration);
+#endif
+                    }
+
+                    var durations = Utils.RemoveOutliers(dataPoints.Select(GetDuration), threshold: 1.5).ToList();
+                    if (durations.Count >= minIterations || stopwatch.Elapsed.TotalMinutes >= _remainingTimeInMinutes)
+                    {
+                        var mean = durations.Average();
+                        var stdev = durations.StandardDeviation();
+                        var stderr = stdev / Math.Sqrt(durations.Count);
+
+                        // Critical t value
+                        var tCritical = StudentT.InvCDF(0, 1, durations.Count - 1, 1 - (1 - confidenceLevel) / 2);
+
+                        // Confidence intervals
+                        var marginOfError = tCritical * stderr;
+                        var confidenceIntervalLower = mean - marginOfError;
+                        var confidenceIntervalUpper = mean + marginOfError;
+                        var relativeWidth = (confidenceIntervalUpper - confidenceIntervalLower) / mean;
+
+                        // Check if the maximum duration is reached
+                        if (stopwatch.Elapsed.TotalMinutes >= _remainingTimeInMinutes)
+                        {
+                            AnsiConsole.WriteLine();
+                            AnsiConsole.MarkupLine(
+                                "    [blueviolet]Maximum duration has been reached. Stopping iterations for this scenario.[/]");
+                            AnsiConsole.MarkupLine("    [blueviolet]N: {0}[/]", durations.Count);
+                            AnsiConsole.MarkupLine("    [blueviolet]Mean: {0}ms[/]",
+                                Math.Round(Utils.FromNanosecondsToMilliseconds(mean), 3));
+                            AnsiConsole.Markup(
+                                "    [blueviolet]Confidence Interval at {0}: [[{1}ms, {2}ms]]. Relative width: {3}%[/]",
+                                confidenceLevel * 100,
+                                Math.Round(Utils.FromNanosecondsToMilliseconds(confidenceIntervalLower), 3),
+                                Math.Round(Utils.FromNanosecondsToMilliseconds(confidenceIntervalUpper), 3),
+                                Math.Round(relativeWidth * 100, 4));
+
+                            break;
+                        }
+
+                        // Check if the statistical criterion is met
+                        if (relativeWidth < _configuration.AcceptableRelativeWidth)
+                        {
+                            AnsiConsole.WriteLine();
+                            AnsiConsole.MarkupLine(
+                                "    [blueviolet]Acceptable relative width criteria met. Stopping iterations for this scenario.[/]");
+                            AnsiConsole.MarkupLine("    [blueviolet]N: {0}[/]", durations.Count);
+                            AnsiConsole.MarkupLine("    [blueviolet]Mean: {0}ms[/]",
+                                Math.Round(Utils.FromNanosecondsToMilliseconds(mean), 3));
+                            AnsiConsole.Markup(
+                                "    [blueviolet]Confidence Interval at {0}: [[{1}ms, {2}ms]]. Relative width: {3}%[/]",
+                                confidenceLevel * 100,
+                                Math.Round(Utils.FromNanosecondsToMilliseconds(confidenceIntervalLower), 3),
+                                Math.Round(Utils.FromNanosecondsToMilliseconds(confidenceIntervalUpper), 3),
+                                Math.Round(relativeWidth * 100, 4));
+                            break;
+                        }
+
+                        // Check for each `evaluationInterval` iteration
+                        if ((durations.Count - minIterations) % _configuration.EvaluationInterval == 0)
+                        {
+                            var errorReduction = (previousRelativeWidth - relativeWidth) / previousRelativeWidth;
+                            if (errorReduction < _configuration.MinimumErrorReduction)
+                            {
+                                AnsiConsole.WriteLine();
+                                AnsiConsole.MarkupLine(
+                                    "    [blueviolet]The error is not decreasing significantly. Stopping iterations for this scenario.[/]");
+                                AnsiConsole.MarkupLine("    [blueviolet]N: {0}[/]", durations.Count);
+                                AnsiConsole.MarkupLine("    [blueviolet]Mean: {0}ms[/]",
+                                    Math.Round(Utils.FromNanosecondsToMilliseconds(mean), 3));
+                                AnsiConsole.MarkupLine(
+                                    "    [blueviolet]Confidence Interval at {0}: [[{1}ms, {2}ms]]. Relative width: {3}%[/]",
+                                    confidenceLevel * 100,
+                                    Math.Round(Utils.FromNanosecondsToMilliseconds(confidenceIntervalLower), 3),
+                                    Math.Round(Utils.FromNanosecondsToMilliseconds(confidenceIntervalUpper), 3),
+                                    Math.Round(relativeWidth * 100, 4));
+                                AnsiConsole.Markup("    [blueviolet]Error reduction: {0}%. Minimal expected: {1}%[/]",
+                                    Math.Round(errorReduction * 100, 4),
+                                    Math.Round(_configuration.MinimumErrorReduction * 100, 4));
+
+                                break;
+                            }
+
+                            previousRelativeWidth = relativeWidth;
+                        }
+                    }
+                }
+            }
+            catch (Exception ex)
+            {
+                AnsiConsole.WriteLine();
+                AnsiConsole.MarkupLine("    [red]Error: {0}[/]", ex.Message);
+                break;
+            }
         }
 
         AnsiConsole.WriteLine();
 
+        if (phase == TimeItPhase.Run)
+        {
+            _remainingTimeInMinutes -= (int)stopwatch.Elapsed.TotalMinutes;
+        }
+
         return dataPoints;
     }
 

src/TimeItSharp.Common/TimeItEngine.cs

@@ -93,7 +93,11 @@ public static async Task<int> RunAsync(Config config, TimeItOptions? options = n
 
         AnsiConsole.Profile.Width = Utils.GetSafeWidth();
         AnsiConsole.MarkupLine("[bold aqua]Warmup count:[/] {0}", config.WarmUpCount);
-        AnsiConsole.MarkupLine("[bold aqua]Count:[/] {0}", config.Count);
+        AnsiConsole.MarkupLine("[bold aqua]Max count:[/] {0}", config.Count);
+        AnsiConsole.MarkupLine("[bold aqua]Acceptable relative width:[/] {0}%", Math.Round(config.AcceptableRelativeWidth * 100, 2));
+        AnsiConsole.MarkupLine("[bold aqua]Confidence level:[/] {0}%", Math.Round(config.ConfidenceLevel * 100, 2));
+        AnsiConsole.MarkupLine("[bold aqua]Minimum error reduction:[/] {0}%", Math.Round(config.MinimumErrorReduction * 100, 2));
+        AnsiConsole.MarkupLine("[bold aqua]Maximum duration:[/] {0}min", config.MaximumDurationInMinutes);
         AnsiConsole.MarkupLine("[bold aqua]Number of Scenarios:[/] {0}", config.Scenarios.Count);
         AnsiConsole.MarkupLine("[bold aqua]Exporters:[/] {0}", string.Join(", ", exporters.Select(e => e.Name)));
         AnsiConsole.MarkupLine("[bold aqua]Assertors:[/] {0}", string.Join(", ", assertors.Select(e => e.Name)));