Skip to content

Commit

Permalink
Smart iteration count based on confidence intervals and error (#64)
Browse files Browse the repository at this point in the history
  • Loading branch information
@tonyredondo
tonyredondo authored Nov 26, 2024
1 parent f6833e1 commit 1b56e3e
Show file tree
Hide file tree
Showing 8 changed files with 229 additions and 25 deletions.
2 changes: 1 addition & 1 deletion src/Directory.Build.props
View file
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
<Project>
<PropertyGroup>
<Version>0.2.1</Version>
<Version>0.3.0</Version>
<Authors>Tony Redondo, Grégory Léocadie</Authors>
<TargetFrameworks>net6.0;net7.0;net8.0;net9.0</TargetFrameworks>
<ImplicitUsings>enable</ImplicitUsings>
Expand Down
55 changes: 55 additions & 0 deletions src/TimeItSharp.Common/Configuration/Builder/ConfigBuilder.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -202,6 +202,61 @@ public ConfigBuilder WithDebugMode()
_configuration.DebugMode = true;
return this;
}

/// <summary>
/// Sets the acceptable relative width for the confidence interval where timeit will consider the results as valid and stop iterating
/// </summary>
/// <param name="acceptableRelativeWidth">Acceptable relative width</param>
/// <returns>Configuration builder instance</returns>
public ConfigBuilder WithAcceptableRelativeWidth(double acceptableRelativeWidth)
{
_configuration.AcceptableRelativeWidth = acceptableRelativeWidth;
return this;
}

/// <summary>
/// Sets the confidence level for the confidence interval where timeit will compare the acceptable relative width
/// </summary>
/// <param name="confidenceLevel">Confidence level</param>
/// <returns>Configuration builder instance</returns>
public ConfigBuilder WithConfidenceLevel(double confidenceLevel)
{
_configuration.ConfidenceLevel = confidenceLevel;
return this;
}

/// <summary>
/// Sets the maximum duration in minutes for all scenarios to run
/// </summary>
/// <param name="maximumDurationInMinutes">Maximum number of minutes</param>
/// <returns>Configuration builder instance</returns>
public ConfigBuilder WithMaximumDurationInMinutes(int maximumDurationInMinutes)
{
_configuration.MaximumDurationInMinutes = maximumDurationInMinutes;
return this;
}

/// <summary>
/// Sets the interval in which timeit will evaluate the results and decide if there's error reductions.
/// </summary>
/// <param name="evaluationInterval">Interval in number of iterations</param>
/// <returns>Configuration builder instance</returns>
public ConfigBuilder WithEvaluationInterval(int evaluationInterval)
{
_configuration.EvaluationInterval = evaluationInterval;
return this;
}

/// <summary>
/// Sets the minimum error reduction required for timeit to consider the results as valid and stop iterating
/// </summary>
/// <param name="minimumErrorReduction">Minimum error reduction required</param>
/// <returns>Configuration builder instance</returns>
public ConfigBuilder WithMinimumErrorReduction(double minimumErrorReduction)
{
_configuration.MinimumErrorReduction = minimumErrorReduction;
return this;
}

#region WithExporter

Expand Down
26 changes: 26 additions & 0 deletions src/TimeItSharp.Common/Configuration/Config.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -56,6 +56,22 @@ public class Config : ProcessData
[JsonPropertyName("debugMode")]
public bool DebugMode { get; set; }

[JsonPropertyName("acceptableRelativeWidth")]
public double AcceptableRelativeWidth { get; set; }

[JsonPropertyName("confidenceLevel")]
public double ConfidenceLevel { get; set; }

[JsonPropertyName("maximumDurationInMinutes")]
public int MaximumDurationInMinutes { get; set; }

[JsonPropertyName("evaluationInterval")]
public int EvaluationInterval { get; set; }

[JsonPropertyName("minimumErrorReduction")]
public double MinimumErrorReduction { get; set; }


public Config()
{
FilePath = string.Empty;
Expand All @@ -75,6 +91,11 @@ public Config()
ProcessFailedDataPoints = false;
ShowStdOutForFirstRun = false;
DebugMode = false;
AcceptableRelativeWidth = 0.01;
ConfidenceLevel = 0.95;
MaximumDurationInMinutes = 45;
EvaluationInterval = 10;
MinimumErrorReduction = 0.0005;
}

public static Config LoadConfiguration(string filePath)
Expand Down Expand Up @@ -133,5 +154,10 @@ public static Config LoadConfiguration(string filePath)
ProcessFailedDataPoints = ProcessFailedDataPoints,
ShowStdOutForFirstRun = ShowStdOutForFirstRun,
DebugMode = DebugMode,
AcceptableRelativeWidth = AcceptableRelativeWidth,
ConfidenceLevel = ConfidenceLevel,
MaximumDurationInMinutes = MaximumDurationInMinutes,
EvaluationInterval = EvaluationInterval,
MinimumErrorReduction = MinimumErrorReduction,
};
}
15 changes: 8 additions & 7 deletions src/TimeItSharp.Common/Exporters/ConsoleExporter.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -30,21 +30,22 @@ public void Export(TimeitResult results)

// ****************************************
// Results table
AnsiConsole.MarkupLine("[aqua bold underline]### Results:[/]");
AnsiConsole.MarkupLine("[aqua bold underline]### Results (last 10):[/]");
var resultsTable = new Table()
.MarkdownBorder();

// Add columns
resultsTable.AddColumns(results.Scenarios.Select(r => new TableColumn($"[dodgerblue1 bold]{r.Name}[/]").Centered()).ToArray());

// Add rows
for (var i = 0; i < _options.Configuration.Count; i++)
var minDurationCount = Math.Min(results.Scenarios.Select(r => r.Durations.Count).Min(), 10);
for (var i = minDurationCount; i > 0; i--)
{
resultsTable.AddRow(results.Scenarios.Select(r =>
{
if (i < r.Durations.Count)
{
return Math.Round(Utils.FromNanosecondsToMilliseconds(r.Durations[i]), 3) + "ms";
return Math.Round(Utils.FromNanosecondsToMilliseconds(r.Durations[^i]), 3) + "ms";
}

return "-";
Expand All @@ -56,10 +57,10 @@ public void Export(TimeitResult results)

// ****************************************
// Outliers table
var maxOutliersCount = results.Scenarios.Select(r => r.Outliers.Count).Max();
var maxOutliersCount = Math.Min(results.Scenarios.Select(r => r.Outliers.Count).Max(), 5);
if (maxOutliersCount > 0)
{
AnsiConsole.MarkupLine("[aqua bold underline]### Outliers:[/]");
AnsiConsole.MarkupLine("[aqua bold underline]### Outliers (last 5):[/]");
var outliersTable = new Table()
.MarkdownBorder();

Expand All @@ -68,13 +69,13 @@ public void Export(TimeitResult results)
.Select(r => new TableColumn($"[dodgerblue1 bold]{r.Name}[/]").Centered()).ToArray());

// Add rows
for (var i = 0; i < maxOutliersCount; i++)
for (var i = maxOutliersCount; i > 0; i--)
{
outliersTable.AddRow(results.Scenarios.Select(r =>
{
if (i < r.Outliers.Count)
{
return Math.Round(Utils.FromNanosecondsToMilliseconds(r.Outliers[i]), 3) + "ms";
return Math.Round(Utils.FromNanosecondsToMilliseconds(r.Outliers[^i]), 3) + "ms";
}

return "-";
Expand Down
130 changes: 128 additions & 2 deletions src/TimeItSharp.Common/ScenarioProcessor.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,7 @@
using CliWrap;
using CliWrap.Buffered;
using DatadogTestLogger.Vendors.Datadog.Trace;
using MathNet.Numerics.Distributions;
using MathNet.Numerics.Statistics;
using Spectre.Console;
using TimeItSharp.Common.Assertors;
Expand All @@ -29,6 +30,8 @@ internal sealed class ScenarioProcessor

private static readonly IDictionary EnvironmentVariables = Environment.GetEnvironmentVariables();

private double _remainingTimeInMinutes;

public ScenarioProcessor(
Config configuration,
TemplateVariables templateVariables,
Expand All @@ -41,6 +44,7 @@ public ScenarioProcessor(
_assertors = assertors;
_services = services;
_callbacksTriggers = callbacksTriggers;
_remainingTimeInMinutes = configuration.MaximumDurationInMinutes;
}

[UnconditionalSuppressMessage("SingleFile", "IL3000:Avoid accessing Assembly file path when publishing as a single file", Justification = "Case is being handled")]
Expand Down Expand Up @@ -212,6 +216,7 @@ public void CleanScenario(Scenario scenario)
AnsiConsole.Markup(" [gold3_1]Warming up[/]");
watch.Restart();
await RunScenarioAsync(_configuration.WarmUpCount, index, scenario, TimeItPhase.WarmUp, false,
stopwatch: watch,
cancellationToken: cancellationToken).ConfigureAwait(false);
watch.Stop();
if (cancellationToken.IsCancellationRequested)
Expand All @@ -225,7 +230,9 @@ await RunScenarioAsync(_configuration.WarmUpCount, index, scenario, TimeItPhase.
AnsiConsole.Markup(" [green3]Run[/]");
var start = DateTime.UtcNow;
watch.Restart();
var dataPoints = await RunScenarioAsync(_configuration.Count, index, scenario, TimeItPhase.Run, true, cancellationToken: cancellationToken).ConfigureAwait(false);
var dataPoints = await RunScenarioAsync(_configuration.Count, index, scenario, TimeItPhase.Run, true,
stopwatch: watch,
cancellationToken: cancellationToken).ConfigureAwait(false);
watch.Stop();
if (cancellationToken.IsCancellationRequested)
{
Expand All @@ -241,6 +248,7 @@ await RunScenarioAsync(_configuration.WarmUpCount, index, scenario, TimeItPhase.
scenario.ParentService = repeat.ServiceAskingForRepeat;
watch.Restart();
await RunScenarioAsync(repeat.Count, index, scenario, TimeItPhase.ExtraRun, false,
stopwatch: watch,
cancellationToken: cancellationToken).ConfigureAwait(false);
watch.Stop();
if (cancellationToken.IsCancellationRequested)
Expand Down Expand Up @@ -416,8 +424,17 @@ await RunScenarioAsync(repeat.Count, index, scenario, TimeItPhase.ExtraRun, fals
return scenarioResult;
}

private async Task<List<DataPoint>> RunScenarioAsync(int count, int index, Scenario scenario, TimeItPhase phase, bool checkShouldContinue, CancellationToken cancellationToken)
private async Task<List<DataPoint>> RunScenarioAsync(int count, int index, Scenario scenario, TimeItPhase phase, bool checkShouldContinue, Stopwatch stopwatch, CancellationToken cancellationToken)
{
var minIterations = count / 3;
minIterations = minIterations < 10 ? 10 : minIterations;
var confidenceLevel = _configuration.ConfidenceLevel;
if (confidenceLevel is <= 0 or >= 1)
{
confidenceLevel = 0.95;
}
var previousRelativeWidth = double.MaxValue;

var dataPoints = new List<DataPoint>();
AnsiConsole.Markup(" ");
for (var i = 0; i < count; i++)
Expand All @@ -440,10 +457,119 @@ private async Task<List<DataPoint>> RunScenarioAsync(int count, int index, Scena
{
break;
}

try
{
// If we are in a run phase, let's do the automatic checks
if (phase == TimeItPhase.Run)
{
static double GetDuration(DataPoint point)
{
#if NET7_0_OR_GREATER
return point.Duration.TotalNanoseconds;
#else
return Utils.FromTimeSpanToNanoseconds(point.Duration);
#endif
}

var durations = Utils.RemoveOutliers(dataPoints.Select(GetDuration), threshold: 1.5).ToList();
if (durations.Count >= minIterations || stopwatch.Elapsed.TotalMinutes >= _remainingTimeInMinutes)
{
var mean = durations.Average();
var stdev = durations.StandardDeviation();
var stderr = stdev / Math.Sqrt(durations.Count);

// Critical t value
var tCritical = StudentT.InvCDF(0, 1, durations.Count - 1, 1 - (1 - confidenceLevel) / 2);

// Confidence intervals
var marginOfError = tCritical * stderr;
var confidenceIntervalLower = mean - marginOfError;
var confidenceIntervalUpper = mean + marginOfError;
var relativeWidth = (confidenceIntervalUpper - confidenceIntervalLower) / mean;

// Check if the maximum duration is reached
if (stopwatch.Elapsed.TotalMinutes >= _remainingTimeInMinutes)
{
AnsiConsole.WriteLine();
AnsiConsole.MarkupLine(
" [blueviolet]Maximum duration has been reached. Stopping iterations for this scenario.[/]");
AnsiConsole.MarkupLine(" [blueviolet]N: {0}[/]", durations.Count);
AnsiConsole.MarkupLine(" [blueviolet]Mean: {0}ms[/]",
Math.Round(Utils.FromNanosecondsToMilliseconds(mean), 3));
AnsiConsole.Markup(
" [blueviolet]Confidence Interval at {0}: [[{1}ms, {2}ms]]. Relative width: {3}%[/]",
confidenceLevel * 100,
Math.Round(Utils.FromNanosecondsToMilliseconds(confidenceIntervalLower), 3),
Math.Round(Utils.FromNanosecondsToMilliseconds(confidenceIntervalUpper), 3),
Math.Round(relativeWidth * 100, 4));

break;
}

// Check if the statistical criterion is met
if (relativeWidth < _configuration.AcceptableRelativeWidth)
{
AnsiConsole.WriteLine();
AnsiConsole.MarkupLine(
" [blueviolet]Acceptable relative width criteria met. Stopping iterations for this scenario.[/]");
AnsiConsole.MarkupLine(" [blueviolet]N: {0}[/]", durations.Count);
AnsiConsole.MarkupLine(" [blueviolet]Mean: {0}ms[/]",
Math.Round(Utils.FromNanosecondsToMilliseconds(mean), 3));
AnsiConsole.Markup(
" [blueviolet]Confidence Interval at {0}: [[{1}ms, {2}ms]]. Relative width: {3}%[/]",
confidenceLevel * 100,
Math.Round(Utils.FromNanosecondsToMilliseconds(confidenceIntervalLower), 3),
Math.Round(Utils.FromNanosecondsToMilliseconds(confidenceIntervalUpper), 3),
Math.Round(relativeWidth * 100, 4));
break;
}

// Check for each `evaluationInterval` iteration
if ((durations.Count - minIterations) % _configuration.EvaluationInterval == 0)
{
var errorReduction = (previousRelativeWidth - relativeWidth) / previousRelativeWidth;
if (errorReduction < _configuration.MinimumErrorReduction)
{
AnsiConsole.WriteLine();
AnsiConsole.MarkupLine(
" [blueviolet]The error is not decreasing significantly. Stopping iterations for this scenario.[/]");
AnsiConsole.MarkupLine(" [blueviolet]N: {0}[/]", durations.Count);
AnsiConsole.MarkupLine(" [blueviolet]Mean: {0}ms[/]",
Math.Round(Utils.FromNanosecondsToMilliseconds(mean), 3));
AnsiConsole.MarkupLine(
" [blueviolet]Confidence Interval at {0}: [[{1}ms, {2}ms]]. Relative width: {3}%[/]",
confidenceLevel * 100,
Math.Round(Utils.FromNanosecondsToMilliseconds(confidenceIntervalLower), 3),
Math.Round(Utils.FromNanosecondsToMilliseconds(confidenceIntervalUpper), 3),
Math.Round(relativeWidth * 100, 4));
AnsiConsole.Markup(" [blueviolet]Error reduction: {0}%. Minimal expected: {1}%[/]",
Math.Round(errorReduction * 100, 4),
Math.Round(_configuration.MinimumErrorReduction * 100, 4));

break;
}

previousRelativeWidth = relativeWidth;
}
}
}
}
catch (Exception ex)
{
AnsiConsole.WriteLine();
AnsiConsole.MarkupLine(" [red]Error: {0}[/]", ex.Message);
break;
}
}

AnsiConsole.WriteLine();

if (phase == TimeItPhase.Run)
{
_remainingTimeInMinutes -= (int)stopwatch.Elapsed.TotalMinutes;
}

return dataPoints;
}

Expand Down
6 changes: 5 additions & 1 deletion src/TimeItSharp.Common/TimeItEngine.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -93,7 +93,11 @@ public static async Task<int> RunAsync(Config config, TimeItOptions? options = n

AnsiConsole.Profile.Width = Utils.GetSafeWidth();
AnsiConsole.MarkupLine("[bold aqua]Warmup count:[/] {0}", config.WarmUpCount);
AnsiConsole.MarkupLine("[bold aqua]Count:[/] {0}", config.Count);
AnsiConsole.MarkupLine("[bold aqua]Max count:[/] {0}", config.Count);
AnsiConsole.MarkupLine("[bold aqua]Acceptable relative width:[/] {0}%", Math.Round(config.AcceptableRelativeWidth * 100, 2));
AnsiConsole.MarkupLine("[bold aqua]Confidence level:[/] {0}%", Math.Round(config.ConfidenceLevel * 100, 2));
AnsiConsole.MarkupLine("[bold aqua]Minimum error reduction:[/] {0}%", Math.Round(config.MinimumErrorReduction * 100, 2));
AnsiConsole.MarkupLine("[bold aqua]Maximum duration:[/] {0}min", config.MaximumDurationInMinutes);
AnsiConsole.MarkupLine("[bold aqua]Number of Scenarios:[/] {0}", config.Scenarios.Count);
AnsiConsole.MarkupLine("[bold aqua]Exporters:[/] {0}", string.Join(", ", exporters.Select(e => e.Name)));
AnsiConsole.MarkupLine("[bold aqua]Assertors:[/] {0}", string.Join(", ", assertors.Select(e => e.Name)));
Expand Down
Loading

0 comments on commit 1b56e3e

Please sign in to comment.