Metrics on failures and debugging. (#57)

src/Directory.Build.props

@@ -1,6 +1,6 @@
 <Project>
     <PropertyGroup>
-        <Version>0.1.17</Version>
+        <Version>0.1.18</Version>
         <Authors>Tony Redondo, Grégory Léocadie</Authors>
         <TargetFrameworks>net6.0;net7.0;net8.0</TargetFrameworks>
         <ImplicitUsings>enable</ImplicitUsings>

src/TimeItSharp.Common/Assertors/DefaultAssertor.cs

@@ -71,10 +71,21 @@ public override AssertResponse ExecutionAssertion(in AssertionData data)
             var message = _sbuilder.ToString();
             _sbuilder.Clear();
             _consecutiveErrorCount++;
-            return new AssertResponse(
-                status: Status.Failed,
-                shouldContinue: _consecutiveErrorCount < 5,
-                message: message);
+
+            if (Options.Configuration.ProcessFailedDataPoints)
+            {
+                return new AssertResponse(
+                    status: Status.Failed,
+                    shouldContinue: true,
+                    message: message);
+            }
+            else
+            {
+                return new AssertResponse(
+                    status: Status.Failed,
+                    shouldContinue: _consecutiveErrorCount < 5,
+                    message: message);
+            }
         }
 
         _consecutiveErrorCount = 0;

src/TimeItSharp.Common/Configuration/Builder/ConfigBuilder.cs

@@ -173,6 +173,26 @@ public ConfigBuilder WithJsonExporterPath(string filePath)
         return this;
     }
 
+    /// <summary>
+    /// Sets if the failed data points should be processed
+    /// </summary>
+    /// <returns>Configuration builder instance</returns>
+    public ConfigBuilder ProcessFailedDataPoints()
+    {
+        _configuration.ProcessFailedDataPoints = true;
+        return this;
+    }
+
+    /// <summary>
+    /// Sets if the standard output should be shown for the first run
+    /// </summary>
+    /// <returns>Configuration builder instance</returns>
+    public ConfigBuilder ShowStdOutForFirstRun()
+    {
+        _configuration.ShowStdOutForFirstRun = true;
+        return this;
+    }
+
     #region WithExporter
 
     /// <summary>

src/TimeItSharp.Common/Configuration/Config.cs

@@ -47,6 +47,12 @@ public class Config : ProcessData
     [JsonPropertyName("services")]
     public List<AssemblyLoadInfo> Services { get; set; }
 
+    [JsonPropertyName("processFailedDataPoints")]
+    public bool ProcessFailedDataPoints { get; set; }
+
+    [JsonPropertyName("showStdOutForFirstRun")]
+    public bool ShowStdOutForFirstRun { get; set; }
+
     public Config()
     {
         FilePath = string.Empty;
@@ -63,6 +69,8 @@ public Config()
         Exporters = new();
         Assertors = new();
         Services = new();
+        ProcessFailedDataPoints = false;
+        ShowStdOutForFirstRun = false;
     }
 
     public static Config LoadConfiguration(string filePath)
@@ -95,7 +103,7 @@ public static Config LoadConfiguration(string filePath)
         return new Config();
     }
 
-    internal override Config Clone() => new Config
+    internal override Config Clone() => new()
     {
         FilePath = FilePath,
         Path = Path,
@@ -118,5 +126,7 @@ public static Config LoadConfiguration(string filePath)
         PathValidations = new List<string>(PathValidations),
         Timeout = Timeout.Clone(),
         Tags = new Dictionary<string, object>(Tags),
+        ProcessFailedDataPoints = ProcessFailedDataPoints,
+        ShowStdOutForFirstRun = ShowStdOutForFirstRun,
     };
 }

src/TimeItSharp.Common/ScenarioProcessor.cs

@@ -258,32 +258,31 @@ await RunScenarioAsync(repeat.Count, index, scenario, TimeItPhase.ExtraRun, fals
         var lastStandardOutput = string.Empty;
         var durations = new List<double>();
         var metricsData = new Dictionary<string, List<double>>();
+        var anyPassedDataPoint = dataPoints.Any(d => d.Status == Status.Passed);
         foreach (var item in dataPoints)
         {
             if (!string.IsNullOrEmpty(item.StandardOutput))
             {
                 lastStandardOutput = item.StandardOutput;
             }
 
-            if (item.Status != Status.Passed)
+            if (item.Status == Status.Passed || _configuration.ProcessFailedDataPoints || !anyPassedDataPoint)
             {
-                continue;
-            }
-
 #if NET7_0_OR_GREATER
-            durations.Add(item.Duration.TotalNanoseconds);
+                durations.Add(item.Duration.TotalNanoseconds);
 #else
-            durations.Add(Utils.FromTimeSpanToNanoseconds(item.Duration));
+                durations.Add(Utils.FromTimeSpanToNanoseconds(item.Duration));
 #endif
-            foreach (var kv in item.Metrics)
-            {
-                if (!metricsData.TryGetValue(kv.Key, out var metricsItem))
+                foreach (var kv in item.Metrics)
                 {
-                    metricsItem = new List<double>();
-                    metricsData[kv.Key] = metricsItem;
-                }
+                    if (!metricsData.TryGetValue(kv.Key, out var metricsItem))
+                    {
+                        metricsItem = new List<double>();
+                        metricsData[kv.Key] = metricsItem;
+                    }
 
-                metricsItem.Add(kv.Value);
+                    metricsItem.Add(kv.Value);
+                }
             }
         }
 
@@ -418,7 +417,7 @@ private async Task<List<DataPoint>> RunScenarioAsync(int count, int index, Scena
         AnsiConsole.Markup(" ");
         for (var i = 0; i < count; i++)
         {
-            var currentRun = await RunCommandAsync(index, scenario, phase, cancellationToken).ConfigureAwait(false);
+            var currentRun = await RunCommandAsync(index, scenario, phase, i, cancellationToken).ConfigureAwait(false);
             if (cancellationToken.IsCancellationRequested)
             {
                 AnsiConsole.Markup("[red]cancelled[/]");
@@ -439,7 +438,7 @@ private async Task<List<DataPoint>> RunScenarioAsync(int count, int index, Scena
         return dataPoints;
     }
 
-    private async Task<DataPoint> RunCommandAsync(int index, Scenario scenario, TimeItPhase phase, CancellationToken cancellationToken)
+    private async Task<DataPoint> RunCommandAsync(int index, Scenario scenario, TimeItPhase phase, int executionId, CancellationToken cancellationToken)
     {
         // Prepare variables
         var cmdString = scenario.ProcessName ?? string.Empty;
@@ -482,6 +481,16 @@ private async Task<DataPoint> RunCommandAsync(int index, Scenario scenario, Time
             cmd = cmd.WithArguments(cmdArguments);
         }
 
+        if (executionId == 0 && _configuration.ShowStdOutForFirstRun)
+        {
+            AnsiConsole.WriteLine();
+            AnsiConsole.WriteLine(new string('-', 80));
+            cmd = cmd.WithStandardOutputPipe(PipeTarget.Merge(cmd.StandardOutputPipe,
+                PipeTarget.ToStream(Console.OpenStandardOutput())));
+            cmd = cmd.WithStandardErrorPipe(PipeTarget.Merge(cmd.StandardErrorPipe,
+                PipeTarget.ToStream(Console.OpenStandardError())));
+        }
+
         // Execute the command
         var dataPoint = new DataPoint
         {
@@ -725,6 +734,13 @@ mainEndDate is not null &&
         }
 
         _callbacksTriggers.ExecutionEnd(dataPoint, phase);
+
+        if (executionId == 0 && _configuration.ShowStdOutForFirstRun)
+        {
+            AnsiConsole.WriteLine(new string('-', 80));
+            AnsiConsole.Write("   ");
+        }
+
         return dataPoint;
     }
 

src/TimeItSharp/Program.cs

@@ -1,6 +1,8 @@
 using Spectre.Console;
 using TimeItSharp.Common;
 using System.CommandLine;
+using System.CommandLine.Binding;
+using System.CommandLine.Invocation;
 using System.Text.Json;
 using TimeItSharp.Common.Configuration;
 using TimeItSharp.Common.Configuration.Builder;
@@ -53,6 +55,8 @@
 var jsonExporter = new Option<bool>("--json-exporter", () => false, "Enable JSON exporter");
 var datadogExporter = new Option<bool>("--datadog-exporter", () => false, "Enable Datadog exporter");
 var datadogProfiler = new Option<bool>("--datadog-profiler", () => false, "Enable Datadog profiler");
+var showStdOutForFistRun = new Option<bool>("--first-run-stdout", () => false, "Show the StdOut and StdErr for the first run");
+var processFailedExecutions = new Option<bool>("--process-failed-executions", () => false, "Include failed executions in the final results");
 
 var root = new RootCommand
 {
@@ -64,17 +68,30 @@
     jsonExporter,
     datadogExporter,
     datadogProfiler,
+    showStdOutForFistRun,
+    processFailedExecutions,
 };
 
-root.SetHandler(async (configFile, templateVariables, countValue, warmupValue, metricsValue, jsonExporterValue, datadogExporterValue, datadogProfilerValue) =>
+root.SetHandler(async (context) =>
 {
+    var argumentValue = GetValueForHandlerParameter(argument, context) ?? string.Empty;
+    var templateVariablesValue = GetValueForHandlerParameter(templateVariables, context);
+    var countValue = GetValueForHandlerParameter(count, context);
+    var warmupValue = GetValueForHandlerParameter(warmup, context);
+    var metricsValue = GetValueForHandlerParameter(metrics, context);
+    var jsonExporterValue = GetValueForHandlerParameter(jsonExporter, context);
+    var datadogExporterValue = GetValueForHandlerParameter(datadogExporter, context);
+    var datadogProfilerValue = GetValueForHandlerParameter(datadogProfiler, context);
+    var showStdOutForFistRunValue = GetValueForHandlerParameter(showStdOutForFistRun, context);
+    var processFailedExecutionsValue = GetValueForHandlerParameter(processFailedExecutions, context);
+
     var isConfigFile = false;
-    if (File.Exists(configFile))
+    if (File.Exists(argumentValue))
     {
         try
         {
-            await using var fstream = File.Open(configFile, FileMode.Open, FileAccess.Read, FileShare.ReadWrite);
-            var config = JsonSerializer.Deserialize<Config>(fstream, ConfigContext.Default.Config);
+            await using var fstream = File.Open(argumentValue, FileMode.Open, FileAccess.Read, FileShare.ReadWrite);
+            var config = JsonSerializer.Deserialize(fstream, ConfigContext.Default.Config);
             isConfigFile = config is not null;
         }
         catch
@@ -90,7 +107,7 @@
     var exitCode = 0;
     if (isConfigFile)
     {
-        var config = Config.LoadConfiguration(configFile);
+        var config = Config.LoadConfiguration(argumentValue);
         config.WarmUpCount = warmupValue ?? config.WarmUpCount;
         config.Count = countValue ?? config.Count;
         var configBuilder = new ConfigBuilder(config);
@@ -104,11 +121,11 @@
             configBuilder.WithExporter<DatadogExporter>();
         }
 
-        exitCode = await TimeItEngine.RunAsync(configBuilder, new TimeItOptions(templateVariables)).ConfigureAwait(false);
+        exitCode = await TimeItEngine.RunAsync(configBuilder, new TimeItOptions(templateVariablesValue)).ConfigureAwait(false);
     }
     else
     {
-        var commandLineArray = configFile.Split(' ', StringSplitOptions.None);
+        var commandLineArray = argumentValue.Split(' ', StringSplitOptions.None);
         var processName = commandLineArray[0];
         var processArgs = string.Empty;
         if (commandLineArray.Length > 1)
@@ -118,7 +135,7 @@
 
         var finalCount = countValue ?? 10;
         var configBuilder = ConfigBuilder.Create()
-            .WithName(configFile)
+            .WithName(argumentValue)
             .WithProcessName(processName)
             .WithProcessArguments(processArgs)
             .WithMetrics(metricsValue)
@@ -128,7 +145,17 @@
             .WithTimeout(t => t.WithMaxDuration((int)TimeSpan.FromMinutes(30).TotalSeconds))
             .WithScenario(s => s.WithName("Default"));
 
-        var timeitOption = new TimeItOptions(templateVariables);
+        if (showStdOutForFistRunValue)
+        {
+            configBuilder = configBuilder.ShowStdOutForFirstRun();
+        }
+
+        if (processFailedExecutionsValue)
+        {
+            configBuilder = configBuilder.ProcessFailedDataPoints();
+        }
+
+        var timeitOption = new TimeItOptions(templateVariablesValue);
 
         if (jsonExporterValue)
         {
@@ -154,6 +181,20 @@
     {
         Environment.Exit(exitCode);
     }
-}, argument, templateVariables, count, warmup, metrics, jsonExporter, datadogExporter, datadogProfiler);
+});
 
 await root.InvokeAsync(args);
+
+static T? GetValueForHandlerParameter<T>(
+    IValueDescriptor<T> symbol,
+    InvocationContext context)
+{
+    return symbol switch
+    {
+        IValueSource valueSource when valueSource.TryGetValue(symbol, context.BindingContext, out var boundValue) &&
+                                      boundValue is T value => value,
+        Argument argument => (T?)context.ParseResult.GetValueForArgument(argument),
+        Option option => (T?)context.ParseResult.GetValueForOption(option),
+        _ => default
+    };
+}