Skip to content

Commit

Permalink
WIP: Implementation of Jenkins (controller) executor.
Browse files Browse the repository at this point in the history
  • Loading branch information
@JasperDeLaat94
JasperDeLaat94 committed Jan 5, 2025
1 parent 822bd6b commit f9040c5
Show file tree
Hide file tree
Showing 7 changed files with 183 additions and 50 deletions.
159 changes: 114 additions & 45 deletions UET/Redpoint.Uet.BuildPipeline.Executors.Jenkins/JenkinsBuildExecutor.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -17,29 +17,35 @@ public class JenkinsBuildExecutor : BuildServerBuildExecutor
{
private readonly ILogger<JenkinsBuildExecutor> _logger;
private static readonly HttpClient _httpClient = new();
private readonly Uri? _gitUri;
private readonly string _gitBranch;

public JenkinsBuildExecutor(
IServiceProvider serviceProvider,
ILogger<JenkinsBuildExecutor> logger,
string buildServerOutputFilePath) : base(
string buildServerOutputFilePath,
Uri? gitUri,
string gitBranch) : base(
serviceProvider,
buildServerOutputFilePath)
{
_logger = logger;

string? controllerUri = Environment.GetEnvironmentVariable("UET_JENKINS_CONTROLLER_URI");
if (string.IsNullOrWhiteSpace(controllerUri))
_gitUri = gitUri;
if (_gitUri == null)
{
throw new InvalidOperationException("Jenkins controller URI is not specified, please specify using UET_JENKINS_CONTROLLER_URI environment variable.");
var gitUriEnv = Environment.GetEnvironmentVariable("GIT_URL");
if (!string.IsNullOrWhiteSpace(gitUriEnv))
{
Uri.TryCreate(gitUriEnv, UriKind.Absolute, out _gitUri);
}
}
_httpClient.BaseAddress = new Uri(controllerUri);

string? authenticationToken = Environment.GetEnvironmentVariable("UET_JENKINS_AUTH");
if (string.IsNullOrWhiteSpace(authenticationToken))
_gitBranch = gitBranch;
if (string.IsNullOrWhiteSpace(_gitBranch))
{
throw new InvalidOperationException("Jenkins authorization token is not specified, please specify using UET_JENKINS_AUTH environment variable, format is: your-user-name:apiToken");
_gitBranch = Environment.GetEnvironmentVariable("GIT_LOCAL_BRANCH") ?? string.Empty;
}
_httpClient.DefaultRequestHeaders.Authorization = new AuthenticationHeaderValue("Basic", Convert.ToBase64String(Encoding.ASCII.GetBytes(authenticationToken)));
}

public override string DiscoverPipelineId()
Expand All @@ -52,6 +58,41 @@ protected override async Task EmitBuildServerSpecificFileAsync(BuildSpecificatio
{
ArgumentNullException.ThrowIfNull(buildServerPipeline);

var prerequisitesPassed = true;
if (_gitUri == null)
{
_logger.LogError("Jenkins executor requires a valid Git URL. Specify using command-line argument or 'GIT_URL' environment variable.");
prerequisitesPassed = false;
}

if (string.IsNullOrWhiteSpace(_gitBranch))
{
_logger.LogError("Jenkins executor requires a valid Git branch. Specify using command-line argument or 'GIT_LOCAL_BRANCH' environment variable.");
prerequisitesPassed = false;
}

string? controllerUri = Environment.GetEnvironmentVariable("UET_JENKINS_CONTROLLER_URL");
if (string.IsNullOrWhiteSpace(controllerUri))
{
_logger.LogError("Jenkins executor requires a valid Jenkins controller URL. Specify using 'UET_JENKINS_CONTROLLER_URL' environment variable.");
prerequisitesPassed = false;
}

string? authenticationToken = Environment.GetEnvironmentVariable("UET_JENKINS_AUTH");
if (string.IsNullOrWhiteSpace(authenticationToken))
{
_logger.LogError("Jenkins executor requires a valid authorization token for Jenkins controller. Specify using 'UET_JENKINS_AUTH' environment variable (example: your-user-name:apiToken)");
prerequisitesPassed = false;
}

if (!prerequisitesPassed)
{
throw new BuildPipelineExecutionFailureException("One or more prerequisite checks have failed, fix and try again.");
}

_httpClient.BaseAddress = new Uri(controllerUri!);
_httpClient.DefaultRequestHeaders.Authorization = new AuthenticationHeaderValue("Basic", Convert.ToBase64String(Encoding.ASCII.GetBytes(authenticationToken!)));

foreach (var stage in buildServerPipeline.Stages)
{
_logger.LogInformation($"Creating and executing Jenkins jobs for stage '{stage}'");
Expand All @@ -63,8 +104,6 @@ protected override async Task EmitBuildServerSpecificFileAsync(BuildSpecificatio
var jobName = sourceJob.Key;
var jobData = sourceJob.Value;

// TODO: Check if job exist. Re-use it?

// Job config data.
using var stringWriter = new StringWriter();
using (var xmlWriter = XmlWriter.Create(stringWriter, new XmlWriterSettings
Expand Down Expand Up @@ -111,6 +150,9 @@ protected override async Task EmitBuildServerSpecificFileAsync(BuildSpecificatio
{
buildCommandString += $"$env:{kv.Key}=\'{kv.Value}\'\n";
}
buildCommandString += $"$env:UET_GIT_URL=\'{_gitUri!.ToString}\'\n";
buildCommandString += $"$env:UET_GIT_REF=\'{_gitBranch}\'\n";

buildCommandString += jobData.Script("jenkins");

xmlWriter.WriteStartElement("builders");
Expand All @@ -128,33 +170,49 @@ protected override async Task EmitBuildServerSpecificFileAsync(BuildSpecificatio
xmlWriter.WriteEndElement(); // project
}

// Create job.
var query = HttpUtility.ParseQueryString(string.Empty);
query["name"] = jobName;
var uriBuilder = new UriBuilder(_httpClient.BaseAddress + "createItem")
// Create or update Jenkins job depending on whether it exists.
var uriBuilder = new UriBuilder(_httpClient.BaseAddress + $"job/{jobName}/api/json");
using var checkResponse = await _httpClient.GetAsync(uriBuilder.Uri).ConfigureAwait(false);
if (checkResponse.IsSuccessStatusCode)
{
Query = query.ToString()
};
// Update existing job.
uriBuilder = new UriBuilder(_httpClient.BaseAddress + $"job/{jobName}/config.xml");

using StringContent xmlContent = new(stringWriter.ToString(), Encoding.UTF8, "application/xml");
using StringContent xmlContent = new(stringWriter.ToString(), Encoding.UTF8, "text/xml");

using var createJobResponse = await _httpClient.PostAsync(uriBuilder.Uri, xmlContent).ConfigureAwait(false);
if (!createJobResponse.IsSuccessStatusCode)
using var updateJobResponse = await _httpClient.PostAsync(uriBuilder.Uri, xmlContent).ConfigureAwait(false);
if (!updateJobResponse.IsSuccessStatusCode)
{
throw new BuildPipelineExecutionFailureException($"Could not update Jenkins job '{jobName}'.");
}
}
else
{
var errorMsg = $"Could not create Jenkins job '{jobName}'.";
_logger.LogError(errorMsg);
throw new InvalidOperationException(errorMsg);
// Create new job.
var query = HttpUtility.ParseQueryString(string.Empty);
query["name"] = jobName;
uriBuilder = new UriBuilder(_httpClient.BaseAddress + "createItem")
{
Query = query.ToString()
};

using StringContent xmlContent = new(stringWriter.ToString(), Encoding.UTF8, "application/xml");

using var createJobResponse = await _httpClient.PostAsync(uriBuilder.Uri, xmlContent).ConfigureAwait(false);
if (!createJobResponse.IsSuccessStatusCode)
{
throw new BuildPipelineExecutionFailureException($"Could not create Jenkins job '{jobName}'.");
}
}

jobs.Add(jobName, new JenkinsJob());

// Submit to build queue.
uriBuilder = new UriBuilder(_httpClient.BaseAddress + $"job/{jobName}/build");
using var buildResponse = await _httpClient.PostAsync(uriBuilder.Uri, null).ConfigureAwait(false);
if (!buildResponse.IsSuccessStatusCode)
{
var errorMsg = $"Could not enqueue Jenkins job '{jobName}'.";
_logger.LogError(errorMsg);
throw new InvalidOperationException(errorMsg);
throw new BuildPipelineExecutionFailureException($"Could not enqueue Jenkins job '{jobName}'.");
}
jobs[jobName].QueueUri = buildResponse.Headers.Location;
jobs[jobName].Status = JenkinsJobStatus.Queued;
Expand All @@ -168,19 +226,17 @@ protected override async Task EmitBuildServerSpecificFileAsync(BuildSpecificatio
var uriBuilder = new UriBuilder(job.Value.QueueUri + "api/json");
using var response = await _httpClient.GetAsync(uriBuilder.Uri).ConfigureAwait(false);
response.EnsureSuccessStatusCode();
var queueResponse = await response.Content.ReadFromJsonAsync(JenkinsJsonSourceGenerationContext.Default.JenkinsQueueItem).ConfigureAwait(false);
ArgumentNullException.ThrowIfNull(queueResponse);
var queueItem = await response.Content.ReadFromJsonAsync(JenkinsJsonSourceGenerationContext.Default.JenkinsQueueItem).ConfigureAwait(false);
ArgumentNullException.ThrowIfNull(queueItem);

if (queueResponse.Cancelled ?? false)
if (queueItem.Cancelled ?? false)
{
var errorMsg = $"Queued job '{job.Key}' was cancelled.";
_logger.LogError(errorMsg);
throw new InvalidOperationException(errorMsg);
throw new BuildPipelineExecutionFailureException($"Queued job '{job.Key}' was cancelled.");
}

if (queueResponse.Executable != null)
if (queueItem.Executable != null)
{
job.Value.ExecutionUri = new Uri(queueResponse.Executable.Url);
job.Value.ExecutionUri = new Uri(queueItem.Executable.Url);
job.Value.Status = JenkinsJobStatus.Executing;
}
}
Expand All @@ -194,33 +250,46 @@ protected override async Task EmitBuildServerSpecificFileAsync(BuildSpecificatio
{
Query = query.ToString()
};
using var response = await _httpClient.GetAsync(uriBuilder.Uri).ConfigureAwait(false);
response.EnsureSuccessStatusCode();
using var progressResponse = await _httpClient.GetAsync(uriBuilder.Uri).ConfigureAwait(false);
progressResponse.EnsureSuccessStatusCode();

// Get offset for the next log query, while at the same time check if we received any new log data.
int newByteOffset = int.Parse(response.Headers.GetValues("X-Text-Size").First(), CultureInfo.InvariantCulture);
int newByteOffset = int.Parse(progressResponse.Headers.GetValues("X-Text-Size").First(), CultureInfo.InvariantCulture);
if (newByteOffset != job.Value.ExecutionLogByteOffset)
{
job.Value.ExecutionLogByteOffset = newByteOffset;

var newLogText = await response.Content.ReadAsStringAsync().ConfigureAwait(false);
_logger.LogInformation(newLogText); // TODO: Make this print completed lines instead of arbitrary amount of text?
var newLogText = await progressResponse.Content.ReadAsStringAsync().ConfigureAwait(false);
foreach (var line in newLogText.Split('\n', StringSplitOptions.RemoveEmptyEntries))
{
LogLevel logLevel = line.Contains("FAILURE", StringComparison.OrdinalIgnoreCase) ? LogLevel.Error : LogLevel.Information;
_logger.Log(logLevel, $"[Remote: {job.Key}] " + line);
}
}

// Check if build is still in progress, header will disappear when build is complete.
bool buildStillInProgress = response.Headers.TryGetValues("X-More-Data", out var values) && bool.Parse(values.FirstOrDefault() ?? bool.FalseString);
bool buildStillInProgress = progressResponse.Headers.TryGetValues("X-More-Data", out var values) && bool.Parse(values.FirstOrDefault() ?? bool.FalseString);
if (!buildStillInProgress)
{
// TODO: Implement: Did build fail or complete successfully?
job.Value.Status = JenkinsJobStatus.Completed;
uriBuilder = new UriBuilder(job.Value.ExecutionUri + "api/json");
using var resultResponse = await _httpClient.GetAsync(uriBuilder.Uri).ConfigureAwait(false);
resultResponse.EnsureSuccessStatusCode();
var buildInfo = await resultResponse.Content.ReadFromJsonAsync(JenkinsJsonSourceGenerationContext.Default.JenkinsBuildInfo).ConfigureAwait(false);
ArgumentNullException.ThrowIfNull(buildInfo);

job.Value.Status = buildInfo.Result.Equals("SUCCESS", StringComparison.OrdinalIgnoreCase) ? JenkinsJobStatus.Succeeded : JenkinsJobStatus.Failed;
}
}

// Delay between polling.
await Task.Delay(500).ConfigureAwait(false);
// Don't poll too frequently.
await Task.Delay(1000).ConfigureAwait(false);
}

// TODO: Stuff when jobs of this stage have finished?
// Don't bother starting the next stage if a prerequisite job has failed.
if (jobs.Any(job => job.Value.Status == JenkinsJobStatus.Failed))
{
throw new BuildPipelineExecutionFailureException("A job has failed, aborting build process.");
}
}

// TODO: Stuff when all jobs have finished?
Expand Down
6 changes: 4 additions & 2 deletions UET/Redpoint.Uet.BuildPipeline.Executors.Jenkins/JenkinsBuildExecutorFactory.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -13,12 +13,14 @@ public JenkinsBuildExecutorFactory(
_serviceProvider = serviceProvider;
}

public IBuildExecutor CreateExecutor(string buildServerOutputFilePath)
public IBuildExecutor CreateExecutor(string buildServerOutputFilePath, Uri? gitUrl, string gitBranch)
{
return new JenkinsBuildExecutor(
_serviceProvider,
_serviceProvider.GetRequiredService<ILogger<JenkinsBuildExecutor>>(),
buildServerOutputFilePath);
buildServerOutputFilePath,
gitUrl,
gitBranch);
}
}
}
10 changes: 10 additions & 0 deletions UET/Redpoint.Uet.BuildPipeline.Executors.Jenkins/JenkinsBuildInfo.cs
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,10 @@
namespace Redpoint.Uet.BuildPipeline.Executors.Jenkins
{
using System.Text.Json.Serialization;

internal class JenkinsBuildInfo
{
[JsonPropertyName("result"), JsonRequired]
public string Result { get; set; } = string.Empty;
}
}
26 changes: 26 additions & 0 deletions UET/Redpoint.Uet.BuildPipeline.Executors.Jenkins/JenkinsBuildNodeExecutor.cs
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,26 @@
namespace Redpoint.Uet.BuildPipeline.Executors.Jenkins
{
using Redpoint.Uet.BuildPipeline.Executors;
using Redpoint.Uet.BuildPipeline.Executors.BuildServer;
using Redpoint.Uet.Configuration.Dynamic;
using Redpoint.Uet.Configuration.Plugin;
using Redpoint.Uet.Configuration.Project;
using System;
using System.Collections.Generic;
using System.Threading;
using System.Threading.Tasks;

public class JenkinsBuildNodeExecutor : IBuildNodeExecutor
{
public string DiscoverPipelineId()
{
// TODO: This might be wrong if the ID is expected to be the same between the main executor and node executors, investigate.
return Environment.GetEnvironmentVariable("BUILD_TAG") ?? string.Empty;
}

public Task<int> ExecuteBuildNodesAsync(BuildSpecification buildSpecification, BuildConfigDynamic<BuildConfigPluginDistribution, IPrepareProvider>[]? preparePlugin, BuildConfigDynamic<BuildConfigProjectDistribution, IPrepareProvider>[]? prepareProject, IBuildExecutionEvents buildExecutionEvents, IReadOnlyList<string> nodeNames, CancellationToken cancellationToken)
{
throw new NotImplementedException();
}
}
}
9 changes: 7 additions & 2 deletions UET/Redpoint.Uet.BuildPipeline.Executors.Jenkins/JenkinsJobStatus.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -18,8 +18,13 @@ public enum JenkinsJobStatus
Executing,

/// <summary>
/// The job has completed building.
/// The job has successfully completed building.
/// </summary>
Completed,
Succeeded,

/// <summary>
/// The job has successfully completed building.
/// </summary>
Failed,
}
}
1 change: 1 addition & 0 deletions UET/Redpoint.Uet.BuildPipeline.Executors.Jenkins/JenkinsJsonSourceGenerationContext.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,7 @@

[JsonSourceGenerationOptions(WriteIndented = true)]
[JsonSerializable(typeof(JenkinsQueueItem))]
[JsonSerializable(typeof(JenkinsBuildInfo))]
internal partial class JenkinsJsonSourceGenerationContext : JsonSerializerContext
{
}
Expand Down
Loading

0 comments on commit f9040c5

Please sign in to comment.