diff --git a/UET/Redpoint.Uet.BuildPipeline.Executors.Jenkins/JenkinsBuildExecutor.cs b/UET/Redpoint.Uet.BuildPipeline.Executors.Jenkins/JenkinsBuildExecutor.cs index e18d3496..4a4dd2f4 100644 --- a/UET/Redpoint.Uet.BuildPipeline.Executors.Jenkins/JenkinsBuildExecutor.cs +++ b/UET/Redpoint.Uet.BuildPipeline.Executors.Jenkins/JenkinsBuildExecutor.cs @@ -17,29 +17,35 @@ public class JenkinsBuildExecutor : BuildServerBuildExecutor { private readonly ILogger _logger; private static readonly HttpClient _httpClient = new(); + private readonly Uri? _gitUri; + private readonly string _gitBranch; public JenkinsBuildExecutor( IServiceProvider serviceProvider, ILogger logger, - string buildServerOutputFilePath) : base( + string buildServerOutputFilePath, + Uri? gitUri, + string gitBranch) : base( serviceProvider, buildServerOutputFilePath) { _logger = logger; - string? controllerUri = Environment.GetEnvironmentVariable("UET_JENKINS_CONTROLLER_URI"); - if (string.IsNullOrWhiteSpace(controllerUri)) + _gitUri = gitUri; + if (_gitUri == null) { - throw new InvalidOperationException("Jenkins controller URI is not specified, please specify using UET_JENKINS_CONTROLLER_URI environment variable."); + var gitUriEnv = Environment.GetEnvironmentVariable("GIT_URL"); + if (!string.IsNullOrWhiteSpace(gitUriEnv)) + { + Uri.TryCreate(gitUriEnv, UriKind.Absolute, out _gitUri); + } } - _httpClient.BaseAddress = new Uri(controllerUri); - string? authenticationToken = Environment.GetEnvironmentVariable("UET_JENKINS_AUTH"); - if (string.IsNullOrWhiteSpace(authenticationToken)) + _gitBranch = gitBranch; + if (string.IsNullOrWhiteSpace(_gitBranch)) { - throw new InvalidOperationException("Jenkins authorization token is not specified, please specify using UET_JENKINS_AUTH environment variable, format is: your-user-name:apiToken"); + _gitBranch = Environment.GetEnvironmentVariable("GIT_LOCAL_BRANCH") ?? string.Empty; } - _httpClient.DefaultRequestHeaders.Authorization = new AuthenticationHeaderValue("Basic", Convert.ToBase64String(Encoding.ASCII.GetBytes(authenticationToken))); } public override string DiscoverPipelineId() @@ -52,6 +58,41 @@ protected override async Task EmitBuildServerSpecificFileAsync(BuildSpecificatio { ArgumentNullException.ThrowIfNull(buildServerPipeline); + var prerequisitesPassed = true; + if (_gitUri == null) + { + _logger.LogError("Jenkins executor requires a valid Git URL. Specify using command-line argument or 'GIT_URL' environment variable."); + prerequisitesPassed = false; + } + + if (string.IsNullOrWhiteSpace(_gitBranch)) + { + _logger.LogError("Jenkins executor requires a valid Git branch. Specify using command-line argument or 'GIT_LOCAL_BRANCH' environment variable."); + prerequisitesPassed = false; + } + + string? controllerUri = Environment.GetEnvironmentVariable("UET_JENKINS_CONTROLLER_URL"); + if (string.IsNullOrWhiteSpace(controllerUri)) + { + _logger.LogError("Jenkins executor requires a valid Jenkins controller URL. Specify using 'UET_JENKINS_CONTROLLER_URL' environment variable."); + prerequisitesPassed = false; + } + + string? authenticationToken = Environment.GetEnvironmentVariable("UET_JENKINS_AUTH"); + if (string.IsNullOrWhiteSpace(authenticationToken)) + { + _logger.LogError("Jenkins executor requires a valid authorization token for Jenkins controller. Specify using 'UET_JENKINS_AUTH' environment variable (example: your-user-name:apiToken)"); + prerequisitesPassed = false; + } + + if (!prerequisitesPassed) + { + throw new BuildPipelineExecutionFailureException("One or more prerequisite checks have failed, fix and try again."); + } + + _httpClient.BaseAddress = new Uri(controllerUri!); + _httpClient.DefaultRequestHeaders.Authorization = new AuthenticationHeaderValue("Basic", Convert.ToBase64String(Encoding.ASCII.GetBytes(authenticationToken!))); + foreach (var stage in buildServerPipeline.Stages) { _logger.LogInformation($"Creating and executing Jenkins jobs for stage '{stage}'"); @@ -63,8 +104,6 @@ protected override async Task EmitBuildServerSpecificFileAsync(BuildSpecificatio var jobName = sourceJob.Key; var jobData = sourceJob.Value; - // TODO: Check if job exist. Re-use it? - // Job config data. using var stringWriter = new StringWriter(); using (var xmlWriter = XmlWriter.Create(stringWriter, new XmlWriterSettings @@ -111,6 +150,9 @@ protected override async Task EmitBuildServerSpecificFileAsync(BuildSpecificatio { buildCommandString += $"$env:{kv.Key}=\'{kv.Value}\'\n"; } + buildCommandString += $"$env:UET_GIT_URL=\'{_gitUri!.ToString}\'\n"; + buildCommandString += $"$env:UET_GIT_REF=\'{_gitBranch}\'\n"; + buildCommandString += jobData.Script("jenkins"); xmlWriter.WriteStartElement("builders"); @@ -128,23 +170,41 @@ protected override async Task EmitBuildServerSpecificFileAsync(BuildSpecificatio xmlWriter.WriteEndElement(); // project } - // Create job. - var query = HttpUtility.ParseQueryString(string.Empty); - query["name"] = jobName; - var uriBuilder = new UriBuilder(_httpClient.BaseAddress + "createItem") + // Create or update Jenkins job depending on whether it exists. + var uriBuilder = new UriBuilder(_httpClient.BaseAddress + $"job/{jobName}/api/json"); + using var checkResponse = await _httpClient.GetAsync(uriBuilder.Uri).ConfigureAwait(false); + if (checkResponse.IsSuccessStatusCode) { - Query = query.ToString() - }; + // Update existing job. + uriBuilder = new UriBuilder(_httpClient.BaseAddress + $"job/{jobName}/config.xml"); - using StringContent xmlContent = new(stringWriter.ToString(), Encoding.UTF8, "application/xml"); + using StringContent xmlContent = new(stringWriter.ToString(), Encoding.UTF8, "text/xml"); - using var createJobResponse = await _httpClient.PostAsync(uriBuilder.Uri, xmlContent).ConfigureAwait(false); - if (!createJobResponse.IsSuccessStatusCode) + using var updateJobResponse = await _httpClient.PostAsync(uriBuilder.Uri, xmlContent).ConfigureAwait(false); + if (!updateJobResponse.IsSuccessStatusCode) + { + throw new BuildPipelineExecutionFailureException($"Could not update Jenkins job '{jobName}'."); + } + } + else { - var errorMsg = $"Could not create Jenkins job '{jobName}'."; - _logger.LogError(errorMsg); - throw new InvalidOperationException(errorMsg); + // Create new job. + var query = HttpUtility.ParseQueryString(string.Empty); + query["name"] = jobName; + uriBuilder = new UriBuilder(_httpClient.BaseAddress + "createItem") + { + Query = query.ToString() + }; + + using StringContent xmlContent = new(stringWriter.ToString(), Encoding.UTF8, "application/xml"); + + using var createJobResponse = await _httpClient.PostAsync(uriBuilder.Uri, xmlContent).ConfigureAwait(false); + if (!createJobResponse.IsSuccessStatusCode) + { + throw new BuildPipelineExecutionFailureException($"Could not create Jenkins job '{jobName}'."); + } } + jobs.Add(jobName, new JenkinsJob()); // Submit to build queue. @@ -152,9 +212,7 @@ protected override async Task EmitBuildServerSpecificFileAsync(BuildSpecificatio using var buildResponse = await _httpClient.PostAsync(uriBuilder.Uri, null).ConfigureAwait(false); if (!buildResponse.IsSuccessStatusCode) { - var errorMsg = $"Could not enqueue Jenkins job '{jobName}'."; - _logger.LogError(errorMsg); - throw new InvalidOperationException(errorMsg); + throw new BuildPipelineExecutionFailureException($"Could not enqueue Jenkins job '{jobName}'."); } jobs[jobName].QueueUri = buildResponse.Headers.Location; jobs[jobName].Status = JenkinsJobStatus.Queued; @@ -168,19 +226,17 @@ protected override async Task EmitBuildServerSpecificFileAsync(BuildSpecificatio var uriBuilder = new UriBuilder(job.Value.QueueUri + "api/json"); using var response = await _httpClient.GetAsync(uriBuilder.Uri).ConfigureAwait(false); response.EnsureSuccessStatusCode(); - var queueResponse = await response.Content.ReadFromJsonAsync(JenkinsJsonSourceGenerationContext.Default.JenkinsQueueItem).ConfigureAwait(false); - ArgumentNullException.ThrowIfNull(queueResponse); + var queueItem = await response.Content.ReadFromJsonAsync(JenkinsJsonSourceGenerationContext.Default.JenkinsQueueItem).ConfigureAwait(false); + ArgumentNullException.ThrowIfNull(queueItem); - if (queueResponse.Cancelled ?? false) + if (queueItem.Cancelled ?? false) { - var errorMsg = $"Queued job '{job.Key}' was cancelled."; - _logger.LogError(errorMsg); - throw new InvalidOperationException(errorMsg); + throw new BuildPipelineExecutionFailureException($"Queued job '{job.Key}' was cancelled."); } - if (queueResponse.Executable != null) + if (queueItem.Executable != null) { - job.Value.ExecutionUri = new Uri(queueResponse.Executable.Url); + job.Value.ExecutionUri = new Uri(queueItem.Executable.Url); job.Value.Status = JenkinsJobStatus.Executing; } } @@ -194,33 +250,46 @@ protected override async Task EmitBuildServerSpecificFileAsync(BuildSpecificatio { Query = query.ToString() }; - using var response = await _httpClient.GetAsync(uriBuilder.Uri).ConfigureAwait(false); - response.EnsureSuccessStatusCode(); + using var progressResponse = await _httpClient.GetAsync(uriBuilder.Uri).ConfigureAwait(false); + progressResponse.EnsureSuccessStatusCode(); // Get offset for the next log query, while at the same time check if we received any new log data. - int newByteOffset = int.Parse(response.Headers.GetValues("X-Text-Size").First(), CultureInfo.InvariantCulture); + int newByteOffset = int.Parse(progressResponse.Headers.GetValues("X-Text-Size").First(), CultureInfo.InvariantCulture); if (newByteOffset != job.Value.ExecutionLogByteOffset) { job.Value.ExecutionLogByteOffset = newByteOffset; - var newLogText = await response.Content.ReadAsStringAsync().ConfigureAwait(false); - _logger.LogInformation(newLogText); // TODO: Make this print completed lines instead of arbitrary amount of text? + var newLogText = await progressResponse.Content.ReadAsStringAsync().ConfigureAwait(false); + foreach (var line in newLogText.Split('\n', StringSplitOptions.RemoveEmptyEntries)) + { + LogLevel logLevel = line.Contains("FAILURE", StringComparison.OrdinalIgnoreCase) ? LogLevel.Error : LogLevel.Information; + _logger.Log(logLevel, $"[Remote: {job.Key}] " + line); + } } // Check if build is still in progress, header will disappear when build is complete. - bool buildStillInProgress = response.Headers.TryGetValues("X-More-Data", out var values) && bool.Parse(values.FirstOrDefault() ?? bool.FalseString); + bool buildStillInProgress = progressResponse.Headers.TryGetValues("X-More-Data", out var values) && bool.Parse(values.FirstOrDefault() ?? bool.FalseString); if (!buildStillInProgress) { - // TODO: Implement: Did build fail or complete successfully? - job.Value.Status = JenkinsJobStatus.Completed; + uriBuilder = new UriBuilder(job.Value.ExecutionUri + "api/json"); + using var resultResponse = await _httpClient.GetAsync(uriBuilder.Uri).ConfigureAwait(false); + resultResponse.EnsureSuccessStatusCode(); + var buildInfo = await resultResponse.Content.ReadFromJsonAsync(JenkinsJsonSourceGenerationContext.Default.JenkinsBuildInfo).ConfigureAwait(false); + ArgumentNullException.ThrowIfNull(buildInfo); + + job.Value.Status = buildInfo.Result.Equals("SUCCESS", StringComparison.OrdinalIgnoreCase) ? JenkinsJobStatus.Succeeded : JenkinsJobStatus.Failed; } } - // Delay between polling. - await Task.Delay(500).ConfigureAwait(false); + // Don't poll too frequently. + await Task.Delay(1000).ConfigureAwait(false); } - // TODO: Stuff when jobs of this stage have finished? + // Don't bother starting the next stage if a prerequisite job has failed. + if (jobs.Any(job => job.Value.Status == JenkinsJobStatus.Failed)) + { + throw new BuildPipelineExecutionFailureException("A job has failed, aborting build process."); + } } // TODO: Stuff when all jobs have finished? diff --git a/UET/Redpoint.Uet.BuildPipeline.Executors.Jenkins/JenkinsBuildExecutorFactory.cs b/UET/Redpoint.Uet.BuildPipeline.Executors.Jenkins/JenkinsBuildExecutorFactory.cs index d075aeca..e46ab0f8 100644 --- a/UET/Redpoint.Uet.BuildPipeline.Executors.Jenkins/JenkinsBuildExecutorFactory.cs +++ b/UET/Redpoint.Uet.BuildPipeline.Executors.Jenkins/JenkinsBuildExecutorFactory.cs @@ -13,12 +13,14 @@ public JenkinsBuildExecutorFactory( _serviceProvider = serviceProvider; } - public IBuildExecutor CreateExecutor(string buildServerOutputFilePath) + public IBuildExecutor CreateExecutor(string buildServerOutputFilePath, Uri? gitUrl, string gitBranch) { return new JenkinsBuildExecutor( _serviceProvider, _serviceProvider.GetRequiredService>(), - buildServerOutputFilePath); + buildServerOutputFilePath, + gitUrl, + gitBranch); } } } diff --git a/UET/Redpoint.Uet.BuildPipeline.Executors.Jenkins/JenkinsBuildInfo.cs b/UET/Redpoint.Uet.BuildPipeline.Executors.Jenkins/JenkinsBuildInfo.cs new file mode 100644 index 00000000..6f818451 --- /dev/null +++ b/UET/Redpoint.Uet.BuildPipeline.Executors.Jenkins/JenkinsBuildInfo.cs @@ -0,0 +1,10 @@ +namespace Redpoint.Uet.BuildPipeline.Executors.Jenkins +{ + using System.Text.Json.Serialization; + + internal class JenkinsBuildInfo + { + [JsonPropertyName("result"), JsonRequired] + public string Result { get; set; } = string.Empty; + } +} diff --git a/UET/Redpoint.Uet.BuildPipeline.Executors.Jenkins/JenkinsBuildNodeExecutor.cs b/UET/Redpoint.Uet.BuildPipeline.Executors.Jenkins/JenkinsBuildNodeExecutor.cs new file mode 100644 index 00000000..1da4165f --- /dev/null +++ b/UET/Redpoint.Uet.BuildPipeline.Executors.Jenkins/JenkinsBuildNodeExecutor.cs @@ -0,0 +1,26 @@ +namespace Redpoint.Uet.BuildPipeline.Executors.Jenkins +{ + using Redpoint.Uet.BuildPipeline.Executors; + using Redpoint.Uet.BuildPipeline.Executors.BuildServer; + using Redpoint.Uet.Configuration.Dynamic; + using Redpoint.Uet.Configuration.Plugin; + using Redpoint.Uet.Configuration.Project; + using System; + using System.Collections.Generic; + using System.Threading; + using System.Threading.Tasks; + + public class JenkinsBuildNodeExecutor : IBuildNodeExecutor + { + public string DiscoverPipelineId() + { + // TODO: This might be wrong if the ID is expected to be the same between the main executor and node executors, investigate. + return Environment.GetEnvironmentVariable("BUILD_TAG") ?? string.Empty; + } + + public Task ExecuteBuildNodesAsync(BuildSpecification buildSpecification, BuildConfigDynamic[]? preparePlugin, BuildConfigDynamic[]? prepareProject, IBuildExecutionEvents buildExecutionEvents, IReadOnlyList nodeNames, CancellationToken cancellationToken) + { + throw new NotImplementedException(); + } + } +} diff --git a/UET/Redpoint.Uet.BuildPipeline.Executors.Jenkins/JenkinsJobStatus.cs b/UET/Redpoint.Uet.BuildPipeline.Executors.Jenkins/JenkinsJobStatus.cs index 12952e60..4c20128d 100644 --- a/UET/Redpoint.Uet.BuildPipeline.Executors.Jenkins/JenkinsJobStatus.cs +++ b/UET/Redpoint.Uet.BuildPipeline.Executors.Jenkins/JenkinsJobStatus.cs @@ -18,8 +18,13 @@ public enum JenkinsJobStatus Executing, /// - /// The job has completed building. + /// The job has successfully completed building. /// - Completed, + Succeeded, + + /// + /// The job has successfully completed building. + /// + Failed, } } diff --git a/UET/Redpoint.Uet.BuildPipeline.Executors.Jenkins/JenkinsJsonSourceGenerationContext.cs b/UET/Redpoint.Uet.BuildPipeline.Executors.Jenkins/JenkinsJsonSourceGenerationContext.cs index d4f13b82..8b52fa8c 100644 --- a/UET/Redpoint.Uet.BuildPipeline.Executors.Jenkins/JenkinsJsonSourceGenerationContext.cs +++ b/UET/Redpoint.Uet.BuildPipeline.Executors.Jenkins/JenkinsJsonSourceGenerationContext.cs @@ -4,6 +4,7 @@ [JsonSourceGenerationOptions(WriteIndented = true)] [JsonSerializable(typeof(JenkinsQueueItem))] + [JsonSerializable(typeof(JenkinsBuildInfo))] internal partial class JenkinsJsonSourceGenerationContext : JsonSerializerContext { } diff --git a/UET/uet/Commands/Build/BuildCommand.cs b/UET/uet/Commands/Build/BuildCommand.cs index b9a292a5..a866b5f1 100644 --- a/UET/uet/Commands/Build/BuildCommand.cs +++ b/UET/uet/Commands/Build/BuildCommand.cs @@ -46,6 +46,8 @@ internal sealed class Options public Option Executor; public Option ExecutorOutputFile; + public Option ExecutorGitUrl; + public Option ExecutorGitBranch; public Option WindowsSharedStoragePath; public Option WindowsSharedGitCachePath; public Option WindowsSdksPath; @@ -204,6 +206,20 @@ Set the plugin package to use this version number instead of the auto-generated ArgumentGroupName = cicdOptions }; + ExecutorGitUrl = new Option( + "--executor-git-url", + description: "If the executor runs the build externally and is not implicitly integrated with git (e.g. Jenkins), the URL to the git repository to clone (e.g. 'https://user-name:access-token@git.example.com/folders/project.git').") + { + ArgumentGroupName = cicdOptions + }; + + ExecutorGitBranch = new Option( + "--executor-git-branch", + description: "If the executor runs the build externally and is not implicitly integrated with git (e.g. Jenkins), the branch of the git repository to checkout.") + { + ArgumentGroupName = cicdOptions + }; + WindowsSharedStoragePath = new Option( "--windows-shared-storage-path", description: "If the build is running across multiple machines (depending on the executor), this is the network share for Windows machines to access.") @@ -298,6 +314,8 @@ public async Task ExecuteAsync(InvocationContext context) var shipping = context.ParseResult.GetValueForOption(_options.Shipping); var executorName = context.ParseResult.GetValueForOption(_options.Executor); var executorOutputFile = context.ParseResult.GetValueForOption(_options.ExecutorOutputFile); + var executorGitUrl = context.ParseResult.GetValueForOption(_options.ExecutorGitUrl); + var executorGitBranch = context.ParseResult.GetValueForOption(_options.ExecutorGitBranch); var windowsSharedStoragePath = context.ParseResult.GetValueForOption(_options.WindowsSharedStoragePath); var windowsSharedGitCachePath = context.ParseResult.GetValueForOption(_options.WindowsSharedGitCachePath); var windowsSdksPath = context.ParseResult.GetValueForOption(_options.WindowsSdksPath); @@ -381,6 +399,8 @@ public async Task ExecuteAsync(InvocationContext context) _logger.LogInformation($"--shipping: {(distribution != null ? "n/a" : (shipping ? "yes" : "no"))}"); _logger.LogInformation($"--executor: {executorName}"); _logger.LogInformation($"--executor-output-file: {executorOutputFile}"); + _logger.LogInformation($"--executor-git-url: {executorGitUrl}"); + _logger.LogInformation($"--executor-git-branch: {executorGitBranch}"); _logger.LogInformation($"--windows-shared-storage-path: {windowsSharedStoragePath}"); _logger.LogInformation($"--windows-shared-git-cache-path: {windowsSharedGitCachePath}"); _logger.LogInformation($"--windows-sdks-path: {windowsSdksPath}"); @@ -451,7 +471,7 @@ public async Task ExecuteAsync(InvocationContext context) { "local" => _localBuildExecutorFactory.CreateExecutor(), "gitlab" => _gitLabBuildExecutorFactory.CreateExecutor(executorOutputFile!), - "jenkins" => _jenkinsBuildExecutorFactory.CreateExecutor(executorOutputFile!), + "jenkins" => _jenkinsBuildExecutorFactory.CreateExecutor(executorOutputFile!, executorGitUrl, executorGitBranch!), _ => throw new NotSupportedException(), };