Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Draft: Add support for building plugin platform extensions #37

Draft
wants to merge 2 commits into
base: main
Choose a base branch
from
Draft

Draft: Add support for building plugin platform extensions #37

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
6d1f677
wip: platform extension
hach-que Jun 17, 2024
1373852
wip: OpenCPP notes
hach-que Oct 31, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 3 additions & 0 deletions UET/Redpoint.Uet.Automation/Worker/Local/LocalEditorWorker.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -159,6 +159,9 @@ await _processExecutor.ExecuteAsync(
]);
}

// Need a unique path per worker, and then unify it afterwards.
// & "C:\Program Files\OpenCppCoverage\OpenCppCoverage.exe" --sources "OnlineSubsystemRedpointEOS*" --modules "*Redpoint*" --export_type cobertura:<path> --

// Start the process in the background.
var st = Stopwatch.StartNew();
var automationTask = Task.Run(async () => await _processExecutor.ExecuteAsync(
Expand Down
21 changes: 14 additions & 7 deletions UET/Redpoint.Uet.BuildPipeline.Executors.BuildServer/BuildServerBuildExecutor.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -495,15 +495,22 @@ public virtual async Task<int> ExecuteBuildAsync(
{
_logger.LogInformation("Generating BuildGraph JSON based on settings...");
buildGraph = await _buildGraphExecutor.GenerateGraphAsync(
engineWorkspace.Path,
temporaryWorkspace.Path,
buildSpecification.UETPath,
buildSpecification.ArtifactExportPath,
new BuildGraphArgumentContext
{
RepositoryRoot = new RepositoryRoot
{
BaseCodePath = temporaryWorkspace.Path,
PlatformCodePath = string.Empty,
},
UetPath = buildSpecification.UETPath,
EnginePath = engineWorkspace.Path,
SharedStoragePath = OperatingSystem.IsWindows()
? buildSpecification.BuildGraphEnvironment.Windows.SharedStorageAbsolutePath
: buildSpecification.BuildGraphEnvironment.Mac!.SharedStorageAbsolutePath,
ArtifactExportPath = buildSpecification.ArtifactExportPath,
},
buildSpecification.BuildGraphScript,
buildSpecification.BuildGraphTarget,
OperatingSystem.IsWindows()
? buildSpecification.BuildGraphEnvironment.Windows.SharedStorageAbsolutePath
: buildSpecification.BuildGraphEnvironment.Mac!.SharedStorageAbsolutePath,
buildSpecification.BuildGraphSettings,
buildSpecification.BuildGraphSettingReplacements,
generationCaptureSpecification,
Expand Down
134 changes: 96 additions & 38 deletions UET/Redpoint.Uet.BuildPipeline.Executors.GitLab/GitLabBuildNodeExecutor.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -15,6 +15,7 @@
using System.Threading;
using System.Threading.Tasks;
using Redpoint.Concurrency;
using Redpoint.Uet.BuildPipeline.MultiWorkspace;

public class GitLabBuildNodeExecutor : IBuildNodeExecutor
{
Expand All @@ -24,6 +25,7 @@ public class GitLabBuildNodeExecutor : IBuildNodeExecutor
private readonly IDynamicWorkspaceProvider _workspaceProvider;
private readonly ISdkSetupForBuildExecutor _sdkSetupForBuildExecutor;
private readonly IBuildGraphArgumentGenerator _buildGraphArgumentGenerator;
private readonly IMultiWorkspaceAllocator _multiWorkspaceAllocator;
private readonly IDynamicProvider<BuildConfigPluginDistribution, IPrepareProvider>[] _pluginPrepare;
private readonly IDynamicProvider<BuildConfigProjectDistribution, IPrepareProvider>[] _projectPrepare;

Expand All @@ -34,14 +36,16 @@ public GitLabBuildNodeExecutor(
IEngineWorkspaceProvider engineWorkspaceProvider,
IDynamicWorkspaceProvider workspaceProvider,
ISdkSetupForBuildExecutor sdkSetupForBuildExecutor,
IBuildGraphArgumentGenerator buildGraphArgumentGenerator)
IBuildGraphArgumentGenerator buildGraphArgumentGenerator,
IMultiWorkspaceAllocator multiWorkspaceAllocator)
{
_logger = logger;
_buildGraphExecutor = buildGraphExecutor;
_engineWorkspaceProvider = engineWorkspaceProvider;
_workspaceProvider = workspaceProvider;
_sdkSetupForBuildExecutor = sdkSetupForBuildExecutor;
_buildGraphArgumentGenerator = buildGraphArgumentGenerator;
_multiWorkspaceAllocator = multiWorkspaceAllocator;
_pluginPrepare = serviceProvider.GetServices<IDynamicProvider<BuildConfigPluginDistribution, IPrepareProvider>>().ToArray();
_projectPrepare = serviceProvider.GetServices<IDynamicProvider<BuildConfigProjectDistribution, IPrepareProvider>>().ToArray();
}
Expand Down Expand Up @@ -72,6 +76,9 @@ public async Task<int> ExecuteBuildNodesAsync(
var commit = System.Environment.GetEnvironmentVariable("CI_COMMIT_SHA")!;
var executingNode = new NodeNameExecutionState();

var baseRepository = System.Environment.GetEnvironmentVariable("BASE_CI_REPOSITORY_URL");
var baseCommit = System.Environment.GetEnvironmentVariable("BASE_CI_COMMIT_SHA");

_logger.LogTrace("Starting execution of nodes...");
try
{
Expand All @@ -80,8 +87,21 @@ public async Task<int> ExecuteBuildNodesAsync(
string.Empty,
cancellationToken).ConfigureAwait(false)).AsAsyncDisposable(out var engineWorkspace).ConfigureAwait(false))
{
async Task<int> ExecuteNodeInWorkspaceAsync(string nodeName, string targetWorkspacePath)
async Task<int> ExecuteNodeInWorkspaceAsync(
string nodeName,
RepositoryRoot repositoryRoot)
{
var buildGraphArgumentContext = new BuildGraphArgumentContext
{
RepositoryRoot = repositoryRoot,
UetPath = buildSpecification.UETPath,
EnginePath = engineWorkspace.Path,
SharedStoragePath = OperatingSystem.IsWindows()
? buildSpecification.BuildGraphEnvironment.Windows.SharedStorageAbsolutePath
: buildSpecification.BuildGraphEnvironment.Mac!.SharedStorageAbsolutePath,
ArtifactExportPath = buildSpecification.ArtifactExportPath,
};

var globalEnvironmentVariablesWithSdk = await _sdkSetupForBuildExecutor.SetupForBuildAsync(
buildSpecification,
nodeName,
Expand All @@ -92,13 +112,7 @@ async Task<int> ExecuteNodeInWorkspaceAsync(string nodeName, string targetWorksp
var preBuildGraphArguments = _buildGraphArgumentGenerator.GeneratePreBuildGraphArguments(
buildSpecification.BuildGraphSettings,
buildSpecification.BuildGraphSettingReplacements,
targetWorkspacePath,
buildSpecification.UETPath,
engineWorkspace.Path,
OperatingSystem.IsWindows()
? buildSpecification.BuildGraphEnvironment.Windows.SharedStorageAbsolutePath
: buildSpecification.BuildGraphEnvironment.Mac!.SharedStorageAbsolutePath,
buildSpecification.ArtifactExportPath);
buildGraphArgumentContext);

if (preparePlugin != null && preparePlugin.Length > 0)
{
Expand All @@ -111,7 +125,7 @@ async Task<int> ExecuteNodeInWorkspaceAsync(string nodeName, string targetWorksp
.First();
var exitCode = await provider.RunBeforeBuildGraphAsync(
byType,
targetWorkspacePath,
repositoryRoot.OutputPath,
preBuildGraphArguments,
cancellationToken).ConfigureAwait(false);
if (exitCode != 0)
Expand All @@ -133,7 +147,7 @@ async Task<int> ExecuteNodeInWorkspaceAsync(string nodeName, string targetWorksp
.First();
var exitCode = await provider.RunBeforeBuildGraphAsync(
byType,
targetWorkspacePath,
repositoryRoot.OutputPath,
preBuildGraphArguments,
cancellationToken).ConfigureAwait(false);
if (exitCode != 0)
Expand All @@ -146,16 +160,10 @@ async Task<int> ExecuteNodeInWorkspaceAsync(string nodeName, string targetWorksp

_logger.LogTrace($"Starting: {nodeName}");
return await _buildGraphExecutor.ExecuteGraphNodeAsync(
engineWorkspace.Path,
targetWorkspacePath,
buildSpecification.UETPath,
buildSpecification.ArtifactExportPath,
buildGraphArgumentContext,
buildSpecification.BuildGraphScript,
buildSpecification.BuildGraphTarget,
nodeName,
OperatingSystem.IsWindows()
? buildSpecification.BuildGraphEnvironment.Windows.SharedStorageAbsolutePath
: buildSpecification.BuildGraphEnvironment.Mac!.SharedStorageAbsolutePath,
buildSpecification.BuildGraphSettings,
buildSpecification.BuildGraphSettingReplacements,
globalEnvironmentVariablesWithSdk,
Expand All @@ -176,13 +184,15 @@ async Task<int> ExecuteNodeInWorkspaceAsync(string nodeName, string targetWorksp
cancellationToken).ConfigureAwait(false);
}

async Task<int> ExecuteNodesInWorkspaceAsync(string targetWorkspacePath)
async Task<int> ExecuteNodesInWorkspaceAsync(RepositoryRoot repositoryRoot)
{
foreach (var nodeName in nodeNames)
{
await buildExecutionEvents.OnNodeStarted(nodeName).ConfigureAwait(false);
executingNode.NodeName = nodeName;
var exitCode = await ExecuteNodeInWorkspaceAsync(nodeName, targetWorkspacePath).ConfigureAwait(false);
var exitCode = await ExecuteNodeInWorkspaceAsync(
nodeName,
repositoryRoot).ConfigureAwait(false);
if (exitCode == 0)
{
_logger.LogTrace($"Finished: {nodeName} = Success");
Expand All @@ -204,31 +214,79 @@ async Task<int> ExecuteNodesInWorkspaceAsync(string targetWorkspacePath)
int overallExitCode;
if (buildSpecification.Engine.IsEngineBuild)
{
overallExitCode = await ExecuteNodesInWorkspaceAsync(engineWorkspace.Path).ConfigureAwait(false);
overallExitCode = await ExecuteNodesInWorkspaceAsync(
new RepositoryRoot
{
BaseCodePath = engineWorkspace.Path,
PlatformCodePath = string.Empty,
}).ConfigureAwait(false);
}
else
{
_logger.LogTrace($"Obtaining workspace for build.");
await using ((await _workspaceProvider.GetWorkspaceAsync(
new GitWorkspaceDescriptor
{
RepositoryUrl = repository,
RepositoryCommitOrRef = commit,
AdditionalFolderLayers = Array.Empty<string>(),
AdditionalFolderZips = Array.Empty<string>(),
WorkspaceDisambiguators = nodeNames,
ProjectFolderName = buildSpecification.ProjectFolderName,
IsEngineBuild = false,
WindowsSharedGitCachePath = null,
MacSharedGitCachePath = null,
},
cancellationToken).ConfigureAwait(false)).AsAsyncDisposable(out var targetWorkspace).ConfigureAwait(false))
await using (await _multiWorkspaceAllocator.AllocateAsync())

var workspaces = new Dictionary<string, IWorkspace?>();
try
{
_logger.LogTrace($"Obtaining workspace for build.");
workspaces.Add("base", await _workspaceProvider.GetWorkspaceAsync(
new GitWorkspaceDescriptor
{
RepositoryUrl = repository,
RepositoryCommitOrRef = commit,
AdditionalFolderLayers = Array.Empty<string>(),
AdditionalFolderZips = Array.Empty<string>(),
WorkspaceDisambiguators = nodeNames,
ProjectFolderName = buildSpecification.ProjectFolderName,
IsEngineBuild = false,
WindowsSharedGitCachePath = null,
MacSharedGitCachePath = null,
},
cancellationToken).ConfigureAwait(false));

if (!string.IsNullOrWhiteSpace(baseRepository) &&
!string.IsNullOrWhiteSpace(baseCommit))
{
_logger.LogTrace($"Obtaining platform-specific workspace for build.");
workspaces.Add("platform", await _workspaceProvider.GetWorkspaceAsync(
new GitWorkspaceDescriptor
{
RepositoryUrl = baseRepository,
RepositoryCommitOrRef = baseCommit,
AdditionalFolderLayers = Array.Empty<string>(),
AdditionalFolderZips = Array.Empty<string>(),
WorkspaceDisambiguators = nodeNames,
ProjectFolderName = buildSpecification.ProjectFolderName,
IsEngineBuild = false,
WindowsSharedGitCachePath = null,
MacSharedGitCachePath = null,
},
cancellationToken).ConfigureAwait(false));
}
else
{
workspaces.Add("platform", null);
}

_logger.LogTrace($"Calling ExecuteNodesInWorkspaceAsync inside allocated workspace.");
overallExitCode = await ExecuteNodesInWorkspaceAsync(targetWorkspace.Path).ConfigureAwait(false);
overallExitCode = await ExecuteNodesInWorkspaceAsync(new RepositoryRoot
{
BaseCodePath = workspaces["base"]!.Path,
PlatformCodePath = workspaces["platform"]?.Path ?? string.Empty,
}).ConfigureAwait(false);
_logger.LogTrace($"Finished ExecuteNodesInWorkspaceAsync with exit code '{overallExitCode}'.");
}
_logger.LogTrace($"Released workspace for build.");
finally
{
foreach (var kv in workspaces)
{
if (kv.Value != null)
{
await kv.Value.DisposeAsync().ConfigureAwait(false);
}
}
_logger.LogTrace($"Released workspace for build.");
}
}
_logger.LogTrace($"Returning overall exit code '{overallExitCode}' from ExecuteBuildNodesAsync.");
return overallExitCode;
Expand Down
Loading
Loading