Ensure Gradle home is completely empty for build, but sync download cache for performance

UET/Redpoint.IO/DirectoryAsync.cs

@@ -44,6 +44,10 @@ await Task.Run(() =>
                     // Now try to delete again.
                     Directory.Delete(path, recursive);
                 }
+                catch (DirectoryNotFoundException)
+                {
+                    // Directory already doesn't exist; ignore.
+                }
             }).ConfigureAwait(false);
         }
 

UET/Redpoint.Uet.BuildPipeline/BuildGraph/DefaultBuildGraphExecutor.cs

@@ -5,6 +5,7 @@
     using Redpoint.IO;
     using Redpoint.ProcessExecution;
     using Redpoint.Uet.BuildPipeline.BuildGraph.Export;
+    using Redpoint.Uet.BuildPipeline.BuildGraph.Gradle;
     using Redpoint.Uet.BuildPipeline.BuildGraph.MobileProvisioning;
     using Redpoint.Uet.BuildPipeline.BuildGraph.Patching;
     using Redpoint.Uet.Configuration.Engine;
@@ -24,21 +25,24 @@ internal class DefaultBuildGraphExecutor : IBuildGraphExecutor
         private readonly IBuildGraphPatcher _buildGraphPatcher;
         private readonly IDynamicWorkspaceProvider _dynamicWorkspaceProvider;
         private readonly IMobileProvisioning _mobileProvisioning;
+        private readonly IGradleWorkspace _gradleWorkspace;
 
         public DefaultBuildGraphExecutor(
             ILogger<DefaultBuildGraphExecutor> logger,
             IUATExecutor uatExecutor,
             IBuildGraphArgumentGenerator buildGraphArgumentGenerator,
             IBuildGraphPatcher buildGraphPatcher,
             IDynamicWorkspaceProvider dynamicWorkspaceProvider,
-            IMobileProvisioning mobileProvisioning)
+            IMobileProvisioning mobileProvisioning,
+            IGradleWorkspace gradleWorkspace)
         {
             _logger = logger;
             _uatExecutor = uatExecutor;
             _buildGraphArgumentGenerator = buildGraphArgumentGenerator;
             _buildGraphPatcher = buildGraphPatcher;
             _dynamicWorkspaceProvider = dynamicWorkspaceProvider;
             _mobileProvisioning = mobileProvisioning;
+            _gradleWorkspace = gradleWorkspace;
         }
 
         public async Task ListGraphAsync(
@@ -93,28 +97,18 @@ public async Task<int> ExecuteGraphNodeAsync(
                 Name = "NuGetPackages"
             }, cancellationToken).ConfigureAwait(false)).AsAsyncDisposable(out var nugetPackages).ConfigureAwait(false))
             {
-                await using ((await _dynamicWorkspaceProvider.GetWorkspaceAsync(new TemporaryWorkspaceDescriptor
+                GradleWorkspaceInstance? gradleInstance = null;
+                if (buildGraphNodeName.Contains("Android", StringComparison.InvariantCultureIgnoreCase) ||
+                    buildGraphNodeName.Contains("MetaQuest", StringComparison.InvariantCultureIgnoreCase) ||
+                    buildGraphNodeName.Contains("GooglePlay", StringComparison.InvariantCultureIgnoreCase))
                 {
-                    Name = "GradleUserHome"
-                }, cancellationToken).ConfigureAwait(false)).AsAsyncDisposable(out var gradleUserHome).ConfigureAwait(false))
+                    // Only set up and tear down the Gradle workspace if the node is for Android,
+                    // since it's quite expensive to copy the cache back and forth (but necessary
+                    // to mitigate Gradle concurrency bugs).
+                    gradleInstance = await _gradleWorkspace.GetGradleWorkspaceInstance(cancellationToken).ConfigureAwait(false);
+                }
+                try
                 {
-                    // Delete the Gradle 'tarnsforms-4' cache folder, since it can become corrupt and then prevent
-                    // any further Android build jobs from working on this machine.
-                    var transformsFolder = Path.Combine(gradleUserHome.Path, "caches", "transforms-4");
-                    if (Directory.Exists(transformsFolder))
-                    {
-                        _logger.LogInformation("Deleting Gradle 'transforms-4' cache...");
-                        try
-                        {
-                            await DirectoryAsync.DeleteAsync(transformsFolder, true);
-                            _logger.LogInformation("Successfully deleted Gradle 'transforms-4' cache.");
-                        }
-                        catch (Exception ex)
-                        {
-                            _logger.LogWarning($"Failed to delete Gradle 'transforms-4' cache: {ex}");
-                        }
-                    }
-
                     var environmentVariables = new Dictionary<string, string>
                     {
                         { "IsBuildMachine", "1" },
@@ -130,9 +124,12 @@ public async Task<int> ExecuteGraphNodeAsync(
                         // Isolate NuGet package restore so that multiple jobs can restore at
                         // the same time.
                         { "NUGET_PACKAGES", nugetPackages.Path },
-                        // Adjust Gradle cache path so that Android packaging works under SYSTEM.
-                        { "GRADLE_USER_HOME", gradleUserHome.Path },
                     };
+                    if (gradleInstance != null)
+                    {
+                        // Adjust Gradle cache path so that Android packaging works under SYSTEM.
+                        environmentVariables["GRADLE_USER_HOME"] = gradleInstance.GradleHomePath;
+                    }
                     if (!string.IsNullOrWhiteSpace(buildGraphRepositoryRootPath))
                     {
                         environmentVariables["BUILD_GRAPH_PROJECT_ROOT"] = buildGraphRepositoryRootPath;
@@ -171,7 +168,7 @@ public async Task<int> ExecuteGraphNodeAsync(
                     }
                     try
                     {
-                        return await InternalRunAsync(
+                        var exitCode = await InternalRunAsync(
                             enginePath,
                             buildGraphRepositoryRootPath,
                             uetPath,
@@ -191,6 +188,11 @@ public async Task<int> ExecuteGraphNodeAsync(
                             mobileProvisions,
                             captureSpecification,
                             cancellationToken).ConfigureAwait(false);
+                        if (exitCode == 0)
+                        {
+                            gradleInstance?.MarkBuildAsSuccessful();
+                        }
+                        return exitCode;
                     }
                     finally
                     {
@@ -200,6 +202,14 @@ public async Task<int> ExecuteGraphNodeAsync(
                         }
                     }
                 }
+                catch
+                {
+                    if (gradleInstance != null)
+                    {
+                        await gradleInstance.DisposeAsync().ConfigureAwait(false);
+                    }
+                    throw;
+                }
             }
         }
 

UET/Redpoint.Uet.BuildPipeline/BuildGraph/Gradle/DefaultGradleWorkspace.cs

@@ -0,0 +1,134 @@
+namespace Redpoint.Uet.BuildPipeline.BuildGraph.Gradle
+{
+    using Microsoft.Extensions.Logging;
+    using Redpoint.IO;
+    using Redpoint.Uet.Workspace;
+    using Redpoint.Uet.Workspace.Descriptors;
+    using System;
+    using System.Threading.Tasks;
+
+    internal class DefaultGradleWorkspace : IGradleWorkspace
+    {
+        private readonly IDynamicWorkspaceProvider _dynamicWorkspaceProvider;
+        private readonly ILogger<DefaultGradleWorkspace> _logger;
+
+        public DefaultGradleWorkspace(
+            IDynamicWorkspaceProvider dynamicWorkspaceProvider,
+            ILogger<DefaultGradleWorkspace> logger)
+        {
+            _dynamicWorkspaceProvider = dynamicWorkspaceProvider;
+            _logger = logger;
+        }
+
+        public async Task<GradleWorkspaceInstance> GetGradleWorkspaceInstance(CancellationToken cancellationToken)
+        {
+            IWorkspace? gradleDownloadCache = null;
+            IWorkspace? gradleTemporaryWorkspace = null;
+
+            var ok = false;
+            try
+            {
+                gradleDownloadCache = await _dynamicWorkspaceProvider.GetWorkspaceAsync(new TemporaryWorkspaceDescriptor
+                {
+                    Name = "GradleDownloadCache",
+                }, cancellationToken).ConfigureAwait(false);
+
+                var gradleTemporaryWorkspaceAttempt = 0;
+                do
+                {
+                    gradleTemporaryWorkspace = await _dynamicWorkspaceProvider.GetWorkspaceAsync(new TemporaryWorkspaceDescriptor
+                    {
+                        Name = $"GradleHome_{Environment.ProcessId}_{gradleTemporaryWorkspaceAttempt}",
+                    }, cancellationToken).ConfigureAwait(false);
+
+                    _logger.LogInformation($"Checking the following Gradle home: {gradleTemporaryWorkspace.Path}");
+
+                    // If the gradle home has any files in it, delete them. The gradle home must be wiped out before/after each
+                    // build.
+                    try
+                    {
+                        foreach (var entry in new DirectoryInfo(gradleTemporaryWorkspace.Path).GetFileSystemInfos())
+                        {
+                            _logger.LogInformation($"Deleting file/directory from previous Gradle home: {entry.FullName}");
+                            if (entry is DirectoryInfo directory)
+                            {
+                                directory.Delete(true);
+                            }
+                            else
+                            {
+                                entry.Delete();
+                            }
+
+                            cancellationToken.ThrowIfCancellationRequested();
+                        }
+                    }
+                    catch
+                    {
+                        _logger.LogInformation($"Unable to use that Gradle home as one or more existing files/directories could not be removed.");
+                        gradleTemporaryWorkspaceAttempt++;
+                        await gradleTemporaryWorkspace.DisposeAsync().ConfigureAwait(false);
+                        gradleTemporaryWorkspace = null;
+                        cancellationToken.ThrowIfCancellationRequested();
+                        continue;
+                    }
+
+                    // We have a Gradle home we can use.
+                    break;
+                } while (true);
+
+                _logger.LogInformation($"Using the following Gradle home: {gradleTemporaryWorkspace.Path}");
+
+                cancellationToken.ThrowIfCancellationRequested();
+
+                var centralCacheSafeToUse = Path.Combine(gradleDownloadCache.Path, "safe-to-use");
+                var centralCacheModules2 = Path.Combine(gradleDownloadCache.Path, "modules-2");
+                var centralCacheWrapper = Path.Combine(gradleDownloadCache.Path, "wrapper");
+                if (File.Exists(centralCacheSafeToUse) && Directory.Exists(centralCacheModules2))
+                {
+                    var homeCacheModules2 = Path.Combine(gradleTemporaryWorkspace.Path, "caches", "modules-2");
+                    var homeCacheWrapper = Path.Combine(gradleTemporaryWorkspace.Path, "wrapper");
+
+                    _logger.LogInformation($"Copying the existing Gradle download cache from '{centralCacheModules2}' to '{homeCacheModules2}': {gradleTemporaryWorkspace.Path}");
+                    await DirectoryAsync.CopyAsync(centralCacheModules2, homeCacheModules2, true);
+
+                    _logger.LogInformation($"Copying the existing Gradle wrapper from '{centralCacheWrapper}' to '{homeCacheWrapper}': {gradleTemporaryWorkspace.Path}");
+                    await DirectoryAsync.CopyAsync(centralCacheWrapper, homeCacheWrapper, true);
+                }
+
+                cancellationToken.ThrowIfCancellationRequested();
+
+                ok = true;
+                return new GradleWorkspaceInstance(_logger, gradleDownloadCache, gradleTemporaryWorkspace);
+            }
+            finally
+            {
+                if (!ok)
+                {
+                    if (gradleDownloadCache != null)
+                    {
+                        try
+                        {
+                            await gradleDownloadCache.DisposeAsync().ConfigureAwait(false);
+                        }
+                        catch (Exception ex)
+                        {
+                            _logger.LogWarning($"Failed to release Gradle download cache: {ex}");
+                        }
+                    }
+
+                    if (gradleTemporaryWorkspace != null)
+                    {
+                        try
+                        {
+                            await gradleTemporaryWorkspace.DisposeAsync().ConfigureAwait(false);
+                        }
+                        catch (Exception ex)
+                        {
+                            _logger.LogWarning($"Failed to release Gradle home: {ex}");
+                        }
+                    }
+                }
+            }
+        }
+    }
+}

UET/Redpoint.Uet.BuildPipeline/BuildGraph/Gradle/GradleWorkspaceInstance.cs

@@ -0,0 +1,121 @@
+namespace Redpoint.Uet.BuildPipeline.BuildGraph.Gradle
+{
+    using Microsoft.Extensions.Logging;
+    using Redpoint.IO;
+    using Redpoint.Uet.Workspace;
+    using System;
+    using System.Threading.Tasks;
+
+    internal class GradleWorkspaceInstance : IAsyncDisposable
+    {
+        private readonly ILogger _logger;
+        private readonly IWorkspace _gradleDownloadCache;
+        private readonly IWorkspace _gradleTemporaryWorkspace;
+        private bool _buildSuccessful;
+
+        public GradleWorkspaceInstance(
+            ILogger logger,
+            IWorkspace gradleDownloadCache,
+            IWorkspace gradleTemporaryWorkspace)
+        {
+            _logger = logger;
+            _gradleDownloadCache = gradleDownloadCache;
+            _gradleTemporaryWorkspace = gradleTemporaryWorkspace;
+            _buildSuccessful = false;
+        }
+
+        public string GradleHomePath => _gradleTemporaryWorkspace.Path;
+
+        public void MarkBuildAsSuccessful()
+        {
+            _buildSuccessful = true;
+        }
+
+        public async ValueTask DisposeAsync()
+        {
+            if (_buildSuccessful)
+            {
+                // If the build was successful, copy the Gradle download cache back.
+                var centralCacheSafeToUse = Path.Combine(_gradleDownloadCache.Path, "safe-to-use");
+                var centralCacheModules2 = Path.Combine(_gradleDownloadCache.Path, "modules-2");
+                var centralCacheWrapper = Path.Combine(_gradleDownloadCache.Path, "wrapper");
+
+                var homeCacheModules2 = Path.Combine(_gradleTemporaryWorkspace.Path, "caches", "modules-2");
+                var homeCacheWrapper = Path.Combine(_gradleTemporaryWorkspace.Path, "wrapper");
+
+                try
+                {
+                    _logger.LogInformation($"Removing existing download cache so we can copy across the new version from the build: {_gradleDownloadCache.Path}");
+                    if (File.Exists(centralCacheSafeToUse))
+                    {
+                        File.Delete(centralCacheSafeToUse);
+                    }
+
+                    do
+                    {
+                        try
+                        {
+                            if (Directory.Exists(homeCacheModules2))
+                            {
+                                await DirectoryAsync.DeleteAsync(centralCacheModules2, true);
+                            }
+                            if (Directory.Exists(homeCacheWrapper))
+                            {
+                                await DirectoryAsync.DeleteAsync(centralCacheWrapper, true);
+                            }
+                            break;
+                        }
+                        catch (IOException ex) when (ex.Message.Contains("The directory is not empty.", StringComparison.Ordinal))
+                        {
+                            _logger.LogInformation("Can't remove existing central cache yet, trying again in 1 second...");
+                            await Task.Delay(1000);
+                            continue;
+                        }
+                    }
+                    while (true);
+
+                    if (Directory.Exists(homeCacheModules2))
+                    {
+                        _logger.LogInformation($"Copying home Gradle download cache from '{homeCacheModules2}' to central download cache at '{centralCacheModules2}'.");
+                        await DirectoryAsync.CopyAsync(homeCacheModules2, centralCacheModules2, true);
+                    }
+
+                    if (Directory.Exists(homeCacheWrapper))
+                    {
+                        _logger.LogInformation($"Copying home Gradle wrapper from '{homeCacheWrapper}' to central download cache at '{centralCacheWrapper}'.");
+                        await DirectoryAsync.CopyAsync(homeCacheWrapper, centralCacheWrapper, true);
+                    }
+
+                    _logger.LogInformation($"Marking central download cache as safe to use.");
+                    File.WriteAllText(centralCacheSafeToUse, "ok");
+                }
+                catch (Exception ex)
+                {
+                    _logger.LogWarning($"Failed to copy download cache back to central download cache; it will not be used until a successful copy occurs again: {ex}");
+                }
+            }
+            else
+            {
+                _logger.LogInformation($"Not syncing download cache to central download cache as the build was not successful.");
+            }
+
+            try
+            {
+                await _gradleDownloadCache.DisposeAsync().ConfigureAwait(false);
+            }
+            catch (Exception ex)
+            {
+                _logger.LogWarning($"Failed to release Gradle download cache: {ex}");
+            }
+
+            try
+            {
+                await _gradleTemporaryWorkspace.DisposeAsync().ConfigureAwait(false);
+            }
+            catch (Exception ex)
+            {
+                _logger.LogWarning($"Failed to release Gradle home: {ex}");
+            }
+        }
+    }
+}