From 06c9ae67e364f29986fd9aa713d5dd14e8dfb186 Mon Sep 17 00:00:00 2001 From: Dirk Eisenberg Date: Sat, 10 Sep 2022 21:04:19 +0200 Subject: [PATCH] Added dedicated backup restore module including verification --- .../IStorageContext.cs | 2 + .../IRestoreContext.cs | 2 +- .../BackupContext.cs | 22 ++- .../BackupService.cs | 5 +- ...s.WindowsAzure.Storage.Table.Backup.csproj | 6 + .../RestoreContext.cs | 156 ++++++++++-------- ...rs.WindowsAzure.Storage.Table.Tests.csproj | 8 + .../ITS21VerifyBackup.cs | 86 ++++++++++ .../Startup.cs | 14 +- .../StorageContextTableManagement.cs | 3 + 10 files changed, 226 insertions(+), 78 deletions(-) create mode 100644 CoreHelpers.WindowsAzure.Storage.Table.Tests/ITS21VerifyBackup.cs diff --git a/CoreHelpers.WindowsAzure.Storage.Table.Abstractions/IStorageContext.cs b/CoreHelpers.WindowsAzure.Storage.Table.Abstractions/IStorageContext.cs index c9e14e0..d8f9bae 100644 --- a/CoreHelpers.WindowsAzure.Storage.Table.Abstractions/IStorageContext.cs +++ b/CoreHelpers.WindowsAzure.Storage.Table.Abstractions/IStorageContext.cs @@ -54,6 +54,8 @@ Task> QueryAsync(string partitionKey, IEnumerable void SetTableNamePrefix(string tableNamePrefix); + string GetTableNamePrefix(); + void OverrideTableName(string table) where T : class, new(); Task MergeOrInsertAsync(IEnumerable models) where T : class, new(); diff --git a/CoreHelpers.WindowsAzure.Storage.Table.Backup.Abstractions/IRestoreContext.cs b/CoreHelpers.WindowsAzure.Storage.Table.Backup.Abstractions/IRestoreContext.cs index d72c774..5261b45 100644 --- a/CoreHelpers.WindowsAzure.Storage.Table.Backup.Abstractions/IRestoreContext.cs +++ b/CoreHelpers.WindowsAzure.Storage.Table.Backup.Abstractions/IRestoreContext.cs @@ -5,7 +5,7 @@ namespace CoreHelpers.WindowsAzure.Storage.Table.Backup.Abstractions { public interface IRestoreContext : IDisposable { - // Task BackupTable(IStorageContext storageContext, string tableName, bool compress = true); + Task Restore(IStorageContext storageContext, string[] excludedTables = null); } } diff --git a/CoreHelpers.WindowsAzure.Storage.Table.Backup/BackupContext.cs b/CoreHelpers.WindowsAzure.Storage.Table.Backup/BackupContext.cs index 3c806c0..aee2977 100644 --- a/CoreHelpers.WindowsAzure.Storage.Table.Backup/BackupContext.cs +++ b/CoreHelpers.WindowsAzure.Storage.Table.Backup/BackupContext.cs @@ -61,12 +61,17 @@ public async Task Backup(IStorageContext storageContext, string[] excludedTables _logger.LogInformation($"Statsfile is under {memoryStatsFile}..."); statsFile.WriteLine($"TableName,PageCounter,ItemCount,MemoryFootprint"); + // calculate the real prefix + var effectiveTableNamePrefix = String.IsNullOrEmpty(_targetTableNamePrefix) ? "" : _targetTableNamePrefix; + if (!String.IsNullOrEmpty(storageContext.GetTableNamePrefix())) + effectiveTableNamePrefix = $"{storageContext.GetTableNamePrefix()}{_targetTableNamePrefix}"; + // visit every table foreach (var tableName in tables) { // filter the table prefix - if (!String.IsNullOrEmpty(_targetTableNamePrefix) && !tableName.StartsWith(_targetTableNamePrefix, StringComparison.CurrentCulture)) + if (!String.IsNullOrEmpty(effectiveTableNamePrefix) && !tableName.StartsWith(effectiveTableNamePrefix, StringComparison.CurrentCulture)) { _logger.LogInformation($"Ignoring table {tableName}..."); continue; @@ -80,9 +85,10 @@ public async Task Backup(IStorageContext storageContext, string[] excludedTables } using (_logger.BeginScope($"Processing backup for table {tableName}...")) - { + { // do the backup - var fileName = $"{tableName}.json"; + var prefixLessTableName = tableName.Remove(0, effectiveTableNamePrefix.Length); + var fileName = $"{prefixLessTableName}.json"; if (!string.IsNullOrEmpty(_targetPath)) { fileName = $"{_targetPath}/{fileName}"; } if (compress) { fileName += ".gz"; } @@ -96,7 +102,7 @@ public async Task Backup(IStorageContext storageContext, string[] excludedTables _logger.LogInformation($"Writing backup to non compressed file"); // do it - using (var backupFileStream = await blobClient.OpenWriteAsync(false)) + using (var backupFileStream = await blobClient.OpenWriteAsync(true)) { using (var contentWriter = new ZippedStreamWriter(backupFileStream, compress)) { @@ -105,7 +111,13 @@ public async Task Backup(IStorageContext storageContext, string[] excludedTables var pageLogScope = default(IDisposable); - await storageContext.ExportToJsonAsync(tableName, contentWriter, (ImportExportOperation operation) => + // get the effective tablename + var effectiveTableName = tableName; + if (!String.IsNullOrEmpty(storageContext.GetTableNamePrefix())) + effectiveTableName = effectiveTableName.Remove(0, storageContext.GetTableNamePrefix().Length); + + // start export + await storageContext.ExportToJsonAsync(effectiveTableName, contentWriter, (ImportExportOperation operation) => { switch (operation) { diff --git a/CoreHelpers.WindowsAzure.Storage.Table.Backup/BackupService.cs b/CoreHelpers.WindowsAzure.Storage.Table.Backup/BackupService.cs index 099a43d..3b18fc8 100644 --- a/CoreHelpers.WindowsAzure.Storage.Table.Backup/BackupService.cs +++ b/CoreHelpers.WindowsAzure.Storage.Table.Backup/BackupService.cs @@ -26,7 +26,10 @@ public async Task OpenBackupContext(string targetBlobStorageConn public async Task OpenRestorContext(string sourceBlobStorageConnectionString, string sourceContainerName, string sourcePath, string tableNamePrefix = null) { await Task.CompletedTask; - return new RestoreContext(); + return new RestoreContext( + _loggerFactory.CreateLogger(), + sourceBlobStorageConnectionString, sourceContainerName, sourcePath, + tableNamePrefix); } } } diff --git a/CoreHelpers.WindowsAzure.Storage.Table.Backup/CoreHelpers.WindowsAzure.Storage.Table.Backup.csproj b/CoreHelpers.WindowsAzure.Storage.Table.Backup/CoreHelpers.WindowsAzure.Storage.Table.Backup.csproj index 4b57c11..8c2f87e 100644 --- a/CoreHelpers.WindowsAzure.Storage.Table.Backup/CoreHelpers.WindowsAzure.Storage.Table.Backup.csproj +++ b/CoreHelpers.WindowsAzure.Storage.Table.Backup/CoreHelpers.WindowsAzure.Storage.Table.Backup.csproj @@ -14,6 +14,12 @@ + + default + + + default + diff --git a/CoreHelpers.WindowsAzure.Storage.Table.Backup/RestoreContext.cs b/CoreHelpers.WindowsAzure.Storage.Table.Backup/RestoreContext.cs index 9565467..bf84875 100644 --- a/CoreHelpers.WindowsAzure.Storage.Table.Backup/RestoreContext.cs +++ b/CoreHelpers.WindowsAzure.Storage.Table.Backup/RestoreContext.cs @@ -1,93 +1,109 @@ using System; +using System.ComponentModel; +using System.IO; +using System.Threading.Tasks; +using Azure.Storage.Blobs; using CoreHelpers.WindowsAzure.Storage.Table.Backup.Abstractions; +using Microsoft.Extensions.Logging; namespace CoreHelpers.WindowsAzure.Storage.Table.Backup { public class RestoreContext : IRestoreContext { - public RestoreContext() + private ILogger _logger; + private BlobServiceClient _blobServiceClient; + + private string _sourceConnectionString; + private string _sourceContainer; + private string _sourcePath; + private string _sourceTableNamePrefix; + + public RestoreContext(ILogger logger, string connectionString, string container, string path, string tableNamePrefix) { + _logger = logger; + _blobServiceClient = new BlobServiceClient(connectionString); + + _sourceConnectionString = connectionString; + _sourceContainer = container; + _sourcePath = path; + _sourceTableNamePrefix = tableNamePrefix; } public void Dispose() - { - throw new NotImplementedException(); + { } - } -} - -/* - * public async Task Restore(string containerName, string srcPath, string tablePrefix = null) { - - // log - storageLogger.LogInformation($"Starting restore procedure..."); - // get all backup files - var blobClient = backupStorageAccount.CreateCloudBlobClient(); - var containerReference = blobClient.GetContainerReference(containerName.ToLower()); - - // check if the container exists - if (!await containerReference.ExistsAsync()) { - storageLogger.LogInformation($"Missing container {containerName.ToLower()}"); - return; - } - - // build the path including prefix - storageLogger.LogInformation($"Search Prefix is {srcPath}"); - - // track the state - var continuationToken = default(BlobContinuationToken); - - do + public async Task Restore(IStorageContext storageContext, string[] excludedTables = null) + { + using (_logger.BeginScope("Starting restore procedure...")) { - // get all blobs - var blobResult = await containerReference.ListBlobsSegmentedAsync(srcPath, true, BlobListingDetails.All, 1000, continuationToken, null, null); - - // process every backup file as table - foreach(var blob in blobResult.Results) { - - // build the name - var blobName = blob.StorageUri.PrimaryUri.AbsolutePath; - blobName = blobName.Remove(0, containerName.Length + 2); - - // get the tablename - var tableName = Path.GetFileNameWithoutExtension(blobName); - var compressed = blobName.EndsWith(".gz", StringComparison.CurrentCultureIgnoreCase); - if (compressed) - tableName = Path.GetFileNameWithoutExtension(tableName); - - // add the prefix - if (!String.IsNullOrEmpty(tablePrefix)) - tableName = $"{tablePrefix}{tableName}"; + // get the container reference + var blobContainerClient = _blobServiceClient.GetBlobContainerClient(_sourceContainer); + if (!await blobContainerClient.ExistsAsync()) + throw new Exception("Container not found"); + + // check if the container exists + if (!await blobContainerClient.ExistsAsync()) + { + _logger.LogInformation($"Missing container {_sourceContainer.ToLower()}"); + return; + } - // log - storageLogger.LogInformation($"Restoring {blobName} to table {tableName} (Compressed: {compressed})"); + // build the path including prefix + _logger.LogInformation($"Search Prefix is {_sourcePath}"); - // build the reference - var blockBlobReference = containerReference.GetBlockBlobReference(blobName); + // get the pages + var blobPages = blobContainerClient.GetBlobsAsync(Azure.Storage.Blobs.Models.BlobTraits.None, Azure.Storage.Blobs.Models.BlobStates.None, _sourcePath).AsPages(); - // open the read stream - using (var readStream = await blockBlobReference.OpenReadAsync()) + // visit every page + await foreach (var page in blobPages) + { + foreach(var blob in page.Values) { - // unzip the stream - using (var contentReader = new ZippedStreamReader(readStream, compressed)) + // build the name + var blobName = blob.Name; + + // get the tablename + var tableName = Path.GetFileNameWithoutExtension(blobName); + var compressed = blobName.EndsWith(".gz", StringComparison.CurrentCultureIgnoreCase); + if (compressed) + tableName = Path.GetFileNameWithoutExtension(tableName); + + // add the prefix + if (!String.IsNullOrEmpty(_sourceTableNamePrefix)) + tableName = $"{_sourceTableNamePrefix}{tableName}"; + + // log + _logger.LogInformation($"Restoring {blobName} to table {tableName} (Compressed: {compressed})"); + + // open the read stream + var blobClient = blobContainerClient.GetBlobClient(blob.Name); + using (var readStream = await blobClient.OpenReadAsync()) { - // import the stream - var pageCounter = 0; - await dataImportService.ImportFromJsonStreamAsync(tableName, contentReader, (c) => { - pageCounter++; - storageLogger.LogInformation($" Processing page #{pageCounter} with #{c} items..."); - }); + // unzip the stream + using (var contentReader = new ZippedStreamReader(readStream, compressed)) + { + // import the stream + var pageCounter = 0; + await storageContext.ImportFromJsonAsync(tableName, contentReader, (c) => + { + switch (c) + { + case ImportExportOperation.processingPage: + _logger.LogInformation($" Processing page #{pageCounter}..."); + break; + case ImportExportOperation.processedPage: + pageCounter++; + break; + } + }); + } } } - } - - // proces the token - continuationToken = blobResult.ContinuationToken; - - } while (continuationToken != null); - - + } + } + await Task.CompletedTask; } -*/ \ No newline at end of file + } +} \ No newline at end of file diff --git a/CoreHelpers.WindowsAzure.Storage.Table.Tests/CoreHelpers.WindowsAzure.Storage.Table.Tests.csproj b/CoreHelpers.WindowsAzure.Storage.Table.Tests/CoreHelpers.WindowsAzure.Storage.Table.Tests.csproj index 729157b..b7a410b 100644 --- a/CoreHelpers.WindowsAzure.Storage.Table.Tests/CoreHelpers.WindowsAzure.Storage.Table.Tests.csproj +++ b/CoreHelpers.WindowsAzure.Storage.Table.Tests/CoreHelpers.WindowsAzure.Storage.Table.Tests.csproj @@ -11,6 +11,10 @@ 4 true + default + + + default @@ -24,6 +28,8 @@ all + + @@ -33,6 +39,8 @@ + + diff --git a/CoreHelpers.WindowsAzure.Storage.Table.Tests/ITS21VerifyBackup.cs b/CoreHelpers.WindowsAzure.Storage.Table.Tests/ITS21VerifyBackup.cs new file mode 100644 index 0000000..b3e2d81 --- /dev/null +++ b/CoreHelpers.WindowsAzure.Storage.Table.Tests/ITS21VerifyBackup.cs @@ -0,0 +1,86 @@ +using System; +using CoreHelpers.WindowsAzure.Storage.Table.Backup; +using CoreHelpers.WindowsAzure.Storage.Table.Tests.Contracts; +using CoreHelpers.WindowsAzure.Storage.Table.Tests.Extensions; +using CoreHelpers.WindowsAzure.Storage.Table.Tests.Models; +using Xunit.DependencyInjection; + +namespace CoreHelpers.WindowsAzure.Storage.Table.Tests +{ + [Startup(typeof(Startup))] + [Collection("Sequential")] + public class ITS21VerifyBackup + { + private readonly IStorageContext _rootContext; + private readonly IBackupService _backupService; + private readonly ITestEnvironment _testEnvironment; + + public ITS21VerifyBackup(IStorageContext context, IBackupService backupService, ITestEnvironment testEnvironment) + { + _rootContext = context; + _backupService = backupService; + _testEnvironment = testEnvironment; + } + + [Fact] + public async Task CreateAndVerifyBackup() + { + var containerName = $"bck{Guid.NewGuid().ToString()}".Replace("_", ""); + var targetPath = $"CreateAndVerifyBackup/{Guid.NewGuid()}"; + + using (var scp = _rootContext.CreateChildContext()) + { + // set the tablename context + scp.SetTableContext(); + + // configure the entity mapper + scp.AddAttributeMapper(typeof(DemoEntityQuery), "BackupDemoEntityQuery"); + + // verify that we have no items + Assert.Empty((await scp.EnableAutoCreateTable().Query().Now())); + + // create items in two different partitions + var modelsP1 = new List() + { + new DemoEntityQuery() {P = "P1", R = "E1", StringField = "Demo01"}, + new DemoEntityQuery() {P = "P1", R = "E2", StringField = "Demo02"}, + }; + + await scp.EnableAutoCreateTable().MergeOrInsertAsync(modelsP1); + + using (var backupContext = await _backupService.OpenBackupContext(_testEnvironment.ConnectionString, containerName, targetPath, "Backup")) + { + await backupContext.Backup(scp, null, true); + } + + await scp.DropTableAsync(); + } + + using (var scp = _rootContext.CreateChildContext()) + { + // set the tablename context + scp.SetTableContext(); + + // configure the entity mapper + scp.AddAttributeMapper(typeof(DemoEntityQuery), "BackupDemoEntityQuery"); + + var itemsBeforeRestore= await scp.EnableAutoCreateTable().Query().Now(); + Assert.Empty(itemsBeforeRestore); + + // verify that we have no items + Assert.Empty((await scp.EnableAutoCreateTable().Query().Now())); + + // restore + using (var restoreContext = await _backupService.OpenRestorContext(_testEnvironment.ConnectionString, containerName, targetPath, "Backup")) + { + await restoreContext.Restore(scp, null); + } + + // verify if we have the values + var items = await scp.EnableAutoCreateTable().Query().Now(); + Assert.Equal(2, items.Count()); + } + } + } +} + diff --git a/CoreHelpers.WindowsAzure.Storage.Table.Tests/Startup.cs b/CoreHelpers.WindowsAzure.Storage.Table.Tests/Startup.cs index ea79e7e..a53bde9 100644 --- a/CoreHelpers.WindowsAzure.Storage.Table.Tests/Startup.cs +++ b/CoreHelpers.WindowsAzure.Storage.Table.Tests/Startup.cs @@ -1,6 +1,8 @@ -using CoreHelpers.WindowsAzure.Storage.Table.Tests.Contracts; +using CoreHelpers.WindowsAzure.Storage.Table.Backup; +using CoreHelpers.WindowsAzure.Storage.Table.Tests.Contracts; using CoreHelpers.WindowsAzure.Storage.Table.Tests.TestEnvironments; using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Logging; namespace CoreHelpers.WindowsAzure.Storage.Table.Tests { @@ -8,13 +10,23 @@ public class Startup { public void ConfigureServices(IServiceCollection services) { + services.AddLogging((lb) => + { + lb.AddDebug(); + }); + services.AddTransient(); services.AddScoped((svp) => { var env = svp.GetService(); + if (env == null) + throw new NullReferenceException(); + return new StorageContext(env.ConnectionString); }); + + services.AddTransient(); } } } diff --git a/CoreHelpers.WindowsAzure.Storage.Table/StorageContextTableManagement.cs b/CoreHelpers.WindowsAzure.Storage.Table/StorageContextTableManagement.cs index e52f51d..123dcfc 100644 --- a/CoreHelpers.WindowsAzure.Storage.Table/StorageContextTableManagement.cs +++ b/CoreHelpers.WindowsAzure.Storage.Table/StorageContextTableManagement.cs @@ -27,6 +27,9 @@ public void SetTableNamePrefix(string tableNamePrefix) _tableNamePrefix = tableNamePrefix; } + public string GetTableNamePrefix() + => _tableNamePrefix; + public void OverrideTableName(string table) where T : class, new() { OverrideTableName(typeof(T), table);