-
Notifications
You must be signed in to change notification settings - Fork 5
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Added dedicated backup restore module including verification
- Loading branch information
Showing
10 changed files
with
226 additions
and
78 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
156 changes: 86 additions & 70 deletions
156
CoreHelpers.WindowsAzure.Storage.Table.Backup/RestoreContext.cs
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -1,93 +1,109 @@ | ||
| using System; | ||
| using System.ComponentModel; | ||
| using System.IO; | ||
| using System.Threading.Tasks; | ||
| using Azure.Storage.Blobs; | ||
| using CoreHelpers.WindowsAzure.Storage.Table.Backup.Abstractions; | ||
| using Microsoft.Extensions.Logging; | ||
|
|
||
| namespace CoreHelpers.WindowsAzure.Storage.Table.Backup | ||
| { | ||
| public class RestoreContext : IRestoreContext | ||
| { | ||
| public RestoreContext() | ||
| private ILogger<RestoreContext> _logger; | ||
| private BlobServiceClient _blobServiceClient; | ||
|
|
||
| private string _sourceConnectionString; | ||
| private string _sourceContainer; | ||
| private string _sourcePath; | ||
| private string _sourceTableNamePrefix; | ||
|
|
||
| public RestoreContext(ILogger<RestoreContext> logger, string connectionString, string container, string path, string tableNamePrefix) | ||
| { | ||
| _logger = logger; | ||
| _blobServiceClient = new BlobServiceClient(connectionString); | ||
|
|
||
| _sourceConnectionString = connectionString; | ||
| _sourceContainer = container; | ||
| _sourcePath = path; | ||
| _sourceTableNamePrefix = tableNamePrefix; | ||
| } | ||
|
|
||
| public void Dispose() | ||
| { | ||
| throw new NotImplementedException(); | ||
| { | ||
| } | ||
| } | ||
| } | ||
|
|
||
| /* | ||
| * public async Task Restore(string containerName, string srcPath, string tablePrefix = null) { | ||
| // log | ||
| storageLogger.LogInformation($"Starting restore procedure..."); | ||
|
|
||
| // get all backup files | ||
| var blobClient = backupStorageAccount.CreateCloudBlobClient(); | ||
| var containerReference = blobClient.GetContainerReference(containerName.ToLower()); | ||
| // check if the container exists | ||
| if (!await containerReference.ExistsAsync()) { | ||
| storageLogger.LogInformation($"Missing container {containerName.ToLower()}"); | ||
| return; | ||
| } | ||
| // build the path including prefix | ||
| storageLogger.LogInformation($"Search Prefix is {srcPath}"); | ||
| // track the state | ||
| var continuationToken = default(BlobContinuationToken); | ||
| do | ||
| public async Task Restore(IStorageContext storageContext, string[] excludedTables = null) | ||
| { | ||
| using (_logger.BeginScope("Starting restore procedure...")) | ||
| { | ||
| // get all blobs | ||
| var blobResult = await containerReference.ListBlobsSegmentedAsync(srcPath, true, BlobListingDetails.All, 1000, continuationToken, null, null); | ||
| // process every backup file as table | ||
| foreach(var blob in blobResult.Results) { | ||
| // build the name | ||
| var blobName = blob.StorageUri.PrimaryUri.AbsolutePath; | ||
| blobName = blobName.Remove(0, containerName.Length + 2); | ||
| // get the tablename | ||
| var tableName = Path.GetFileNameWithoutExtension(blobName); | ||
| var compressed = blobName.EndsWith(".gz", StringComparison.CurrentCultureIgnoreCase); | ||
| if (compressed) | ||
| tableName = Path.GetFileNameWithoutExtension(tableName); | ||
| // add the prefix | ||
| if (!String.IsNullOrEmpty(tablePrefix)) | ||
| tableName = $"{tablePrefix}{tableName}"; | ||
| // get the container reference | ||
| var blobContainerClient = _blobServiceClient.GetBlobContainerClient(_sourceContainer); | ||
| if (!await blobContainerClient.ExistsAsync()) | ||
| throw new Exception("Container not found"); | ||
|
|
||
| // check if the container exists | ||
| if (!await blobContainerClient.ExistsAsync()) | ||
| { | ||
| _logger.LogInformation($"Missing container {_sourceContainer.ToLower()}"); | ||
| return; | ||
| } | ||
|
|
||
| // log | ||
| storageLogger.LogInformation($"Restoring {blobName} to table {tableName} (Compressed: {compressed})"); | ||
| // build the path including prefix | ||
| _logger.LogInformation($"Search Prefix is {_sourcePath}"); | ||
|
|
||
| // build the reference | ||
| var blockBlobReference = containerReference.GetBlockBlobReference(blobName); | ||
| // get the pages | ||
| var blobPages = blobContainerClient.GetBlobsAsync(Azure.Storage.Blobs.Models.BlobTraits.None, Azure.Storage.Blobs.Models.BlobStates.None, _sourcePath).AsPages(); | ||
|
|
||
| // open the read stream | ||
| using (var readStream = await blockBlobReference.OpenReadAsync()) | ||
| // visit every page | ||
| await foreach (var page in blobPages) | ||
| { | ||
| foreach(var blob in page.Values) | ||
| { | ||
| // unzip the stream | ||
| using (var contentReader = new ZippedStreamReader(readStream, compressed)) | ||
| // build the name | ||
| var blobName = blob.Name; | ||
|
|
||
| // get the tablename | ||
| var tableName = Path.GetFileNameWithoutExtension(blobName); | ||
| var compressed = blobName.EndsWith(".gz", StringComparison.CurrentCultureIgnoreCase); | ||
| if (compressed) | ||
| tableName = Path.GetFileNameWithoutExtension(tableName); | ||
|
|
||
| // add the prefix | ||
| if (!String.IsNullOrEmpty(_sourceTableNamePrefix)) | ||
| tableName = $"{_sourceTableNamePrefix}{tableName}"; | ||
|
|
||
| // log | ||
| _logger.LogInformation($"Restoring {blobName} to table {tableName} (Compressed: {compressed})"); | ||
|
|
||
| // open the read stream | ||
| var blobClient = blobContainerClient.GetBlobClient(blob.Name); | ||
| using (var readStream = await blobClient.OpenReadAsync()) | ||
| { | ||
| // import the stream | ||
| var pageCounter = 0; | ||
| await dataImportService.ImportFromJsonStreamAsync(tableName, contentReader, (c) => { | ||
| pageCounter++; | ||
| storageLogger.LogInformation($" Processing page #{pageCounter} with #{c} items..."); | ||
| }); | ||
| // unzip the stream | ||
| using (var contentReader = new ZippedStreamReader(readStream, compressed)) | ||
| { | ||
| // import the stream | ||
| var pageCounter = 0; | ||
| await storageContext.ImportFromJsonAsync(tableName, contentReader, (c) => | ||
| { | ||
| switch (c) | ||
| { | ||
| case ImportExportOperation.processingPage: | ||
| _logger.LogInformation($" Processing page #{pageCounter}..."); | ||
| break; | ||
| case ImportExportOperation.processedPage: | ||
| pageCounter++; | ||
| break; | ||
| } | ||
| }); | ||
| } | ||
| } | ||
| } | ||
| } | ||
| // proces the token | ||
| continuationToken = blobResult.ContinuationToken; | ||
| } while (continuationToken != null); | ||
| } | ||
| } | ||
|
|
||
| await Task.CompletedTask; | ||
| } | ||
| */ | ||
| } | ||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
86 changes: 86 additions & 0 deletions
86
CoreHelpers.WindowsAzure.Storage.Table.Tests/ITS21VerifyBackup.cs
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,86 @@ | ||
| using System; | ||
| using CoreHelpers.WindowsAzure.Storage.Table.Backup; | ||
| using CoreHelpers.WindowsAzure.Storage.Table.Tests.Contracts; | ||
| using CoreHelpers.WindowsAzure.Storage.Table.Tests.Extensions; | ||
| using CoreHelpers.WindowsAzure.Storage.Table.Tests.Models; | ||
| using Xunit.DependencyInjection; | ||
|
|
||
| namespace CoreHelpers.WindowsAzure.Storage.Table.Tests | ||
| { | ||
| [Startup(typeof(Startup))] | ||
| [Collection("Sequential")] | ||
| public class ITS21VerifyBackup | ||
| { | ||
| private readonly IStorageContext _rootContext; | ||
| private readonly IBackupService _backupService; | ||
| private readonly ITestEnvironment _testEnvironment; | ||
|
|
||
| public ITS21VerifyBackup(IStorageContext context, IBackupService backupService, ITestEnvironment testEnvironment) | ||
| { | ||
| _rootContext = context; | ||
| _backupService = backupService; | ||
| _testEnvironment = testEnvironment; | ||
| } | ||
|
|
||
| [Fact] | ||
| public async Task CreateAndVerifyBackup() | ||
| { | ||
| var containerName = $"bck{Guid.NewGuid().ToString()}".Replace("_", ""); | ||
| var targetPath = $"CreateAndVerifyBackup/{Guid.NewGuid()}"; | ||
|
|
||
| using (var scp = _rootContext.CreateChildContext()) | ||
| { | ||
| // set the tablename context | ||
| scp.SetTableContext(); | ||
|
|
||
| // configure the entity mapper | ||
| scp.AddAttributeMapper(typeof(DemoEntityQuery), "BackupDemoEntityQuery"); | ||
|
|
||
| // verify that we have no items | ||
| Assert.Empty((await scp.EnableAutoCreateTable().Query<DemoEntityQuery>().Now())); | ||
|
|
||
| // create items in two different partitions | ||
| var modelsP1 = new List<DemoEntityQuery>() | ||
| { | ||
| new DemoEntityQuery() {P = "P1", R = "E1", StringField = "Demo01"}, | ||
| new DemoEntityQuery() {P = "P1", R = "E2", StringField = "Demo02"}, | ||
| }; | ||
|
|
||
| await scp.EnableAutoCreateTable().MergeOrInsertAsync<DemoEntityQuery>(modelsP1); | ||
|
|
||
| using (var backupContext = await _backupService.OpenBackupContext(_testEnvironment.ConnectionString, containerName, targetPath, "Backup")) | ||
| { | ||
| await backupContext.Backup(scp, null, true); | ||
| } | ||
|
|
||
| await scp.DropTableAsync<DemoEntityQuery>(); | ||
| } | ||
|
|
||
| using (var scp = _rootContext.CreateChildContext()) | ||
| { | ||
| // set the tablename context | ||
| scp.SetTableContext(); | ||
|
|
||
| // configure the entity mapper | ||
| scp.AddAttributeMapper(typeof(DemoEntityQuery), "BackupDemoEntityQuery"); | ||
|
|
||
| var itemsBeforeRestore= await scp.EnableAutoCreateTable().Query<DemoEntityQuery>().Now(); | ||
| Assert.Empty(itemsBeforeRestore); | ||
|
|
||
| // verify that we have no items | ||
| Assert.Empty((await scp.EnableAutoCreateTable().Query<DemoEntityQuery>().Now())); | ||
|
|
||
| // restore | ||
| using (var restoreContext = await _backupService.OpenRestorContext(_testEnvironment.ConnectionString, containerName, targetPath, "Backup")) | ||
| { | ||
| await restoreContext.Restore(scp, null); | ||
| } | ||
|
|
||
| // verify if we have the values | ||
| var items = await scp.EnableAutoCreateTable().Query<DemoEntityQuery>().Now(); | ||
| Assert.Equal(2, items.Count()); | ||
| } | ||
| } | ||
| } | ||
| } | ||
|
|
Oops, something went wrong.