Started extracting backup capabilities in separate NUGET

CoreHelpers · Sep 4, 2022 · 528ac0c · 528ac0c
1 parent d7c68fb
commit 528ac0c
Show file tree

Hide file tree

Showing 40 changed files with 627 additions and 1,295 deletions.
diff --git a/CoreHelpers.WindowsAzure.Storage.Table.Abstractions/IStorageContext.cs b/CoreHelpers.WindowsAzure.Storage.Table.Abstractions/IStorageContext.cs
@@ -1,11 +1,19 @@
 using System;
 using System.Collections.Generic;
+using System.IO;
 using System.Linq;
 using System.Threading.Tasks;
 using CoreHelpers.WindowsAzure.Storage.Table.Abstractions;
 
 namespace CoreHelpers.WindowsAzure.Storage.Table
 {
+    public enum ImportExportOperation
+    {
+        processingItem,
+        processingPage,
+        processedPage
+    }
+
     public interface IStorageContext : IDisposable
     {
         void AddAttributeMapper();
@@ -36,7 +44,7 @@ Task<IEnumerable<T>> QueryAsync<T>(string partitionKey, IEnumerable<QueryFilter>
             where T : class, new();
 
         Task<IEnumerable<T>> QueryAsync<T>(int maxItems = 0) where T : class, new();
-        
+
         Task DeleteAsync<T>(T model) where T : class, new();
 
         Task DeleteAsync<T>(IEnumerable<T> models, bool allowMultiPartionRemoval = false) where T : class, new();
@@ -53,10 +61,14 @@ Task<IEnumerable<T>> QueryAsync<T>(string partitionKey, IEnumerable<QueryFilter>
 
         void CreateTable<T>(bool ignoreErrorIfExists = true);
 
-        Task <bool> ExistsTableAsync<T>();
+        Task<bool> ExistsTableAsync<T>();
 
         Task DropTableAsync<T>(bool ignoreErrorIfNotExists = true);
 
-        void DropTable<T>(bool ignoreErrorIfNotExists = true);        
+        void DropTable<T>(bool ignoreErrorIfNotExists = true);
+
+        Task<List<string>> QueryTableList();
+
+        Task ExportToJsonAsync(string tableName, TextWriter writer, Action<ImportExportOperation> onOperation);
     }
 }
diff --git a/CoreHelpers.WindowsAzure.Storage.Table.Abstractions/IStorageContextDelegate.cs b/CoreHelpers.WindowsAzure.Storage.Table.Abstractions/IStorageContextDelegate.cs
@@ -3,7 +3,16 @@
 
 namespace CoreHelpers.WindowsAzure.Storage.Table
 {
-	public interface IStorageContextDelegate
+    public enum nStoreOperation
+    {
+        insertOperation,
+        insertOrReplaceOperation,
+        mergeOperation,
+        mergeOrInserOperation,
+        delete
+    }
+
+    public interface IStorageContextDelegate
 	{
 		void OnQuerying(Type modelType, string filter, int maxItems, bool isContinuationQuery);
 

diff --git a/CoreHelpers.WindowsAzure.Storage.Table.Abstractions/StoreOperations.cs b/CoreHelpers.WindowsAzure.Storage.Table.Abstractions/StoreOperations.cs
diff --git a/...ble.Backup.Abstractions/CoreHelpers.WindowsAzure.Storage.Table.Backup.Abstractions.csproj b/...ble.Backup.Abstractions/CoreHelpers.WindowsAzure.Storage.Table.Backup.Abstractions.csproj
@@ -0,0 +1,20 @@
+<Project Sdk="Microsoft.NET.Sdk">
+
+  <PropertyGroup>
+    <TargetFrameworks>netstandard2.0;netstandard2.1;net48</TargetFrameworks>
+    <GenerateAssemblyInfo>false</GenerateAssemblyInfo>
+    <LangVersion>6</LangVersion>
+    <PackageLicenseFile>LICENSE</PackageLicenseFile>
+    <PackageProjectUrl>https://github.com/CoreHelpers/AzureStorageTable</PackageProjectUrl>   
+    <RepositoryType>git</RepositoryType>
+    <RepositoryUrl>https://github.com/CoreHelpers/AzureStorageTable.git</RepositoryUrl>    
+    <Description>This projects implements an abstraction for Azure Storage Tables to use POCOs because deriving every entity from ITableEntity or TableEntity looks like a step backwards. The current implementation is intended to be an abstraction to store every existing entity into Azure Table Store.</Description>
+    <PackageTags>poco dotnet-core dotnet azure azure-storage azure-table-storage</PackageTags>
+    <Copyright>(c) Dirk Eisenberg</Copyright>    
+  </PropertyGroup>
+
+
+  <ItemGroup>
+    <ProjectReference Include="..\CoreHelpers.WindowsAzure.Storage.Table.Abstractions\CoreHelpers.WindowsAzure.Storage.Table.Abstractions.csproj" />
+  </ItemGroup>
+</Project>
diff --git a/CoreHelpers.WindowsAzure.Storage.Table.Backup.Abstractions/IBackupContext.cs b/CoreHelpers.WindowsAzure.Storage.Table.Backup.Abstractions/IBackupContext.cs
@@ -0,0 +1,11 @@
+using System;
+using System.Threading.Tasks;
+
+namespace CoreHelpers.WindowsAzure.Storage.Table.Backup.Abstractions
+{
+    public interface IBackupContext : IDisposable
+    {
+        Task Backup(IStorageContext storageContext, string[] excludedTables = null, bool compress = true);
+    }
+}
+
diff --git a/CoreHelpers.WindowsAzure.Storage.Table.Backup.Abstractions/IBackupService.cs b/CoreHelpers.WindowsAzure.Storage.Table.Backup.Abstractions/IBackupService.cs
@@ -0,0 +1,14 @@
+using System;
+using System.Threading.Tasks;
+using CoreHelpers.WindowsAzure.Storage.Table.Backup.Abstractions;
+
+namespace CoreHelpers.WindowsAzure.Storage.Table.Backup
+{
+    public interface IBackupService
+    {
+        Task<IBackupContext> OpenBackupContext(string targetBlobStorageConnectionString, string targetContainerName, string targetPath, string tableNamePrefix = null);
+
+        Task<IRestoreContext> OpenRestorContext(string sourceBlobStorageConnectionString, string sourceContainerName, string sourcePath, string tableNamePrefix = null);
+    }
+}
+
diff --git a/CoreHelpers.WindowsAzure.Storage.Table.Backup.Abstractions/IRestoreContext.cs b/CoreHelpers.WindowsAzure.Storage.Table.Backup.Abstractions/IRestoreContext.cs
@@ -0,0 +1,11 @@
+using System;
+using System.Threading.Tasks;
+
+namespace CoreHelpers.WindowsAzure.Storage.Table.Backup.Abstractions
+{
+    public interface IRestoreContext : IDisposable
+    {
+        // Task BackupTable(IStorageContext storageContext, string tableName, bool compress = true);
+    }
+}
+
diff --git a/CoreHelpers.WindowsAzure.Storage.Table.Backup/BackupContext.cs b/CoreHelpers.WindowsAzure.Storage.Table.Backup/BackupContext.cs
@@ -0,0 +1,154 @@
+using System;
+using System.Collections.Generic;
+using System.Diagnostics;
+using System.IO;
+using System.Threading.Tasks;
+using Azure.Storage.Blobs;
+using CoreHelpers.WindowsAzure.Storage.Table.Backup.Abstractions;
+using Microsoft.Extensions.Logging;
+
+namespace CoreHelpers.WindowsAzure.Storage.Table.Backup
+{
+    public class BackupContext : IBackupContext
+    {
+        private ILogger<BackupContext> _logger;
+
+        private string _targetConnectionString;
+        private string _targetContainer;
+        private string _targetPath;
+        private string _targetTableNamePrefix;
+        private BlobServiceClient _blobServiceClient;
+
+        public BackupContext(ILogger<BackupContext> logger, string connectionString, string container, string path, string tableNamePrefix)
+        {
+            _logger = logger;
+            _targetConnectionString = connectionString;
+            _targetContainer = container;
+            _targetPath = path;
+            _targetTableNamePrefix = tableNamePrefix;
+
+            _blobServiceClient = new BlobServiceClient(_targetConnectionString);
+        }
+
+        public async Task Backup(IStorageContext storageContext, string[] excludedTables = null, bool compress = true)
+        {
+            using (_logger.BeginScope("Starting backup procedure..."))
+            {
+
+                // generate the excludeTables
+                var excludedTablesList = new List<string>();
+                if (excludedTables != null)
+                {
+                    foreach (var tbl in excludedTables)
+                        excludedTablesList.Add(tbl.ToLower());
+                }
+
+                // get all tables 
+                var tables = await storageContext.QueryTableList();
+                _logger.LogInformation($"Processing {tables.Count} tables");
+
+                // prepare the backup container
+                _logger.LogInformation($"Creating target container {_targetContainer} if needed");
+                var blobContainerClient = _blobServiceClient.GetBlobContainerClient(_targetContainer);
+                if (!await blobContainerClient.ExistsAsync())
+                    await blobContainerClient.CreateIfNotExistsAsync();
+
+                // prepare the memory stats file
+                var memoryStatsFile = $"{Path.GetTempFileName()}.csv";
+                using (var statsFile = new StreamWriter(memoryStatsFile))
+                {
+                    // use the statfile
+                    _logger.LogInformation($"Statsfile is under {memoryStatsFile}...");
+                    statsFile.WriteLine($"TableName,PageCounter,ItemCount,MemoryFootprint");
+
+                    // visit every table
+                    foreach (var tableName in tables)
+                    {
+
+                        // filter the table prefix
+                        if (!String.IsNullOrEmpty(_targetTableNamePrefix) && !tableName.StartsWith(_targetTableNamePrefix, StringComparison.CurrentCulture))
+                        {
+                            _logger.LogInformation($"Ignoring table {tableName}...");
+                            continue;
+                        }
+
+                        // check the  excluded tables
+                        if (excludedTablesList.Contains(tableName.ToLower()))
+                        {
+                            _logger.LogInformation($"Ignoring table {tableName} (is part of excluded tables)...");
+                            continue;
+                        }
+
+                        using (_logger.BeginScope($"Processing backup for table {tableName}..."))
+                        {                            
+                            // do the backup
+                            var fileName = $"{tableName}.json";
+                            if (!string.IsNullOrEmpty(_targetPath)) { fileName = $"{_targetPath}/{fileName}"; }
+                            if (compress) { fileName += ".gz"; }
+
+                            // open block blog reference
+                            var blobClient = blobContainerClient.GetBlobClient(fileName);
+
+                            // open the file stream 
+                            if (compress)
+                                _logger.LogInformation($"Writing backup to compressed file");
+                            else
+                                _logger.LogInformation($"Writing backup to non compressed file");
+
+                            // do it                    
+                            using (var backupFileStream = await blobClient.OpenWriteAsync(false))
+                            {
+                                using (var contentWriter = new ZippedStreamWriter(backupFileStream, compress))
+                                {
+                                    var pageCounter = 0;
+                                    var itemCounter = 0;
+
+                                    var pageLogScope = default(IDisposable);
+
+                                    await storageContext.ExportToJsonAsync(tableName, contentWriter, (ImportExportOperation operation) =>
+                                    {
+                                        switch (operation)
+                                        {
+                                            case ImportExportOperation.processingPage:
+                                                {
+                                                    pageCounter++;
+                                                    pageLogScope = _logger.BeginScope($"Processing page #{pageCounter}");
+                                                    break;
+                                                }
+                                            case ImportExportOperation.processingItem:
+                                                {
+                                                    itemCounter++;
+                                                    break;
+                                                }
+                                            case ImportExportOperation.processedPage:
+                                                {
+                                                    _logger.LogInformation($"#{itemCounter} processed!");
+                                                    statsFile.WriteLine($"{tableName},{pageCounter},{itemCounter},{Process.GetCurrentProcess().WorkingSet64}");
+                                                    pageLogScope.Dispose();
+                                                    pageLogScope = null;
+                                                    break;
+                                                }                                                
+                                        }                                     
+                                    });
+                                }
+                            }
+
+                            // ensure we clean up the memory beause sometimes 
+                            // we have to much referenced data
+                            GC.Collect();
+
+                            // flush the statfile 
+                            await statsFile.FlushAsync();
+                        }
+                    }
+                }
+            }
+        }
+
+        public void Dispose()
+        {
+
+        }
+    }
+}
+
diff --git a/CoreHelpers.WindowsAzure.Storage.Table.Backup/BackupService.cs b/CoreHelpers.WindowsAzure.Storage.Table.Backup/BackupService.cs
@@ -0,0 +1,33 @@
+using System;
+using System.Threading.Tasks;
+using CoreHelpers.WindowsAzure.Storage.Table.Backup.Abstractions;
+using Microsoft.Extensions.Logging;
+
+namespace CoreHelpers.WindowsAzure.Storage.Table.Backup
+{
+    public class BackupService : IBackupService
+    {        
+        private ILoggerFactory _loggerFactory;        
+
+        public BackupService(ILoggerFactory loggerFactory)
+        {
+            _loggerFactory = loggerFactory;            
+        }
+
+        public async Task<IBackupContext> OpenBackupContext(string targetBlobStorageConnectionString, string targetContainerName, string targetPath, string tableNamePrefix = null)
+        {
+            await Task.CompletedTask;
+            return new BackupContext(
+                _loggerFactory.CreateLogger<BackupContext>(),                
+                targetBlobStorageConnectionString, targetContainerName, targetPath,
+                tableNamePrefix);
+        }
+
+        public async Task<IRestoreContext> OpenRestorContext(string sourceBlobStorageConnectionString, string sourceContainerName, string sourcePath, string tableNamePrefix = null)
+        {
+            await Task.CompletedTask;
+            return new RestoreContext();
+        }
+    }
+}
+
diff --git a/...rs.WindowsAzure.Storage.Table.Backup/CoreHelpers.WindowsAzure.Storage.Table.Backup.csproj b/...rs.WindowsAzure.Storage.Table.Backup/CoreHelpers.WindowsAzure.Storage.Table.Backup.csproj
@@ -0,0 +1,28 @@
+<Project Sdk="Microsoft.NET.Sdk">
+
+  <PropertyGroup>
+    <TargetFrameworks>netstandard2.0;netstandard2.1;net48</TargetFrameworks>
+    <GenerateAssemblyInfo>false</GenerateAssemblyInfo>
+    <LangVersion>6</LangVersion>
+    <PackageLicenseFile>LICENSE</PackageLicenseFile>
+    <PackageProjectUrl>https://github.com/CoreHelpers/AzureStorageTable</PackageProjectUrl>   
+    <RepositoryType>git</RepositoryType>
+    <RepositoryUrl>https://github.com/CoreHelpers/AzureStorageTable.git</RepositoryUrl>    
+    <Description>This projects implements an abstraction for Azure Storage Tables to use POCOs because deriving every entity from ITableEntity or TableEntity looks like a step backwards. The current implementation is intended to be an abstraction to store every existing entity into Azure Table Store.</Description>
+    <PackageTags>poco dotnet-core dotnet azure azure-storage azure-table-storage</PackageTags>
+    <Copyright>(c) Dirk Eisenberg</Copyright>    
+  </PropertyGroup>
+
+
+  <ItemGroup>
+    <ProjectReference Include="..\CoreHelpers.WindowsAzure.Storage.Table.Backup.Abstractions\CoreHelpers.WindowsAzure.Storage.Table.Backup.Abstractions.csproj" />
+  </ItemGroup>
+  <ItemGroup>
+    <None Remove="Microsoft.Extensions.Logging.Abstractions" />
+    <None Remove="Azure.Storage.Blobs" />
+  </ItemGroup>
+  <ItemGroup>
+    <PackageReference Include="Microsoft.Extensions.Logging.Abstractions" Version="6.0.1" />
+    <PackageReference Include="Azure.Storage.Blobs" Version="12.13.1" />
+  </ItemGroup>
+</Project>