Fix initial size + encryption litedb-org#929

LiteDB.Demo/Program.cs

@@ -1,90 +1,130 @@
-using LiteDB;
-using System;
-using System.Collections.Generic;
+using System;
 using System.Diagnostics;
 using System.IO;
 using System.Linq;
-using System.Threading;
-using System.Threading.Tasks;
+using LiteDB;
 
-namespace LiteDB.Demo
+namespace litedb_test
 {
-    class Program
+    /// <summary>
+    /// Test record from desktop app
+    /// </summary>
+
+    public class UnreadNotificationRecord
     {
-        static void Main(string[] args)
+        public enum NotificationTypeEnum
         {
-            var timer = new Stopwatch();
-            ITest test = new LiteDB_Paging();
-            //ITest test = new SQLite_Paging();
+            Info,
+            Error
+        }
 
-            Console.WriteLine("Testing: {0}", test.GetType().Name);
+        [BsonId]
+        public int Id { get; set; }
 
-            test.Init();
+        public Guid UserId { get; set; }
 
-            Console.WriteLine("Populating 100.000 documents...");
+        public string Title { get; set; }
+        public string Message { get; set; }
+        public NotificationTypeEnum NotificationType { get; set; }
 
-            timer.Start();
-            test.Populate(ReadDocuments());
-            timer.Stop();
+        public DateTime When { get; set; }
+    }
 
-            Console.WriteLine("Done in {0}ms", timer.ElapsedMilliseconds);
 
-            timer.Restart();
-            var counter = test.Count();
-            timer.Stop();
-
-            Console.WriteLine("Result query counter: {0} ({1}ms)", counter, timer.ElapsedMilliseconds);
-
-            var input = "0";
-
-            while (input != "")
-            {
-                var skip = Convert.ToInt32(input);
-                var limit = 10;
-
-                timer.Restart();
-                var result = test.Fetch(skip, limit);
-                timer.Stop();
-
-                foreach(var doc in result)
-                {
-                    Console.WriteLine(
-                        doc["_id"].AsString.PadRight(6) + " - " +
-                        doc["name"].AsString.PadRight(30) + "  -> " +
-                        doc["age"].AsInt32);
-                }
-
-                Console.Write("\n({0}ms) => Enter skip index: ", timer.ElapsedMilliseconds);
-                input = Console.ReadLine();
-            }
-
-            Console.WriteLine("End");
-            Console.ReadKey();
-        }
 
-        static IEnumerable<BsonDocument> ReadDocuments()
+    /// <summary>
+    /// 
+    /// </summary>
+
+    internal class Program1
+    {
+
+        /// <summary>
+        /// Defines the entry point of the application.
+        /// </summary>
+        /// <param name="args">The args.</param>
+        [STAThread]
+        static void Main0(string[] args)
         {
-            using (var s = File.OpenRead(@"datagen.txt"))
+            /*
+             * Important!: 
+             * `connectionString`
+             * has to be path to **NON-SSD** drive
+             */
+
+            const string connectionString = "d://generated.litedb";
+
+            //Some GUID keys to share across all processes
+            Guid[] sharedGuids = {
+                Guid.Parse("B9321547-D4BE-461F-B7F9-2E2600839428"),
+                Guid.Parse("1F0689E8-121A-414D-80D1-2A54B516A6AC")
+            };
+
+            // main start point
+            if (args.Length == 0)
             {
-                var r = new StreamReader(s);
+                File.Delete(connectionString);
 
-                while(!r.EndOfStream)
+                const int processCount = 15;
+
+                Console.WriteLine($"Spawning {processCount} child processes");
+
+                for (int i = 0; i < processCount; i++)
+                    Process.Start(Process.GetCurrentProcess().MainModule.FileName, $"child_{i}");
+
+                return;
+            }
+
+            var procId = args[0];
+
+            Console.WriteLine($"Running as `{procId}`");
+
+            try
+            {
+                using (var database = new LiteDatabase(connectionString))
                 {
-                    var line = r.ReadLine();
 
-                    if (!string.IsNullOrEmpty(line))
+                    database.Shrink();
+
+                    var collection = database.GetCollection<UnreadNotificationRecord>();
+                    collection.EnsureIndex(x => x.UserId);
+
+                    for (int i = 0; i < 50; i++)
                     {
-                        var row = line.Split(',');
+                        var random = new Random();
 
-                        yield return new BsonDocument
+                        var record = new UnreadNotificationRecord
                         {
-                            ["_id"] = Convert.ToInt32(row[0]),
-                            ["name"] = row[1],
-                            ["age"] = Convert.ToInt32(row[2])
+                            UserId = sharedGuids[random.Next() % sharedGuids.Length],
                         };
+
+                        Console.WriteLine($"Item[{i}]: {procId}");
+
+                        //Every 2nd iteration run some query that actually has to yield some results
+
+                        if (i % 2 == 0)
+                            collection.
+                                Find(Query.EQ(nameof(UnreadNotificationRecord.UserId), sharedGuids[random.Next() % sharedGuids.Length])).
+                                ToArray();
+
+                        //Every iteration insert new record
+
+                        collection.Insert(record);
                     }
+
+                    Console.WriteLine($"{procId} process finished");
+
                 }
             }
+            catch (Exception ex)
+            {
+                Console.ForegroundColor = ConsoleColor.Red;
+                Console.WriteLine(ex.Message);
+                Console.WriteLine(ex.StackTrace);
+                Console.ReadKey();
+            }
         }
     }
+
+
 }

LiteDB.Demo/Program1.cs

@@ -0,0 +1,45 @@
+using System;
+using System.Diagnostics;
+using System.IO;
+using System.Linq;
+using LiteDB;
+
+namespace litedb_test
+{
+    internal class Program
+    {
+        public class EntityInt { public int Id { get; set; } public string Name { get; set; } }
+
+
+        /// <summary>
+        /// Defines the entry point of the application.
+        /// </summary>
+        /// <param name="args">The args.</param>
+        [STAThread]
+        static void Main(string[] args)
+        {
+            var r = new { name = "John", Age = 40 };
+
+            var d = BsonMapper.Global.ToDocument(r);
+
+            var j = JsonSerializer.Serialize(d);
+
+            File.Delete("d:\\Test.db");
+            var db = new LiteDatabase("d:\\Test.db");
+            var col = db.GetCollection<EntityInt>("col1");
+            for (int i = 0; i < 50; i++)
+                col.Upsert(new EntityInt { Name = i.ToString() });
+
+            for (int i = 0; i < 10; i++)
+                col.Delete(i);
+
+            db.Shrink();
+
+            for (int i = 0; i < 5; i++)
+                col.Upsert(new EntityInt { Name = i.ToString() }); //Cannot insert duplicate key in unique index '_id'. The duplicate value is '42'.
+
+        }
+    }
+
+
+}

LiteDB.Tests/Engine/Create_Database_Tests.cs

@@ -34,5 +34,32 @@ public void Create_Database_With_Initial_Size()
                 Assert.AreEqual(minimal, file.Size);
             }
         }
+
+        [TestMethod]
+        public void Create_Database_With_Initial_Size_Encrypted()
+        {
+            var initial = 40 * 1024; // initial size: 40kb
+            var minimal = 4096 * 5; // 1 header + 1 lock + 1 collection + 1 data + 1 index = 5 pages minimal
+
+            using (var file = new TempFile(checkIntegrity: false))
+            using (var db = new LiteDatabase(file.Conn("initial size=40kb; password=123")))
+            {
+                // just ensure open datafile
+                var uv = db.Engine.UserVersion;
+
+                // test if file has 40kb
+                Assert.AreEqual(initial, file.Size);
+
+                // simple insert to test if datafile still with 40kb
+                db.Engine.Run("db.col1.insert {a:1}"); // use 3 pages to this
+
+                Assert.AreEqual(initial, file.Size);
+
+                // ok, now shrink and test if file are minimal size
+                db.Shrink();
+
+                Assert.AreEqual(minimal, file.Size);
+            }
+        }
     }
 }

LiteDB.Tests/Utils/TempFile.cs

@@ -68,7 +68,10 @@ protected virtual void Dispose(bool disposing)
             }
 
             // check file integrity
-            this.CheckIntegrity();
+            if (_checkIntegrity)
+            {
+                this.CheckIntegrity();
+            }
 
             File.Delete(this.Filename);
 

LiteDB/Database/LiteDatabase.cs

@@ -209,7 +209,7 @@ public bool RenameCollection(string oldName, string newName)
         /// </summary>
         public long Shrink()
         {
-            return this.Shrink(_connectionString == null ? null : _connectionString.Password);
+            return this.Shrink(_connectionString?.Password);
         }
 
         /// <summary>

LiteDB/Engine/Engine/Shrink.cs

@@ -65,6 +65,8 @@ public long Shrink(string password = null, IDiskService tempDisk = null)
                 }
 
                 // create/destroy crypto class
+                if (_crypto != null) _crypto.Dispose();
+
                 _crypto = password == null ? null : new AesEncryption(password, header.Salt);
 
                 // initialize all services again (crypto can be changed)

LiteDB/Engine/LiteEngine.cs

@@ -261,6 +261,9 @@ public static void CreateDatabase(Stream stream, string password = null, long in
             // write second page as an empty AREA (it's not a page) just to use as lock control
             stream.Write(new byte[BasePage.PAGE_SIZE], 0, BasePage.PAGE_SIZE);
 
+            // create crypto class if has password
+            var crypto = password != null ? new AesEncryption(password, header.Salt) : null;
+
             // if initial size is defined, lets create empty pages in a linked list
             if (emptyPages > 0)
             {
@@ -276,9 +279,18 @@ public static void CreateDatabase(Stream stream, string password = null, long in
                         NextPageID = pageID == emptyPages + 1 ? uint.MaxValue : pageID + 1
                     };
 
-                    stream.Write(empty.WritePage(), 0, BasePage.PAGE_SIZE);
+                    var bytes = empty.WritePage();
+
+                    if (password != null)
+                    {
+                        bytes = crypto.Encrypt(bytes);
+                    }
+
+                    stream.Write(bytes, 0, BasePage.PAGE_SIZE);
                 }
             }
+
+            if (crypto != null) crypto.Dispose();
         }
     }
 }

LiteDB/Utils/AesEncryption.cs

@@ -17,13 +17,12 @@ public AesEncryption(string password, byte[] salt)
             _aes = Aes.Create();
             _aes.Padding = PaddingMode.Zeros;
 
+            var pdb = new Rfc2898DeriveBytes(password, salt);
+
+            using (pdb as IDisposable)
             {
-                var pdb = new Rfc2898DeriveBytes(password, salt);
-                using (pdb as IDisposable)
-                {
-                    _aes.Key = pdb.GetBytes(32);
-                    _aes.IV = pdb.GetBytes(16);
-                }
+                _aes.Key = pdb.GetBytes(32);
+                _aes.IV = pdb.GetBytes(16);
             }
         }