Merge remote-tracking branch 'refs/remotes/origin/dev'

inikep · Jan 5, 2016 · 55d665b · 55d665b
2 parents 9e94145 + 0dbb715
commit 55d665b
Show file tree

Hide file tree

Showing 8 changed files with 669 additions and 75 deletions.
diff --git a/NEWS b/NEWS
@@ -1,3 +1,8 @@
+v1.3.3
+- added: new levels from 11 to 16 (maximum compression ratio)
+- added: a new parser: LZ5HC_optimal_price
+- fixed: buffer-overflow during decompression (thanks to m^2)
+
 r132
 - improved compression ratio
 - added: new parsers: LZ5HC_fast, LZ5HC_price_fast, LZ5HC_lowest_price

diff --git a/README.md b/README.md
@@ -2,18 +2,35 @@ Introduction
 -------------------------
 
 LZ5 is a modification of [LZ4] which gives a better ratio at cost of slower compression and decompression speed. 
-This is caused mainly because of 22-bit dictionary instead of 16-bit in LZ4.
+The improvement in compression ratio is caused mainly because of:
+- 22-bit dictionary instead of 16-bit in LZ4
+- using 4 new parsers (including an optimal parser) optimized for a bigger dictionary
+- support for 3-byte long matches (MINMATCH = 3)
+- a special 1-byte codeword for the last occured offset
+
+**In my experiments there is no open-source bytewise compressor that gives better ratio than lz5hc.**
+
+[LZ4]: https://github.com/Cyan4973/lz4
+
+
+The codewords description
+-------------------------
 
 LZ5 uses different output codewords and is not compatible with LZ4. LZ4 output codewords are 3 byte long (24-bit) and look as follows:
 - LLLL_MMMM OOOOOOOO OOOOOOOO - 16-bit offset, 4-bit match length, 4-bit literal length 
 
 LZ5 uses 4 types of codewords from 1 to 4 bytes long:
-- 1_OO_LL_MMM OOOOOOOO - 10-bit offset, 3-bit match length, 2-bit literal length
-- 00_LLL_MMM OOOOOOOO OOOOOOOO - 16-bit offset, 3-bit match length, 3-bit literal length
-- 010_LL_MMM OOOOOOOO OOOOOOOO OOOOOOOO - 24-bit offset, 3-bit match length, 2-bit literal length 
-- 011_LL_MMM - last offset, 3-bit match length, 2-bit literal length
+- [1_OO_LL_MMM] [OOOOOOOO] - 10-bit offset, 3-bit match length, 2-bit literal length
+- [00_LLL_MMM] [OOOOOOOO] [OOOOOOOO] - 16-bit offset, 3-bit match length, 3-bit literal length
+- [010_LL_MMM] [OOOOOOOO] [OOOOOOOO] [OOOOOOOO] - 24-bit offset, 3-bit match length, 2-bit literal length
+- [011_LL_MMM] - last offset, 3-bit match length, 2-bit literal length
+
+1, 00, 010, 011 can be seen as Huffman codes and are selected according to frequences of given codewords for my test files. 
+Match lengths have always 3-bits (MMM) and literal lengths are usually 2-bits (LL) because it gives better ratio than any other division of 5-bits remaining bits. 
+So we can encode values 0-7 (3-bits) for matches (what means length of 3-10 for MINMATCH=3). But 7 is reserved as a flag signaling that a match is equal or longer
+that 10 bytes. So e.g. 30 is encoded as a flag 7 (match length=10) and a next byte 30-10=20. I tried many different variants (e.g. separate match lenghts and literal lenghts)
+but these codewords were the best. 
 
-[LZ4]: https://github.com/Cyan4973/lz4
 
 Benchmarks
 -------------------------
@@ -33,28 +50,30 @@ With the compresion ratio is opposite: LZ5 is better than LZ4 but worse than zst
 | lz4hc r131 -11              |    20 MB/s |  1969 MB/s |    54751363 | 52.21 |
 | lz4hc r131 -13              |    17 MB/s |  1969 MB/s |    54744790 | 52.21 |
 | lz4hc r131 -15              |    14 MB/s |  2007 MB/s |    54741827 | 52.21 |
-| lz5 r132                    |   180 MB/s |   877 MB/s |    56183327 | 53.58 |
-| lz5hc r132 level 1          |   453 MB/s |  1649 MB/s |    68770655 | 65.58 |
-| lz5hc r132 level 2          |   341 MB/s |  1533 MB/s |    65201626 | 62.18 |
-| lz5hc r132 level 3          |   222 MB/s |  1267 MB/s |    61423270 | 58.58 |
-| lz5hc r132 level 4          |   122 MB/s |   892 MB/s |    55011906 | 52.46 |
-| lz5hc r132 level 5          |    92 MB/s |   784 MB/s |    52790905 | 50.35 |
-| lz5hc r132 level 6          |    40 MB/s |   872 MB/s |    52561673 | 50.13 |
-| lz5hc r132 level 7          |    30 MB/s |   825 MB/s |    50947061 | 48.59 |
-| lz5hc r132 level 8          |    21 MB/s |   771 MB/s |    50049555 | 47.73 |
-| lz5hc r132 level 9          |    16 MB/s |   702 MB/s |    48718531 | 46.46 |
-| lz5hc r132 level 10         |    12 MB/s |   670 MB/s |    48109030 | 45.88 |
-| lz5hc r132 level 11         |  6.60 MB/s |   592 MB/s |    47639520 | 45.43 |
-| lz5hc r132 level 12         |  3.22 MB/s |   670 MB/s |    47461368 | 45.26 |
-| zstd_HC v0.3.6 level 1      |   250 MB/s |   529 MB/s |    51230550 | 48.86 |
-| zstd_HC v0.3.6 level 2      |   186 MB/s |   498 MB/s |    49678572 | 47.38 |
-| zstd_HC v0.3.6 level 3      |    90 MB/s |   484 MB/s |    48838293 | 46.58 |
-| zstd_HC v0.3.6 level 5      |    61 MB/s |   467 MB/s |    46480999 | 44.33 |
-| zstd_HC v0.3.6 level 7      |    28 MB/s |   480 MB/s |    44803941 | 42.73 |
-| zstd_HC v0.3.6 level 9      |    15 MB/s |   497 MB/s |    43899996 | 41.87 |
-| zstd_HC v0.3.6 level 12     |    11 MB/s |   505 MB/s |    42402232 | 40.44 |
-| zstd_HC v0.3.6 level 16     |  2.29 MB/s |   499 MB/s |    42122327 | 40.17 |
-| zstd_HC v0.3.6 level 20     |  1.65 MB/s |   454 MB/s |    41884658 | 39.94 |
+| lz5 v1.3.3                  |   191 MB/s |   892 MB/s |    56183327 | 53.58 |
+| lz5hc v1.3.3 level 1        |   468 MB/s |  1682 MB/s |    68770655 | 65.58 |
+| lz5hc v1.3.3 level 2        |   337 MB/s |  1574 MB/s |    65201626 | 62.18 |
+| lz5hc v1.3.3 level 3        |   232 MB/s |  1330 MB/s |    61423270 | 58.58 |
+| lz5hc v1.3.3 level 4        |   129 MB/s |   894 MB/s |    55011906 | 52.46 |
+| lz5hc v1.3.3 level 5        |    99 MB/s |   840 MB/s |    52790905 | 50.35 |
+| lz5hc v1.3.3 level 6        |    41 MB/s |   894 MB/s |    52561673 | 50.13 |
+| lz5hc v1.3.3 level 7        |    35 MB/s |   875 MB/s |    50947061 | 48.59 |
+| lz5hc v1.3.3 level 8        |    23 MB/s |   812 MB/s |    50049555 | 47.73 |
+| lz5hc v1.3.3 level 9        |    17 MB/s |   727 MB/s |    48718531 | 46.46 |
+| lz5hc v1.3.3 level 10       |    13 MB/s |   728 MB/s |    48109030 | 45.88 |
+| lz5hc v1.3.3 level 11       |  9.18 MB/s |   719 MB/s |    47438817 | 45.24 |
+| lz5hc v1.3.3 level 12       |  7.96 MB/s |   752 MB/s |    47063261 | 44.88 |
+| lz5hc v1.3.3 level 13       |  5.43 MB/s |   762 MB/s |    46718698 | 44.55 |
+| lz5hc v1.3.3 level 14       |  4.34 MB/s |   756 MB/s |    46484969 | 44.33 |
+| lz5hc v1.3.3 level 15       |  1.96 MB/s |   760 MB/s |    46227364 | 44.09 |
+| lz5hc v1.3.3 level 16       |  0.81 MB/s |   681 MB/s |    46125742 | 43.99 |
+| zstd v0.4.1 level 1         |   249 MB/s |   537 MB/s |    51160301 | 48.79 |
+| zstd v0.4.1 level 2         |   183 MB/s |   505 MB/s |    49719335 | 47.42 |
+| zstd v0.4.1 level 5         |    72 MB/s |   461 MB/s |    46389082 | 44.24 |
+| zstd v0.4.1 level 9         |    17 MB/s |   474 MB/s |    43892280 | 41.86 |
+| zstd v0.4.1 level 13        |    10 MB/s |   487 MB/s |    42321163 | 40.36 |
+| zstd v0.4.1 level 17        |  1.97 MB/s |   476 MB/s |    42009876 | 40.06 |
+| zstd v0.4.1 level 20        |  1.70 MB/s |   459 MB/s |    41880158 | 39.94 |
 | brotli 2015-10-29 -1        |    86 MB/s |   208 MB/s |    47882059 | 45.66 |
 | brotli 2015-10-29 -3        |    60 MB/s |   214 MB/s |    47451223 | 45.25 |
 | brotli 2015-10-29 -5        |    17 MB/s |   217 MB/s |    43363897 | 41.36 |

diff --git a/lib/lz5.c b/lib/lz5.c
@@ -743,12 +743,12 @@ int LZ5_loadDict (LZ5_stream_t* LZ5_dict, const char* dictionary, int dictSize)
     if ((dict->initCheck) || (dict->currentOffset > 1 GB))  /* Uninitialized structure, or reuse overflow */
         LZ5_resetStream(LZ5_dict);
 
-    if (dictSize < (int)HASH_UNIT)
+ /*   if (dictSize < (int)HASH_UNIT)
     {
         dict->dictionary = NULL;
         dict->dictSize = 0;
         return 0;
-    }
+    }*/
 
     if ((dictEnd - p) > LZ5_DICT_SIZE) p = dictEnd - LZ5_DICT_SIZE;
     dict->currentOffset += LZ5_DICT_SIZE;
@@ -863,6 +863,8 @@ int LZ5_compress_forceExtDict (LZ5_stream_t* LZ5_dict, const char* source, char*
 int LZ5_saveDict (LZ5_stream_t* LZ5_dict, char* safeBuffer, int dictSize)
 {
     LZ5_stream_t_internal* dict = (LZ5_stream_t_internal*) LZ5_dict;
+    if (!dict->dictionary)
+        return 0;
     const BYTE* previousDictEnd = dict->dictionary + dict->dictSize;
 
     if ((U32)dictSize > LZ5_DICT_SIZE) dictSize = LZ5_DICT_SIZE;   /* useless to define a dictionary > LZ5_DICT_SIZE */
@@ -970,7 +972,7 @@ FORCE_INLINE int LZ5_decompress_generic(
 
         /* copy literals */
         cpy = op+length;
-        if (((endOnInput) && ((cpy>(partialDecoding?oexit:oend-MFLIMIT)) || (ip+length>iend-(0+1+LASTLITERALS))) )
+        if (((endOnInput) && ((cpy>(partialDecoding?oexit:oend-WILDCOPYLENGTH)) || (ip+length>iend-(0+1+LASTLITERALS))) )
             || ((!endOnInput) && (cpy>oend-WILDCOPYLENGTH)))
         {
             if (partialDecoding)
@@ -1092,7 +1094,7 @@ FORCE_INLINE int LZ5_decompress_generic(
         } else { MEM_copy8(op, match); match+=8; }
         op += 8;
 
-        if (unlikely(cpy>oend-12))
+        if (unlikely(cpy>oend-(16-MINMATCH)))
         {
             BYTE* const oCopyLimit = oend-(WILDCOPYLENGTH-1);
             if (cpy > oend-LASTLITERALS) goto _output_error;    /* Error : last LASTLITERALS bytes must be literals (uncompressed) */

diff --git a/lib/lz5.h b/lib/lz5.h
@@ -49,7 +49,7 @@ extern "C" {
 **************************************/
 #define LZ5_VERSION_MAJOR    1    /* for breaking interface changes  */
 #define LZ5_VERSION_MINOR    3    /* for new (non-breaking) interface capabilities */
-#define LZ5_VERSION_RELEASE  2    /* for tweaks, bug-fixes, or development */
+#define LZ5_VERSION_RELEASE  3    /* for tweaks, bug-fixes, or development */
 #define LZ5_VERSION_NUMBER (LZ5_VERSION_MAJOR *100*100 + LZ5_VERSION_MINOR *100 + LZ5_VERSION_RELEASE)
 int LZ5_versionNumber (void);
 

diff --git a/lib/lz5common.h b/lib/lz5common.h
@@ -182,7 +182,12 @@ static size_t LZ5HC_hashPtr(const void* p, U32 hBits, U32 mls)
 *  HC Local Macros
 **************************************/
 #define LZ5HC_DEBUG(fmt, args...) ; //printf(fmt, ##args)
+#define LZ5_LOG_PARSER(fmt, args...) ;//printf(fmt, ##args)
+#define LZ5_LOG_PRICE(fmt, args...) ;//printf(fmt, ##args)
+#define LZ5_LOG_ENCODE(fmt, args...) ; // if ((char*)ip-source > 5711448) printf(fmt, ##args)
+
 #define MAX(a,b) ((a)>(b))?(a):(b)
+#define LZ5_OPT_NUM   (1<<12)
 
 #define LZ5_SHORT_LITERALS          ((1<<RUN_BITS2)-1)
 #define LZ5_LITERALS                ((1<<RUN_BITS)-1)
@@ -191,16 +196,21 @@ static size_t LZ5HC_hashPtr(const void* p, U32 hBits, U32 mls)
 #define LZ5_LEN_COST(len)           (len<LZ5_LITERALS ? 0 : (len-LZ5_LITERALS < 255 ? 1 : (len-LZ5_LITERALS-255 < (1<<7) ? 2 : 3)))
 
 static size_t LZ5_LIT_COST(size_t len, size_t offset){ return (len)+(((offset > LZ5_MID_OFFSET_DISTANCE) || (offset<LZ5_SHORT_OFFSET_DISTANCE)) ? LZ5_SHORT_LITLEN_COST(len) : LZ5_LEN_COST(len)); }
-static size_t LZ5_MATCH_COST(size_t mlen, size_t offset) { return LZ5_LEN_COST(mlen) + ((offset == 0) ? 1 : (offset<LZ5_SHORT_OFFSET_DISTANCE ? 2 : (offset<(1 << 16) ? 3 : 4))); }
+static size_t LZ5_MATCH_COST(size_t mlen, size_t offset) { return LZ5_LEN_COST(mlen) + ((offset == 0) ? 1 : (offset<LZ5_SHORT_OFFSET_DISTANCE ? 2 : (offset<LZ5_MID_OFFSET_DISTANCE ? 3 : 4))); }
 
 #define LZ5_CODEWORD_COST(litlen,offset,mlen)   (LZ5_MATCH_COST(mlen,offset) + LZ5_LIT_COST(litlen,offset))
-#define LZ5_LIT_ONLY_COST(len)                  ((len)+(LZ5_LEN_COST(len)))
+#define LZ5_LIT_ONLY_COST(len)                  ((len)+(LZ5_LEN_COST(len))+1)
 
 #define LZ5_NORMAL_MATCH_COST(mlen,offset)  (LZ5_MATCH_COST(mlen,offset))
 #define LZ5_NORMAL_LIT_COST(len)            (len)
 
 
 
+FORCE_INLINE uint32_t LZ5HC_get_price(uint32_t litlen, uint32_t offset, uint32_t mlen)
+{
+	return LZ5_CODEWORD_COST(litlen, offset, mlen);
+}
+
 FORCE_INLINE int LZ5HC_better_price(uint32_t best_off, uint32_t best_common, uint32_t off, uint32_t common, uint32_t last_off)
 {
   return LZ5_NORMAL_MATCH_COST(common - MINMATCH, (off == last_off) ? 0 : off) < LZ5_NORMAL_MATCH_COST(best_common - MINMATCH, (best_off == last_off) ? 0 : best_off) + (LZ5_NORMAL_LIT_COST(common - best_common) );
@@ -226,7 +236,7 @@ FORCE_INLINE int LZ5HC_more_profitable(uint32_t best_off, uint32_t best_common,
 *  HC Types
 ***************************************/
 /** from faster to stronger */
-typedef enum { LZ5HC_fast, LZ5HC_price_fast, LZ5HC_lowest_price } LZ5HC_strategy;
+typedef enum { LZ5HC_fast, LZ5HC_price_fast, LZ5HC_lowest_price, LZ5HC_optimal_price } LZ5HC_strategy;
 
 typedef struct
 {
@@ -236,6 +246,8 @@ typedef struct
     U32 hashLog3;      /* dispatch table : larger == more memory, faster*/
     U32 searchNum;     /* nb of searches : larger == more compression, slower*/
     U32 searchLength;  /* size of matches : larger == faster decompression */
+    U32 sufficientLength;  /* used only by optimal parser: size of matches which is acceptable: larger == more compression, slower */
+    U32 fullSearch;    /* used only by optimal parser: perform full search of matches: 1 == more compression, slower */
     LZ5HC_strategy strategy;
 } LZ5HC_parameters;
 
@@ -258,44 +270,59 @@ struct LZ5HC_Data_s
     LZ5HC_parameters params;
 };
 
+typedef struct
+{
+	int off;
+	int len;
+	int back;
+} LZ5HC_match_t;
+
+typedef struct
+{
+	int price;
+	int off;
+	int mlen;
+	int litlen;
+   	int rep;
+} LZ5HC_optimal_t;
 
 /* *************************************
 *  HC Pre-defined compression levels
 ***************************************/
-#define LZ5HC_MAX_CLEVEL 13
+#define LZ5HC_MAX_CLEVEL 18
 
 static const int g_maxCompressionLevel = LZ5HC_MAX_CLEVEL;
 static const int LZ5HC_compressionLevel_default = 6;
 
 static const LZ5HC_parameters LZ5HC_defaultParameters[LZ5HC_MAX_CLEVEL+1] =
 {
-    /* W,  C,  H, H3,  S,  L, strat */
-    {  0,  0,  0,  0,  0,  0, LZ5HC_fast         },  // level 0 - never used
-    { 22, 22, 13,  0,  4,  6, LZ5HC_fast         },  // level 1
- //   { 22, 22, 14,  0,  4,  6, LZ5HC_fast         },  // level 2
-    { 22, 22, 13,  0,  2,  6, LZ5HC_fast         },  // level 3
- //   { 22, 22, 14,  0,  2,  6, LZ5HC_fast         },  // level 4
- //   { 22, 22, 13,  0,  2,  5, LZ5HC_fast         },  // level 5
- //   { 22, 22, 14,  0,  2,  5, LZ5HC_fast         },  // level 6
-    { 22, 22, 13,  0,  1,  5, LZ5HC_fast         },  // level 7
- //   { 22, 22, 14,  0,  1,  5, LZ5HC_fast         },  // level 8
- //   { 22, 22, 15,  0,  1,  5, LZ5HC_fast         },  // level 9
- //   { 22, 22, 17,  0,  1,  5, LZ5HC_fast         },  // level 10
- //   { 22, 22, 14, 13,  4,  6, LZ5HC_price_fast   },  // level 12
- //   { 22, 22, 14, 13,  2,  5, LZ5HC_price_fast   },  // level 13
-    { 22, 22, 14, 13,  1,  4, LZ5HC_price_fast   },  // level 14
-    { 22, 22, 17, 13,  1,  4, LZ5HC_price_fast   },  // level 15
-    { 22, 22, 15, 13,  1,  4, LZ5HC_lowest_price },  // level 16
-    { 22, 22, 17, 13,  1,  4, LZ5HC_lowest_price },  // level 17
-    { 22, 22, 19, 16,  1,  4, LZ5HC_lowest_price },  // level 18
-    { 22, 22, 23, 16,  3,  4, LZ5HC_lowest_price },  // level 19
-    { 22, 22, 23, 16,  8,  4, LZ5HC_lowest_price },  // level 20
-    { 22, 22, 23, 16, 32,  4, LZ5HC_lowest_price },  // level 21
-    { 22, 22, 23, 16, 128, 4, LZ5HC_lowest_price },  // level 22
-    { 22, 22, 23, 16, 1024, 4, LZ5HC_lowest_price },  // level 23
+    /* W,  C,  H, H3,  Snum, SL, SuffL, FS, Strategy */
+    {  0,  0,  0,  0,     0,  0,     0,  0, LZ5HC_fast         },  // level 0 - never used
+    { 22, 22, 13,  0,     4,  6,     0,  0, LZ5HC_fast         },  // level 1
+    { 22, 22, 13,  0,     2,  6,     0,  0, LZ5HC_fast         },  // level 2
+    { 22, 22, 13,  0,     1,  5,     0,  0, LZ5HC_fast         },  // level 3
+    { 22, 22, 14, 13,     1,  4,     0,  0, LZ5HC_price_fast   },  // level 4
+    { 22, 22, 17, 13,     1,  4,     0,  0, LZ5HC_price_fast   },  // level 5
+    { 22, 22, 15, 13,     1,  4,     0,  0, LZ5HC_lowest_price },  // level 6
+    { 22, 22, 17, 13,     1,  4,     0,  0, LZ5HC_lowest_price },  // level 7
+    { 22, 22, 19, 16,     1,  4,     0,  0, LZ5HC_lowest_price },  // level 8
+    { 22, 22, 23, 16,     3,  4,     0,  0, LZ5HC_lowest_price },  // level 9
+    { 22, 22, 23, 16,     8,  4,     0,  0, LZ5HC_lowest_price },  // level 10
+    { 22, 22, 23, 16,     8,  4,    12,  0, LZ5HC_optimal_price }, // level 11
+    { 22, 22, 23, 16,     8,  4,    64,  0, LZ5HC_optimal_price }, // level 12
+    { 22, 22, 23, 16,     8,  4,    64,  1, LZ5HC_optimal_price }, // level 13
+    { 22, 22, 23, 16,    32,  4,    64,  0, LZ5HC_optimal_price }, // level 14
+    { 22, 22, 23, 16,   128,  4,    64,  0, LZ5HC_optimal_price }, // level 15
+    { 22, 22, 23, 16,   512,  4,    64,  0, LZ5HC_optimal_price }, // level 16
+    { 22, 22, 23, 16,   512,  4,    64,  1, LZ5HC_optimal_price }, // level 17
+    { 22, 22, 28, 24, 1<<10,  4, 1<<10,  1, LZ5HC_optimal_price }, // level 18
+//  { 10, 10, 10,  0,     0,  4,     0,  0, LZ5HC_fast          }, // min values
+//  { 24, 24, 28, 24, 1<<24,  7, 1<<24,  1, LZ5HC_optimal_price }, // max values
 };
 
 
+
+
 #if defined (__cplusplus)
 }
 #endif