3-Using-Compression.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {
    "deletable": true,
    "editable": true
   },
   "source": [
    "# Using compression"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "deletable": true,
    "editable": true
   },
   "source": [
    "> Objectives:\n",
    "> * How to compress chunked datasets\n",
    "> * Learn how to fine-tune the HDF5 compression pipeline to suit your needs\n",
    "> * How to use pandas for reading CSV files"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "deletable": true,
    "editable": true
   },
   "source": [
    "## Load movielens datasets"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": true,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "import os\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "import tables"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "# Import CSV files via pandas\n",
    "dset = 'movielens-1m'\n",
    "fdata = os.path.join(dset, 'ratings.dat.gz')\n",
    "fitem = os.path.join(dset, 'movies.dat.gz')\n",
    "\n",
    "# pass in column names for each CSV\n",
    "r_cols = ['user_id', 'movie_id', 'rating', 'unix_timestamp']\n",
    "ratings = pd.read_csv(fdata, sep=';', names=r_cols)\n",
    "\n",
    "m_cols = ['movie_id', 'title', 'genres']\n",
    "movies = pd.read_csv(fitem, sep=';', names=m_cols,\n",
    "                     dtype={'title': object, 'genres': object})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "movie_id     int64:dense\n",
       "title       object:dense\n",
       "genres      object:dense\n",
       "dtype: object"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "movies.ftypes"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "user_id           int64:dense\n",
       "movie_id          int64:dense\n",
       "rating            int64:dense\n",
       "unix_timestamp    int64:dense\n",
       "dtype: object"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "ratings.ftypes"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "deletable": true,
    "editable": true
   },
   "source": [
    "## Storing in HDF5/PyTables in compressed form"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "collapsed": true,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "import os\n",
    "import shutil\n",
    "data_dir = \"compression\"\n",
    "if os.path.exists(data_dir):\n",
    "    shutil.rmtree(data_dir)\n",
    "os.mkdir(data_dir)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "collapsed": true,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "def to_hdf5(ratings, movies, filters):\n",
    "    \n",
    "    class Ratings(tables.IsDescription):\n",
    "        user_id = tables.Int32Col(pos=0)\n",
    "        movie_id = tables.Int32Col(pos=1)\n",
    "        rating = tables.Int8Col(pos=2)\n",
    "        unix_timestamp = tables.Int64Col(pos=3)\n",
    "    \n",
    "    class Movies(tables.IsDescription):\n",
    "        movie_id = tables.Int32Col(pos=0)\n",
    "        title = tables.StringCol(100, pos=1)\n",
    "        genres = tables.StringCol(50, pos=2)\n",
    "    \n",
    "    def get_filename(filters):\n",
    "        if filters.complevel != 0:\n",
    "            complib = filters.complib if \":\" not in filters.complib else filters.complib.replace(\":\", \"-\")\n",
    "            shuffle = \"shuffle\" if filters.shuffle else \"noshuffle\"\n",
    "            filename = \"%s/%s-%d-%s.h5\" % (data_dir, complib, filters.complevel, shuffle)\n",
    "        else:\n",
    "            filename = \"%s/no-compressed.h5\" % (data_dir,)\n",
    "        return filename\n",
    "\n",
    "    filename = get_filename(filters)\n",
    "    print(\"Creating file:\", filename)\n",
    "    with tables.open_file(filename, \"w\") as f:\n",
    "        table_ratings = f.create_table(f.root, \"ratings\", Ratings, filters=filters, expectedrows=len(ratings))\n",
    "        table_ratings.append([ratings[col].values for col in ratings.ftypes.keys()])\n",
    "        table_movies = f.create_table(f.root, \"movies\", Movies, filters=filters, expectedrows=len(movies))\n",
    "        table_movies.append([movies[col].values for col in movies.ftypes.keys()])\n",
    "    return filename"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Creating file: compression/zlib-5-shuffle.h5\n",
      "CPU times: user 389 ms, sys: 26.7 ms, total: 416 ms\n",
      "Wall time: 455 ms\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "filters = tables.Filters(complevel=5)\n",
    "h5file = to_hdf5(ratings, movies, filters)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "/ (RootGroup) ''\r\n",
      "/movies (Table(3883,), shuffle, zlib(5)) ''\r\n",
      "  description := {\r\n",
      "  \"movie_id\": Int32Col(shape=(), dflt=0, pos=0),\r\n",
      "  \"title\": StringCol(itemsize=100, shape=(), dflt=b'', pos=1),\r\n",
      "  \"genres\": StringCol(itemsize=50, shape=(), dflt=b'', pos=2)}\r\n",
      "  byteorder := 'little'\r\n",
      "  chunkshape := (425,)\r\n",
      "/ratings (Table(1000209,), shuffle, zlib(5)) ''\r\n",
      "  description := {\r\n",
      "  \"user_id\": Int32Col(shape=(), dflt=0, pos=0),\r\n",
      "  \"movie_id\": Int32Col(shape=(), dflt=0, pos=1),\r\n",
      "  \"rating\": Int8Col(shape=(), dflt=0, pos=2),\r\n",
      "  \"unix_timestamp\": Int64Col(shape=(), dflt=0, pos=3)}\r\n",
      "  byteorder := 'little'\r\n",
      "  chunkshape := (7710,)\r\n"
     ]
    }
   ],
   "source": [
    "!ptdump -v {h5file}"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "deletable": true,
    "editable": true
   },
   "source": [
    "### Exercise 1\n",
    "\n",
    "PyTables comes with out-of-box support for a series of codecs.  Do a quick comparison between \"zlib\", \"bzip2\", and \"blosc\" for compression levels of 1 (fastest), 5 and 9 (slowest).  Which one compresses best?  Which one compresses faster?\n",
    "\n",
    "Also, Blosc being a meta-compressor, it has support for different codecs internally that can be selected from PyTables in the \"blosc:`codec`\" form.  Do another comparison between internal Blosc codecs, namely, \"blosc:blosclz\" (the default), \"blosc:lz4\", \"blosc:lz4hc\", \"blosc:snappy\", \"blosc:zlib\" and \"blosc:zstd\".\n",
    "\n",
    "Finally, avoid any compression totally (`complevel=0`).  How fast it is compared with existing codecs?"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Creating file: compression/zlib-5-shuffle.h5\n",
      "CPU times: user 373 ms, sys: 21.7 ms, total: 395 ms\n",
      "Wall time: 396 ms\n",
      "Creating file: compression/bzip2-5-shuffle.h5\n",
      "CPU times: user 2.37 s, sys: 18.7 ms, total: 2.39 s\n",
      "Wall time: 2.4 s\n",
      "Creating file: compression/blosc-5-shuffle.h5\n",
      "CPU times: user 68.7 ms, sys: 12.7 ms, total: 81.4 ms\n",
      "Wall time: 82.2 ms\n"
     ]
    }
   ],
   "source": [
    "for complib in (\"zlib\", \"bzip2\", \"blosc\"):\n",
    "    filters = tables.Filters(complevel=5, complib=complib)\n",
    "    %time to_hdf5(ratings, movies, filters)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "total 27152\r\n",
      "-rw-r--r--  1 faltet  staff   5.0M May 20 13:07 blosc-5-shuffle.h5\r\n",
      "-rw-r--r--  1 faltet  staff   4.1M May 20 13:07 bzip2-5-shuffle.h5\r\n",
      "-rw-r--r--  1 faltet  staff   4.2M May 20 13:07 zlib-5-shuffle.h5\r\n"
     ]
    }
   ],
   "source": [
    "%ls -lh {data_dir}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Creating file: compression/blosc-blosclz-5-shuffle.h5\n",
      "CPU times: user 88.4 ms, sys: 24.1 ms, total: 112 ms\n",
      "Wall time: 116 ms\n",
      "Creating file: compression/blosc-lz4-5-shuffle.h5\n",
      "CPU times: user 37.7 ms, sys: 12.7 ms, total: 50.5 ms\n",
      "Wall time: 51.9 ms\n",
      "Creating file: compression/blosc-lz4hc-5-shuffle.h5\n",
      "CPU times: user 340 ms, sys: 14.1 ms, total: 354 ms\n",
      "Wall time: 357 ms\n",
      "Creating file: compression/blosc-snappy-5-shuffle.h5\n",
      "CPU times: user 39.1 ms, sys: 10.9 ms, total: 50 ms\n",
      "Wall time: 51.2 ms\n",
      "Creating file: compression/blosc-zlib-5-shuffle.h5\n",
      "CPU times: user 306 ms, sys: 15.4 ms, total: 321 ms\n",
      "Wall time: 327 ms\n",
      "Creating file: compression/blosc-zstd-5-shuffle.h5\n",
      "CPU times: user 179 ms, sys: 11.8 ms, total: 191 ms\n",
      "Wall time: 192 ms\n"
     ]
    }
   ],
   "source": [
    "for complib in (\"blosc:blosclz\", \"blosc:lz4\", \"blosc:lz4hc\", \"blosc:snappy\", \"blosc:zlib\", \"blosc:zstd\"):\n",
    "    filters = tables.Filters(complevel=5, complib=complib)\n",
    "    %time to_hdf5(ratings, movies, filters)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Creating file: compression/no-compressed.h5\n",
      "CPU times: user 30.6 ms, sys: 30.2 ms, total: 60.7 ms\n",
      "Wall time: 69 ms\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "'compression/no-compressed.h5'"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Finally, the uncompressed case\n",
    "filters = tables.Filters(complevel=0)\n",
    "%time to_hdf5(ratings, movies, filters)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "total 121712\r\n",
      "-rw-r--r--  1 faltet  staff   5.0M May 20 13:07 blosc-5-shuffle.h5\r\n",
      "-rw-r--r--  1 faltet  staff   5.0M May 20 13:07 blosc-blosclz-5-shuffle.h5\r\n",
      "-rw-r--r--  1 faltet  staff   5.5M May 20 13:07 blosc-lz4-5-shuffle.h5\r\n",
      "-rw-r--r--  1 faltet  staff   4.8M May 20 13:07 blosc-lz4hc-5-shuffle.h5\r\n",
      "-rw-r--r--  1 faltet  staff   5.5M May 20 13:07 blosc-snappy-5-shuffle.h5\r\n",
      "-rw-r--r--  1 faltet  staff   4.3M May 20 13:07 blosc-zlib-5-shuffle.h5\r\n",
      "-rw-r--r--  1 faltet  staff   4.3M May 20 13:07 blosc-zstd-5-shuffle.h5\r\n",
      "-rw-r--r--  1 faltet  staff   4.1M May 20 13:07 bzip2-5-shuffle.h5\r\n",
      "-rw-r--r--  1 faltet  staff    17M May 20 13:07 no-compressed.h5\r\n",
      "-rw-r--r--  1 faltet  staff   4.2M May 20 13:07 zlib-5-shuffle.h5\r\n"
     ]
    }
   ],
   "source": [
    "%ls -lh {data_dir}"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "deletable": true,
    "editable": true
   },
   "source": [
    "## Reading compressed datasets"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "files = list(os.walk(data_dir))[0][2]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['blosc-5-shuffle.h5',\n",
       " 'blosc-blosclz-5-shuffle.h5',\n",
       " 'blosc-lz4-5-shuffle.h5',\n",
       " 'blosc-lz4hc-5-shuffle.h5',\n",
       " 'blosc-snappy-5-shuffle.h5',\n",
       " 'blosc-zlib-5-shuffle.h5',\n",
       " 'blosc-zstd-5-shuffle.h5',\n",
       " 'bzip2-5-shuffle.h5',\n",
       " 'no-compressed.h5',\n",
       " 'zlib-5-shuffle.h5']"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "files"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Reading file: blosc-5-shuffle.h5\n",
      "CPU times: user 42.1 ms, sys: 13.1 ms, total: 55.2 ms\n",
      "Wall time: 54.1 ms\n",
      "Reading file: blosc-blosclz-5-shuffle.h5\n",
      "CPU times: user 41.1 ms, sys: 11.9 ms, total: 52.9 ms\n",
      "Wall time: 59.9 ms\n",
      "Reading file: blosc-lz4-5-shuffle.h5\n",
      "CPU times: user 23.2 ms, sys: 7.52 ms, total: 30.7 ms\n",
      "Wall time: 31.5 ms\n",
      "Reading file: blosc-lz4hc-5-shuffle.h5\n",
      "CPU times: user 25.7 ms, sys: 8.77 ms, total: 34.4 ms\n",
      "Wall time: 38.1 ms\n",
      "Reading file: blosc-snappy-5-shuffle.h5\n",
      "CPU times: user 28.7 ms, sys: 7.67 ms, total: 36.3 ms\n",
      "Wall time: 38.9 ms\n",
      "Reading file: blosc-zlib-5-shuffle.h5\n",
      "CPU times: user 84.3 ms, sys: 8.91 ms, total: 93.2 ms\n",
      "Wall time: 98 ms\n",
      "Reading file: blosc-zstd-5-shuffle.h5\n",
      "CPU times: user 41.7 ms, sys: 6.29 ms, total: 48 ms\n",
      "Wall time: 48.6 ms\n",
      "Reading file: bzip2-5-shuffle.h5\n",
      "CPU times: user 683 ms, sys: 11.6 ms, total: 694 ms\n",
      "Wall time: 700 ms\n",
      "Reading file: no-compressed.h5\n",
      "CPU times: user 7.35 ms, sys: 9.98 ms, total: 17.3 ms\n",
      "Wall time: 17.5 ms\n",
      "Reading file: zlib-5-shuffle.h5\n",
      "CPU times: user 72.7 ms, sys: 6.63 ms, total: 79.3 ms\n",
      "Wall time: 78.9 ms\n"
     ]
    }
   ],
   "source": [
    "for f in files:\n",
    "    print(\"Reading file:\", f)\n",
    "    with tables.open_file(os.path.join(data_dir, f)) as h5f:\n",
    "        %time h5f.root.ratings[:]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "deletable": true,
    "editable": true
   },
   "source": [
    "### Exercise 2\n",
    "\n",
    "Which codec and compression level can read the fastest?  How it does compare with reading an uncompressed dataset?"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "deletable": true,
    "editable": true
   },
   "source": [
    "### Exercise 3\n",
    "\n",
    "Blosc can use multithreading for compressing/decompressing, although it is disabled by default.  You can enable a multithreaded Blosc in a series of ways, but perhaps the easiest is to set the \"BLOSC_NTHREADS\" environment variable to the desired number of threads (typically the available number of cores in your computer).\n",
    "\n",
    "Execute the cell below and re-do the reading benchmarks and look at how the reading speed vary.  Pay special attention to the difference between the CPU times and wall times."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {
    "collapsed": true,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "os.environ[\"BLOSC_NTHREADS\"] = \"4\"  # set to any other number you prefer"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Reading file: blosc-5-shuffle.h5\n",
      "CPU times: user 37.8 ms, sys: 10.4 ms, total: 48.3 ms\n",
      "Wall time: 50.6 ms\n",
      "Reading file: blosc-blosclz-5-shuffle.h5\n",
      "CPU times: user 41.3 ms, sys: 17.2 ms, total: 58.5 ms\n",
      "Wall time: 63.3 ms\n",
      "Reading file: blosc-lz4-5-shuffle.h5\n",
      "CPU times: user 28.3 ms, sys: 19.3 ms, total: 47.6 ms\n",
      "Wall time: 34.8 ms\n",
      "Reading file: blosc-lz4hc-5-shuffle.h5\n",
      "CPU times: user 25.4 ms, sys: 17.4 ms, total: 42.8 ms\n",
      "Wall time: 32 ms\n",
      "Reading file: blosc-snappy-5-shuffle.h5\n",
      "CPU times: user 34.1 ms, sys: 18.6 ms, total: 52.7 ms\n",
      "Wall time: 39.2 ms\n",
      "Reading file: blosc-zlib-5-shuffle.h5\n",
      "CPU times: user 81.5 ms, sys: 17.5 ms, total: 99 ms\n",
      "Wall time: 59.6 ms\n",
      "Reading file: blosc-zstd-5-shuffle.h5\n",
      "CPU times: user 50.8 ms, sys: 18.2 ms, total: 68.9 ms\n",
      "Wall time: 44.7 ms\n",
      "Reading file: bzip2-5-shuffle.h5\n",
      "CPU times: user 659 ms, sys: 10.8 ms, total: 670 ms\n",
      "Wall time: 676 ms\n",
      "Reading file: no-compressed.h5\n",
      "CPU times: user 6.77 ms, sys: 9.6 ms, total: 16.4 ms\n",
      "Wall time: 16.1 ms\n",
      "Reading file: zlib-5-shuffle.h5\n",
      "CPU times: user 66.9 ms, sys: 5.59 ms, total: 72.5 ms\n",
      "Wall time: 72.3 ms\n"
     ]
    }
   ],
   "source": [
    "for f in files:\n",
    "    print(\"Reading file:\", f)\n",
    "    with tables.open_file(os.path.join(data_dir, f)) as h5f:\n",
    "        %time h5f.root.ratings[:]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.1"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}