SampleTPCHDataset.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import time\n",
    "import random\n",
    "import numpy as np\n",
    "import pandas as pd # for batch data loading, in generating sampled dataset\n",
    "from numpy import genfromtxt\n",
    "import matplotlib.pyplot as plt"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def process_chunk_sampling(row, domains, sampled_subset, prob_threshold, total_dims):\n",
    "    prob = random.uniform(0, 1)\n",
    "    row_numpy = row.to_numpy()  \n",
    "    for i in range(len(domains)):\n",
    "        if row_numpy[i] > domains[i][1]:\n",
    "            domains[i][1] = row_numpy[i]\n",
    "        if row_numpy[i] < domains[i][0]:\n",
    "            domains[i][0] = row_numpy[i]\n",
    "    if prob <= prob_threshold:    \n",
    "        sampled_subset.append(row_numpy[0:total_dims].tolist())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def generate_dataset_and_save(original_table_path, prob_threshold,chunk_size = 100000, total_dims = 8):\n",
    "    '''\n",
    "    refer to TPCH tools to generate the original dataset (.tbl)\n",
    "    this function is used to process the .tbl file with given sampling rate to generate a .csv file\n",
    "    consider the possible table size, this function is implemented in a batch processing manner\n",
    "    '''\n",
    "    sampled_subset = []\n",
    "    domains = [[float('Infinity'), float('-Infinity')] for i in range(total_dims)] # indicate min, max\n",
    "\n",
    "    col_names = ['_c'+str(i) for i in range(total_dims)]\n",
    "    cols = [i for i in range(total_dims)]\n",
    "\n",
    "    start_time = time.time()\n",
    "\n",
    "    batch_count = 0\n",
    "    for chunk in pd.read_table(original_table_path, delimiter='|', usecols=cols, names=col_names, chunksize=chunk_size):\n",
    "        print('current chunk: ', batch_count)\n",
    "        chunk.apply(lambda row: process_chunk_sampling(row, domains, sampled_subset, prob_threshold, total_dims), axis=1)\n",
    "        batch_count += 1\n",
    "\n",
    "    end_time = time.time()\n",
    "    print('total processing time: ', end_time - start_time)\n",
    "\n",
    "    sampled_subset = np.array(sampled_subset)\n",
    "    domains = np.array(domains)\n",
    "    np.savetxt('C:/Users/Cloud/iCloudDrive/NORA_experiments/dataset/lineitem_50_0.02.csv', sampled_subset, delimiter=',')\n",
    "    np.savetxt('C:/Users/Cloud/iCloudDrive/NORA_experiments/dataset/lineitem_50_0.02_domains.csv', domains, delimiter=',')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "current chunk:  0\n",
      "current chunk:  1\n",
      "current chunk:  2\n",
      "current chunk:  3\n",
      "current chunk:  4\n",
      "current chunk:  5\n",
      "current chunk:  6\n",
      "current chunk:  7\n",
      "current chunk:  8\n",
      "current chunk:  9\n",
      "current chunk:  10\n",
      "current chunk:  11\n",
      "current chunk:  12\n",
      "current chunk:  13\n",
      "current chunk:  14\n",
      "current chunk:  15\n",
      "current chunk:  16\n",
      "current chunk:  17\n",
      "current chunk:  18\n",
      "current chunk:  19\n",
      "current chunk:  20\n",
      "current chunk:  21\n",
      "current chunk:  22\n",
      "current chunk:  23\n",
      "current chunk:  24\n",
      "current chunk:  25\n",
      "current chunk:  26\n",
      "current chunk:  27\n",
      "current chunk:  28\n",
      "current chunk:  29\n",
      "current chunk:  30\n",
      "current chunk:  31\n",
      "current chunk:  32\n",
      "current chunk:  33\n",
      "current chunk:  34\n",
      "current chunk:  35\n",
      "current chunk:  36\n",
      "current chunk:  37\n",
      "current chunk:  38\n",
      "current chunk:  39\n",
      "current chunk:  40\n",
      "current chunk:  41\n",
      "current chunk:  42\n",
      "current chunk:  43\n",
      "current chunk:  44\n",
      "current chunk:  45\n",
      "current chunk:  46\n",
      "current chunk:  47\n",
      "current chunk:  48\n",
      "current chunk:  49\n",
      "current chunk:  50\n",
      "current chunk:  51\n",
      "current chunk:  52\n",
      "current chunk:  53\n",
      "current chunk:  54\n",
      "current chunk:  55\n",
      "current chunk:  56\n",
      "current chunk:  57\n",
      "current chunk:  58\n",
      "current chunk:  59\n",
      "current chunk:  60\n",
      "current chunk:  61\n",
      "current chunk:  62\n",
      "current chunk:  63\n",
      "current chunk:  64\n",
      "current chunk:  65\n",
      "current chunk:  66\n",
      "current chunk:  67\n",
      "current chunk:  68\n",
      "current chunk:  69\n",
      "current chunk:  70\n",
      "current chunk:  71\n",
      "current chunk:  72\n",
      "current chunk:  73\n",
      "current chunk:  74\n",
      "current chunk:  75\n",
      "current chunk:  76\n",
      "current chunk:  77\n",
      "current chunk:  78\n",
      "current chunk:  79\n",
      "current chunk:  80\n",
      "current chunk:  81\n",
      "current chunk:  82\n",
      "current chunk:  83\n",
      "current chunk:  84\n",
      "current chunk:  85\n",
      "current chunk:  86\n",
      "current chunk:  87\n",
      "current chunk:  88\n",
      "current chunk:  89\n",
      "current chunk:  90\n",
      "current chunk:  91\n",
      "current chunk:  92\n",
      "current chunk:  93\n",
      "current chunk:  94\n",
      "current chunk:  95\n",
      "current chunk:  96\n",
      "current chunk:  97\n",
      "current chunk:  98\n",
      "current chunk:  99\n",
      "current chunk:  100\n",
      "current chunk:  101\n",
      "current chunk:  102\n",
      "current chunk:  103\n",
      "current chunk:  104\n",
      "current chunk:  105\n",
      "current chunk:  106\n",
      "current chunk:  107\n",
      "current chunk:  108\n",
      "current chunk:  109\n",
      "current chunk:  110\n",
      "current chunk:  111\n",
      "current chunk:  112\n",
      "current chunk:  113\n",
      "current chunk:  114\n",
      "current chunk:  115\n",
      "current chunk:  116\n",
      "current chunk:  117\n",
      "current chunk:  118\n",
      "current chunk:  119\n",
      "current chunk:  120\n",
      "current chunk:  121\n",
      "current chunk:  122\n",
      "current chunk:  123\n",
      "current chunk:  124\n",
      "current chunk:  125\n",
      "current chunk:  126\n",
      "current chunk:  127\n",
      "current chunk:  128\n",
      "current chunk:  129\n",
      "current chunk:  130\n",
      "current chunk:  131\n",
      "current chunk:  132\n",
      "current chunk:  133\n",
      "current chunk:  134\n",
      "current chunk:  135\n",
      "current chunk:  136\n",
      "current chunk:  137\n",
      "current chunk:  138\n",
      "current chunk:  139\n",
      "current chunk:  140\n",
      "current chunk:  141\n",
      "current chunk:  142\n",
      "current chunk:  143\n",
      "current chunk:  144\n",
      "current chunk:  145\n",
      "current chunk:  146\n",
      "current chunk:  147\n",
      "current chunk:  148\n",
      "current chunk:  149\n",
      "current chunk:  150\n",
      "current chunk:  151\n",
      "current chunk:  152\n",
      "current chunk:  153\n",
      "current chunk:  154\n",
      "current chunk:  155\n",
      "current chunk:  156\n",
      "current chunk:  157\n",
      "current chunk:  158\n",
      "current chunk:  159\n",
      "current chunk:  160\n",
      "current chunk:  161\n",
      "current chunk:  162\n",
      "current chunk:  163\n",
      "current chunk:  164\n",
      "current chunk:  165\n",
      "current chunk:  166\n",
      "current chunk:  167\n",
      "current chunk:  168\n",
      "current chunk:  169\n",
      "current chunk:  170\n",
      "current chunk:  171\n",
      "current chunk:  172\n",
      "current chunk:  173\n",
      "current chunk:  174\n",
      "current chunk:  175\n",
      "current chunk:  176\n",
      "current chunk:  177\n",
      "current chunk:  178\n",
      "current chunk:  179\n",
      "current chunk:  180\n",
      "current chunk:  181\n",
      "current chunk:  182\n",
      "current chunk:  183\n",
      "current chunk:  184\n",
      "current chunk:  185\n",
      "current chunk:  186\n",
      "current chunk:  187\n",
      "current chunk:  188\n",
      "current chunk:  189\n",
      "current chunk:  190\n",
      "current chunk:  191\n",
      "current chunk:  192\n",
      "current chunk:  193\n",
      "current chunk:  194\n",
      "current chunk:  195\n",
      "current chunk:  196\n",
      "current chunk:  197\n",
      "current chunk:  198\n",
      "current chunk:  199\n",
      "current chunk:  200\n",
      "current chunk:  201\n",
      "current chunk:  202\n",
      "current chunk:  203\n",
      "current chunk:  204\n",
      "current chunk:  205\n",
      "current chunk:  206\n",
      "current chunk:  207\n",
      "current chunk:  208\n",
      "current chunk:  209\n",
      "current chunk:  210\n",
      "current chunk:  211\n",
      "current chunk:  212\n",
      "current chunk:  213\n",
      "current chunk:  214\n",
      "current chunk:  215\n",
      "current chunk:  216\n",
      "current chunk:  217\n",
      "current chunk:  218\n",
      "current chunk:  219\n",
      "current chunk:  220\n",
      "current chunk:  221\n",
      "current chunk:  222\n",
      "current chunk:  223\n",
      "current chunk:  224\n",
      "current chunk:  225\n",
      "current chunk:  226\n",
      "current chunk:  227\n",
      "current chunk:  228\n",
      "current chunk:  229\n",
      "current chunk:  230\n",
      "current chunk:  231\n",
      "current chunk:  232\n",
      "current chunk:  233\n",
      "current chunk:  234\n",
      "current chunk:  235\n",
      "current chunk:  236\n",
      "current chunk:  237\n",
      "current chunk:  238\n",
      "current chunk:  239\n",
      "current chunk:  240\n",
      "current chunk:  241\n",
      "current chunk:  242\n",
      "current chunk:  243\n",
      "current chunk:  244\n",
      "current chunk:  245\n",
      "current chunk:  246\n",
      "current chunk:  247\n",
      "current chunk:  248\n",
      "current chunk:  249\n",
      "current chunk:  250\n",
      "current chunk:  251\n",
      "current chunk:  252\n",
      "current chunk:  253\n",
      "current chunk:  254\n",
      "current chunk:  255\n",
      "current chunk:  256\n",
      "current chunk:  257\n",
      "current chunk:  258\n",
      "current chunk:  259\n",
      "current chunk:  260\n",
      "current chunk:  261\n",
      "current chunk:  262\n",
      "current chunk:  263\n",
      "current chunk:  264\n",
      "current chunk:  265\n",
      "current chunk:  266\n",
      "current chunk:  267\n",
      "current chunk:  268\n",
      "current chunk:  269\n",
      "current chunk:  270\n",
      "current chunk:  271\n",
      "current chunk:  272\n",
      "current chunk:  273\n",
      "current chunk:  274\n",
      "current chunk:  275\n",
      "current chunk:  276\n",
      "current chunk:  277\n",
      "current chunk:  278\n",
      "current chunk:  279\n",
      "current chunk:  280\n",
      "current chunk:  281\n",
      "current chunk:  282\n",
      "current chunk:  283\n",
      "current chunk:  284\n",
      "current chunk:  285\n",
      "current chunk:  286\n",
      "current chunk:  287\n",
      "current chunk:  288\n",
      "current chunk:  289\n",
      "current chunk:  290\n",
      "current chunk:  291\n",
      "current chunk:  292\n",
      "current chunk:  293\n",
      "current chunk:  294\n",
      "current chunk:  295\n",
      "current chunk:  296\n",
      "current chunk:  297\n",
      "current chunk:  298\n",
      "current chunk:  299\n",
      "current chunk:  300\n",
      "current chunk:  301\n",
      "current chunk:  302\n",
      "current chunk:  303\n",
      "current chunk:  304\n",
      "current chunk:  305\n",
      "current chunk:  306\n",
      "current chunk:  307\n",
      "current chunk:  308\n",
      "current chunk:  309\n",
      "current chunk:  310\n",
      "current chunk:  311\n",
      "current chunk:  312\n",
      "current chunk:  313\n",
      "current chunk:  314\n",
      "current chunk:  315\n",
      "current chunk:  316\n",
      "current chunk:  317\n",
      "current chunk:  318\n",
      "current chunk:  319\n",
      "current chunk:  320\n",
      "current chunk:  321\n",
      "current chunk:  322\n",
      "current chunk:  323\n",
      "current chunk:  324\n",
      "current chunk:  325\n",
      "current chunk:  326\n",
      "current chunk:  327\n",
      "current chunk:  328\n",
      "current chunk:  329\n",
      "current chunk:  330\n",
      "current chunk:  331\n",
      "current chunk:  332\n",
      "current chunk:  333\n",
      "current chunk:  334\n",
      "current chunk:  335\n",
      "current chunk:  336\n",
      "current chunk:  337\n",
      "current chunk:  338\n",
      "current chunk:  339\n",
      "current chunk:  340\n",
      "current chunk:  341\n",
      "current chunk:  342\n",
      "current chunk:  343\n",
      "current chunk:  344\n",
      "current chunk:  345\n",
      "current chunk:  346\n",
      "current chunk:  347\n",
      "current chunk:  348\n",
      "current chunk:  349\n",
      "current chunk:  350\n",
      "current chunk:  351\n",
      "current chunk:  352\n",
      "current chunk:  353\n",
      "current chunk:  354\n",
      "current chunk:  355\n",
      "current chunk:  356\n",
      "current chunk:  357\n",
      "current chunk:  358\n",
      "current chunk:  359\n",
      "current chunk:  360\n",
      "current chunk:  361\n",
      "current chunk:  362\n",
      "current chunk:  363\n",
      "current chunk:  364\n",
      "current chunk:  365\n",
      "current chunk:  366\n",
      "current chunk:  367\n",
      "current chunk:  368\n",
      "current chunk:  369\n",
      "current chunk:  370\n",
      "current chunk:  371\n",
      "current chunk:  372\n",
      "current chunk:  373\n",
      "current chunk:  374\n",
      "current chunk:  375\n",
      "current chunk:  376\n",
      "current chunk:  377\n",
      "current chunk:  378\n",
      "current chunk:  379\n",
      "current chunk:  380\n",
      "current chunk:  381\n",
      "current chunk:  382\n",
      "current chunk:  383\n",
      "current chunk:  384\n",
      "current chunk:  385\n",
      "current chunk:  386\n",
      "current chunk:  387\n",
      "current chunk:  388\n",
      "current chunk:  389\n",
      "current chunk:  390\n",
      "current chunk:  391\n",
      "current chunk:  392\n",
      "current chunk:  393\n",
      "current chunk:  394\n",
      "current chunk:  395\n",
      "current chunk:  396\n",
      "current chunk:  397\n",
      "current chunk:  398\n",
      "current chunk:  399\n",
      "current chunk:  400\n",
      "current chunk:  401\n",
      "current chunk:  402\n",
      "current chunk:  403\n",
      "current chunk:  404\n",
      "current chunk:  405\n",
      "current chunk:  406\n",
      "current chunk:  407\n",
      "current chunk:  408\n",
      "current chunk:  409\n",
      "current chunk:  410\n",
      "current chunk:  411\n",
      "current chunk:  412\n",
      "current chunk:  413\n",
      "current chunk:  414\n",
      "current chunk:  415\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "current chunk:  416\n",
      "current chunk:  417\n",
      "current chunk:  418\n",
      "current chunk:  419\n",
      "current chunk:  420\n",
      "current chunk:  421\n",
      "current chunk:  422\n",
      "current chunk:  423\n",
      "current chunk:  424\n",
      "current chunk:  425\n",
      "current chunk:  426\n",
      "current chunk:  427\n",
      "current chunk:  428\n",
      "current chunk:  429\n",
      "current chunk:  430\n",
      "current chunk:  431\n",
      "current chunk:  432\n",
      "current chunk:  433\n",
      "current chunk:  434\n",
      "current chunk:  435\n",
      "current chunk:  436\n",
      "current chunk:  437\n",
      "current chunk:  438\n",
      "current chunk:  439\n",
      "current chunk:  440\n",
      "current chunk:  441\n",
      "current chunk:  442\n",
      "current chunk:  443\n",
      "current chunk:  444\n",
      "current chunk:  445\n",
      "current chunk:  446\n",
      "current chunk:  447\n",
      "current chunk:  448\n",
      "current chunk:  449\n",
      "current chunk:  450\n",
      "current chunk:  451\n",
      "current chunk:  452\n",
      "current chunk:  453\n",
      "current chunk:  454\n",
      "current chunk:  455\n",
      "current chunk:  456\n",
      "current chunk:  457\n",
      "current chunk:  458\n",
      "current chunk:  459\n",
      "current chunk:  460\n",
      "current chunk:  461\n",
      "current chunk:  462\n",
      "current chunk:  463\n",
      "current chunk:  464\n",
      "current chunk:  465\n",
      "current chunk:  466\n",
      "current chunk:  467\n",
      "current chunk:  468\n",
      "current chunk:  469\n",
      "current chunk:  470\n",
      "current chunk:  471\n",
      "current chunk:  472\n",
      "current chunk:  473\n",
      "current chunk:  474\n",
      "current chunk:  475\n",
      "current chunk:  476\n",
      "current chunk:  477\n",
      "current chunk:  478\n",
      "current chunk:  479\n",
      "current chunk:  480\n",
      "current chunk:  481\n",
      "current chunk:  482\n",
      "current chunk:  483\n",
      "current chunk:  484\n",
      "current chunk:  485\n",
      "current chunk:  486\n",
      "current chunk:  487\n",
      "current chunk:  488\n",
      "current chunk:  489\n",
      "current chunk:  490\n",
      "current chunk:  491\n",
      "current chunk:  492\n",
      "current chunk:  493\n",
      "current chunk:  494\n",
      "current chunk:  495\n",
      "current chunk:  496\n",
      "current chunk:  497\n",
      "current chunk:  498\n",
      "current chunk:  499\n",
      "current chunk:  500\n",
      "current chunk:  501\n",
      "current chunk:  502\n",
      "current chunk:  503\n",
      "current chunk:  504\n",
      "current chunk:  505\n",
      "current chunk:  506\n",
      "current chunk:  507\n",
      "current chunk:  508\n",
      "current chunk:  509\n",
      "current chunk:  510\n",
      "current chunk:  511\n",
      "current chunk:  512\n",
      "current chunk:  513\n",
      "current chunk:  514\n",
      "current chunk:  515\n",
      "current chunk:  516\n",
      "current chunk:  517\n",
      "current chunk:  518\n",
      "current chunk:  519\n",
      "current chunk:  520\n",
      "current chunk:  521\n",
      "current chunk:  522\n",
      "current chunk:  523\n",
      "current chunk:  524\n",
      "current chunk:  525\n",
      "current chunk:  526\n",
      "current chunk:  527\n",
      "current chunk:  528\n",
      "current chunk:  529\n",
      "current chunk:  530\n",
      "current chunk:  531\n",
      "current chunk:  532\n",
      "current chunk:  533\n",
      "current chunk:  534\n",
      "current chunk:  535\n",
      "current chunk:  536\n",
      "current chunk:  537\n",
      "current chunk:  538\n",
      "current chunk:  539\n",
      "current chunk:  540\n",
      "current chunk:  541\n",
      "current chunk:  542\n",
      "current chunk:  543\n",
      "current chunk:  544\n",
      "current chunk:  545\n",
      "current chunk:  546\n",
      "current chunk:  547\n",
      "current chunk:  548\n",
      "current chunk:  549\n",
      "current chunk:  550\n",
      "current chunk:  551\n",
      "current chunk:  552\n",
      "current chunk:  553\n",
      "current chunk:  554\n",
      "current chunk:  555\n",
      "current chunk:  556\n",
      "current chunk:  557\n",
      "current chunk:  558\n",
      "current chunk:  559\n",
      "current chunk:  560\n",
      "current chunk:  561\n",
      "current chunk:  562\n",
      "current chunk:  563\n",
      "current chunk:  564\n",
      "current chunk:  565\n",
      "current chunk:  566\n",
      "current chunk:  567\n",
      "current chunk:  568\n",
      "current chunk:  569\n",
      "current chunk:  570\n",
      "current chunk:  571\n",
      "current chunk:  572\n",
      "current chunk:  573\n",
      "current chunk:  574\n",
      "current chunk:  575\n",
      "current chunk:  576\n",
      "current chunk:  577\n",
      "current chunk:  578\n",
      "current chunk:  579\n",
      "current chunk:  580\n",
      "current chunk:  581\n",
      "current chunk:  582\n",
      "current chunk:  583\n",
      "current chunk:  584\n",
      "current chunk:  585\n",
      "current chunk:  586\n",
      "current chunk:  587\n",
      "current chunk:  588\n",
      "current chunk:  589\n",
      "current chunk:  590\n",
      "current chunk:  591\n",
      "current chunk:  592\n",
      "current chunk:  593\n",
      "current chunk:  594\n",
      "current chunk:  595\n",
      "current chunk:  596\n",
      "current chunk:  597\n",
      "current chunk:  598\n",
      "current chunk:  599\n",
      "current chunk:  600\n",
      "current chunk:  601\n",
      "current chunk:  602\n",
      "current chunk:  603\n",
      "current chunk:  604\n",
      "current chunk:  605\n",
      "current chunk:  606\n",
      "current chunk:  607\n",
      "current chunk:  608\n",
      "current chunk:  609\n",
      "current chunk:  610\n",
      "current chunk:  611\n",
      "current chunk:  612\n",
      "current chunk:  613\n",
      "current chunk:  614\n",
      "current chunk:  615\n",
      "current chunk:  616\n",
      "current chunk:  617\n",
      "current chunk:  618\n",
      "current chunk:  619\n",
      "current chunk:  620\n",
      "current chunk:  621\n",
      "current chunk:  622\n",
      "current chunk:  623\n",
      "current chunk:  624\n",
      "current chunk:  625\n",
      "current chunk:  626\n",
      "current chunk:  627\n",
      "current chunk:  628\n",
      "current chunk:  629\n",
      "current chunk:  630\n",
      "current chunk:  631\n",
      "current chunk:  632\n",
      "current chunk:  633\n",
      "current chunk:  634\n",
      "current chunk:  635\n",
      "current chunk:  636\n",
      "current chunk:  637\n",
      "current chunk:  638\n",
      "current chunk:  639\n",
      "current chunk:  640\n",
      "current chunk:  641\n",
      "current chunk:  642\n",
      "current chunk:  643\n",
      "current chunk:  644\n",
      "current chunk:  645\n",
      "current chunk:  646\n",
      "current chunk:  647\n",
      "current chunk:  648\n",
      "current chunk:  649\n",
      "current chunk:  650\n",
      "current chunk:  651\n",
      "current chunk:  652\n",
      "current chunk:  653\n",
      "current chunk:  654\n",
      "current chunk:  655\n",
      "current chunk:  656\n",
      "current chunk:  657\n",
      "current chunk:  658\n",
      "current chunk:  659\n",
      "current chunk:  660\n",
      "current chunk:  661\n",
      "current chunk:  662\n",
      "current chunk:  663\n",
      "current chunk:  664\n",
      "current chunk:  665\n",
      "current chunk:  666\n",
      "current chunk:  667\n",
      "current chunk:  668\n",
      "current chunk:  669\n",
      "current chunk:  670\n",
      "current chunk:  671\n",
      "current chunk:  672\n",
      "current chunk:  673\n",
      "current chunk:  674\n",
      "current chunk:  675\n",
      "current chunk:  676\n",
      "current chunk:  677\n",
      "current chunk:  678\n",
      "current chunk:  679\n",
      "current chunk:  680\n",
      "current chunk:  681\n",
      "current chunk:  682\n",
      "current chunk:  683\n",
      "current chunk:  684\n",
      "current chunk:  685\n",
      "current chunk:  686\n",
      "current chunk:  687\n",
      "current chunk:  688\n",
      "current chunk:  689\n",
      "current chunk:  690\n",
      "current chunk:  691\n",
      "current chunk:  692\n",
      "current chunk:  693\n",
      "current chunk:  694\n",
      "current chunk:  695\n",
      "current chunk:  696\n",
      "current chunk:  697\n",
      "current chunk:  698\n",
      "current chunk:  699\n",
      "current chunk:  700\n",
      "current chunk:  701\n",
      "current chunk:  702\n",
      "current chunk:  703\n",
      "current chunk:  704\n",
      "current chunk:  705\n",
      "current chunk:  706\n",
      "current chunk:  707\n",
      "current chunk:  708\n",
      "current chunk:  709\n",
      "current chunk:  710\n",
      "current chunk:  711\n",
      "current chunk:  712\n",
      "current chunk:  713\n",
      "current chunk:  714\n",
      "current chunk:  715\n",
      "current chunk:  716\n",
      "current chunk:  717\n",
      "current chunk:  718\n",
      "current chunk:  719\n",
      "current chunk:  720\n",
      "current chunk:  721\n",
      "current chunk:  722\n",
      "current chunk:  723\n",
      "current chunk:  724\n",
      "current chunk:  725\n",
      "current chunk:  726\n",
      "current chunk:  727\n",
      "current chunk:  728\n",
      "current chunk:  729\n",
      "current chunk:  730\n",
      "current chunk:  731\n",
      "current chunk:  732\n",
      "current chunk:  733\n",
      "current chunk:  734\n",
      "current chunk:  735\n",
      "current chunk:  736\n",
      "current chunk:  737\n",
      "current chunk:  738\n",
      "current chunk:  739\n",
      "current chunk:  740\n",
      "current chunk:  741\n",
      "current chunk:  742\n",
      "current chunk:  743\n",
      "current chunk:  744\n",
      "current chunk:  745\n",
      "current chunk:  746\n",
      "current chunk:  747\n",
      "current chunk:  748\n",
      "current chunk:  749\n",
      "current chunk:  750\n",
      "current chunk:  751\n",
      "current chunk:  752\n",
      "current chunk:  753\n",
      "current chunk:  754\n",
      "current chunk:  755\n",
      "current chunk:  756\n",
      "current chunk:  757\n",
      "current chunk:  758\n",
      "current chunk:  759\n",
      "current chunk:  760\n",
      "current chunk:  761\n",
      "current chunk:  762\n",
      "current chunk:  763\n",
      "current chunk:  764\n",
      "current chunk:  765\n",
      "current chunk:  766\n",
      "current chunk:  767\n",
      "current chunk:  768\n",
      "current chunk:  769\n",
      "current chunk:  770\n",
      "current chunk:  771\n",
      "current chunk:  772\n",
      "current chunk:  773\n",
      "current chunk:  774\n",
      "current chunk:  775\n",
      "current chunk:  776\n",
      "current chunk:  777\n",
      "current chunk:  778\n",
      "current chunk:  779\n",
      "current chunk:  780\n",
      "current chunk:  781\n",
      "current chunk:  782\n",
      "current chunk:  783\n",
      "current chunk:  784\n",
      "current chunk:  785\n",
      "current chunk:  786\n",
      "current chunk:  787\n",
      "current chunk:  788\n",
      "current chunk:  789\n",
      "current chunk:  790\n",
      "current chunk:  791\n",
      "current chunk:  792\n",
      "current chunk:  793\n",
      "current chunk:  794\n",
      "current chunk:  795\n",
      "current chunk:  796\n",
      "current chunk:  797\n",
      "current chunk:  798\n",
      "current chunk:  799\n",
      "current chunk:  800\n",
      "current chunk:  801\n",
      "current chunk:  802\n",
      "current chunk:  803\n",
      "current chunk:  804\n",
      "current chunk:  805\n",
      "current chunk:  806\n",
      "current chunk:  807\n",
      "current chunk:  808\n",
      "current chunk:  809\n",
      "current chunk:  810\n",
      "current chunk:  811\n",
      "current chunk:  812\n",
      "current chunk:  813\n",
      "current chunk:  814\n",
      "current chunk:  815\n",
      "current chunk:  816\n",
      "current chunk:  817\n",
      "current chunk:  818\n",
      "current chunk:  819\n",
      "current chunk:  820\n",
      "current chunk:  821\n",
      "current chunk:  822\n",
      "current chunk:  823\n",
      "current chunk:  824\n",
      "current chunk:  825\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "current chunk:  826\n",
      "current chunk:  827\n",
      "current chunk:  828\n",
      "current chunk:  829\n",
      "current chunk:  830\n",
      "current chunk:  831\n",
      "current chunk:  832\n",
      "current chunk:  833\n",
      "current chunk:  834\n",
      "current chunk:  835\n",
      "current chunk:  836\n",
      "current chunk:  837\n",
      "current chunk:  838\n",
      "current chunk:  839\n",
      "current chunk:  840\n",
      "current chunk:  841\n",
      "current chunk:  842\n",
      "current chunk:  843\n",
      "current chunk:  844\n",
      "current chunk:  845\n",
      "current chunk:  846\n",
      "current chunk:  847\n",
      "current chunk:  848\n",
      "current chunk:  849\n",
      "current chunk:  850\n",
      "current chunk:  851\n",
      "current chunk:  852\n",
      "current chunk:  853\n",
      "current chunk:  854\n",
      "current chunk:  855\n",
      "current chunk:  856\n",
      "current chunk:  857\n",
      "current chunk:  858\n",
      "current chunk:  859\n",
      "current chunk:  860\n",
      "current chunk:  861\n",
      "current chunk:  862\n",
      "current chunk:  863\n",
      "current chunk:  864\n",
      "current chunk:  865\n",
      "current chunk:  866\n",
      "current chunk:  867\n",
      "current chunk:  868\n",
      "current chunk:  869\n",
      "current chunk:  870\n",
      "current chunk:  871\n",
      "current chunk:  872\n",
      "current chunk:  873\n",
      "current chunk:  874\n",
      "current chunk:  875\n",
      "current chunk:  876\n",
      "current chunk:  877\n",
      "current chunk:  878\n",
      "current chunk:  879\n",
      "current chunk:  880\n",
      "current chunk:  881\n",
      "current chunk:  882\n",
      "current chunk:  883\n",
      "current chunk:  884\n",
      "current chunk:  885\n",
      "current chunk:  886\n",
      "current chunk:  887\n",
      "current chunk:  888\n",
      "current chunk:  889\n",
      "current chunk:  890\n",
      "current chunk:  891\n",
      "current chunk:  892\n",
      "current chunk:  893\n",
      "current chunk:  894\n",
      "current chunk:  895\n",
      "current chunk:  896\n",
      "current chunk:  897\n",
      "current chunk:  898\n",
      "current chunk:  899\n",
      "current chunk:  900\n",
      "current chunk:  901\n",
      "current chunk:  902\n",
      "current chunk:  903\n",
      "current chunk:  904\n",
      "current chunk:  905\n",
      "current chunk:  906\n",
      "current chunk:  907\n",
      "current chunk:  908\n",
      "current chunk:  909\n",
      "current chunk:  910\n",
      "current chunk:  911\n",
      "current chunk:  912\n",
      "current chunk:  913\n",
      "current chunk:  914\n",
      "current chunk:  915\n",
      "current chunk:  916\n",
      "current chunk:  917\n",
      "current chunk:  918\n",
      "current chunk:  919\n",
      "current chunk:  920\n",
      "current chunk:  921\n",
      "current chunk:  922\n",
      "current chunk:  923\n",
      "current chunk:  924\n",
      "current chunk:  925\n",
      "current chunk:  926\n",
      "current chunk:  927\n",
      "current chunk:  928\n",
      "current chunk:  929\n",
      "current chunk:  930\n",
      "current chunk:  931\n",
      "current chunk:  932\n",
      "current chunk:  933\n",
      "current chunk:  934\n",
      "current chunk:  935\n",
      "current chunk:  936\n",
      "current chunk:  937\n",
      "current chunk:  938\n",
      "current chunk:  939\n",
      "current chunk:  940\n",
      "current chunk:  941\n",
      "current chunk:  942\n",
      "current chunk:  943\n",
      "current chunk:  944\n",
      "current chunk:  945\n",
      "current chunk:  946\n",
      "current chunk:  947\n",
      "current chunk:  948\n",
      "current chunk:  949\n",
      "current chunk:  950\n",
      "current chunk:  951\n",
      "current chunk:  952\n",
      "current chunk:  953\n",
      "current chunk:  954\n",
      "current chunk:  955\n",
      "current chunk:  956\n",
      "current chunk:  957\n",
      "current chunk:  958\n",
      "current chunk:  959\n",
      "current chunk:  960\n",
      "current chunk:  961\n",
      "current chunk:  962\n",
      "current chunk:  963\n",
      "current chunk:  964\n",
      "current chunk:  965\n",
      "current chunk:  966\n",
      "current chunk:  967\n",
      "current chunk:  968\n",
      "current chunk:  969\n",
      "current chunk:  970\n",
      "current chunk:  971\n",
      "current chunk:  972\n",
      "current chunk:  973\n",
      "current chunk:  974\n",
      "current chunk:  975\n",
      "current chunk:  976\n",
      "current chunk:  977\n",
      "current chunk:  978\n",
      "current chunk:  979\n",
      "current chunk:  980\n",
      "current chunk:  981\n",
      "current chunk:  982\n",
      "current chunk:  983\n",
      "current chunk:  984\n",
      "current chunk:  985\n",
      "current chunk:  986\n",
      "current chunk:  987\n",
      "current chunk:  988\n",
      "current chunk:  989\n",
      "current chunk:  990\n",
      "current chunk:  991\n",
      "current chunk:  992\n",
      "current chunk:  993\n",
      "current chunk:  994\n",
      "current chunk:  995\n",
      "current chunk:  996\n",
      "current chunk:  997\n",
      "current chunk:  998\n",
      "current chunk:  999\n",
      "current chunk:  1000\n",
      "current chunk:  1001\n",
      "current chunk:  1002\n",
      "current chunk:  1003\n",
      "current chunk:  1004\n",
      "current chunk:  1005\n",
      "current chunk:  1006\n",
      "current chunk:  1007\n",
      "current chunk:  1008\n",
      "current chunk:  1009\n",
      "current chunk:  1010\n",
      "current chunk:  1011\n",
      "current chunk:  1012\n",
      "current chunk:  1013\n",
      "current chunk:  1014\n",
      "current chunk:  1015\n",
      "current chunk:  1016\n",
      "current chunk:  1017\n",
      "current chunk:  1018\n",
      "current chunk:  1019\n",
      "current chunk:  1020\n",
      "current chunk:  1021\n",
      "current chunk:  1022\n",
      "current chunk:  1023\n",
      "current chunk:  1024\n",
      "current chunk:  1025\n",
      "current chunk:  1026\n",
      "current chunk:  1027\n",
      "current chunk:  1028\n",
      "current chunk:  1029\n",
      "current chunk:  1030\n",
      "current chunk:  1031\n",
      "current chunk:  1032\n",
      "current chunk:  1033\n",
      "current chunk:  1034\n",
      "current chunk:  1035\n",
      "current chunk:  1036\n",
      "current chunk:  1037\n",
      "current chunk:  1038\n",
      "current chunk:  1039\n",
      "current chunk:  1040\n",
      "current chunk:  1041\n",
      "current chunk:  1042\n",
      "current chunk:  1043\n",
      "current chunk:  1044\n",
      "current chunk:  1045\n",
      "current chunk:  1046\n",
      "current chunk:  1047\n",
      "current chunk:  1048\n",
      "current chunk:  1049\n",
      "current chunk:  1050\n",
      "current chunk:  1051\n",
      "current chunk:  1052\n",
      "current chunk:  1053\n",
      "current chunk:  1054\n",
      "current chunk:  1055\n",
      "current chunk:  1056\n",
      "current chunk:  1057\n",
      "current chunk:  1058\n",
      "current chunk:  1059\n",
      "current chunk:  1060\n",
      "current chunk:  1061\n",
      "current chunk:  1062\n",
      "current chunk:  1063\n",
      "current chunk:  1064\n",
      "current chunk:  1065\n",
      "current chunk:  1066\n",
      "current chunk:  1067\n",
      "current chunk:  1068\n",
      "current chunk:  1069\n",
      "current chunk:  1070\n",
      "current chunk:  1071\n",
      "current chunk:  1072\n",
      "current chunk:  1073\n",
      "current chunk:  1074\n",
      "current chunk:  1075\n",
      "current chunk:  1076\n",
      "current chunk:  1077\n",
      "current chunk:  1078\n",
      "current chunk:  1079\n",
      "current chunk:  1080\n",
      "current chunk:  1081\n",
      "current chunk:  1082\n",
      "current chunk:  1083\n",
      "current chunk:  1084\n",
      "current chunk:  1085\n",
      "current chunk:  1086\n",
      "current chunk:  1087\n",
      "current chunk:  1088\n",
      "current chunk:  1089\n",
      "current chunk:  1090\n",
      "current chunk:  1091\n",
      "current chunk:  1092\n",
      "current chunk:  1093\n",
      "current chunk:  1094\n",
      "current chunk:  1095\n",
      "current chunk:  1096\n",
      "current chunk:  1097\n",
      "current chunk:  1098\n",
      "current chunk:  1099\n",
      "current chunk:  1100\n",
      "current chunk:  1101\n",
      "current chunk:  1102\n",
      "current chunk:  1103\n",
      "current chunk:  1104\n",
      "current chunk:  1105\n",
      "current chunk:  1106\n",
      "current chunk:  1107\n",
      "current chunk:  1108\n",
      "current chunk:  1109\n",
      "current chunk:  1110\n",
      "current chunk:  1111\n",
      "current chunk:  1112\n",
      "current chunk:  1113\n",
      "current chunk:  1114\n",
      "current chunk:  1115\n",
      "current chunk:  1116\n",
      "current chunk:  1117\n",
      "current chunk:  1118\n",
      "current chunk:  1119\n",
      "current chunk:  1120\n",
      "current chunk:  1121\n",
      "current chunk:  1122\n",
      "current chunk:  1123\n",
      "current chunk:  1124\n",
      "current chunk:  1125\n",
      "current chunk:  1126\n",
      "current chunk:  1127\n",
      "current chunk:  1128\n",
      "current chunk:  1129\n",
      "current chunk:  1130\n",
      "current chunk:  1131\n",
      "current chunk:  1132\n",
      "current chunk:  1133\n",
      "current chunk:  1134\n",
      "current chunk:  1135\n",
      "current chunk:  1136\n",
      "current chunk:  1137\n",
      "current chunk:  1138\n",
      "current chunk:  1139\n",
      "current chunk:  1140\n",
      "current chunk:  1141\n",
      "current chunk:  1142\n",
      "current chunk:  1143\n",
      "current chunk:  1144\n",
      "current chunk:  1145\n",
      "current chunk:  1146\n",
      "current chunk:  1147\n",
      "current chunk:  1148\n",
      "current chunk:  1149\n",
      "current chunk:  1150\n",
      "current chunk:  1151\n",
      "current chunk:  1152\n",
      "current chunk:  1153\n",
      "current chunk:  1154\n",
      "current chunk:  1155\n",
      "current chunk:  1156\n",
      "current chunk:  1157\n",
      "current chunk:  1158\n",
      "current chunk:  1159\n",
      "current chunk:  1160\n",
      "current chunk:  1161\n",
      "current chunk:  1162\n",
      "current chunk:  1163\n",
      "current chunk:  1164\n",
      "current chunk:  1165\n",
      "current chunk:  1166\n",
      "current chunk:  1167\n",
      "current chunk:  1168\n",
      "current chunk:  1169\n",
      "current chunk:  1170\n",
      "current chunk:  1171\n",
      "current chunk:  1172\n",
      "current chunk:  1173\n",
      "current chunk:  1174\n",
      "current chunk:  1175\n",
      "current chunk:  1176\n",
      "current chunk:  1177\n",
      "current chunk:  1178\n",
      "current chunk:  1179\n",
      "current chunk:  1180\n",
      "current chunk:  1181\n",
      "current chunk:  1182\n",
      "current chunk:  1183\n",
      "current chunk:  1184\n",
      "current chunk:  1185\n",
      "current chunk:  1186\n",
      "current chunk:  1187\n",
      "current chunk:  1188\n",
      "current chunk:  1189\n",
      "current chunk:  1190\n",
      "current chunk:  1191\n",
      "current chunk:  1192\n",
      "current chunk:  1193\n",
      "current chunk:  1194\n",
      "current chunk:  1195\n",
      "current chunk:  1196\n",
      "current chunk:  1197\n",
      "current chunk:  1198\n",
      "current chunk:  1199\n",
      "current chunk:  1200\n",
      "current chunk:  1201\n",
      "current chunk:  1202\n",
      "current chunk:  1203\n",
      "current chunk:  1204\n",
      "current chunk:  1205\n",
      "current chunk:  1206\n",
      "current chunk:  1207\n",
      "current chunk:  1208\n",
      "current chunk:  1209\n",
      "current chunk:  1210\n",
      "current chunk:  1211\n",
      "current chunk:  1212\n",
      "current chunk:  1213\n",
      "current chunk:  1214\n",
      "current chunk:  1215\n",
      "current chunk:  1216\n",
      "current chunk:  1217\n",
      "current chunk:  1218\n",
      "current chunk:  1219\n",
      "current chunk:  1220\n",
      "current chunk:  1221\n",
      "current chunk:  1222\n",
      "current chunk:  1223\n",
      "current chunk:  1224\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "current chunk:  1225\n",
      "current chunk:  1226\n",
      "current chunk:  1227\n",
      "current chunk:  1228\n",
      "current chunk:  1229\n",
      "current chunk:  1230\n",
      "current chunk:  1231\n",
      "current chunk:  1232\n",
      "current chunk:  1233\n",
      "current chunk:  1234\n",
      "current chunk:  1235\n",
      "current chunk:  1236\n",
      "current chunk:  1237\n",
      "current chunk:  1238\n",
      "current chunk:  1239\n",
      "current chunk:  1240\n",
      "current chunk:  1241\n",
      "current chunk:  1242\n",
      "current chunk:  1243\n",
      "current chunk:  1244\n",
      "current chunk:  1245\n",
      "current chunk:  1246\n",
      "current chunk:  1247\n",
      "current chunk:  1248\n",
      "current chunk:  1249\n",
      "current chunk:  1250\n",
      "current chunk:  1251\n",
      "current chunk:  1252\n",
      "current chunk:  1253\n",
      "current chunk:  1254\n",
      "current chunk:  1255\n",
      "current chunk:  1256\n",
      "current chunk:  1257\n",
      "current chunk:  1258\n",
      "current chunk:  1259\n",
      "current chunk:  1260\n",
      "current chunk:  1261\n",
      "current chunk:  1262\n",
      "current chunk:  1263\n",
      "current chunk:  1264\n",
      "current chunk:  1265\n",
      "current chunk:  1266\n",
      "current chunk:  1267\n",
      "current chunk:  1268\n",
      "current chunk:  1269\n",
      "current chunk:  1270\n",
      "current chunk:  1271\n",
      "current chunk:  1272\n",
      "current chunk:  1273\n",
      "current chunk:  1274\n",
      "current chunk:  1275\n",
      "current chunk:  1276\n",
      "current chunk:  1277\n",
      "current chunk:  1278\n",
      "current chunk:  1279\n",
      "current chunk:  1280\n",
      "current chunk:  1281\n",
      "current chunk:  1282\n",
      "current chunk:  1283\n",
      "current chunk:  1284\n",
      "current chunk:  1285\n",
      "current chunk:  1286\n",
      "current chunk:  1287\n",
      "current chunk:  1288\n",
      "current chunk:  1289\n",
      "current chunk:  1290\n",
      "current chunk:  1291\n",
      "current chunk:  1292\n",
      "current chunk:  1293\n",
      "current chunk:  1294\n",
      "current chunk:  1295\n",
      "current chunk:  1296\n",
      "current chunk:  1297\n",
      "current chunk:  1298\n",
      "current chunk:  1299\n",
      "current chunk:  1300\n",
      "current chunk:  1301\n",
      "current chunk:  1302\n",
      "current chunk:  1303\n",
      "current chunk:  1304\n",
      "current chunk:  1305\n",
      "current chunk:  1306\n",
      "current chunk:  1307\n",
      "current chunk:  1308\n",
      "current chunk:  1309\n",
      "current chunk:  1310\n",
      "current chunk:  1311\n",
      "current chunk:  1312\n",
      "current chunk:  1313\n",
      "current chunk:  1314\n",
      "current chunk:  1315\n",
      "current chunk:  1316\n",
      "current chunk:  1317\n",
      "current chunk:  1318\n",
      "current chunk:  1319\n",
      "current chunk:  1320\n",
      "current chunk:  1321\n",
      "current chunk:  1322\n",
      "current chunk:  1323\n",
      "current chunk:  1324\n",
      "current chunk:  1325\n",
      "current chunk:  1326\n",
      "current chunk:  1327\n",
      "current chunk:  1328\n",
      "current chunk:  1329\n",
      "current chunk:  1330\n",
      "current chunk:  1331\n",
      "current chunk:  1332\n",
      "current chunk:  1333\n",
      "current chunk:  1334\n",
      "current chunk:  1335\n",
      "current chunk:  1336\n",
      "current chunk:  1337\n",
      "current chunk:  1338\n",
      "current chunk:  1339\n",
      "current chunk:  1340\n",
      "current chunk:  1341\n",
      "current chunk:  1342\n",
      "current chunk:  1343\n",
      "current chunk:  1344\n",
      "current chunk:  1345\n",
      "current chunk:  1346\n",
      "current chunk:  1347\n",
      "current chunk:  1348\n",
      "current chunk:  1349\n",
      "current chunk:  1350\n",
      "current chunk:  1351\n",
      "current chunk:  1352\n",
      "current chunk:  1353\n",
      "current chunk:  1354\n",
      "current chunk:  1355\n",
      "current chunk:  1356\n",
      "current chunk:  1357\n",
      "current chunk:  1358\n",
      "current chunk:  1359\n",
      "current chunk:  1360\n",
      "current chunk:  1361\n",
      "current chunk:  1362\n",
      "current chunk:  1363\n",
      "current chunk:  1364\n",
      "current chunk:  1365\n",
      "current chunk:  1366\n",
      "current chunk:  1367\n",
      "current chunk:  1368\n",
      "current chunk:  1369\n",
      "current chunk:  1370\n",
      "current chunk:  1371\n",
      "current chunk:  1372\n",
      "current chunk:  1373\n",
      "current chunk:  1374\n",
      "current chunk:  1375\n",
      "current chunk:  1376\n",
      "current chunk:  1377\n",
      "current chunk:  1378\n",
      "current chunk:  1379\n",
      "current chunk:  1380\n",
      "current chunk:  1381\n",
      "current chunk:  1382\n",
      "current chunk:  1383\n",
      "current chunk:  1384\n",
      "current chunk:  1385\n",
      "current chunk:  1386\n",
      "current chunk:  1387\n",
      "current chunk:  1388\n",
      "current chunk:  1389\n",
      "current chunk:  1390\n",
      "current chunk:  1391\n",
      "current chunk:  1392\n",
      "current chunk:  1393\n",
      "current chunk:  1394\n",
      "current chunk:  1395\n",
      "current chunk:  1396\n",
      "current chunk:  1397\n",
      "current chunk:  1398\n",
      "current chunk:  1399\n",
      "current chunk:  1400\n",
      "current chunk:  1401\n",
      "current chunk:  1402\n",
      "current chunk:  1403\n",
      "current chunk:  1404\n",
      "current chunk:  1405\n",
      "current chunk:  1406\n",
      "current chunk:  1407\n",
      "current chunk:  1408\n",
      "current chunk:  1409\n",
      "current chunk:  1410\n",
      "current chunk:  1411\n",
      "current chunk:  1412\n",
      "current chunk:  1413\n",
      "current chunk:  1414\n",
      "current chunk:  1415\n",
      "current chunk:  1416\n",
      "current chunk:  1417\n",
      "current chunk:  1418\n",
      "current chunk:  1419\n",
      "current chunk:  1420\n",
      "current chunk:  1421\n",
      "current chunk:  1422\n",
      "current chunk:  1423\n",
      "current chunk:  1424\n",
      "current chunk:  1425\n",
      "current chunk:  1426\n",
      "current chunk:  1427\n",
      "current chunk:  1428\n",
      "current chunk:  1429\n",
      "current chunk:  1430\n",
      "current chunk:  1431\n",
      "current chunk:  1432\n",
      "current chunk:  1433\n",
      "current chunk:  1434\n",
      "current chunk:  1435\n",
      "current chunk:  1436\n",
      "current chunk:  1437\n",
      "current chunk:  1438\n",
      "current chunk:  1439\n",
      "current chunk:  1440\n",
      "current chunk:  1441\n",
      "current chunk:  1442\n",
      "current chunk:  1443\n",
      "current chunk:  1444\n",
      "current chunk:  1445\n",
      "current chunk:  1446\n",
      "current chunk:  1447\n",
      "current chunk:  1448\n",
      "current chunk:  1449\n",
      "current chunk:  1450\n",
      "current chunk:  1451\n",
      "current chunk:  1452\n",
      "current chunk:  1453\n",
      "current chunk:  1454\n",
      "current chunk:  1455\n",
      "current chunk:  1456\n",
      "current chunk:  1457\n",
      "current chunk:  1458\n",
      "current chunk:  1459\n",
      "current chunk:  1460\n",
      "current chunk:  1461\n",
      "current chunk:  1462\n",
      "current chunk:  1463\n",
      "current chunk:  1464\n",
      "current chunk:  1465\n",
      "current chunk:  1466\n",
      "current chunk:  1467\n",
      "current chunk:  1468\n",
      "current chunk:  1469\n",
      "current chunk:  1470\n",
      "current chunk:  1471\n",
      "current chunk:  1472\n",
      "current chunk:  1473\n",
      "current chunk:  1474\n",
      "current chunk:  1475\n",
      "current chunk:  1476\n",
      "current chunk:  1477\n",
      "current chunk:  1478\n",
      "current chunk:  1479\n",
      "current chunk:  1480\n",
      "current chunk:  1481\n",
      "current chunk:  1482\n",
      "current chunk:  1483\n",
      "current chunk:  1484\n",
      "current chunk:  1485\n",
      "current chunk:  1486\n",
      "current chunk:  1487\n",
      "current chunk:  1488\n",
      "current chunk:  1489\n",
      "current chunk:  1490\n",
      "current chunk:  1491\n",
      "current chunk:  1492\n",
      "current chunk:  1493\n",
      "current chunk:  1494\n",
      "current chunk:  1495\n",
      "current chunk:  1496\n",
      "current chunk:  1497\n",
      "current chunk:  1498\n",
      "current chunk:  1499\n",
      "current chunk:  1500\n",
      "current chunk:  1501\n",
      "current chunk:  1502\n",
      "current chunk:  1503\n",
      "current chunk:  1504\n",
      "current chunk:  1505\n",
      "current chunk:  1506\n",
      "current chunk:  1507\n",
      "current chunk:  1508\n",
      "current chunk:  1509\n",
      "current chunk:  1510\n",
      "current chunk:  1511\n",
      "current chunk:  1512\n",
      "current chunk:  1513\n",
      "current chunk:  1514\n",
      "current chunk:  1515\n",
      "current chunk:  1516\n",
      "current chunk:  1517\n",
      "current chunk:  1518\n",
      "current chunk:  1519\n",
      "current chunk:  1520\n",
      "current chunk:  1521\n",
      "current chunk:  1522\n",
      "current chunk:  1523\n",
      "current chunk:  1524\n",
      "current chunk:  1525\n",
      "current chunk:  1526\n",
      "current chunk:  1527\n",
      "current chunk:  1528\n",
      "current chunk:  1529\n",
      "current chunk:  1530\n",
      "current chunk:  1531\n",
      "current chunk:  1532\n",
      "current chunk:  1533\n",
      "current chunk:  1534\n",
      "current chunk:  1535\n",
      "current chunk:  1536\n",
      "current chunk:  1537\n",
      "current chunk:  1538\n",
      "current chunk:  1539\n",
      "current chunk:  1540\n",
      "current chunk:  1541\n",
      "current chunk:  1542\n",
      "current chunk:  1543\n",
      "current chunk:  1544\n",
      "current chunk:  1545\n",
      "current chunk:  1546\n",
      "current chunk:  1547\n",
      "current chunk:  1548\n",
      "current chunk:  1549\n",
      "current chunk:  1550\n",
      "current chunk:  1551\n",
      "current chunk:  1552\n",
      "current chunk:  1553\n",
      "current chunk:  1554\n",
      "current chunk:  1555\n",
      "current chunk:  1556\n",
      "current chunk:  1557\n",
      "current chunk:  1558\n",
      "current chunk:  1559\n",
      "current chunk:  1560\n",
      "current chunk:  1561\n",
      "current chunk:  1562\n",
      "current chunk:  1563\n",
      "current chunk:  1564\n",
      "current chunk:  1565\n",
      "current chunk:  1566\n",
      "current chunk:  1567\n",
      "current chunk:  1568\n",
      "current chunk:  1569\n",
      "current chunk:  1570\n",
      "current chunk:  1571\n",
      "current chunk:  1572\n",
      "current chunk:  1573\n",
      "current chunk:  1574\n",
      "current chunk:  1575\n",
      "current chunk:  1576\n",
      "current chunk:  1577\n",
      "current chunk:  1578\n",
      "current chunk:  1579\n",
      "current chunk:  1580\n",
      "current chunk:  1581\n",
      "current chunk:  1582\n",
      "current chunk:  1583\n",
      "current chunk:  1584\n",
      "current chunk:  1585\n",
      "current chunk:  1586\n",
      "current chunk:  1587\n",
      "current chunk:  1588\n",
      "current chunk:  1589\n",
      "current chunk:  1590\n",
      "current chunk:  1591\n",
      "current chunk:  1592\n",
      "current chunk:  1593\n",
      "current chunk:  1594\n",
      "current chunk:  1595\n",
      "current chunk:  1596\n",
      "current chunk:  1597\n",
      "current chunk:  1598\n",
      "current chunk:  1599\n",
      "current chunk:  1600\n",
      "current chunk:  1601\n",
      "current chunk:  1602\n",
      "current chunk:  1603\n",
      "current chunk:  1604\n",
      "current chunk:  1605\n",
      "current chunk:  1606\n",
      "current chunk:  1607\n",
      "current chunk:  1608\n",
      "current chunk:  1609\n",
      "current chunk:  1610\n",
      "current chunk:  1611\n",
      "current chunk:  1612\n",
      "current chunk:  1613\n",
      "current chunk:  1614\n",
      "current chunk:  1615\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "current chunk:  1616\n",
      "current chunk:  1617\n",
      "current chunk:  1618\n",
      "current chunk:  1619\n",
      "current chunk:  1620\n",
      "current chunk:  1621\n",
      "current chunk:  1622\n",
      "current chunk:  1623\n",
      "current chunk:  1624\n",
      "current chunk:  1625\n",
      "current chunk:  1626\n",
      "current chunk:  1627\n",
      "current chunk:  1628\n",
      "current chunk:  1629\n",
      "current chunk:  1630\n",
      "current chunk:  1631\n",
      "current chunk:  1632\n",
      "current chunk:  1633\n",
      "current chunk:  1634\n",
      "current chunk:  1635\n",
      "current chunk:  1636\n",
      "current chunk:  1637\n",
      "current chunk:  1638\n",
      "current chunk:  1639\n",
      "current chunk:  1640\n",
      "current chunk:  1641\n",
      "current chunk:  1642\n",
      "current chunk:  1643\n",
      "current chunk:  1644\n",
      "current chunk:  1645\n",
      "current chunk:  1646\n",
      "current chunk:  1647\n",
      "current chunk:  1648\n",
      "current chunk:  1649\n",
      "current chunk:  1650\n",
      "current chunk:  1651\n",
      "current chunk:  1652\n",
      "current chunk:  1653\n",
      "current chunk:  1654\n",
      "current chunk:  1655\n",
      "current chunk:  1656\n",
      "current chunk:  1657\n",
      "current chunk:  1658\n",
      "current chunk:  1659\n",
      "current chunk:  1660\n",
      "current chunk:  1661\n",
      "current chunk:  1662\n",
      "current chunk:  1663\n",
      "current chunk:  1664\n",
      "current chunk:  1665\n",
      "current chunk:  1666\n",
      "current chunk:  1667\n",
      "current chunk:  1668\n",
      "current chunk:  1669\n",
      "current chunk:  1670\n",
      "current chunk:  1671\n",
      "current chunk:  1672\n",
      "current chunk:  1673\n",
      "current chunk:  1674\n",
      "current chunk:  1675\n",
      "current chunk:  1676\n",
      "current chunk:  1677\n",
      "current chunk:  1678\n",
      "current chunk:  1679\n",
      "current chunk:  1680\n",
      "current chunk:  1681\n",
      "current chunk:  1682\n",
      "current chunk:  1683\n",
      "current chunk:  1684\n",
      "current chunk:  1685\n",
      "current chunk:  1686\n",
      "current chunk:  1687\n",
      "current chunk:  1688\n",
      "current chunk:  1689\n",
      "current chunk:  1690\n",
      "current chunk:  1691\n",
      "current chunk:  1692\n",
      "current chunk:  1693\n",
      "current chunk:  1694\n",
      "current chunk:  1695\n",
      "current chunk:  1696\n",
      "current chunk:  1697\n",
      "current chunk:  1698\n",
      "current chunk:  1699\n",
      "current chunk:  1700\n",
      "current chunk:  1701\n",
      "current chunk:  1702\n",
      "current chunk:  1703\n",
      "current chunk:  1704\n",
      "current chunk:  1705\n",
      "current chunk:  1706\n",
      "current chunk:  1707\n",
      "current chunk:  1708\n",
      "current chunk:  1709\n",
      "current chunk:  1710\n",
      "current chunk:  1711\n",
      "current chunk:  1712\n",
      "current chunk:  1713\n",
      "current chunk:  1714\n",
      "current chunk:  1715\n",
      "current chunk:  1716\n",
      "current chunk:  1717\n",
      "current chunk:  1718\n",
      "current chunk:  1719\n",
      "current chunk:  1720\n",
      "current chunk:  1721\n",
      "current chunk:  1722\n",
      "current chunk:  1723\n",
      "current chunk:  1724\n",
      "current chunk:  1725\n",
      "current chunk:  1726\n",
      "current chunk:  1727\n",
      "current chunk:  1728\n",
      "current chunk:  1729\n",
      "current chunk:  1730\n",
      "current chunk:  1731\n",
      "current chunk:  1732\n",
      "current chunk:  1733\n",
      "current chunk:  1734\n",
      "current chunk:  1735\n",
      "current chunk:  1736\n",
      "current chunk:  1737\n",
      "current chunk:  1738\n",
      "current chunk:  1739\n",
      "current chunk:  1740\n",
      "current chunk:  1741\n",
      "current chunk:  1742\n",
      "current chunk:  1743\n",
      "current chunk:  1744\n",
      "current chunk:  1745\n",
      "current chunk:  1746\n",
      "current chunk:  1747\n",
      "current chunk:  1748\n",
      "current chunk:  1749\n",
      "current chunk:  1750\n",
      "current chunk:  1751\n",
      "current chunk:  1752\n",
      "current chunk:  1753\n",
      "current chunk:  1754\n",
      "current chunk:  1755\n",
      "current chunk:  1756\n",
      "current chunk:  1757\n",
      "current chunk:  1758\n",
      "current chunk:  1759\n",
      "current chunk:  1760\n",
      "current chunk:  1761\n",
      "current chunk:  1762\n",
      "current chunk:  1763\n",
      "current chunk:  1764\n",
      "current chunk:  1765\n",
      "current chunk:  1766\n",
      "current chunk:  1767\n",
      "current chunk:  1768\n",
      "current chunk:  1769\n",
      "current chunk:  1770\n",
      "current chunk:  1771\n",
      "current chunk:  1772\n",
      "current chunk:  1773\n",
      "current chunk:  1774\n",
      "current chunk:  1775\n",
      "current chunk:  1776\n",
      "current chunk:  1777\n",
      "current chunk:  1778\n",
      "current chunk:  1779\n",
      "current chunk:  1780\n",
      "current chunk:  1781\n",
      "current chunk:  1782\n",
      "current chunk:  1783\n",
      "current chunk:  1784\n",
      "current chunk:  1785\n",
      "current chunk:  1786\n",
      "current chunk:  1787\n",
      "current chunk:  1788\n",
      "current chunk:  1789\n",
      "current chunk:  1790\n",
      "current chunk:  1791\n",
      "current chunk:  1792\n",
      "current chunk:  1793\n",
      "current chunk:  1794\n",
      "current chunk:  1795\n",
      "current chunk:  1796\n",
      "current chunk:  1797\n",
      "current chunk:  1798\n",
      "current chunk:  1799\n",
      "current chunk:  1800\n",
      "current chunk:  1801\n",
      "current chunk:  1802\n",
      "current chunk:  1803\n",
      "current chunk:  1804\n",
      "current chunk:  1805\n",
      "current chunk:  1806\n",
      "current chunk:  1807\n",
      "current chunk:  1808\n",
      "current chunk:  1809\n",
      "current chunk:  1810\n",
      "current chunk:  1811\n",
      "current chunk:  1812\n",
      "current chunk:  1813\n",
      "current chunk:  1814\n",
      "current chunk:  1815\n",
      "current chunk:  1816\n",
      "current chunk:  1817\n",
      "current chunk:  1818\n",
      "current chunk:  1819\n",
      "current chunk:  1820\n",
      "current chunk:  1821\n",
      "current chunk:  1822\n",
      "current chunk:  1823\n",
      "current chunk:  1824\n",
      "current chunk:  1825\n",
      "current chunk:  1826\n",
      "current chunk:  1827\n",
      "current chunk:  1828\n",
      "current chunk:  1829\n",
      "current chunk:  1830\n",
      "current chunk:  1831\n",
      "current chunk:  1832\n",
      "current chunk:  1833\n",
      "current chunk:  1834\n",
      "current chunk:  1835\n",
      "current chunk:  1836\n",
      "current chunk:  1837\n",
      "current chunk:  1838\n",
      "current chunk:  1839\n",
      "current chunk:  1840\n",
      "current chunk:  1841\n",
      "current chunk:  1842\n",
      "current chunk:  1843\n",
      "current chunk:  1844\n",
      "current chunk:  1845\n",
      "current chunk:  1846\n",
      "current chunk:  1847\n",
      "current chunk:  1848\n",
      "current chunk:  1849\n",
      "current chunk:  1850\n",
      "current chunk:  1851\n",
      "current chunk:  1852\n",
      "current chunk:  1853\n",
      "current chunk:  1854\n",
      "current chunk:  1855\n",
      "current chunk:  1856\n",
      "current chunk:  1857\n",
      "current chunk:  1858\n",
      "current chunk:  1859\n",
      "current chunk:  1860\n",
      "current chunk:  1861\n",
      "current chunk:  1862\n",
      "current chunk:  1863\n",
      "current chunk:  1864\n",
      "current chunk:  1865\n",
      "current chunk:  1866\n",
      "current chunk:  1867\n",
      "current chunk:  1868\n",
      "current chunk:  1869\n",
      "current chunk:  1870\n",
      "current chunk:  1871\n",
      "current chunk:  1872\n",
      "current chunk:  1873\n",
      "current chunk:  1874\n",
      "current chunk:  1875\n",
      "current chunk:  1876\n",
      "current chunk:  1877\n",
      "current chunk:  1878\n",
      "current chunk:  1879\n",
      "current chunk:  1880\n",
      "current chunk:  1881\n",
      "current chunk:  1882\n",
      "current chunk:  1883\n",
      "current chunk:  1884\n",
      "current chunk:  1885\n",
      "current chunk:  1886\n",
      "current chunk:  1887\n",
      "current chunk:  1888\n",
      "current chunk:  1889\n",
      "current chunk:  1890\n",
      "current chunk:  1891\n",
      "current chunk:  1892\n",
      "current chunk:  1893\n",
      "current chunk:  1894\n",
      "current chunk:  1895\n",
      "current chunk:  1896\n",
      "current chunk:  1897\n",
      "current chunk:  1898\n",
      "current chunk:  1899\n",
      "current chunk:  1900\n",
      "current chunk:  1901\n",
      "current chunk:  1902\n",
      "current chunk:  1903\n",
      "current chunk:  1904\n",
      "current chunk:  1905\n",
      "current chunk:  1906\n",
      "current chunk:  1907\n",
      "current chunk:  1908\n",
      "current chunk:  1909\n",
      "current chunk:  1910\n",
      "current chunk:  1911\n",
      "current chunk:  1912\n",
      "current chunk:  1913\n",
      "current chunk:  1914\n",
      "current chunk:  1915\n",
      "current chunk:  1916\n",
      "current chunk:  1917\n",
      "current chunk:  1918\n",
      "current chunk:  1919\n",
      "current chunk:  1920\n",
      "current chunk:  1921\n",
      "current chunk:  1922\n",
      "current chunk:  1923\n",
      "current chunk:  1924\n",
      "current chunk:  1925\n",
      "current chunk:  1926\n",
      "current chunk:  1927\n",
      "current chunk:  1928\n",
      "current chunk:  1929\n",
      "current chunk:  1930\n",
      "current chunk:  1931\n",
      "current chunk:  1932\n",
      "current chunk:  1933\n",
      "current chunk:  1934\n",
      "current chunk:  1935\n",
      "current chunk:  1936\n",
      "current chunk:  1937\n",
      "current chunk:  1938\n",
      "current chunk:  1939\n",
      "current chunk:  1940\n",
      "current chunk:  1941\n",
      "current chunk:  1942\n",
      "current chunk:  1943\n",
      "current chunk:  1944\n",
      "current chunk:  1945\n",
      "current chunk:  1946\n",
      "current chunk:  1947\n",
      "current chunk:  1948\n",
      "current chunk:  1949\n",
      "current chunk:  1950\n",
      "current chunk:  1951\n",
      "current chunk:  1952\n",
      "current chunk:  1953\n",
      "current chunk:  1954\n",
      "current chunk:  1955\n",
      "current chunk:  1956\n",
      "current chunk:  1957\n",
      "current chunk:  1958\n",
      "current chunk:  1959\n",
      "current chunk:  1960\n",
      "current chunk:  1961\n",
      "current chunk:  1962\n",
      "current chunk:  1963\n",
      "current chunk:  1964\n",
      "current chunk:  1965\n",
      "current chunk:  1966\n",
      "current chunk:  1967\n",
      "current chunk:  1968\n",
      "current chunk:  1969\n",
      "current chunk:  1970\n",
      "current chunk:  1971\n",
      "current chunk:  1972\n",
      "current chunk:  1973\n",
      "current chunk:  1974\n",
      "current chunk:  1975\n",
      "current chunk:  1976\n",
      "current chunk:  1977\n",
      "current chunk:  1978\n",
      "current chunk:  1979\n",
      "current chunk:  1980\n",
      "current chunk:  1981\n",
      "current chunk:  1982\n",
      "current chunk:  1983\n",
      "current chunk:  1984\n",
      "current chunk:  1985\n",
      "current chunk:  1986\n",
      "current chunk:  1987\n",
      "current chunk:  1988\n",
      "current chunk:  1989\n",
      "current chunk:  1990\n",
      "current chunk:  1991\n",
      "current chunk:  1992\n",
      "current chunk:  1993\n",
      "current chunk:  1994\n",
      "current chunk:  1995\n",
      "current chunk:  1996\n",
      "current chunk:  1997\n",
      "current chunk:  1998\n",
      "current chunk:  1999\n",
      "current chunk:  2000\n",
      "current chunk:  2001\n",
      "current chunk:  2002\n",
      "current chunk:  2003\n",
      "current chunk:  2004\n",
      "current chunk:  2005\n",
      "current chunk:  2006\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "current chunk:  2007\n",
      "current chunk:  2008\n",
      "current chunk:  2009\n",
      "current chunk:  2010\n",
      "current chunk:  2011\n",
      "current chunk:  2012\n",
      "current chunk:  2013\n",
      "current chunk:  2014\n",
      "current chunk:  2015\n",
      "current chunk:  2016\n",
      "current chunk:  2017\n",
      "current chunk:  2018\n",
      "current chunk:  2019\n",
      "current chunk:  2020\n",
      "current chunk:  2021\n",
      "current chunk:  2022\n",
      "current chunk:  2023\n",
      "current chunk:  2024\n",
      "current chunk:  2025\n",
      "current chunk:  2026\n",
      "current chunk:  2027\n",
      "current chunk:  2028\n",
      "current chunk:  2029\n",
      "current chunk:  2030\n",
      "current chunk:  2031\n",
      "current chunk:  2032\n",
      "current chunk:  2033\n",
      "current chunk:  2034\n",
      "current chunk:  2035\n",
      "current chunk:  2036\n",
      "current chunk:  2037\n",
      "current chunk:  2038\n",
      "current chunk:  2039\n",
      "current chunk:  2040\n",
      "current chunk:  2041\n",
      "current chunk:  2042\n",
      "current chunk:  2043\n",
      "current chunk:  2044\n",
      "current chunk:  2045\n",
      "current chunk:  2046\n",
      "current chunk:  2047\n",
      "current chunk:  2048\n",
      "current chunk:  2049\n",
      "current chunk:  2050\n",
      "current chunk:  2051\n",
      "current chunk:  2052\n",
      "current chunk:  2053\n",
      "current chunk:  2054\n",
      "current chunk:  2055\n",
      "current chunk:  2056\n",
      "current chunk:  2057\n",
      "current chunk:  2058\n",
      "current chunk:  2059\n",
      "current chunk:  2060\n",
      "current chunk:  2061\n",
      "current chunk:  2062\n",
      "current chunk:  2063\n",
      "current chunk:  2064\n",
      "current chunk:  2065\n",
      "current chunk:  2066\n",
      "current chunk:  2067\n",
      "current chunk:  2068\n",
      "current chunk:  2069\n",
      "current chunk:  2070\n",
      "current chunk:  2071\n",
      "current chunk:  2072\n",
      "current chunk:  2073\n",
      "current chunk:  2074\n",
      "current chunk:  2075\n",
      "current chunk:  2076\n",
      "current chunk:  2077\n",
      "current chunk:  2078\n",
      "current chunk:  2079\n",
      "current chunk:  2080\n",
      "current chunk:  2081\n",
      "current chunk:  2082\n",
      "current chunk:  2083\n",
      "current chunk:  2084\n",
      "current chunk:  2085\n",
      "current chunk:  2086\n",
      "current chunk:  2087\n",
      "current chunk:  2088\n",
      "current chunk:  2089\n",
      "current chunk:  2090\n",
      "current chunk:  2091\n",
      "current chunk:  2092\n",
      "current chunk:  2093\n",
      "current chunk:  2094\n",
      "current chunk:  2095\n",
      "current chunk:  2096\n",
      "current chunk:  2097\n",
      "current chunk:  2098\n",
      "current chunk:  2099\n",
      "current chunk:  2100\n",
      "current chunk:  2101\n",
      "current chunk:  2102\n",
      "current chunk:  2103\n",
      "current chunk:  2104\n",
      "current chunk:  2105\n",
      "current chunk:  2106\n",
      "current chunk:  2107\n",
      "current chunk:  2108\n",
      "current chunk:  2109\n",
      "current chunk:  2110\n",
      "current chunk:  2111\n",
      "current chunk:  2112\n",
      "current chunk:  2113\n",
      "current chunk:  2114\n",
      "current chunk:  2115\n",
      "current chunk:  2116\n",
      "current chunk:  2117\n",
      "current chunk:  2118\n",
      "current chunk:  2119\n",
      "current chunk:  2120\n",
      "current chunk:  2121\n",
      "current chunk:  2122\n",
      "current chunk:  2123\n",
      "current chunk:  2124\n",
      "current chunk:  2125\n",
      "current chunk:  2126\n",
      "current chunk:  2127\n",
      "current chunk:  2128\n",
      "current chunk:  2129\n",
      "current chunk:  2130\n",
      "current chunk:  2131\n",
      "current chunk:  2132\n",
      "current chunk:  2133\n",
      "current chunk:  2134\n",
      "current chunk:  2135\n",
      "current chunk:  2136\n",
      "current chunk:  2137\n",
      "current chunk:  2138\n",
      "current chunk:  2139\n",
      "current chunk:  2140\n",
      "current chunk:  2141\n",
      "current chunk:  2142\n",
      "current chunk:  2143\n",
      "current chunk:  2144\n",
      "current chunk:  2145\n",
      "current chunk:  2146\n",
      "current chunk:  2147\n",
      "current chunk:  2148\n",
      "current chunk:  2149\n",
      "current chunk:  2150\n",
      "current chunk:  2151\n",
      "current chunk:  2152\n",
      "current chunk:  2153\n",
      "current chunk:  2154\n",
      "current chunk:  2155\n",
      "current chunk:  2156\n",
      "current chunk:  2157\n",
      "current chunk:  2158\n",
      "current chunk:  2159\n",
      "current chunk:  2160\n",
      "current chunk:  2161\n",
      "current chunk:  2162\n",
      "current chunk:  2163\n",
      "current chunk:  2164\n",
      "current chunk:  2165\n",
      "current chunk:  2166\n",
      "current chunk:  2167\n",
      "current chunk:  2168\n",
      "current chunk:  2169\n",
      "current chunk:  2170\n",
      "current chunk:  2171\n",
      "current chunk:  2172\n",
      "current chunk:  2173\n",
      "current chunk:  2174\n",
      "current chunk:  2175\n",
      "current chunk:  2176\n",
      "current chunk:  2177\n",
      "current chunk:  2178\n",
      "current chunk:  2179\n",
      "current chunk:  2180\n",
      "current chunk:  2181\n",
      "current chunk:  2182\n",
      "current chunk:  2183\n",
      "current chunk:  2184\n",
      "current chunk:  2185\n",
      "current chunk:  2186\n",
      "current chunk:  2187\n",
      "current chunk:  2188\n",
      "current chunk:  2189\n",
      "current chunk:  2190\n",
      "current chunk:  2191\n",
      "current chunk:  2192\n",
      "current chunk:  2193\n",
      "current chunk:  2194\n",
      "current chunk:  2195\n",
      "current chunk:  2196\n",
      "current chunk:  2197\n",
      "current chunk:  2198\n",
      "current chunk:  2199\n",
      "current chunk:  2200\n",
      "current chunk:  2201\n",
      "current chunk:  2202\n",
      "current chunk:  2203\n",
      "current chunk:  2204\n",
      "current chunk:  2205\n",
      "current chunk:  2206\n",
      "current chunk:  2207\n",
      "current chunk:  2208\n",
      "current chunk:  2209\n",
      "current chunk:  2210\n",
      "current chunk:  2211\n",
      "current chunk:  2212\n",
      "current chunk:  2213\n",
      "current chunk:  2214\n",
      "current chunk:  2215\n",
      "current chunk:  2216\n",
      "current chunk:  2217\n",
      "current chunk:  2218\n",
      "current chunk:  2219\n",
      "current chunk:  2220\n",
      "current chunk:  2221\n",
      "current chunk:  2222\n",
      "current chunk:  2223\n",
      "current chunk:  2224\n",
      "current chunk:  2225\n",
      "current chunk:  2226\n",
      "current chunk:  2227\n",
      "current chunk:  2228\n",
      "current chunk:  2229\n",
      "current chunk:  2230\n",
      "current chunk:  2231\n",
      "current chunk:  2232\n",
      "current chunk:  2233\n",
      "current chunk:  2234\n",
      "current chunk:  2235\n",
      "current chunk:  2236\n",
      "current chunk:  2237\n",
      "current chunk:  2238\n",
      "current chunk:  2239\n",
      "current chunk:  2240\n",
      "current chunk:  2241\n",
      "current chunk:  2242\n",
      "current chunk:  2243\n",
      "current chunk:  2244\n",
      "current chunk:  2245\n",
      "current chunk:  2246\n",
      "current chunk:  2247\n",
      "current chunk:  2248\n",
      "current chunk:  2249\n",
      "current chunk:  2250\n",
      "current chunk:  2251\n",
      "current chunk:  2252\n",
      "current chunk:  2253\n",
      "current chunk:  2254\n",
      "current chunk:  2255\n",
      "current chunk:  2256\n",
      "current chunk:  2257\n",
      "current chunk:  2258\n",
      "current chunk:  2259\n",
      "current chunk:  2260\n",
      "current chunk:  2261\n",
      "current chunk:  2262\n",
      "current chunk:  2263\n",
      "current chunk:  2264\n",
      "current chunk:  2265\n",
      "current chunk:  2266\n",
      "current chunk:  2267\n",
      "current chunk:  2268\n",
      "current chunk:  2269\n",
      "current chunk:  2270\n",
      "current chunk:  2271\n",
      "current chunk:  2272\n",
      "current chunk:  2273\n",
      "current chunk:  2274\n",
      "current chunk:  2275\n",
      "current chunk:  2276\n",
      "current chunk:  2277\n",
      "current chunk:  2278\n",
      "current chunk:  2279\n",
      "current chunk:  2280\n",
      "current chunk:  2281\n",
      "current chunk:  2282\n",
      "current chunk:  2283\n",
      "current chunk:  2284\n",
      "current chunk:  2285\n",
      "current chunk:  2286\n",
      "current chunk:  2287\n",
      "current chunk:  2288\n",
      "current chunk:  2289\n",
      "current chunk:  2290\n",
      "current chunk:  2291\n",
      "current chunk:  2292\n",
      "current chunk:  2293\n",
      "current chunk:  2294\n",
      "current chunk:  2295\n",
      "current chunk:  2296\n",
      "current chunk:  2297\n",
      "current chunk:  2298\n",
      "current chunk:  2299\n",
      "current chunk:  2300\n",
      "current chunk:  2301\n",
      "current chunk:  2302\n",
      "current chunk:  2303\n",
      "current chunk:  2304\n",
      "current chunk:  2305\n",
      "current chunk:  2306\n",
      "current chunk:  2307\n",
      "current chunk:  2308\n",
      "current chunk:  2309\n",
      "current chunk:  2310\n",
      "current chunk:  2311\n",
      "current chunk:  2312\n",
      "current chunk:  2313\n",
      "current chunk:  2314\n",
      "current chunk:  2315\n",
      "current chunk:  2316\n",
      "current chunk:  2317\n",
      "current chunk:  2318\n",
      "current chunk:  2319\n",
      "current chunk:  2320\n",
      "current chunk:  2321\n",
      "current chunk:  2322\n",
      "current chunk:  2323\n",
      "current chunk:  2324\n",
      "current chunk:  2325\n",
      "current chunk:  2326\n",
      "current chunk:  2327\n",
      "current chunk:  2328\n",
      "current chunk:  2329\n",
      "current chunk:  2330\n",
      "current chunk:  2331\n",
      "current chunk:  2332\n",
      "current chunk:  2333\n",
      "current chunk:  2334\n",
      "current chunk:  2335\n",
      "current chunk:  2336\n",
      "current chunk:  2337\n",
      "current chunk:  2338\n",
      "current chunk:  2339\n",
      "current chunk:  2340\n",
      "current chunk:  2341\n",
      "current chunk:  2342\n",
      "current chunk:  2343\n",
      "current chunk:  2344\n",
      "current chunk:  2345\n",
      "current chunk:  2346\n",
      "current chunk:  2347\n",
      "current chunk:  2348\n",
      "current chunk:  2349\n",
      "current chunk:  2350\n",
      "current chunk:  2351\n",
      "current chunk:  2352\n",
      "current chunk:  2353\n",
      "current chunk:  2354\n",
      "current chunk:  2355\n",
      "current chunk:  2356\n",
      "current chunk:  2357\n",
      "current chunk:  2358\n",
      "current chunk:  2359\n",
      "current chunk:  2360\n",
      "current chunk:  2361\n",
      "current chunk:  2362\n",
      "current chunk:  2363\n",
      "current chunk:  2364\n",
      "current chunk:  2365\n",
      "current chunk:  2366\n",
      "current chunk:  2367\n",
      "current chunk:  2368\n",
      "current chunk:  2369\n",
      "current chunk:  2370\n",
      "current chunk:  2371\n",
      "current chunk:  2372\n",
      "current chunk:  2373\n",
      "current chunk:  2374\n",
      "current chunk:  2375\n",
      "current chunk:  2376\n",
      "current chunk:  2377\n",
      "current chunk:  2378\n",
      "current chunk:  2379\n",
      "current chunk:  2380\n",
      "current chunk:  2381\n",
      "current chunk:  2382\n",
      "current chunk:  2383\n",
      "current chunk:  2384\n",
      "current chunk:  2385\n",
      "current chunk:  2386\n",
      "current chunk:  2387\n",
      "current chunk:  2388\n",
      "current chunk:  2389\n",
      "current chunk:  2390\n",
      "current chunk:  2391\n",
      "current chunk:  2392\n",
      "current chunk:  2393\n",
      "current chunk:  2394\n",
      "current chunk:  2395\n",
      "current chunk:  2396\n",
      "current chunk:  2397\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "current chunk:  2398\n",
      "current chunk:  2399\n",
      "current chunk:  2400\n",
      "current chunk:  2401\n",
      "current chunk:  2402\n",
      "current chunk:  2403\n",
      "current chunk:  2404\n",
      "current chunk:  2405\n",
      "current chunk:  2406\n",
      "current chunk:  2407\n",
      "current chunk:  2408\n",
      "current chunk:  2409\n",
      "current chunk:  2410\n",
      "current chunk:  2411\n",
      "current chunk:  2412\n",
      "current chunk:  2413\n",
      "current chunk:  2414\n",
      "current chunk:  2415\n",
      "current chunk:  2416\n",
      "current chunk:  2417\n",
      "current chunk:  2418\n",
      "current chunk:  2419\n",
      "current chunk:  2420\n",
      "current chunk:  2421\n",
      "current chunk:  2422\n",
      "current chunk:  2423\n",
      "current chunk:  2424\n",
      "current chunk:  2425\n",
      "current chunk:  2426\n",
      "current chunk:  2427\n",
      "current chunk:  2428\n",
      "current chunk:  2429\n",
      "current chunk:  2430\n",
      "current chunk:  2431\n",
      "current chunk:  2432\n",
      "current chunk:  2433\n",
      "current chunk:  2434\n",
      "current chunk:  2435\n",
      "current chunk:  2436\n",
      "current chunk:  2437\n",
      "current chunk:  2438\n",
      "current chunk:  2439\n",
      "current chunk:  2440\n",
      "current chunk:  2441\n",
      "current chunk:  2442\n",
      "current chunk:  2443\n",
      "current chunk:  2444\n",
      "current chunk:  2445\n",
      "current chunk:  2446\n",
      "current chunk:  2447\n",
      "current chunk:  2448\n",
      "current chunk:  2449\n",
      "current chunk:  2450\n",
      "current chunk:  2451\n",
      "current chunk:  2452\n",
      "current chunk:  2453\n",
      "current chunk:  2454\n",
      "current chunk:  2455\n",
      "current chunk:  2456\n",
      "current chunk:  2457\n",
      "current chunk:  2458\n",
      "current chunk:  2459\n",
      "current chunk:  2460\n",
      "current chunk:  2461\n",
      "current chunk:  2462\n",
      "current chunk:  2463\n",
      "current chunk:  2464\n",
      "current chunk:  2465\n",
      "current chunk:  2466\n",
      "current chunk:  2467\n",
      "current chunk:  2468\n",
      "current chunk:  2469\n",
      "current chunk:  2470\n",
      "current chunk:  2471\n",
      "current chunk:  2472\n",
      "current chunk:  2473\n",
      "current chunk:  2474\n",
      "current chunk:  2475\n",
      "current chunk:  2476\n",
      "current chunk:  2477\n",
      "current chunk:  2478\n",
      "current chunk:  2479\n",
      "current chunk:  2480\n",
      "current chunk:  2481\n",
      "current chunk:  2482\n",
      "current chunk:  2483\n",
      "current chunk:  2484\n",
      "current chunk:  2485\n",
      "current chunk:  2486\n",
      "current chunk:  2487\n",
      "current chunk:  2488\n",
      "current chunk:  2489\n",
      "current chunk:  2490\n",
      "current chunk:  2491\n",
      "current chunk:  2492\n",
      "current chunk:  2493\n",
      "current chunk:  2494\n",
      "current chunk:  2495\n",
      "current chunk:  2496\n",
      "current chunk:  2497\n",
      "current chunk:  2498\n",
      "current chunk:  2499\n",
      "current chunk:  2500\n",
      "current chunk:  2501\n",
      "current chunk:  2502\n",
      "current chunk:  2503\n",
      "current chunk:  2504\n",
      "current chunk:  2505\n",
      "current chunk:  2506\n",
      "current chunk:  2507\n",
      "current chunk:  2508\n",
      "current chunk:  2509\n",
      "current chunk:  2510\n",
      "current chunk:  2511\n",
      "current chunk:  2512\n",
      "current chunk:  2513\n",
      "current chunk:  2514\n",
      "current chunk:  2515\n",
      "current chunk:  2516\n",
      "current chunk:  2517\n",
      "current chunk:  2518\n",
      "current chunk:  2519\n",
      "current chunk:  2520\n",
      "current chunk:  2521\n",
      "current chunk:  2522\n",
      "current chunk:  2523\n",
      "current chunk:  2524\n",
      "current chunk:  2525\n",
      "current chunk:  2526\n",
      "current chunk:  2527\n",
      "current chunk:  2528\n",
      "current chunk:  2529\n",
      "current chunk:  2530\n",
      "current chunk:  2531\n",
      "current chunk:  2532\n",
      "current chunk:  2533\n",
      "current chunk:  2534\n",
      "current chunk:  2535\n",
      "current chunk:  2536\n",
      "current chunk:  2537\n",
      "current chunk:  2538\n",
      "current chunk:  2539\n",
      "current chunk:  2540\n",
      "current chunk:  2541\n",
      "current chunk:  2542\n",
      "current chunk:  2543\n",
      "current chunk:  2544\n",
      "current chunk:  2545\n",
      "current chunk:  2546\n",
      "current chunk:  2547\n",
      "current chunk:  2548\n",
      "current chunk:  2549\n",
      "current chunk:  2550\n",
      "current chunk:  2551\n",
      "current chunk:  2552\n",
      "current chunk:  2553\n",
      "current chunk:  2554\n",
      "current chunk:  2555\n",
      "current chunk:  2556\n",
      "current chunk:  2557\n",
      "current chunk:  2558\n",
      "current chunk:  2559\n",
      "current chunk:  2560\n",
      "current chunk:  2561\n",
      "current chunk:  2562\n",
      "current chunk:  2563\n",
      "current chunk:  2564\n",
      "current chunk:  2565\n",
      "current chunk:  2566\n",
      "current chunk:  2567\n",
      "current chunk:  2568\n",
      "current chunk:  2569\n",
      "current chunk:  2570\n",
      "current chunk:  2571\n",
      "current chunk:  2572\n",
      "current chunk:  2573\n",
      "current chunk:  2574\n",
      "current chunk:  2575\n",
      "current chunk:  2576\n",
      "current chunk:  2577\n",
      "current chunk:  2578\n",
      "current chunk:  2579\n",
      "current chunk:  2580\n",
      "current chunk:  2581\n",
      "current chunk:  2582\n",
      "current chunk:  2583\n",
      "current chunk:  2584\n",
      "current chunk:  2585\n",
      "current chunk:  2586\n",
      "current chunk:  2587\n",
      "current chunk:  2588\n",
      "current chunk:  2589\n",
      "current chunk:  2590\n",
      "current chunk:  2591\n",
      "current chunk:  2592\n",
      "current chunk:  2593\n",
      "current chunk:  2594\n",
      "current chunk:  2595\n",
      "current chunk:  2596\n",
      "current chunk:  2597\n",
      "current chunk:  2598\n",
      "current chunk:  2599\n",
      "current chunk:  2600\n",
      "current chunk:  2601\n",
      "current chunk:  2602\n",
      "current chunk:  2603\n",
      "current chunk:  2604\n",
      "current chunk:  2605\n",
      "current chunk:  2606\n",
      "current chunk:  2607\n",
      "current chunk:  2608\n",
      "current chunk:  2609\n",
      "current chunk:  2610\n",
      "current chunk:  2611\n",
      "current chunk:  2612\n",
      "current chunk:  2613\n",
      "current chunk:  2614\n",
      "current chunk:  2615\n",
      "current chunk:  2616\n",
      "current chunk:  2617\n",
      "current chunk:  2618\n",
      "current chunk:  2619\n",
      "current chunk:  2620\n",
      "current chunk:  2621\n",
      "current chunk:  2622\n",
      "current chunk:  2623\n",
      "current chunk:  2624\n",
      "current chunk:  2625\n",
      "current chunk:  2626\n",
      "current chunk:  2627\n",
      "current chunk:  2628\n",
      "current chunk:  2629\n",
      "current chunk:  2630\n",
      "current chunk:  2631\n",
      "current chunk:  2632\n",
      "current chunk:  2633\n",
      "current chunk:  2634\n",
      "current chunk:  2635\n",
      "current chunk:  2636\n",
      "current chunk:  2637\n",
      "current chunk:  2638\n",
      "current chunk:  2639\n",
      "current chunk:  2640\n",
      "current chunk:  2641\n",
      "current chunk:  2642\n",
      "current chunk:  2643\n",
      "current chunk:  2644\n",
      "current chunk:  2645\n",
      "current chunk:  2646\n",
      "current chunk:  2647\n",
      "current chunk:  2648\n",
      "current chunk:  2649\n",
      "current chunk:  2650\n",
      "current chunk:  2651\n",
      "current chunk:  2652\n",
      "current chunk:  2653\n",
      "current chunk:  2654\n",
      "current chunk:  2655\n",
      "current chunk:  2656\n",
      "current chunk:  2657\n",
      "current chunk:  2658\n",
      "current chunk:  2659\n",
      "current chunk:  2660\n",
      "current chunk:  2661\n",
      "current chunk:  2662\n",
      "current chunk:  2663\n",
      "current chunk:  2664\n",
      "current chunk:  2665\n",
      "current chunk:  2666\n",
      "current chunk:  2667\n",
      "current chunk:  2668\n",
      "current chunk:  2669\n",
      "current chunk:  2670\n",
      "current chunk:  2671\n",
      "current chunk:  2672\n",
      "current chunk:  2673\n",
      "current chunk:  2674\n",
      "current chunk:  2675\n",
      "current chunk:  2676\n",
      "current chunk:  2677\n",
      "current chunk:  2678\n",
      "current chunk:  2679\n",
      "current chunk:  2680\n",
      "current chunk:  2681\n",
      "current chunk:  2682\n",
      "current chunk:  2683\n",
      "current chunk:  2684\n",
      "current chunk:  2685\n",
      "current chunk:  2686\n",
      "current chunk:  2687\n",
      "current chunk:  2688\n",
      "current chunk:  2689\n",
      "current chunk:  2690\n",
      "current chunk:  2691\n",
      "current chunk:  2692\n",
      "current chunk:  2693\n",
      "current chunk:  2694\n",
      "current chunk:  2695\n",
      "current chunk:  2696\n",
      "current chunk:  2697\n",
      "current chunk:  2698\n",
      "current chunk:  2699\n",
      "current chunk:  2700\n",
      "current chunk:  2701\n",
      "current chunk:  2702\n",
      "current chunk:  2703\n",
      "current chunk:  2704\n",
      "current chunk:  2705\n",
      "current chunk:  2706\n",
      "current chunk:  2707\n",
      "current chunk:  2708\n",
      "current chunk:  2709\n",
      "current chunk:  2710\n",
      "current chunk:  2711\n",
      "current chunk:  2712\n",
      "current chunk:  2713\n",
      "current chunk:  2714\n",
      "current chunk:  2715\n",
      "current chunk:  2716\n",
      "current chunk:  2717\n",
      "current chunk:  2718\n",
      "current chunk:  2719\n",
      "current chunk:  2720\n",
      "current chunk:  2721\n",
      "current chunk:  2722\n",
      "current chunk:  2723\n",
      "current chunk:  2724\n",
      "current chunk:  2725\n",
      "current chunk:  2726\n",
      "current chunk:  2727\n",
      "current chunk:  2728\n",
      "current chunk:  2729\n",
      "current chunk:  2730\n",
      "current chunk:  2731\n",
      "current chunk:  2732\n",
      "current chunk:  2733\n",
      "current chunk:  2734\n",
      "current chunk:  2735\n",
      "current chunk:  2736\n",
      "current chunk:  2737\n",
      "current chunk:  2738\n",
      "current chunk:  2739\n",
      "current chunk:  2740\n",
      "current chunk:  2741\n",
      "current chunk:  2742\n",
      "current chunk:  2743\n",
      "current chunk:  2744\n",
      "current chunk:  2745\n",
      "current chunk:  2746\n",
      "current chunk:  2747\n",
      "current chunk:  2748\n",
      "current chunk:  2749\n",
      "current chunk:  2750\n",
      "current chunk:  2751\n",
      "current chunk:  2752\n",
      "current chunk:  2753\n",
      "current chunk:  2754\n",
      "current chunk:  2755\n",
      "current chunk:  2756\n",
      "current chunk:  2757\n",
      "current chunk:  2758\n",
      "current chunk:  2759\n",
      "current chunk:  2760\n",
      "current chunk:  2761\n",
      "current chunk:  2762\n",
      "current chunk:  2763\n",
      "current chunk:  2764\n",
      "current chunk:  2765\n",
      "current chunk:  2766\n",
      "current chunk:  2767\n",
      "current chunk:  2768\n",
      "current chunk:  2769\n",
      "current chunk:  2770\n",
      "current chunk:  2771\n",
      "current chunk:  2772\n",
      "current chunk:  2773\n",
      "current chunk:  2774\n",
      "current chunk:  2775\n",
      "current chunk:  2776\n",
      "current chunk:  2777\n",
      "current chunk:  2778\n",
      "current chunk:  2779\n",
      "current chunk:  2780\n",
      "current chunk:  2781\n",
      "current chunk:  2782\n",
      "current chunk:  2783\n",
      "current chunk:  2784\n",
      "current chunk:  2785\n",
      "current chunk:  2786\n",
      "current chunk:  2787\n",
      "current chunk:  2788\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "current chunk:  2789\n",
      "current chunk:  2790\n",
      "current chunk:  2791\n",
      "current chunk:  2792\n",
      "current chunk:  2793\n",
      "current chunk:  2794\n",
      "current chunk:  2795\n",
      "current chunk:  2796\n",
      "current chunk:  2797\n",
      "current chunk:  2798\n",
      "current chunk:  2799\n",
      "current chunk:  2800\n",
      "current chunk:  2801\n",
      "current chunk:  2802\n",
      "current chunk:  2803\n",
      "current chunk:  2804\n",
      "current chunk:  2805\n",
      "current chunk:  2806\n",
      "current chunk:  2807\n",
      "current chunk:  2808\n",
      "current chunk:  2809\n",
      "current chunk:  2810\n",
      "current chunk:  2811\n",
      "current chunk:  2812\n",
      "current chunk:  2813\n",
      "current chunk:  2814\n",
      "current chunk:  2815\n",
      "current chunk:  2816\n",
      "current chunk:  2817\n",
      "current chunk:  2818\n",
      "current chunk:  2819\n",
      "current chunk:  2820\n",
      "current chunk:  2821\n",
      "current chunk:  2822\n",
      "current chunk:  2823\n",
      "current chunk:  2824\n",
      "current chunk:  2825\n",
      "current chunk:  2826\n",
      "current chunk:  2827\n",
      "current chunk:  2828\n",
      "current chunk:  2829\n",
      "current chunk:  2830\n",
      "current chunk:  2831\n",
      "current chunk:  2832\n",
      "current chunk:  2833\n",
      "current chunk:  2834\n",
      "current chunk:  2835\n",
      "current chunk:  2836\n",
      "current chunk:  2837\n",
      "current chunk:  2838\n",
      "current chunk:  2839\n",
      "current chunk:  2840\n",
      "current chunk:  2841\n",
      "current chunk:  2842\n",
      "current chunk:  2843\n",
      "current chunk:  2844\n",
      "current chunk:  2845\n",
      "current chunk:  2846\n",
      "current chunk:  2847\n",
      "current chunk:  2848\n",
      "current chunk:  2849\n",
      "current chunk:  2850\n",
      "current chunk:  2851\n",
      "current chunk:  2852\n",
      "current chunk:  2853\n",
      "current chunk:  2854\n",
      "current chunk:  2855\n",
      "current chunk:  2856\n",
      "current chunk:  2857\n",
      "current chunk:  2858\n",
      "current chunk:  2859\n",
      "current chunk:  2860\n",
      "current chunk:  2861\n",
      "current chunk:  2862\n",
      "current chunk:  2863\n",
      "current chunk:  2864\n",
      "current chunk:  2865\n",
      "current chunk:  2866\n",
      "current chunk:  2867\n",
      "current chunk:  2868\n",
      "current chunk:  2869\n",
      "current chunk:  2870\n",
      "current chunk:  2871\n",
      "current chunk:  2872\n",
      "current chunk:  2873\n",
      "current chunk:  2874\n",
      "current chunk:  2875\n",
      "current chunk:  2876\n",
      "current chunk:  2877\n",
      "current chunk:  2878\n",
      "current chunk:  2879\n",
      "current chunk:  2880\n",
      "current chunk:  2881\n",
      "current chunk:  2882\n",
      "current chunk:  2883\n",
      "current chunk:  2884\n",
      "current chunk:  2885\n",
      "current chunk:  2886\n",
      "current chunk:  2887\n",
      "current chunk:  2888\n",
      "current chunk:  2889\n",
      "current chunk:  2890\n",
      "current chunk:  2891\n",
      "current chunk:  2892\n",
      "current chunk:  2893\n",
      "current chunk:  2894\n",
      "current chunk:  2895\n",
      "current chunk:  2896\n",
      "current chunk:  2897\n",
      "current chunk:  2898\n",
      "current chunk:  2899\n",
      "current chunk:  2900\n",
      "current chunk:  2901\n",
      "current chunk:  2902\n",
      "current chunk:  2903\n",
      "current chunk:  2904\n",
      "current chunk:  2905\n",
      "current chunk:  2906\n",
      "current chunk:  2907\n",
      "current chunk:  2908\n",
      "current chunk:  2909\n",
      "current chunk:  2910\n",
      "current chunk:  2911\n",
      "current chunk:  2912\n",
      "current chunk:  2913\n",
      "current chunk:  2914\n",
      "current chunk:  2915\n",
      "current chunk:  2916\n",
      "current chunk:  2917\n",
      "current chunk:  2918\n",
      "current chunk:  2919\n",
      "current chunk:  2920\n",
      "current chunk:  2921\n",
      "current chunk:  2922\n",
      "current chunk:  2923\n",
      "current chunk:  2924\n",
      "current chunk:  2925\n",
      "current chunk:  2926\n",
      "current chunk:  2927\n",
      "current chunk:  2928\n",
      "current chunk:  2929\n",
      "current chunk:  2930\n",
      "current chunk:  2931\n",
      "current chunk:  2932\n",
      "current chunk:  2933\n",
      "current chunk:  2934\n",
      "current chunk:  2935\n",
      "current chunk:  2936\n",
      "current chunk:  2937\n",
      "current chunk:  2938\n",
      "current chunk:  2939\n",
      "current chunk:  2940\n",
      "current chunk:  2941\n",
      "current chunk:  2942\n",
      "current chunk:  2943\n",
      "current chunk:  2944\n",
      "current chunk:  2945\n",
      "current chunk:  2946\n",
      "current chunk:  2947\n",
      "current chunk:  2948\n",
      "current chunk:  2949\n",
      "current chunk:  2950\n",
      "current chunk:  2951\n",
      "current chunk:  2952\n",
      "current chunk:  2953\n",
      "current chunk:  2954\n",
      "current chunk:  2955\n",
      "current chunk:  2956\n",
      "current chunk:  2957\n",
      "current chunk:  2958\n",
      "current chunk:  2959\n",
      "current chunk:  2960\n",
      "current chunk:  2961\n",
      "current chunk:  2962\n",
      "current chunk:  2963\n",
      "current chunk:  2964\n",
      "current chunk:  2965\n",
      "current chunk:  2966\n",
      "current chunk:  2967\n",
      "current chunk:  2968\n",
      "current chunk:  2969\n",
      "current chunk:  2970\n",
      "current chunk:  2971\n",
      "current chunk:  2972\n",
      "current chunk:  2973\n",
      "current chunk:  2974\n",
      "current chunk:  2975\n",
      "current chunk:  2976\n",
      "current chunk:  2977\n",
      "current chunk:  2978\n",
      "current chunk:  2979\n",
      "current chunk:  2980\n",
      "current chunk:  2981\n",
      "current chunk:  2982\n",
      "current chunk:  2983\n",
      "current chunk:  2984\n",
      "current chunk:  2985\n",
      "current chunk:  2986\n",
      "current chunk:  2987\n",
      "current chunk:  2988\n",
      "current chunk:  2989\n",
      "current chunk:  2990\n",
      "current chunk:  2991\n",
      "current chunk:  2992\n",
      "current chunk:  2993\n",
      "current chunk:  2994\n",
      "current chunk:  2995\n",
      "current chunk:  2996\n",
      "current chunk:  2997\n",
      "current chunk:  2998\n",
      "current chunk:  2999\n",
      "current chunk:  3000\n",
      "total processing time:  2533.0605084896088\n"
     ]
    }
   ],
   "source": [
    "generate_dataset_and_save(\"D:/NORA_Experiments/TPCH/dbgen/lineitem.tbl\", 0.02)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Util to convert the records in to scan ratio\n",
    "\n",
    "first_folder = \"skewed_queries/\"\n",
    "second_folder = \"OSM/\"\n",
    "method = \"uniform_skewed_prob2\"\n",
    "\n",
    "# for method in [\"kdtree\", \"Qdtree\", \"NORA\", \"LBCost\"]:\n",
    "# for method in [\"kdtree_delta0\", \"Qdtree_delta0\", \"PAW_delta0\", \"LBCost_delta0\"]:\n",
    "original_results = genfromtxt(\"C:/Users/Cloud/iCloudDrive/NORA_experiments/plots/\"+first_folder+second_folder+method, delimiter=' ')\n",
    "for record in original_results:\n",
    "    record[1] = record[1] / 6001309 * 100\n",
    "    record[2] = record[2] / 6001309 * 100\n",
    "np.savetxt('C:/Users/Cloud/iCloudDrive/NORA_experiments/plots/'+first_folder+second_folder+\"scan_ratio/\"+method, original_results, \n",
    "           delimiter=' ', fmt='%s %1.3f %1.3f')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.10"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}