extract_transform_dataset.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import csv\n",
    "import numpy as np\n",
    "import matplotlib.pyplot as plt\n",
    "import seaborn as sns\n",
    "import os\n",
    "import time\n",
    "import pickle\n",
    "import math\n",
    "\n",
    "from pprint import pprint\n",
    "from datetime import datetime"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Uncomment the next cell if running for the first time."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "# !wget https://userinterfaces.aalto.fi/136Mkeystrokes/resources/136m-keystrokes.zip\n",
    "# !unzip 136m-keystrokes.zip\n",
    "# !mv Keystrokes/files data/"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Set seed for reproducibility\n",
    "np.random.seed(58)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "data_folder = 'data'\n",
    "output_folder = 'preprocessed_feather'\n",
    "\n",
    "# Get participants list from metadata file (quicker than listing subfolder!!!)\n",
    "participants = pd.read_table(f'{data_folder}/metadata_participants.txt')['PARTICIPANT_ID']\n",
    "\n",
    "# remove participant 3 (does not exist) and 127969 as it has an non-parseable file!!!\n",
    "participants = participants.drop(participants[participants == 3].index)\n",
    "participants = participants.drop(participants[participants == 127969].index)\n",
    "\n",
    "participants = np.array(participants)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "## Feature extraction\n",
    "# Hold Latency (HL)\n",
    "# Inter-key latency (IL)\n",
    "# Press latency (PL)\n",
    "# Release latency (RL)\n",
    "\n",
    "def keystroke_seq_feat_extr(keystroke_seq_section):\n",
    "    # This function assumes that there is a single test section per participant\n",
    "    keystroke_seq_section = keystroke_seq_section.copy(deep=False)\n",
    "\n",
    "    keystroke_seq_section['HL'] = keystroke_seq_section['RELEASE_TIME'] - keystroke_seq_section['PRESS_TIME']\n",
    "    keystroke_seq_section['IL'] = keystroke_seq_section['PRESS_TIME'].shift(-1) - keystroke_seq_section['RELEASE_TIME']\n",
    "    keystroke_seq_section['PL'] = keystroke_seq_section['HL'] + keystroke_seq_section['IL']\n",
    "    keystroke_seq_section['RL'] = keystroke_seq_section['RELEASE_TIME'].shift(-1) - keystroke_seq_section['RELEASE_TIME']\n",
    "\n",
    "    # There is a NaN for the last keystroke due to shifting, we can drop it\n",
    "    keystroke_seq_section.dropna(inplace=True)\n",
    "\n",
    "    # Drop features we are not going to need anymore (we already used them to calculate the new features)\n",
    "    keystroke_seq_section.drop(['PRESS_TIME', 'RELEASE_TIME', 'TEST_SECTION_ID'], axis='columns', inplace=True)\n",
    "\n",
    "    # Normalize the features as described in the paper\n",
    "    keystroke_seq_section[['HL', 'IL', 'PL', 'RL']] = keystroke_seq_section[['HL', 'IL', 'PL', 'RL']].div(1000)\n",
    "    keystroke_seq_section['KEYCODE'] = keystroke_seq_section['KEYCODE'].div(255)\n",
    "\n",
    "    # Clip values bigger than one or lower than zero\n",
    "    keystroke_seq_section[['HL', 'IL', 'PL', 'RL', 'KEYCODE']] = keystroke_seq_section[['HL', 'IL', 'PL', 'RL', 'KEYCODE']].clip(lower=0, upper=1)\n",
    "\n",
    "    return keystroke_seq_section"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "os.makedirs(output_folder, exist_ok=True)\n",
    "\n",
    "def preprocess_participant(participant_id):\n",
    "    try:\n",
    "        # ISO-8859-15, encoding that supports finnish characters as the dataset contains finnish participants.\n",
    "        participant_data = pd.read_table(f'{data_folder}/{participant_id}_keystrokes.txt', encoding='ISO-8859-15', quoting=csv.QUOTE_NONE, usecols=[\n",
    "                                        'TEST_SECTION_ID', 'KEYCODE', 'PRESS_TIME', 'RELEASE_TIME'], on_bad_lines='skip')\n",
    "        \n",
    "        participant_data = participant_data.groupby('TEST_SECTION_ID')\n",
    "        for section_id in participant_data.groups:\n",
    "            group_dataframe = participant_data.get_group(section_id)\n",
    "            features = keystroke_seq_feat_extr(group_dataframe)\n",
    "            \n",
    "            os.makedirs(f'{output_folder}/{participant_id}/', exist_ok=True)\n",
    "            features.reset_index().to_feather(\n",
    "                f'{output_folder}/{participant_id}/{participant_id}_section_{section_id}_keystrokes.feather')\n",
    "\n",
    "        with open(f'{output_folder}/METADATA_participant_sections.tsv', 'a') as f:\n",
    "            f.write(f'{participant_id}\\t{list(participant_data.groups.keys())}\\n')\n",
    "    except Exception as e:\n",
    "        print(e)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Parallel(n_jobs=-1)]: Using backend LokyBackend with 8 concurrent workers.\n",
      "[Parallel(n_jobs=-1)]: Done  34 tasks      | elapsed:    5.5s\n",
      "[Parallel(n_jobs=-1)]: Done 184 tasks      | elapsed:   13.5s\n",
      "[Parallel(n_jobs=-1)]: Done 434 tasks      | elapsed:   29.4s\n",
      "[Parallel(n_jobs=-1)]: Done 784 tasks      | elapsed:   50.0s\n",
      "[Parallel(n_jobs=-1)]: Done 1234 tasks      | elapsed:  1.2min\n",
      "[Parallel(n_jobs=-1)]: Done 1784 tasks      | elapsed:  1.7min\n",
      "[Parallel(n_jobs=-1)]: Done 2434 tasks      | elapsed:  2.3min\n",
      "[Parallel(n_jobs=-1)]: Done 3184 tasks      | elapsed:  2.8min\n",
      "[Parallel(n_jobs=-1)]: Done 4034 tasks      | elapsed:  3.5min\n",
      "[Parallel(n_jobs=-1)]: Done 4984 tasks      | elapsed:  4.2min\n",
      "[Parallel(n_jobs=-1)]: Done 6034 tasks      | elapsed:  5.1min\n",
      "[Parallel(n_jobs=-1)]: Done 7184 tasks      | elapsed:  6.0min\n",
      "[Parallel(n_jobs=-1)]: Done 8434 tasks      | elapsed:  7.1min\n",
      "[Parallel(n_jobs=-1)]: Done 9784 tasks      | elapsed:  8.2min\n",
      "[Parallel(n_jobs=-1)]: Done 11234 tasks      | elapsed:  9.5min\n",
      "[Parallel(n_jobs=-1)]: Done 12784 tasks      | elapsed: 10.8min\n",
      "[Parallel(n_jobs=-1)]: Done 14434 tasks      | elapsed: 12.1min\n",
      "[Parallel(n_jobs=-1)]: Done 16184 tasks      | elapsed: 13.4min\n",
      "[Parallel(n_jobs=-1)]: Done 18034 tasks      | elapsed: 14.8min\n",
      "[Parallel(n_jobs=-1)]: Done 19984 tasks      | elapsed: 16.2min\n",
      "[Parallel(n_jobs=-1)]: Done 22034 tasks      | elapsed: 17.8min\n",
      "[Parallel(n_jobs=-1)]: Done 24184 tasks      | elapsed: 19.4min\n",
      "[Parallel(n_jobs=-1)]: Done 26434 tasks      | elapsed: 21.0min\n",
      "[Parallel(n_jobs=-1)]: Done 28784 tasks      | elapsed: 22.8min\n",
      "[Parallel(n_jobs=-1)]: Done 31234 tasks      | elapsed: 24.7min\n",
      "[Parallel(n_jobs=-1)]: Done 33784 tasks      | elapsed: 26.6min\n",
      "[Parallel(n_jobs=-1)]: Done 36434 tasks      | elapsed: 28.8min\n",
      "[Parallel(n_jobs=-1)]: Done 39184 tasks      | elapsed: 31.1min\n",
      "[Parallel(n_jobs=-1)]: Done 42034 tasks      | elapsed: 33.5min\n",
      "[Parallel(n_jobs=-1)]: Done 44984 tasks      | elapsed: 35.9min\n",
      "[Parallel(n_jobs=-1)]: Done 48034 tasks      | elapsed: 38.4min\n",
      "[Parallel(n_jobs=-1)]: Done 51184 tasks      | elapsed: 41.3min\n",
      "[Parallel(n_jobs=-1)]: Done 54434 tasks      | elapsed: 43.9min\n",
      "[Parallel(n_jobs=-1)]: Done 57784 tasks      | elapsed: 47.0min\n",
      "[Parallel(n_jobs=-1)]: Done 61234 tasks      | elapsed: 50.1min\n",
      "[Parallel(n_jobs=-1)]: Done 64784 tasks      | elapsed: 53.2min\n",
      "[Parallel(n_jobs=-1)]: Done 68434 tasks      | elapsed: 56.8min\n",
      "[Parallel(n_jobs=-1)]: Done 72184 tasks      | elapsed: 59.9min\n",
      "[Parallel(n_jobs=-1)]: Done 76034 tasks      | elapsed: 63.3min\n",
      "[Parallel(n_jobs=-1)]: Done 79984 tasks      | elapsed: 66.7min\n",
      "[Parallel(n_jobs=-1)]: Done 84034 tasks      | elapsed: 69.9min\n",
      "[Parallel(n_jobs=-1)]: Done 88184 tasks      | elapsed: 73.3min\n",
      "[Parallel(n_jobs=-1)]: Done 92434 tasks      | elapsed: 76.9min\n",
      "[Parallel(n_jobs=-1)]: Done 96784 tasks      | elapsed: 80.5min\n",
      "[Parallel(n_jobs=-1)]: Done 101234 tasks      | elapsed: 84.2min\n",
      "[Parallel(n_jobs=-1)]: Done 105784 tasks      | elapsed: 87.9min\n",
      "[Parallel(n_jobs=-1)]: Done 110434 tasks      | elapsed: 91.8min\n",
      "[Parallel(n_jobs=-1)]: Done 115184 tasks      | elapsed: 95.7min\n",
      "[Parallel(n_jobs=-1)]: Done 120034 tasks      | elapsed: 99.7min\n",
      "[Parallel(n_jobs=-1)]: Done 124984 tasks      | elapsed: 103.8min\n",
      "[Parallel(n_jobs=-1)]: Done 130034 tasks      | elapsed: 108.0min\n",
      "[Parallel(n_jobs=-1)]: Done 135184 tasks      | elapsed: 112.2min\n",
      "[Parallel(n_jobs=-1)]: Done 140434 tasks      | elapsed: 116.6min\n",
      "[Parallel(n_jobs=-1)]: Done 145784 tasks      | elapsed: 121.1min\n",
      "[Parallel(n_jobs=-1)]: Done 151234 tasks      | elapsed: 125.6min\n",
      "[Parallel(n_jobs=-1)]: Done 156784 tasks      | elapsed: 130.1min\n",
      "[Parallel(n_jobs=-1)]: Done 162434 tasks      | elapsed: 134.8min\n",
      "[Parallel(n_jobs=-1)]: Done 168184 tasks      | elapsed: 139.6min\n",
      "[Parallel(n_jobs=-1)]: Done 168592 out of 168592 | elapsed: 139.9min finished\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "[None,\n",
       " None,\n",
       " ...]"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from joblib import Parallel, delayed\n",
    "\n",
    "Parallel(n_jobs=-1, verbose=1)(delayed(preprocess_participant)(participant_id) for participant_id in participants)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>5</th>\n",
       "      <th>[7, 10, 15, 21, 25, 29, 33, 39, 43, 48, 52, 60, 71, 73, 83]</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>23</td>\n",
       "      <td>[113, 128, 142, 148, 158, 177, 202, 231, 256, ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>7</td>\n",
       "      <td>[11, 24, 35, 42, 50, 68, 85, 97, 111, 114, 126...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>24</td>\n",
       "      <td>[116, 130, 137, 143, 147, 159, 167, 176, 183, ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>32</td>\n",
       "      <td>[173, 192, 223, 241, 261, 276, 293, 307, 318, ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>30</td>\n",
       "      <td>[144, 163, 211, 243, 349, 428, 517, 583, 615, ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>168178</th>\n",
       "      <td>517928</td>\n",
       "      <td>[5579266, 5579278, 5579284, 5579288, 5579293, ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>168179</th>\n",
       "      <td>517936</td>\n",
       "      <td>[5579367, 5579377, 5579393, 5579397, 5579404, ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>168180</th>\n",
       "      <td>517943</td>\n",
       "      <td>[5579477, 5579482, 5579488, 5579493, 5579498, ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>168181</th>\n",
       "      <td>517944</td>\n",
       "      <td>[5579484, 5579490, 5579492, 5579497, 5579503, ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>168182</th>\n",
       "      <td>517947</td>\n",
       "      <td>[5579521, 5579529, 5579536, 5579543, 5579549, ...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>168183 rows × 2 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "             5 [7, 10, 15, 21, 25, 29, 33, 39, 43, 48, 52, 60, 71, 73, 83]\n",
       "0           23  [113, 128, 142, 148, 158, 177, 202, 231, 256, ...         \n",
       "1            7  [11, 24, 35, 42, 50, 68, 85, 97, 111, 114, 126...         \n",
       "2           24  [116, 130, 137, 143, 147, 159, 167, 176, 183, ...         \n",
       "3           32  [173, 192, 223, 241, 261, 276, 293, 307, 318, ...         \n",
       "4           30  [144, 163, 211, 243, 349, 428, 517, 583, 615, ...         \n",
       "...        ...                                                ...         \n",
       "168178  517928  [5579266, 5579278, 5579284, 5579288, 5579293, ...         \n",
       "168179  517936  [5579367, 5579377, 5579393, 5579397, 5579404, ...         \n",
       "168180  517943  [5579477, 5579482, 5579488, 5579493, 5579498, ...         \n",
       "168181  517944  [5579484, 5579490, 5579492, 5579497, 5579503, ...         \n",
       "168182  517947  [5579521, 5579529, 5579536, 5579543, 5579549, ...         \n",
       "\n",
       "[168183 rows x 2 columns]"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "pd.read_table(f'preprocessed_feather/METADATA_participant_sections.tsv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>KEYCODE</th>\n",
       "      <th>HL</th>\n",
       "      <th>IL</th>\n",
       "      <th>PL</th>\n",
       "      <th>RL</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0.329412</td>\n",
       "      <td>0.152</td>\n",
       "      <td>0.000</td>\n",
       "      <td>0.152</td>\n",
       "      <td>0.144</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>0.282353</td>\n",
       "      <td>0.144</td>\n",
       "      <td>0.000</td>\n",
       "      <td>0.096</td>\n",
       "      <td>0.081</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>0.270588</td>\n",
       "      <td>0.129</td>\n",
       "      <td>0.000</td>\n",
       "      <td>0.104</td>\n",
       "      <td>0.159</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>0.321569</td>\n",
       "      <td>0.184</td>\n",
       "      <td>0.000</td>\n",
       "      <td>0.112</td>\n",
       "      <td>0.128</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>0.270588</td>\n",
       "      <td>0.200</td>\n",
       "      <td>0.000</td>\n",
       "      <td>0.160</td>\n",
       "      <td>0.128</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>0.125490</td>\n",
       "      <td>0.168</td>\n",
       "      <td>0.000</td>\n",
       "      <td>0.168</td>\n",
       "      <td>0.167</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>0.341176</td>\n",
       "      <td>0.167</td>\n",
       "      <td>0.097</td>\n",
       "      <td>0.264</td>\n",
       "      <td>0.224</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>0.270588</td>\n",
       "      <td>0.127</td>\n",
       "      <td>0.000</td>\n",
       "      <td>0.056</td>\n",
       "      <td>0.224</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>0.321569</td>\n",
       "      <td>0.295</td>\n",
       "      <td>0.000</td>\n",
       "      <td>0.191</td>\n",
       "      <td>0.001</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>0.125490</td>\n",
       "      <td>0.105</td>\n",
       "      <td>0.560</td>\n",
       "      <td>0.665</td>\n",
       "      <td>0.744</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10</th>\n",
       "      <td>0.031373</td>\n",
       "      <td>0.184</td>\n",
       "      <td>0.047</td>\n",
       "      <td>0.231</td>\n",
       "      <td>0.248</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11</th>\n",
       "      <td>0.270588</td>\n",
       "      <td>0.201</td>\n",
       "      <td>0.000</td>\n",
       "      <td>0.201</td>\n",
       "      <td>0.152</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12</th>\n",
       "      <td>0.125490</td>\n",
       "      <td>0.152</td>\n",
       "      <td>0.720</td>\n",
       "      <td>0.872</td>\n",
       "      <td>0.896</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>13</th>\n",
       "      <td>0.403922</td>\n",
       "      <td>0.176</td>\n",
       "      <td>0.000</td>\n",
       "      <td>0.064</td>\n",
       "      <td>0.071</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>14</th>\n",
       "      <td>0.407843</td>\n",
       "      <td>0.183</td>\n",
       "      <td>1.000</td>\n",
       "      <td>1.000</td>\n",
       "      <td>1.000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>15</th>\n",
       "      <td>0.125490</td>\n",
       "      <td>0.127</td>\n",
       "      <td>0.144</td>\n",
       "      <td>0.271</td>\n",
       "      <td>0.289</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>16</th>\n",
       "      <td>0.270588</td>\n",
       "      <td>0.145</td>\n",
       "      <td>0.128</td>\n",
       "      <td>0.273</td>\n",
       "      <td>0.367</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>17</th>\n",
       "      <td>0.337255</td>\n",
       "      <td>0.239</td>\n",
       "      <td>0.000</td>\n",
       "      <td>0.151</td>\n",
       "      <td>0.096</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>18</th>\n",
       "      <td>0.254902</td>\n",
       "      <td>0.184</td>\n",
       "      <td>0.424</td>\n",
       "      <td>0.608</td>\n",
       "      <td>0.608</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>19</th>\n",
       "      <td>0.262745</td>\n",
       "      <td>0.184</td>\n",
       "      <td>0.752</td>\n",
       "      <td>0.936</td>\n",
       "      <td>0.865</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>20</th>\n",
       "      <td>0.333333</td>\n",
       "      <td>0.113</td>\n",
       "      <td>0.001</td>\n",
       "      <td>0.114</td>\n",
       "      <td>0.119</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>21</th>\n",
       "      <td>0.270588</td>\n",
       "      <td>0.118</td>\n",
       "      <td>0.097</td>\n",
       "      <td>0.215</td>\n",
       "      <td>0.233</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>22</th>\n",
       "      <td>0.270588</td>\n",
       "      <td>0.136</td>\n",
       "      <td>0.127</td>\n",
       "      <td>0.263</td>\n",
       "      <td>0.287</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>23</th>\n",
       "      <td>0.325490</td>\n",
       "      <td>0.160</td>\n",
       "      <td>0.088</td>\n",
       "      <td>0.248</td>\n",
       "      <td>0.200</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>24</th>\n",
       "      <td>0.125490</td>\n",
       "      <td>0.112</td>\n",
       "      <td>0.040</td>\n",
       "      <td>0.152</td>\n",
       "      <td>0.192</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>25</th>\n",
       "      <td>0.309804</td>\n",
       "      <td>0.152</td>\n",
       "      <td>0.000</td>\n",
       "      <td>0.096</td>\n",
       "      <td>0.112</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>26</th>\n",
       "      <td>0.305882</td>\n",
       "      <td>0.168</td>\n",
       "      <td>0.000</td>\n",
       "      <td>0.088</td>\n",
       "      <td>0.024</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>27</th>\n",
       "      <td>0.125490</td>\n",
       "      <td>0.104</td>\n",
       "      <td>0.088</td>\n",
       "      <td>0.192</td>\n",
       "      <td>0.304</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>28</th>\n",
       "      <td>0.254902</td>\n",
       "      <td>0.216</td>\n",
       "      <td>0.153</td>\n",
       "      <td>0.369</td>\n",
       "      <td>0.248</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>29</th>\n",
       "      <td>0.031373</td>\n",
       "      <td>0.095</td>\n",
       "      <td>0.120</td>\n",
       "      <td>0.215</td>\n",
       "      <td>0.288</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>30</th>\n",
       "      <td>0.309804</td>\n",
       "      <td>0.168</td>\n",
       "      <td>0.000</td>\n",
       "      <td>0.104</td>\n",
       "      <td>0.105</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>31</th>\n",
       "      <td>0.305882</td>\n",
       "      <td>0.169</td>\n",
       "      <td>0.000</td>\n",
       "      <td>0.024</td>\n",
       "      <td>0.055</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>32</th>\n",
       "      <td>0.270588</td>\n",
       "      <td>0.200</td>\n",
       "      <td>0.000</td>\n",
       "      <td>0.145</td>\n",
       "      <td>0.072</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>33</th>\n",
       "      <td>0.125490</td>\n",
       "      <td>0.127</td>\n",
       "      <td>0.056</td>\n",
       "      <td>0.183</td>\n",
       "      <td>0.136</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>34</th>\n",
       "      <td>0.313725</td>\n",
       "      <td>0.080</td>\n",
       "      <td>0.120</td>\n",
       "      <td>0.200</td>\n",
       "      <td>0.248</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>35</th>\n",
       "      <td>0.298039</td>\n",
       "      <td>0.128</td>\n",
       "      <td>0.000</td>\n",
       "      <td>0.072</td>\n",
       "      <td>0.056</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>36</th>\n",
       "      <td>0.254902</td>\n",
       "      <td>0.112</td>\n",
       "      <td>0.112</td>\n",
       "      <td>0.224</td>\n",
       "      <td>0.216</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>37</th>\n",
       "      <td>0.305882</td>\n",
       "      <td>0.104</td>\n",
       "      <td>0.000</td>\n",
       "      <td>0.080</td>\n",
       "      <td>0.152</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>38</th>\n",
       "      <td>0.270588</td>\n",
       "      <td>0.176</td>\n",
       "      <td>0.000</td>\n",
       "      <td>0.136</td>\n",
       "      <td>0.072</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>39</th>\n",
       "      <td>0.125490</td>\n",
       "      <td>0.112</td>\n",
       "      <td>0.088</td>\n",
       "      <td>0.200</td>\n",
       "      <td>0.216</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>40</th>\n",
       "      <td>0.254902</td>\n",
       "      <td>0.128</td>\n",
       "      <td>0.000</td>\n",
       "      <td>0.128</td>\n",
       "      <td>0.072</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>41</th>\n",
       "      <td>0.305882</td>\n",
       "      <td>0.072</td>\n",
       "      <td>0.000</td>\n",
       "      <td>0.072</td>\n",
       "      <td>0.152</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>42</th>\n",
       "      <td>0.266667</td>\n",
       "      <td>0.152</td>\n",
       "      <td>0.000</td>\n",
       "      <td>0.096</td>\n",
       "      <td>0.057</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>43</th>\n",
       "      <td>0.125490</td>\n",
       "      <td>0.113</td>\n",
       "      <td>1.000</td>\n",
       "      <td>1.000</td>\n",
       "      <td>1.000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>44</th>\n",
       "      <td>0.403922</td>\n",
       "      <td>0.120</td>\n",
       "      <td>0.152</td>\n",
       "      <td>0.272</td>\n",
       "      <td>0.336</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>45</th>\n",
       "      <td>0.388235</td>\n",
       "      <td>0.184</td>\n",
       "      <td>0.104</td>\n",
       "      <td>0.288</td>\n",
       "      <td>0.208</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>46</th>\n",
       "      <td>0.125490</td>\n",
       "      <td>0.104</td>\n",
       "      <td>0.472</td>\n",
       "      <td>0.576</td>\n",
       "      <td>0.632</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>47</th>\n",
       "      <td>0.309804</td>\n",
       "      <td>0.160</td>\n",
       "      <td>0.000</td>\n",
       "      <td>0.104</td>\n",
       "      <td>0.096</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>48</th>\n",
       "      <td>0.305882</td>\n",
       "      <td>0.152</td>\n",
       "      <td>0.000</td>\n",
       "      <td>0.080</td>\n",
       "      <td>0.056</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>49</th>\n",
       "      <td>0.125490</td>\n",
       "      <td>0.128</td>\n",
       "      <td>0.001</td>\n",
       "      <td>0.129</td>\n",
       "      <td>0.128</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>50</th>\n",
       "      <td>0.329412</td>\n",
       "      <td>0.127</td>\n",
       "      <td>0.000</td>\n",
       "      <td>0.103</td>\n",
       "      <td>0.096</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>51</th>\n",
       "      <td>0.282353</td>\n",
       "      <td>0.120</td>\n",
       "      <td>0.000</td>\n",
       "      <td>0.072</td>\n",
       "      <td>0.104</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>52</th>\n",
       "      <td>0.270588</td>\n",
       "      <td>0.152</td>\n",
       "      <td>0.000</td>\n",
       "      <td>0.096</td>\n",
       "      <td>0.056</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>53</th>\n",
       "      <td>0.125490</td>\n",
       "      <td>0.112</td>\n",
       "      <td>0.056</td>\n",
       "      <td>0.168</td>\n",
       "      <td>0.200</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>54</th>\n",
       "      <td>0.309804</td>\n",
       "      <td>0.144</td>\n",
       "      <td>0.049</td>\n",
       "      <td>0.193</td>\n",
       "      <td>0.184</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>55</th>\n",
       "      <td>0.329412</td>\n",
       "      <td>0.135</td>\n",
       "      <td>0.000</td>\n",
       "      <td>0.135</td>\n",
       "      <td>0.088</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>56</th>\n",
       "      <td>0.282353</td>\n",
       "      <td>0.088</td>\n",
       "      <td>0.001</td>\n",
       "      <td>0.089</td>\n",
       "      <td>0.160</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>57</th>\n",
       "      <td>0.270588</td>\n",
       "      <td>0.159</td>\n",
       "      <td>0.000</td>\n",
       "      <td>0.071</td>\n",
       "      <td>0.112</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>58</th>\n",
       "      <td>0.321569</td>\n",
       "      <td>0.200</td>\n",
       "      <td>0.160</td>\n",
       "      <td>0.360</td>\n",
       "      <td>0.296</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>59</th>\n",
       "      <td>0.745098</td>\n",
       "      <td>0.136</td>\n",
       "      <td>0.072</td>\n",
       "      <td>0.208</td>\n",
       "      <td>0.168</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "     KEYCODE     HL     IL     PL     RL\n",
       "0   0.329412  0.152  0.000  0.152  0.144\n",
       "1   0.282353  0.144  0.000  0.096  0.081\n",
       "2   0.270588  0.129  0.000  0.104  0.159\n",
       "3   0.321569  0.184  0.000  0.112  0.128\n",
       "4   0.270588  0.200  0.000  0.160  0.128\n",
       "5   0.125490  0.168  0.000  0.168  0.167\n",
       "6   0.341176  0.167  0.097  0.264  0.224\n",
       "7   0.270588  0.127  0.000  0.056  0.224\n",
       "8   0.321569  0.295  0.000  0.191  0.001\n",
       "9   0.125490  0.105  0.560  0.665  0.744\n",
       "10  0.031373  0.184  0.047  0.231  0.248\n",
       "11  0.270588  0.201  0.000  0.201  0.152\n",
       "12  0.125490  0.152  0.720  0.872  0.896\n",
       "13  0.403922  0.176  0.000  0.064  0.071\n",
       "14  0.407843  0.183  1.000  1.000  1.000\n",
       "15  0.125490  0.127  0.144  0.271  0.289\n",
       "16  0.270588  0.145  0.128  0.273  0.367\n",
       "17  0.337255  0.239  0.000  0.151  0.096\n",
       "18  0.254902  0.184  0.424  0.608  0.608\n",
       "19  0.262745  0.184  0.752  0.936  0.865\n",
       "20  0.333333  0.113  0.001  0.114  0.119\n",
       "21  0.270588  0.118  0.097  0.215  0.233\n",
       "22  0.270588  0.136  0.127  0.263  0.287\n",
       "23  0.325490  0.160  0.088  0.248  0.200\n",
       "24  0.125490  0.112  0.040  0.152  0.192\n",
       "25  0.309804  0.152  0.000  0.096  0.112\n",
       "26  0.305882  0.168  0.000  0.088  0.024\n",
       "27  0.125490  0.104  0.088  0.192  0.304\n",
       "28  0.254902  0.216  0.153  0.369  0.248\n",
       "29  0.031373  0.095  0.120  0.215  0.288\n",
       "30  0.309804  0.168  0.000  0.104  0.105\n",
       "31  0.305882  0.169  0.000  0.024  0.055\n",
       "32  0.270588  0.200  0.000  0.145  0.072\n",
       "33  0.125490  0.127  0.056  0.183  0.136\n",
       "34  0.313725  0.080  0.120  0.200  0.248\n",
       "35  0.298039  0.128  0.000  0.072  0.056\n",
       "36  0.254902  0.112  0.112  0.224  0.216\n",
       "37  0.305882  0.104  0.000  0.080  0.152\n",
       "38  0.270588  0.176  0.000  0.136  0.072\n",
       "39  0.125490  0.112  0.088  0.200  0.216\n",
       "40  0.254902  0.128  0.000  0.128  0.072\n",
       "41  0.305882  0.072  0.000  0.072  0.152\n",
       "42  0.266667  0.152  0.000  0.096  0.057\n",
       "43  0.125490  0.113  1.000  1.000  1.000\n",
       "44  0.403922  0.120  0.152  0.272  0.336\n",
       "45  0.388235  0.184  0.104  0.288  0.208\n",
       "46  0.125490  0.104  0.472  0.576  0.632\n",
       "47  0.309804  0.160  0.000  0.104  0.096\n",
       "48  0.305882  0.152  0.000  0.080  0.056\n",
       "49  0.125490  0.128  0.001  0.129  0.128\n",
       "50  0.329412  0.127  0.000  0.103  0.096\n",
       "51  0.282353  0.120  0.000  0.072  0.104\n",
       "52  0.270588  0.152  0.000  0.096  0.056\n",
       "53  0.125490  0.112  0.056  0.168  0.200\n",
       "54  0.309804  0.144  0.049  0.193  0.184\n",
       "55  0.329412  0.135  0.000  0.135  0.088\n",
       "56  0.282353  0.088  0.001  0.089  0.160\n",
       "57  0.270588  0.159  0.000  0.071  0.112\n",
       "58  0.321569  0.200  0.160  0.360  0.296\n",
       "59  0.745098  0.136  0.072  0.208  0.168"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "pd.read_feather('preprocessed_feather/517928/517928_section_5579266_keystrokes.feather', columns=['KEYCODE', 'HL', 'IL', 'PL', 'RL'])"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "thesis-py38",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.15"
  },
  "orig_nbformat": 4,
  "vscode": {
   "interpreter": {
    "hash": "8f0f55c3d23b2225915cbba887ea83d839f63d573f0d593bb8eab01cd7137428"
   }
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}