Synthethic Test data.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'\\nThe concept of the creation of the test data in order to evaluate the similarities, a synthethic data creation is introduced.\\nThe idea is as follows:\\n1. Outlier Instances.\\nThese are the users who deviate from the other in the data. \\nIdentifying them would of interest to understand how they behave against most of the dataset\\nThe outlier/dissimilar users are identified through the outlier_detection concept we have introduced in this work.\\nIdentified outliers are [8,20,27,149] -- Of them [8,27,149] are more suspectful\\n4 - Outlier Instances.\\n2. Twin Instances.\\nCreating an instance with similar properties as in the data. This is a virtual data having same properties.\\nFor the time series following thing needs to be done. A sample from a start_date to end_date to create such a data in range from 0 to 1.\\n1. A random set of user are first extracted. We extract 3 for example.\\n2. Then we create exact same features for a virtual user and change their identity such as user_id... (only static properties)\\n3. Virtual user would also require to have their time series recordings. \\nFor this following steps are done:\\n1. For a randomly choosen start date between (first timestamp) to (end time)\\n\\n3. Normal Instances.\\nThe usual test instances.\\n'"
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "\"\"\"\n",
    "The concept of the creation of the test data in order to evaluate the similarities, a synthethic data creation is introduced.\n",
    "The idea is as follows:\n",
    "1. Outlier Instances.\n",
    "These are the users who deviate from the other in the data. \n",
    "Identifying them would of interest to understand how they behave against most of the dataset\n",
    "The outlier/dissimilar users are identified through the outlier_detection concept we have introduced in this work.\n",
    "Identified outliers are [8,20,27,149] -- Of them [8,27,149] are more suspectful\n",
    "4 - Outlier Instances.\n",
    "2. Twin Instances.\n",
    "Creating an instance with similar properties as in the data. This is a virtual data having same properties.\n",
    "For the time series following thing needs to be done. A sample from a start_date to end_date to create such a data in range from 0 to 1.\n",
    "1. A random set of user are first extracted. We extract 3 for example.\n",
    "2. Then we create exact same features for a virtual user and change their identity such as user_id... (only static properties)\n",
    "3. Virtual user would also require to have their time series recordings. \n",
    "For this following steps are done:\n",
    "1. For a randomly choosen start date between (first timestamp) to (end time)\n",
    "\n",
    "3. Normal Instances.\n",
    "The usual test instances.\n",
    "\"\"\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import numpy as np\n",
    "import seaborn as sns"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "static_data_tchq = pd.read_csv(\"data/input_csv/3_q.csv\")\n",
    "static_data_hq = pd.read_csv(\"data/input_csv/4_q.csv\")\n",
    "tyt_data = pd.read_csv(\"data/input_csv/1_q.csv\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "drop_user_ids = [54, 60, 140, 170, 4, 6, 7, 9, 12, 19, 25, 39, 53, 59, 128, 130, 144, 145, 148, 156, 166, 167]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [],
   "source": [
    "valid_static_data_tchq = static_data_tchq[~static_data_tchq[\"user_id\"].isin(drop_user_ids)]\n",
    "valid_static_data_hq = static_data_hq[~static_data_hq[\"user_id\"].isin(drop_user_ids)]\n",
    "valid_tyt_data = tyt_data[~tyt_data[\"user_id\"].isin(drop_user_ids)]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 138,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Unnamed: 0</th>\n",
       "      <th>user_id</th>\n",
       "      <th>questionnaire_id</th>\n",
       "      <th>created_at</th>\n",
       "      <th>tschq01</th>\n",
       "      <th>tschq02</th>\n",
       "      <th>tschq03</th>\n",
       "      <th>tschq04-1</th>\n",
       "      <th>tschq05</th>\n",
       "      <th>tschq06</th>\n",
       "      <th>...</th>\n",
       "      <th>tschq28</th>\n",
       "      <th>tschq29</th>\n",
       "      <th>tschq30</th>\n",
       "      <th>tschq31</th>\n",
       "      <th>tschq32</th>\n",
       "      <th>tschq33</th>\n",
       "      <th>tschq34</th>\n",
       "      <th>tschq35</th>\n",
       "      <th>tschq04-2</th>\n",
       "      <th>tschq07-2</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>3</td>\n",
       "      <td>8</td>\n",
       "      <td>3</td>\n",
       "      <td>2018-08-21 14:09:25</td>\n",
       "      <td>768693600</td>\n",
       "      <td>FEMALE</td>\n",
       "      <td>RIGHT</td>\n",
       "      <td>NO</td>\n",
       "      <td>MORETHAN10YEARS</td>\n",
       "      <td>ABRUPT</td>\n",
       "      <td>...</td>\n",
       "      <td>SOMETIMES</td>\n",
       "      <td>IDONTKNOW</td>\n",
       "      <td>YES</td>\n",
       "      <td>YES</td>\n",
       "      <td>YES</td>\n",
       "      <td>YES</td>\n",
       "      <td>NO</td>\n",
       "      <td>NO</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>5</td>\n",
       "      <td>10</td>\n",
       "      <td>3</td>\n",
       "      <td>2018-08-24 18:20:51</td>\n",
       "      <td>267231600</td>\n",
       "      <td>MALE</td>\n",
       "      <td>RIGHT</td>\n",
       "      <td>NO</td>\n",
       "      <td>3-5YEARS</td>\n",
       "      <td>GRADUAL</td>\n",
       "      <td>...</td>\n",
       "      <td>RARELY</td>\n",
       "      <td>NO</td>\n",
       "      <td>YES</td>\n",
       "      <td>NO</td>\n",
       "      <td>NO</td>\n",
       "      <td>YES</td>\n",
       "      <td>NO</td>\n",
       "      <td>NO</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>6</td>\n",
       "      <td>11</td>\n",
       "      <td>3</td>\n",
       "      <td>2018-09-04 13:38:21</td>\n",
       "      <td>-350787600</td>\n",
       "      <td>FEMALE</td>\n",
       "      <td>LEFT</td>\n",
       "      <td>NO</td>\n",
       "      <td>3-5YEARS</td>\n",
       "      <td>ABRUPT</td>\n",
       "      <td>...</td>\n",
       "      <td>ALWAYS</td>\n",
       "      <td>YES</td>\n",
       "      <td>NO</td>\n",
       "      <td>NO</td>\n",
       "      <td>NO</td>\n",
       "      <td>NO</td>\n",
       "      <td>NO</td>\n",
       "      <td>YES</td>\n",
       "      <td>NaN</td>\n",
       "      <td>Familienmitglied an Krebs erkrankt</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>8</td>\n",
       "      <td>14</td>\n",
       "      <td>3</td>\n",
       "      <td>2018-09-27 14:44:47</td>\n",
       "      <td>1968-11-08</td>\n",
       "      <td>MALE</td>\n",
       "      <td>RIGHT</td>\n",
       "      <td>NO</td>\n",
       "      <td>MORETHAN10YEARS</td>\n",
       "      <td>ABRUPT</td>\n",
       "      <td>...</td>\n",
       "      <td>USUALLY</td>\n",
       "      <td>YES</td>\n",
       "      <td>NO</td>\n",
       "      <td>NO</td>\n",
       "      <td>YES</td>\n",
       "      <td>YES</td>\n",
       "      <td>NO</td>\n",
       "      <td>YES</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>9</td>\n",
       "      <td>22</td>\n",
       "      <td>3</td>\n",
       "      <td>2018-10-07 20:00:33</td>\n",
       "      <td>14.06.1964</td>\n",
       "      <td>MALE</td>\n",
       "      <td>RIGHT</td>\n",
       "      <td>NO</td>\n",
       "      <td>MORETHAN10YEARS</td>\n",
       "      <td>GRADUAL</td>\n",
       "      <td>...</td>\n",
       "      <td>USUALLY</td>\n",
       "      <td>YES</td>\n",
       "      <td>YES</td>\n",
       "      <td>YES</td>\n",
       "      <td>YES</td>\n",
       "      <td>YES</td>\n",
       "      <td>YES</td>\n",
       "      <td>YES</td>\n",
       "      <td>NaN</td>\n",
       "      <td>Beginn meiner Tinnitus Krankheit am Montag den...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 39 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "   Unnamed: 0  user_id  questionnaire_id           created_at     tschq01  \\\n",
       "3           3        8                 3  2018-08-21 14:09:25   768693600   \n",
       "5           5       10                 3  2018-08-24 18:20:51   267231600   \n",
       "6           6       11                 3  2018-09-04 13:38:21  -350787600   \n",
       "8           8       14                 3  2018-09-27 14:44:47  1968-11-08   \n",
       "9           9       22                 3  2018-10-07 20:00:33  14.06.1964   \n",
       "\n",
       "  tschq02 tschq03 tschq04-1          tschq05  tschq06  ...    tschq28  \\\n",
       "3  FEMALE   RIGHT        NO  MORETHAN10YEARS   ABRUPT  ...  SOMETIMES   \n",
       "5    MALE   RIGHT        NO         3-5YEARS  GRADUAL  ...     RARELY   \n",
       "6  FEMALE    LEFT        NO         3-5YEARS   ABRUPT  ...     ALWAYS   \n",
       "8    MALE   RIGHT        NO  MORETHAN10YEARS   ABRUPT  ...    USUALLY   \n",
       "9    MALE   RIGHT        NO  MORETHAN10YEARS  GRADUAL  ...    USUALLY   \n",
       "\n",
       "     tschq29 tschq30 tschq31 tschq32  tschq33 tschq34 tschq35 tschq04-2  \\\n",
       "3  IDONTKNOW     YES     YES     YES      YES      NO      NO       NaN   \n",
       "5         NO     YES      NO      NO      YES      NO      NO       NaN   \n",
       "6        YES      NO      NO      NO       NO      NO     YES       NaN   \n",
       "8        YES      NO      NO     YES      YES      NO     YES       NaN   \n",
       "9        YES     YES     YES     YES      YES     YES     YES       NaN   \n",
       "\n",
       "                                           tschq07-2  \n",
       "3                                                NaN  \n",
       "5                                                NaN  \n",
       "6                Familienmitglied an Krebs erkrankt   \n",
       "8                                                NaN  \n",
       "9  Beginn meiner Tinnitus Krankheit am Montag den...  \n",
       "\n",
       "[5 rows x 39 columns]"
      ]
     },
     "execution_count": 138,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "valid_static_data_tchq.head() "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 139,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<class 'pandas.core.frame.DataFrame'>\n",
      "Int64Index: 50 entries, 3 to 70\n",
      "Data columns (total 39 columns):\n",
      "Unnamed: 0          50 non-null int64\n",
      "user_id             50 non-null int64\n",
      "questionnaire_id    50 non-null int64\n",
      "created_at          50 non-null object\n",
      "tschq01             50 non-null object\n",
      "tschq02             50 non-null object\n",
      "tschq03             50 non-null object\n",
      "tschq04-1           50 non-null object\n",
      "tschq05             50 non-null object\n",
      "tschq06             50 non-null object\n",
      "tschq07-1           50 non-null object\n",
      "tschq08             50 non-null object\n",
      "tschq09             50 non-null object\n",
      "tschq10             50 non-null object\n",
      "tschq11             50 non-null object\n",
      "tschq12             50 non-null int64\n",
      "tschq13             49 non-null object\n",
      "tschq14             50 non-null object\n",
      "tschq15             50 non-null object\n",
      "tschq16             50 non-null int64\n",
      "tschq17             50 non-null int64\n",
      "tschq18             50 non-null object\n",
      "tschq19             50 non-null object\n",
      "tschq20             50 non-null object\n",
      "tschq21             50 non-null object\n",
      "tschq22             50 non-null object\n",
      "tschq23             50 non-null object\n",
      "tschq24             50 non-null object\n",
      "tschq25             33 non-null object\n",
      "tschq28             50 non-null object\n",
      "tschq29             50 non-null object\n",
      "tschq30             50 non-null object\n",
      "tschq31             50 non-null object\n",
      "tschq32             50 non-null object\n",
      "tschq33             50 non-null object\n",
      "tschq34             50 non-null object\n",
      "tschq35             50 non-null object\n",
      "tschq04-2           11 non-null object\n",
      "tschq07-2           21 non-null object\n",
      "dtypes: int64(6), object(33)\n",
      "memory usage: 15.6+ KB\n"
     ]
    }
   ],
   "source": [
    "valid_static_data_tchq.info()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([14, 24, 31], dtype=int64)"
      ]
     },
     "execution_count": 38,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "user_ids=valid_static_data_tchq[\"user_id\"].to_numpy()\n",
    "np.random.choice(user_ids, 3, replace=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Unnamed: 0</th>\n",
       "      <th>user_id</th>\n",
       "      <th>questionnaire_id</th>\n",
       "      <th>created_at</th>\n",
       "      <th>tschq01</th>\n",
       "      <th>tschq02</th>\n",
       "      <th>tschq03</th>\n",
       "      <th>tschq04-1</th>\n",
       "      <th>tschq05</th>\n",
       "      <th>tschq06</th>\n",
       "      <th>...</th>\n",
       "      <th>tschq28</th>\n",
       "      <th>tschq29</th>\n",
       "      <th>tschq30</th>\n",
       "      <th>tschq31</th>\n",
       "      <th>tschq32</th>\n",
       "      <th>tschq33</th>\n",
       "      <th>tschq34</th>\n",
       "      <th>tschq35</th>\n",
       "      <th>tschq04-2</th>\n",
       "      <th>tschq07-2</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>3</td>\n",
       "      <td>8</td>\n",
       "      <td>3</td>\n",
       "      <td>2018-08-21 14:09:25</td>\n",
       "      <td>768693600</td>\n",
       "      <td>FEMALE</td>\n",
       "      <td>RIGHT</td>\n",
       "      <td>NO</td>\n",
       "      <td>MORETHAN10YEARS</td>\n",
       "      <td>ABRUPT</td>\n",
       "      <td>...</td>\n",
       "      <td>SOMETIMES</td>\n",
       "      <td>IDONTKNOW</td>\n",
       "      <td>YES</td>\n",
       "      <td>YES</td>\n",
       "      <td>YES</td>\n",
       "      <td>YES</td>\n",
       "      <td>NO</td>\n",
       "      <td>NO</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11</th>\n",
       "      <td>11</td>\n",
       "      <td>20</td>\n",
       "      <td>3</td>\n",
       "      <td>2018-10-10 17:19:39</td>\n",
       "      <td>-94093200</td>\n",
       "      <td>MALE</td>\n",
       "      <td>RIGHT</td>\n",
       "      <td>YES</td>\n",
       "      <td>3-5YEARS</td>\n",
       "      <td>GRADUAL</td>\n",
       "      <td>...</td>\n",
       "      <td>SOMETIMES</td>\n",
       "      <td>NO</td>\n",
       "      <td>NO</td>\n",
       "      <td>NO</td>\n",
       "      <td>YES</td>\n",
       "      <td>NO</td>\n",
       "      <td>NO</td>\n",
       "      <td>NO</td>\n",
       "      <td>['PARENTS']</td>\n",
       "      <td>kein herausragendes Ereignis</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>15</th>\n",
       "      <td>15</td>\n",
       "      <td>27</td>\n",
       "      <td>3</td>\n",
       "      <td>2018-10-12 08:39:32</td>\n",
       "      <td>-40611600</td>\n",
       "      <td>MALE</td>\n",
       "      <td>RIGHT</td>\n",
       "      <td>NO</td>\n",
       "      <td>5-10YEARS</td>\n",
       "      <td>ABRUPT</td>\n",
       "      <td>...</td>\n",
       "      <td>SOMETIMES</td>\n",
       "      <td>YES</td>\n",
       "      <td>YES</td>\n",
       "      <td>YES</td>\n",
       "      <td>YES</td>\n",
       "      <td>YES</td>\n",
       "      <td>YES</td>\n",
       "      <td>NO</td>\n",
       "      <td>NaN</td>\n",
       "      <td>OP Akkustikusneurinom</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>56</th>\n",
       "      <td>56</td>\n",
       "      <td>149</td>\n",
       "      <td>3</td>\n",
       "      <td>2019-06-16 14:11:46</td>\n",
       "      <td>-762832800</td>\n",
       "      <td>FEMALE</td>\n",
       "      <td>RIGHT</td>\n",
       "      <td>NO</td>\n",
       "      <td>MORETHAN10YEARS</td>\n",
       "      <td>GRADUAL</td>\n",
       "      <td>...</td>\n",
       "      <td>SOMETIMES</td>\n",
       "      <td>NO</td>\n",
       "      <td>NO</td>\n",
       "      <td>NO</td>\n",
       "      <td>NO</td>\n",
       "      <td>NO</td>\n",
       "      <td>YES</td>\n",
       "      <td>NO</td>\n",
       "      <td>NaN</td>\n",
       "      <td>Hörsturz</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>4 rows × 39 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "    Unnamed: 0  user_id  questionnaire_id           created_at     tschq01  \\\n",
       "3            3        8                 3  2018-08-21 14:09:25   768693600   \n",
       "11          11       20                 3  2018-10-10 17:19:39   -94093200   \n",
       "15          15       27                 3  2018-10-12 08:39:32   -40611600   \n",
       "56          56      149                 3  2019-06-16 14:11:46  -762832800   \n",
       "\n",
       "   tschq02 tschq03 tschq04-1          tschq05  tschq06  ...    tschq28  \\\n",
       "3   FEMALE   RIGHT        NO  MORETHAN10YEARS   ABRUPT  ...  SOMETIMES   \n",
       "11    MALE   RIGHT       YES         3-5YEARS  GRADUAL  ...  SOMETIMES   \n",
       "15    MALE   RIGHT        NO        5-10YEARS   ABRUPT  ...  SOMETIMES   \n",
       "56  FEMALE   RIGHT        NO  MORETHAN10YEARS  GRADUAL  ...  SOMETIMES   \n",
       "\n",
       "      tschq29 tschq30 tschq31 tschq32  tschq33 tschq34 tschq35    tschq04-2  \\\n",
       "3   IDONTKNOW     YES     YES     YES      YES      NO      NO          NaN   \n",
       "11         NO      NO      NO     YES       NO      NO      NO  ['PARENTS']   \n",
       "15        YES     YES     YES     YES      YES     YES      NO          NaN   \n",
       "56         NO      NO      NO      NO       NO     YES      NO          NaN   \n",
       "\n",
       "                        tschq07-2  \n",
       "3                             NaN  \n",
       "11  kein herausragendes Ereignis   \n",
       "15          OP Akkustikusneurinom  \n",
       "56                       Hörsturz  \n",
       "\n",
       "[4 rows x 39 columns]"
      ]
     },
     "execution_count": 35,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "valid_static_data_tchq[valid_static_data_tchq[\"user_id\"].isin([8,20,27,149])]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Unnamed: 0</th>\n",
       "      <th>user_id</th>\n",
       "      <th>questionnaire_id</th>\n",
       "      <th>created_at</th>\n",
       "      <th>hq01</th>\n",
       "      <th>hq02</th>\n",
       "      <th>hq03</th>\n",
       "      <th>hq04</th>\n",
       "      <th>hq06</th>\n",
       "      <th>hq05</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>3</td>\n",
       "      <td>8</td>\n",
       "      <td>4</td>\n",
       "      <td>2018-08-21 14:09:25</td>\n",
       "      <td>NEVER</td>\n",
       "      <td>NOTATALL</td>\n",
       "      <td>NO</td>\n",
       "      <td>NO</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11</th>\n",
       "      <td>11</td>\n",
       "      <td>20</td>\n",
       "      <td>4</td>\n",
       "      <td>2018-10-10 17:19:39</td>\n",
       "      <td>RARELY</td>\n",
       "      <td>NOTATALL</td>\n",
       "      <td>YES</td>\n",
       "      <td>NO</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>15</th>\n",
       "      <td>15</td>\n",
       "      <td>27</td>\n",
       "      <td>4</td>\n",
       "      <td>2018-10-12 08:39:32</td>\n",
       "      <td>OFTEN</td>\n",
       "      <td>SOMETIMES</td>\n",
       "      <td>YES</td>\n",
       "      <td>RIGHT</td>\n",
       "      <td>REGULARLY</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>56</th>\n",
       "      <td>56</td>\n",
       "      <td>149</td>\n",
       "      <td>4</td>\n",
       "      <td>2019-06-16 14:11:46</td>\n",
       "      <td>ALWAYS</td>\n",
       "      <td>NOTATALL</td>\n",
       "      <td>YES</td>\n",
       "      <td>LEFT</td>\n",
       "      <td>REGULARLY</td>\n",
       "      <td>Fa. Kind</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "    Unnamed: 0  user_id  questionnaire_id           created_at    hq01  \\\n",
       "3            3        8                 4  2018-08-21 14:09:25   NEVER   \n",
       "11          11       20                 4  2018-10-10 17:19:39  RARELY   \n",
       "15          15       27                 4  2018-10-12 08:39:32   OFTEN   \n",
       "56          56      149                 4  2019-06-16 14:11:46  ALWAYS   \n",
       "\n",
       "         hq02 hq03   hq04       hq06      hq05  \n",
       "3    NOTATALL   NO     NO        NaN       NaN  \n",
       "11   NOTATALL  YES     NO        NaN       NaN  \n",
       "15  SOMETIMES  YES  RIGHT  REGULARLY       NaN  \n",
       "56   NOTATALL  YES   LEFT  REGULARLY  Fa. Kind  "
      ]
     },
     "execution_count": 36,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "valid_static_data_hq[valid_static_data_hq[\"user_id\"].isin([8,20,27,149])]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Unnamed: 0</th>\n",
       "      <th>user_id</th>\n",
       "      <th>questionnaire_id</th>\n",
       "      <th>created_at</th>\n",
       "      <th>hq01</th>\n",
       "      <th>hq02</th>\n",
       "      <th>hq03</th>\n",
       "      <th>hq04</th>\n",
       "      <th>hq06</th>\n",
       "      <th>hq05</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>3</td>\n",
       "      <td>8</td>\n",
       "      <td>4</td>\n",
       "      <td>2018-08-21 14:09:25</td>\n",
       "      <td>NEVER</td>\n",
       "      <td>NOTATALL</td>\n",
       "      <td>NO</td>\n",
       "      <td>NO</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11</th>\n",
       "      <td>11</td>\n",
       "      <td>20</td>\n",
       "      <td>4</td>\n",
       "      <td>2018-10-10 17:19:39</td>\n",
       "      <td>RARELY</td>\n",
       "      <td>NOTATALL</td>\n",
       "      <td>YES</td>\n",
       "      <td>NO</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>15</th>\n",
       "      <td>15</td>\n",
       "      <td>27</td>\n",
       "      <td>4</td>\n",
       "      <td>2018-10-12 08:39:32</td>\n",
       "      <td>OFTEN</td>\n",
       "      <td>SOMETIMES</td>\n",
       "      <td>YES</td>\n",
       "      <td>RIGHT</td>\n",
       "      <td>REGULARLY</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>56</th>\n",
       "      <td>56</td>\n",
       "      <td>149</td>\n",
       "      <td>4</td>\n",
       "      <td>2019-06-16 14:11:46</td>\n",
       "      <td>ALWAYS</td>\n",
       "      <td>NOTATALL</td>\n",
       "      <td>YES</td>\n",
       "      <td>LEFT</td>\n",
       "      <td>REGULARLY</td>\n",
       "      <td>Fa. Kind</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "    Unnamed: 0  user_id  questionnaire_id           created_at    hq01  \\\n",
       "3            3        8                 4  2018-08-21 14:09:25   NEVER   \n",
       "11          11       20                 4  2018-10-10 17:19:39  RARELY   \n",
       "15          15       27                 4  2018-10-12 08:39:32   OFTEN   \n",
       "56          56      149                 4  2019-06-16 14:11:46  ALWAYS   \n",
       "\n",
       "         hq02 hq03   hq04       hq06      hq05  \n",
       "3    NOTATALL   NO     NO        NaN       NaN  \n",
       "11   NOTATALL  YES     NO        NaN       NaN  \n",
       "15  SOMETIMES  YES  RIGHT  REGULARLY       NaN  \n",
       "56   NOTATALL  YES   LEFT  REGULARLY  Fa. Kind  "
      ]
     },
     "execution_count": 37,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "valid_static_data_hq[valid_static_data_hq[\"user_id\"].isin([8,20,27,149,])]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 88,
   "metadata": {},
   "outputs": [],
   "source": [
    "d1 = [str(date) for date in pd.date_range(start='10-01-2018', end='05-09-2019', periods=100)]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {},
   "outputs": [],
   "source": [
    "def generate_synthetic_tyt_data(user_id, start_date, end_date, sampling_data, sample_length=80):\n",
    "    created_at = [str(date) for date in pd.date_range(start=start_date, end=end_date, periods=sample_length)]\n",
    "    u_id_list = [user_id for _ in range(sample_length)]\n",
    "    q_id = [3 for _ in range(sample_length)]\n",
    "    columns=[\"s01\",\"s02\",\"s03\",\"s04\",\"s05\",\"s06\",\"s07\", \"s08\"]\n",
    "    synthetic_data = sampling_data[columns].sample(n=sample_length)\n",
    "    synthetic_data[\"user_id\"] = u_id_list\n",
    "    synthetic_data[\"questionnaire_id\"] = q_id\n",
    "    synthetic_data[\"created_at\"] = created_at\n",
    "    return synthetic_data[[\"user_id\",\"questionnaire_id\",\"created_at\",\"s01\",\"s02\",\"s03\",\"s04\",\"s05\",\"s06\",\"s07\",\"s08\"]]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 62,
   "metadata": {},
   "outputs": [],
   "source": [
    "m1 = generate_synthetic_tyt_data(\"44428\", \"10-10-2018\",\"05-09-2019\",valid_tyt_data)\n",
    "m2 = generate_synthetic_tyt_data(\"444154\", \"11-11-2018\",\"05-07-2019\",valid_tyt_data, sample_length=60)\n",
    "m3 = generate_synthetic_tyt_data(\"444133\", \"11-11-2018\",\"05-07-2019\",valid_tyt_data, sample_length=60)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 63,
   "metadata": {},
   "outputs": [],
   "source": [
    "DT_tyt_data = m1.append([m2,m3])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 64,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>user_id</th>\n",
       "      <th>questionnaire_id</th>\n",
       "      <th>created_at</th>\n",
       "      <th>s01</th>\n",
       "      <th>s02</th>\n",
       "      <th>s03</th>\n",
       "      <th>s04</th>\n",
       "      <th>s05</th>\n",
       "      <th>s06</th>\n",
       "      <th>s07</th>\n",
       "      <th>s08</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>5261</th>\n",
       "      <td>44428</td>\n",
       "      <td>3</td>\n",
       "      <td>2018-10-10 00:00:00</td>\n",
       "      <td>YES</td>\n",
       "      <td>84</td>\n",
       "      <td>62.0</td>\n",
       "      <td>88.0</td>\n",
       "      <td>30.0</td>\n",
       "      <td>45.0</td>\n",
       "      <td>66.0</td>\n",
       "      <td>NO</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1226</th>\n",
       "      <td>44428</td>\n",
       "      <td>3</td>\n",
       "      <td>2018-10-12 16:06:04.556962025</td>\n",
       "      <td>YES</td>\n",
       "      <td>55</td>\n",
       "      <td>28.0</td>\n",
       "      <td>88.0</td>\n",
       "      <td>4.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>NO</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5134</th>\n",
       "      <td>44428</td>\n",
       "      <td>3</td>\n",
       "      <td>2018-10-15 08:12:09.113924050</td>\n",
       "      <td>YES</td>\n",
       "      <td>24</td>\n",
       "      <td>19.0</td>\n",
       "      <td>53.0</td>\n",
       "      <td>8.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>20.0</td>\n",
       "      <td>NO</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2027</th>\n",
       "      <td>44428</td>\n",
       "      <td>3</td>\n",
       "      <td>2018-10-18 00:18:13.670886076</td>\n",
       "      <td>YES</td>\n",
       "      <td>31</td>\n",
       "      <td>32.0</td>\n",
       "      <td>67.0</td>\n",
       "      <td>20.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>YES</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2262</th>\n",
       "      <td>44428</td>\n",
       "      <td>3</td>\n",
       "      <td>2018-10-20 16:24:18.227848101</td>\n",
       "      <td>YES</td>\n",
       "      <td>75</td>\n",
       "      <td>76.0</td>\n",
       "      <td>49.0</td>\n",
       "      <td>45.0</td>\n",
       "      <td>61.0</td>\n",
       "      <td>60.0</td>\n",
       "      <td>NO</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "     user_id  questionnaire_id                     created_at  s01  s02   s03  \\\n",
       "5261   44428                 3            2018-10-10 00:00:00  YES   84  62.0   \n",
       "1226   44428                 3  2018-10-12 16:06:04.556962025  YES   55  28.0   \n",
       "5134   44428                 3  2018-10-15 08:12:09.113924050  YES   24  19.0   \n",
       "2027   44428                 3  2018-10-18 00:18:13.670886076  YES   31  32.0   \n",
       "2262   44428                 3  2018-10-20 16:24:18.227848101  YES   75  76.0   \n",
       "\n",
       "       s04   s05   s06   s07  s08  \n",
       "5261  88.0  30.0  45.0  66.0   NO  \n",
       "1226  88.0   4.0   0.0   0.0   NO  \n",
       "5134  53.0   8.0   0.0  20.0   NO  \n",
       "2027  67.0  20.0   0.0   0.0  YES  \n",
       "2262  49.0  45.0  61.0  60.0   NO  "
      ]
     },
     "execution_count": 64,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "DT_tyt_data.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 65,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Unnamed: 0</th>\n",
       "      <th>user_id</th>\n",
       "      <th>questionnaire_id</th>\n",
       "      <th>created_at</th>\n",
       "      <th>tschq01</th>\n",
       "      <th>tschq02</th>\n",
       "      <th>tschq03</th>\n",
       "      <th>tschq04-1</th>\n",
       "      <th>tschq05</th>\n",
       "      <th>tschq06</th>\n",
       "      <th>...</th>\n",
       "      <th>tschq28</th>\n",
       "      <th>tschq29</th>\n",
       "      <th>tschq30</th>\n",
       "      <th>tschq31</th>\n",
       "      <th>tschq32</th>\n",
       "      <th>tschq33</th>\n",
       "      <th>tschq34</th>\n",
       "      <th>tschq35</th>\n",
       "      <th>tschq04-2</th>\n",
       "      <th>tschq07-2</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>20</th>\n",
       "      <td>20</td>\n",
       "      <td>28</td>\n",
       "      <td>3</td>\n",
       "      <td>2018-10-16 17:30:41</td>\n",
       "      <td>-206326800</td>\n",
       "      <td>FEMALE</td>\n",
       "      <td>LEFT</td>\n",
       "      <td>NO</td>\n",
       "      <td>5-10YEARS</td>\n",
       "      <td>ABRUPT</td>\n",
       "      <td>...</td>\n",
       "      <td>ALWAYS</td>\n",
       "      <td>YES</td>\n",
       "      <td>NO</td>\n",
       "      <td>YES</td>\n",
       "      <td>NO</td>\n",
       "      <td>YES</td>\n",
       "      <td>NO</td>\n",
       "      <td>NO</td>\n",
       "      <td>NaN</td>\n",
       "      <td>Ibuprofen und eine Erkältung</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>57</th>\n",
       "      <td>57</td>\n",
       "      <td>154</td>\n",
       "      <td>3</td>\n",
       "      <td>2019-06-16 15:33:05</td>\n",
       "      <td>743292000</td>\n",
       "      <td>MALE</td>\n",
       "      <td>RIGHT</td>\n",
       "      <td>YES</td>\n",
       "      <td>1-2YEARS</td>\n",
       "      <td>ABRUPT</td>\n",
       "      <td>...</td>\n",
       "      <td>SOMETIMES</td>\n",
       "      <td>YES</td>\n",
       "      <td>YES</td>\n",
       "      <td>NO</td>\n",
       "      <td>NO</td>\n",
       "      <td>NO</td>\n",
       "      <td>NO</td>\n",
       "      <td>NO</td>\n",
       "      <td>['PARENTS']</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>44</th>\n",
       "      <td>44</td>\n",
       "      <td>133</td>\n",
       "      <td>3</td>\n",
       "      <td>2019-05-21 15:34:06</td>\n",
       "      <td>371689200</td>\n",
       "      <td>MALE</td>\n",
       "      <td>RIGHT</td>\n",
       "      <td>NO</td>\n",
       "      <td>MORETHAN10YEARS</td>\n",
       "      <td>ABRUPT</td>\n",
       "      <td>...</td>\n",
       "      <td>ALWAYS</td>\n",
       "      <td>YES</td>\n",
       "      <td>YES</td>\n",
       "      <td>NO</td>\n",
       "      <td>NO</td>\n",
       "      <td>NO</td>\n",
       "      <td>YES</td>\n",
       "      <td>YES</td>\n",
       "      <td>NaN</td>\n",
       "      <td>Operation linkes Trommelfell</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>3 rows × 39 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "    Unnamed: 0  user_id  questionnaire_id           created_at     tschq01  \\\n",
       "20          20       28                 3  2018-10-16 17:30:41  -206326800   \n",
       "57          57      154                 3  2019-06-16 15:33:05   743292000   \n",
       "44          44      133                 3  2019-05-21 15:34:06   371689200   \n",
       "\n",
       "   tschq02 tschq03 tschq04-1          tschq05 tschq06  ...    tschq28 tschq29  \\\n",
       "20  FEMALE    LEFT        NO        5-10YEARS  ABRUPT  ...     ALWAYS     YES   \n",
       "57    MALE   RIGHT       YES         1-2YEARS  ABRUPT  ...  SOMETIMES     YES   \n",
       "44    MALE   RIGHT        NO  MORETHAN10YEARS  ABRUPT  ...     ALWAYS     YES   \n",
       "\n",
       "   tschq30 tschq31 tschq32  tschq33 tschq34 tschq35    tschq04-2  \\\n",
       "20      NO     YES      NO      YES      NO      NO          NaN   \n",
       "57     YES      NO      NO       NO      NO      NO  ['PARENTS']   \n",
       "44     YES      NO      NO       NO     YES     YES          NaN   \n",
       "\n",
       "                        tschq07-2  \n",
       "20  Ibuprofen und eine Erkältung   \n",
       "57                            NaN  \n",
       "44   Operation linkes Trommelfell  \n",
       "\n",
       "[3 rows x 39 columns]"
      ]
     },
     "execution_count": 65,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "valid_static_data_tchq.sample(n=3,random_state=42)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 96,
   "metadata": {},
   "outputs": [],
   "source": [
    "DT_static_data = valid_static_data_tchq.sample(n=3,random_state=42)\n",
    "DT_static_data[\"user_id\"] = DT_static_data[\"user_id\"].apply(lambda x: int(\"\".join(\"444\" + str(x)))) \n",
    "DT_static_data[\"Unnamed: 0\"] = DT_static_data[\"Unnamed: 0\"].apply(lambda x: int(\"\".join(\"444\" + str(x)))) "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 97,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Unnamed: 0</th>\n",
       "      <th>user_id</th>\n",
       "      <th>questionnaire_id</th>\n",
       "      <th>created_at</th>\n",
       "      <th>tschq01</th>\n",
       "      <th>tschq02</th>\n",
       "      <th>tschq03</th>\n",
       "      <th>tschq04-1</th>\n",
       "      <th>tschq05</th>\n",
       "      <th>tschq06</th>\n",
       "      <th>...</th>\n",
       "      <th>tschq28</th>\n",
       "      <th>tschq29</th>\n",
       "      <th>tschq30</th>\n",
       "      <th>tschq31</th>\n",
       "      <th>tschq32</th>\n",
       "      <th>tschq33</th>\n",
       "      <th>tschq34</th>\n",
       "      <th>tschq35</th>\n",
       "      <th>tschq04-2</th>\n",
       "      <th>tschq07-2</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>20</th>\n",
       "      <td>44420</td>\n",
       "      <td>44428</td>\n",
       "      <td>3</td>\n",
       "      <td>2018-10-16 17:30:41</td>\n",
       "      <td>-206326800</td>\n",
       "      <td>FEMALE</td>\n",
       "      <td>LEFT</td>\n",
       "      <td>NO</td>\n",
       "      <td>5-10YEARS</td>\n",
       "      <td>ABRUPT</td>\n",
       "      <td>...</td>\n",
       "      <td>ALWAYS</td>\n",
       "      <td>YES</td>\n",
       "      <td>NO</td>\n",
       "      <td>YES</td>\n",
       "      <td>NO</td>\n",
       "      <td>YES</td>\n",
       "      <td>NO</td>\n",
       "      <td>NO</td>\n",
       "      <td>NaN</td>\n",
       "      <td>Ibuprofen und eine Erkältung</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>57</th>\n",
       "      <td>44457</td>\n",
       "      <td>444154</td>\n",
       "      <td>3</td>\n",
       "      <td>2019-06-16 15:33:05</td>\n",
       "      <td>743292000</td>\n",
       "      <td>MALE</td>\n",
       "      <td>RIGHT</td>\n",
       "      <td>YES</td>\n",
       "      <td>1-2YEARS</td>\n",
       "      <td>ABRUPT</td>\n",
       "      <td>...</td>\n",
       "      <td>SOMETIMES</td>\n",
       "      <td>YES</td>\n",
       "      <td>YES</td>\n",
       "      <td>NO</td>\n",
       "      <td>NO</td>\n",
       "      <td>NO</td>\n",
       "      <td>NO</td>\n",
       "      <td>NO</td>\n",
       "      <td>['PARENTS']</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>44</th>\n",
       "      <td>44444</td>\n",
       "      <td>444133</td>\n",
       "      <td>3</td>\n",
       "      <td>2019-05-21 15:34:06</td>\n",
       "      <td>371689200</td>\n",
       "      <td>MALE</td>\n",
       "      <td>RIGHT</td>\n",
       "      <td>NO</td>\n",
       "      <td>MORETHAN10YEARS</td>\n",
       "      <td>ABRUPT</td>\n",
       "      <td>...</td>\n",
       "      <td>ALWAYS</td>\n",
       "      <td>YES</td>\n",
       "      <td>YES</td>\n",
       "      <td>NO</td>\n",
       "      <td>NO</td>\n",
       "      <td>NO</td>\n",
       "      <td>YES</td>\n",
       "      <td>YES</td>\n",
       "      <td>NaN</td>\n",
       "      <td>Operation linkes Trommelfell</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>3 rows × 39 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "    Unnamed: 0  user_id  questionnaire_id           created_at     tschq01  \\\n",
       "20       44420    44428                 3  2018-10-16 17:30:41  -206326800   \n",
       "57       44457   444154                 3  2019-06-16 15:33:05   743292000   \n",
       "44       44444   444133                 3  2019-05-21 15:34:06   371689200   \n",
       "\n",
       "   tschq02 tschq03 tschq04-1          tschq05 tschq06  ...    tschq28 tschq29  \\\n",
       "20  FEMALE    LEFT        NO        5-10YEARS  ABRUPT  ...     ALWAYS     YES   \n",
       "57    MALE   RIGHT       YES         1-2YEARS  ABRUPT  ...  SOMETIMES     YES   \n",
       "44    MALE   RIGHT        NO  MORETHAN10YEARS  ABRUPT  ...     ALWAYS     YES   \n",
       "\n",
       "   tschq30 tschq31 tschq32  tschq33 tschq34 tschq35    tschq04-2  \\\n",
       "20      NO     YES      NO      YES      NO      NO          NaN   \n",
       "57     YES      NO      NO       NO      NO      NO  ['PARENTS']   \n",
       "44     YES      NO      NO       NO     YES     YES          NaN   \n",
       "\n",
       "                        tschq07-2  \n",
       "20  Ibuprofen und eine Erkältung   \n",
       "57                            NaN  \n",
       "44   Operation linkes Trommelfell  \n",
       "\n",
       "[3 rows x 39 columns]"
      ]
     },
     "execution_count": 97,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "DT_static_data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 98,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Unnamed: 0</th>\n",
       "      <th>user_id</th>\n",
       "      <th>questionnaire_id</th>\n",
       "      <th>created_at</th>\n",
       "      <th>tschq01</th>\n",
       "      <th>tschq02</th>\n",
       "      <th>tschq03</th>\n",
       "      <th>tschq04-1</th>\n",
       "      <th>tschq05</th>\n",
       "      <th>tschq06</th>\n",
       "      <th>...</th>\n",
       "      <th>tschq28</th>\n",
       "      <th>tschq29</th>\n",
       "      <th>tschq30</th>\n",
       "      <th>tschq31</th>\n",
       "      <th>tschq32</th>\n",
       "      <th>tschq33</th>\n",
       "      <th>tschq34</th>\n",
       "      <th>tschq35</th>\n",
       "      <th>tschq04-2</th>\n",
       "      <th>tschq07-2</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>20</th>\n",
       "      <td>44420</td>\n",
       "      <td>44428</td>\n",
       "      <td>3</td>\n",
       "      <td>2018-10-16 17:30:41</td>\n",
       "      <td>-206326800</td>\n",
       "      <td>FEMALE</td>\n",
       "      <td>LEFT</td>\n",
       "      <td>NO</td>\n",
       "      <td>5-10YEARS</td>\n",
       "      <td>ABRUPT</td>\n",
       "      <td>...</td>\n",
       "      <td>ALWAYS</td>\n",
       "      <td>YES</td>\n",
       "      <td>NO</td>\n",
       "      <td>YES</td>\n",
       "      <td>NO</td>\n",
       "      <td>YES</td>\n",
       "      <td>NO</td>\n",
       "      <td>NO</td>\n",
       "      <td>NaN</td>\n",
       "      <td>Ibuprofen und eine Erkältung</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>57</th>\n",
       "      <td>44457</td>\n",
       "      <td>444154</td>\n",
       "      <td>3</td>\n",
       "      <td>2019-06-16 15:33:05</td>\n",
       "      <td>743292000</td>\n",
       "      <td>MALE</td>\n",
       "      <td>RIGHT</td>\n",
       "      <td>YES</td>\n",
       "      <td>1-2YEARS</td>\n",
       "      <td>ABRUPT</td>\n",
       "      <td>...</td>\n",
       "      <td>SOMETIMES</td>\n",
       "      <td>YES</td>\n",
       "      <td>YES</td>\n",
       "      <td>NO</td>\n",
       "      <td>NO</td>\n",
       "      <td>NO</td>\n",
       "      <td>NO</td>\n",
       "      <td>NO</td>\n",
       "      <td>['PARENTS']</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>44</th>\n",
       "      <td>44444</td>\n",
       "      <td>444133</td>\n",
       "      <td>3</td>\n",
       "      <td>2019-05-21 15:34:06</td>\n",
       "      <td>371689200</td>\n",
       "      <td>MALE</td>\n",
       "      <td>RIGHT</td>\n",
       "      <td>NO</td>\n",
       "      <td>MORETHAN10YEARS</td>\n",
       "      <td>ABRUPT</td>\n",
       "      <td>...</td>\n",
       "      <td>ALWAYS</td>\n",
       "      <td>YES</td>\n",
       "      <td>YES</td>\n",
       "      <td>NO</td>\n",
       "      <td>NO</td>\n",
       "      <td>NO</td>\n",
       "      <td>YES</td>\n",
       "      <td>YES</td>\n",
       "      <td>NaN</td>\n",
       "      <td>Operation linkes Trommelfell</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>3 rows × 39 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "    Unnamed: 0  user_id  questionnaire_id           created_at     tschq01  \\\n",
       "20       44420    44428                 3  2018-10-16 17:30:41  -206326800   \n",
       "57       44457   444154                 3  2019-06-16 15:33:05   743292000   \n",
       "44       44444   444133                 3  2019-05-21 15:34:06   371689200   \n",
       "\n",
       "   tschq02 tschq03 tschq04-1          tschq05 tschq06  ...    tschq28 tschq29  \\\n",
       "20  FEMALE    LEFT        NO        5-10YEARS  ABRUPT  ...     ALWAYS     YES   \n",
       "57    MALE   RIGHT       YES         1-2YEARS  ABRUPT  ...  SOMETIMES     YES   \n",
       "44    MALE   RIGHT        NO  MORETHAN10YEARS  ABRUPT  ...     ALWAYS     YES   \n",
       "\n",
       "   tschq30 tschq31 tschq32  tschq33 tschq34 tschq35    tschq04-2  \\\n",
       "20      NO     YES      NO      YES      NO      NO          NaN   \n",
       "57     YES      NO      NO       NO      NO      NO  ['PARENTS']   \n",
       "44     YES      NO      NO       NO     YES     YES          NaN   \n",
       "\n",
       "                        tschq07-2  \n",
       "20  Ibuprofen und eine Erkältung   \n",
       "57                            NaN  \n",
       "44   Operation linkes Trommelfell  \n",
       "\n",
       "[3 rows x 39 columns]"
      ]
     },
     "execution_count": 98,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "DT_static_data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 99,
   "metadata": {},
   "outputs": [],
   "source": [
    "DT_static_data_hq = valid_static_data_hq.sample(n=3,random_state=42)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 100,
   "metadata": {},
   "outputs": [],
   "source": [
    "DT_static_data_hq[\"user_id\"] = DT_static_data_hq[\"user_id\"].apply(lambda x: int(\"\".join(\"444\" + str(x)))) \n",
    "DT_static_data_hq[\"Unnamed: 0\"] = DT_static_data_hq[\"Unnamed: 0\"].apply(lambda x: int(\"\".join(\"444\" + str(x))))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 131,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "numpy.int64"
      ]
     },
     "execution_count": 131,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "type(DT_static_data_hq[\"user_id\"].iloc[0])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 128,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[135, 29, 23]"
      ]
     },
     "execution_count": 128,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "valid_static_data_tchq[\"user_id\"].sample(n=3,random_state=0).to_list()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 78,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Final Simulated Data for 3 users .\n",
    "#1. Static Data - DT_tyt_data\n",
    "#2. Static Data Hq - DT_static_data\n",
    "#3. Time Series Data - DT_static_data_hq"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 102,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "200"
      ]
     },
     "execution_count": 102,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(DT_tyt_data)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 103,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "3"
      ]
     },
     "execution_count": 103,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(DT_static_data)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 104,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "3"
      ]
     },
     "execution_count": 104,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(DT_static_data_hq)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 143,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Unnamed: 0</th>\n",
       "      <th>user_id</th>\n",
       "      <th>questionnaire_id</th>\n",
       "      <th>created_at</th>\n",
       "      <th>tschq01</th>\n",
       "      <th>tschq02</th>\n",
       "      <th>tschq03</th>\n",
       "      <th>tschq04-1</th>\n",
       "      <th>tschq05</th>\n",
       "      <th>tschq06</th>\n",
       "      <th>...</th>\n",
       "      <th>tschq28</th>\n",
       "      <th>tschq29</th>\n",
       "      <th>tschq30</th>\n",
       "      <th>tschq31</th>\n",
       "      <th>tschq32</th>\n",
       "      <th>tschq33</th>\n",
       "      <th>tschq34</th>\n",
       "      <th>tschq35</th>\n",
       "      <th>tschq04-2</th>\n",
       "      <th>tschq07-2</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>20</th>\n",
       "      <td>44420</td>\n",
       "      <td>44428</td>\n",
       "      <td>3</td>\n",
       "      <td>2018-10-16 17:30:41</td>\n",
       "      <td>-206326800</td>\n",
       "      <td>FEMALE</td>\n",
       "      <td>LEFT</td>\n",
       "      <td>NO</td>\n",
       "      <td>5-10YEARS</td>\n",
       "      <td>ABRUPT</td>\n",
       "      <td>...</td>\n",
       "      <td>ALWAYS</td>\n",
       "      <td>YES</td>\n",
       "      <td>NO</td>\n",
       "      <td>YES</td>\n",
       "      <td>NO</td>\n",
       "      <td>YES</td>\n",
       "      <td>NO</td>\n",
       "      <td>NO</td>\n",
       "      <td>NaN</td>\n",
       "      <td>Ibuprofen und eine Erkältung</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>57</th>\n",
       "      <td>44457</td>\n",
       "      <td>444154</td>\n",
       "      <td>3</td>\n",
       "      <td>2019-06-16 15:33:05</td>\n",
       "      <td>743292000</td>\n",
       "      <td>MALE</td>\n",
       "      <td>RIGHT</td>\n",
       "      <td>YES</td>\n",
       "      <td>1-2YEARS</td>\n",
       "      <td>ABRUPT</td>\n",
       "      <td>...</td>\n",
       "      <td>SOMETIMES</td>\n",
       "      <td>YES</td>\n",
       "      <td>YES</td>\n",
       "      <td>NO</td>\n",
       "      <td>NO</td>\n",
       "      <td>NO</td>\n",
       "      <td>NO</td>\n",
       "      <td>NO</td>\n",
       "      <td>['PARENTS']</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>44</th>\n",
       "      <td>44444</td>\n",
       "      <td>444133</td>\n",
       "      <td>3</td>\n",
       "      <td>2019-05-21 15:34:06</td>\n",
       "      <td>371689200</td>\n",
       "      <td>MALE</td>\n",
       "      <td>RIGHT</td>\n",
       "      <td>NO</td>\n",
       "      <td>MORETHAN10YEARS</td>\n",
       "      <td>ABRUPT</td>\n",
       "      <td>...</td>\n",
       "      <td>ALWAYS</td>\n",
       "      <td>YES</td>\n",
       "      <td>YES</td>\n",
       "      <td>NO</td>\n",
       "      <td>NO</td>\n",
       "      <td>NO</td>\n",
       "      <td>YES</td>\n",
       "      <td>YES</td>\n",
       "      <td>NaN</td>\n",
       "      <td>Operation linkes Trommelfell</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>3 rows × 39 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "    Unnamed: 0  user_id  questionnaire_id           created_at     tschq01  \\\n",
       "20       44420    44428                 3  2018-10-16 17:30:41  -206326800   \n",
       "57       44457   444154                 3  2019-06-16 15:33:05   743292000   \n",
       "44       44444   444133                 3  2019-05-21 15:34:06   371689200   \n",
       "\n",
       "   tschq02 tschq03 tschq04-1          tschq05 tschq06  ...    tschq28 tschq29  \\\n",
       "20  FEMALE    LEFT        NO        5-10YEARS  ABRUPT  ...     ALWAYS     YES   \n",
       "57    MALE   RIGHT       YES         1-2YEARS  ABRUPT  ...  SOMETIMES     YES   \n",
       "44    MALE   RIGHT        NO  MORETHAN10YEARS  ABRUPT  ...     ALWAYS     YES   \n",
       "\n",
       "   tschq30 tschq31 tschq32  tschq33 tschq34 tschq35    tschq04-2  \\\n",
       "20      NO     YES      NO      YES      NO      NO          NaN   \n",
       "57     YES      NO      NO       NO      NO      NO  ['PARENTS']   \n",
       "44     YES      NO      NO       NO     YES     YES          NaN   \n",
       "\n",
       "                        tschq07-2  \n",
       "20  Ibuprofen und eine Erkältung   \n",
       "57                            NaN  \n",
       "44   Operation linkes Trommelfell  \n",
       "\n",
       "[3 rows x 39 columns]"
      ]
     },
     "execution_count": 143,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "DT_static_data.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 162,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "\"['SIBLINGS']\""
      ]
     },
     "execution_count": 162,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "static_data_tchq[\"tschq04-2\"].iloc[31]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 106,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Unnamed: 0</th>\n",
       "      <th>user_id</th>\n",
       "      <th>questionnaire_id</th>\n",
       "      <th>created_at</th>\n",
       "      <th>hq01</th>\n",
       "      <th>hq02</th>\n",
       "      <th>hq03</th>\n",
       "      <th>hq04</th>\n",
       "      <th>hq06</th>\n",
       "      <th>hq05</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>20</th>\n",
       "      <td>44420</td>\n",
       "      <td>44428</td>\n",
       "      <td>4</td>\n",
       "      <td>2018-10-16 17:30:42</td>\n",
       "      <td>OFTEN</td>\n",
       "      <td>OFTEN</td>\n",
       "      <td>YES</td>\n",
       "      <td>BOTH</td>\n",
       "      <td>REGULARLY</td>\n",
       "      <td>gerade neue, von Widex</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>57</th>\n",
       "      <td>44457</td>\n",
       "      <td>444154</td>\n",
       "      <td>4</td>\n",
       "      <td>2019-06-16 15:33:05</td>\n",
       "      <td>RARELY</td>\n",
       "      <td>RARELY</td>\n",
       "      <td>NO</td>\n",
       "      <td>NO</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>44</th>\n",
       "      <td>44444</td>\n",
       "      <td>444133</td>\n",
       "      <td>4</td>\n",
       "      <td>2019-05-21 15:34:06</td>\n",
       "      <td>SOMETIMES</td>\n",
       "      <td>RARELY</td>\n",
       "      <td>NO</td>\n",
       "      <td>NO</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "    Unnamed: 0  user_id  questionnaire_id           created_at       hq01  \\\n",
       "20       44420    44428                 4  2018-10-16 17:30:42      OFTEN   \n",
       "57       44457   444154                 4  2019-06-16 15:33:05     RARELY   \n",
       "44       44444   444133                 4  2019-05-21 15:34:06  SOMETIMES   \n",
       "\n",
       "      hq02 hq03  hq04       hq06                    hq05  \n",
       "20   OFTEN  YES  BOTH  REGULARLY  gerade neue, von Widex  \n",
       "57  RARELY   NO    NO        NaN                     NaN  \n",
       "44  RARELY   NO    NO        NaN                     NaN  "
      ]
     },
     "execution_count": 106,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "DT_static_data_hq.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 109,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>user_id</th>\n",
       "      <th>questionnaire_id</th>\n",
       "      <th>created_at</th>\n",
       "      <th>s01</th>\n",
       "      <th>s02</th>\n",
       "      <th>s03</th>\n",
       "      <th>s04</th>\n",
       "      <th>s05</th>\n",
       "      <th>s06</th>\n",
       "      <th>s07</th>\n",
       "      <th>s08</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>5261</th>\n",
       "      <td>44428</td>\n",
       "      <td>3</td>\n",
       "      <td>2018-10-10 00:00:00</td>\n",
       "      <td>YES</td>\n",
       "      <td>84</td>\n",
       "      <td>62.0</td>\n",
       "      <td>88.0</td>\n",
       "      <td>30.0</td>\n",
       "      <td>45.0</td>\n",
       "      <td>66.0</td>\n",
       "      <td>NO</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1226</th>\n",
       "      <td>44428</td>\n",
       "      <td>3</td>\n",
       "      <td>2018-10-12 16:06:04.556962025</td>\n",
       "      <td>YES</td>\n",
       "      <td>55</td>\n",
       "      <td>28.0</td>\n",
       "      <td>88.0</td>\n",
       "      <td>4.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>NO</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5134</th>\n",
       "      <td>44428</td>\n",
       "      <td>3</td>\n",
       "      <td>2018-10-15 08:12:09.113924050</td>\n",
       "      <td>YES</td>\n",
       "      <td>24</td>\n",
       "      <td>19.0</td>\n",
       "      <td>53.0</td>\n",
       "      <td>8.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>20.0</td>\n",
       "      <td>NO</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2027</th>\n",
       "      <td>44428</td>\n",
       "      <td>3</td>\n",
       "      <td>2018-10-18 00:18:13.670886076</td>\n",
       "      <td>YES</td>\n",
       "      <td>31</td>\n",
       "      <td>32.0</td>\n",
       "      <td>67.0</td>\n",
       "      <td>20.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>YES</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2262</th>\n",
       "      <td>44428</td>\n",
       "      <td>3</td>\n",
       "      <td>2018-10-20 16:24:18.227848101</td>\n",
       "      <td>YES</td>\n",
       "      <td>75</td>\n",
       "      <td>76.0</td>\n",
       "      <td>49.0</td>\n",
       "      <td>45.0</td>\n",
       "      <td>61.0</td>\n",
       "      <td>60.0</td>\n",
       "      <td>NO</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "     user_id  questionnaire_id                     created_at  s01  s02   s03  \\\n",
       "5261   44428                 3            2018-10-10 00:00:00  YES   84  62.0   \n",
       "1226   44428                 3  2018-10-12 16:06:04.556962025  YES   55  28.0   \n",
       "5134   44428                 3  2018-10-15 08:12:09.113924050  YES   24  19.0   \n",
       "2027   44428                 3  2018-10-18 00:18:13.670886076  YES   31  32.0   \n",
       "2262   44428                 3  2018-10-20 16:24:18.227848101  YES   75  76.0   \n",
       "\n",
       "       s04   s05   s06   s07  s08  \n",
       "5261  88.0  30.0  45.0  66.0   NO  \n",
       "1226  88.0   4.0   0.0   0.0   NO  \n",
       "5134  53.0   8.0   0.0  20.0   NO  \n",
       "2027  67.0  20.0   0.0   0.0  YES  \n",
       "2262  49.0  45.0  61.0  60.0   NO  "
      ]
     },
     "execution_count": 109,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "DT_tyt_data.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 136,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "float"
      ]
     },
     "execution_count": 136,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "type(DT_static_data[\"tschq04-2\"].iloc[0])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 163,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "\"['PARENTS']\""
      ]
     },
     "execution_count": 163,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "valid_static_data_tchq[\"tschq04-2\"][11]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 165,
   "metadata": {},
   "outputs": [],
   "source": [
    "import ast\n",
    "# Converting string to list \n",
    "res = ast.literal_eval(valid_static_data_tchq[\"tschq04-2\"][11]) "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 172,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'PARENTS_CHILDREN'"
      ]
     },
     "execution_count": 172,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "res\n",
    "list_to_str = \"_\".join([val for val in res + [\"CHILDREN\"]])\n",
    "list_to_str"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 175,
   "metadata": {},
   "outputs": [],
   "source": [
    "res = \"[1,2,3]\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 177,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "True"
      ]
     },
     "execution_count": 177,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "isinstance(res, str)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 174,
   "metadata": {},
   "outputs": [
    {
     "ename": "ValueError",
     "evalue": "malformed node or string: [1, 2, 3]",
     "traceback": [
      "\u001b[1;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[1;31mValueError\u001b[0m                                Traceback (most recent call last)",
      "\u001b[1;32m<ipython-input-174-b9421118a3be>\u001b[0m in \u001b[0;36m<module>\u001b[1;34m()\u001b[0m\n\u001b[1;32m----> 1\u001b[1;33m \u001b[0mast\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mliteral_eval\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mres\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m",
      "\u001b[1;32m~\\Anaconda3\\envs\\python-cvcourse\\lib\\ast.py\u001b[0m in \u001b[0;36mliteral_eval\u001b[1;34m(node_or_string)\u001b[0m\n\u001b[0;32m     83\u001b[0m                     \u001b[1;32mreturn\u001b[0m \u001b[0mleft\u001b[0m \u001b[1;33m-\u001b[0m \u001b[0mright\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m     84\u001b[0m         \u001b[1;32mraise\u001b[0m \u001b[0mValueError\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;34m'malformed node or string: '\u001b[0m \u001b[1;33m+\u001b[0m \u001b[0mrepr\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mnode\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m---> 85\u001b[1;33m     \u001b[1;32mreturn\u001b[0m \u001b[0m_convert\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mnode_or_string\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m     86\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m     87\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32m~\\Anaconda3\\envs\\python-cvcourse\\lib\\ast.py\u001b[0m in \u001b[0;36m_convert\u001b[1;34m(node)\u001b[0m\n\u001b[0;32m     82\u001b[0m                 \u001b[1;32melse\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m     83\u001b[0m                     \u001b[1;32mreturn\u001b[0m \u001b[0mleft\u001b[0m \u001b[1;33m-\u001b[0m \u001b[0mright\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m---> 84\u001b[1;33m         \u001b[1;32mraise\u001b[0m \u001b[0mValueError\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;34m'malformed node or string: '\u001b[0m \u001b[1;33m+\u001b[0m \u001b[0mrepr\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mnode\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m     85\u001b[0m     \u001b[1;32mreturn\u001b[0m \u001b[0m_convert\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mnode_or_string\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m     86\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;31mValueError\u001b[0m: malformed node or string: [1, 2, 3]"
     ],
     "output_type": "error"
    }
   ],
   "source": [
    "ast.literal_eval(res)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 119,
   "metadata": {},
   "outputs": [],
   "source": [
    "## Save all the simulated files as csv and pickle\n",
    "DT_static_data.iloc[:,1:].to_csv(\"data/simulate/3_q_sim.csv\")\n",
    "DT_static_data_hq.iloc[:,1:].to_csv(\"data/simulate/4_q_sim.csv\")\n",
    "DT_tyt_data.to_csv(\"data/simulate/1_q_sim.csv\")\n",
    "DT_static_data.iloc[:,1:].to_pickle(\"data/simulate/3_q_sim.pckl\")\n",
    "DT_static_data_hq.iloc[:,1:].to_pickle(\"data/simulate/4_q_sim.pckl\")\n",
    "DT_tyt_data.to_pickle(\"data/simulate/1_q_sim.pckl\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}