Yewen.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# PE Validation Fall 20"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "# import necessary modules \n",
    "# please run this everytime you open it\n",
    "import pandas as pd \n",
    "import numpy as np \n",
    "import matplotlib as plt \n",
    "import seaborn as sns\n",
    "import re\n",
    "import os"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Experimentation ground below: "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Charlie"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "specialist_dict = {}\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [],
   "source": [
    "specialist_dict[\"Argument Relevance\"] = {\"mc\":{\"ordinal\": [1, 2, 3]}, \"tx\":[4]}\n",
    "\n",
    "specialist_dict[\"Evidence Specialist\"] = {\"mc\":{\"ordinal\":[4, 5, 8, 10, 11, 12, 14, 15], \"categorical\":[6], \"Signal, Noise, and Statistical & Systematic Uncertainty\":[9]}, \"cl\":{\"categorical\":[1, 2]}, \"tx\":[3, 7]}\n",
    "\n",
    "specialist_dict[\"Holistic Evaluation of Artical\"] = {\"mc\":{\"ordinal\":[5, 7, 8, 11, 12, 13, 17, 18], \"categorical\":[1, 14]}, \"tx\":[2, 4, 16], \"cl\":[3, 6, 9, 10, 15, 19]}\n",
    "\n",
    "specialist_dict[\"Language Specialist\"] = {\"mc\":{\"ordinal\":[2, 3, 4, 5, 8, 9, 10, 11, 12, 13, 14, 15]}, \"cl\":[1, 6], \"tx\":[7, 16]}\n",
    "\n",
    "specialist_dict[\"Probability Specialist\"] = {\"mc\":{\"ordinal\":[1, 2, 4, 5, 7, 8, 9, 10, 11, 14, 15], \"categorical\":[6]}, \"cl\":[12], \"tx\":[16]}\n",
    "\n",
    "specialist_dict[\"Reasoning Specialist\"] = {\"mc\":{\"ordinal\":[7, 10, 11]}, \"cl\":[1, 2, 3, 6], \"tx\":[8, 12]}\n",
    "\n",
    "specialist_dict[\"Quote Source Relevance\"] = {\"mc\":{\"categorical\":[2, 3, 8], \"ordinal\":[10, 11]}, \"cl\":[1, 4, 5, 6, 7], \"tx\":[12]}\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'Argument relevance': {'mc': {'ordinal': [1, 2, 3]}, 'tx': [4]},\n",
       " 'Evidence Specialist': {'mc': {'ordinal': [4, 5, 8, 10, 11, 12, 14, 15],\n",
       "   'categorical': [6],\n",
       "   'Signal, Noise, and Statistical & Systematic Uncertainty': [9]},\n",
       "  'cl': {'categorical': [1, 2]},\n",
       "  'tx': [3, 7]},\n",
       " 'Holistic Evaluation of Artical': {'mc': {'ordinal': [5,\n",
       "    7,\n",
       "    8,\n",
       "    11,\n",
       "    12,\n",
       "    13,\n",
       "    17,\n",
       "    18],\n",
       "   'categorical': [1, 14]},\n",
       "  'tx': [2, 4, 16],\n",
       "  'cl': [3, 6, 9, 10, 15, 19]},\n",
       " 'Language Specialist': {'mc': {'ordinal': [2,\n",
       "    3,\n",
       "    4,\n",
       "    5,\n",
       "    8,\n",
       "    9,\n",
       "    10,\n",
       "    11,\n",
       "    12,\n",
       "    13,\n",
       "    14,\n",
       "    15]},\n",
       "  'cl': [1, 6],\n",
       "  'tx': [7, 16]},\n",
       " 'Probability Specialist': {'mc': {'ordinal': [1,\n",
       "    2,\n",
       "    4,\n",
       "    5,\n",
       "    7,\n",
       "    8,\n",
       "    9,\n",
       "    10,\n",
       "    11,\n",
       "    14,\n",
       "    15],\n",
       "   'categorical': [6]},\n",
       "  'cl': [12],\n",
       "  'tx': [16]},\n",
       " 'Argument Relevance': {'mc': {'ordinal': [1, 2, 3]}, 'tx': [4]},\n",
       " 'Reasoning Specialist': {'mc': {'ordinal': [7, 10, 11]},\n",
       "  'cl': [1, 2, 3, 6],\n",
       "  'tx': [8, 12]},\n",
       " 'Quote Source Relevance': {'mc': {'categorical': [2, 3, 8],\n",
       "   'ordinal': [10, 11]},\n",
       "  'cl': [1, 4, 5, 6, 7],\n",
       "  'tx': [12]}}"
      ]
     },
     "execution_count": 20,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "specialist_dict"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Katherine"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Yewen"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Questions: "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "# # read json lines file \n",
    "# data = pd.read_json('2020 data/2020-09-25_pe_webhooks.jsonl', lines=True)\n",
    "# data.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "# source.columns"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "# source[source[\"contributor_uuid\"] == \"00f548b7-6b63-4b47-828e-8e416b6ca0e2\"][[\"contributor_uuid\", \"created\", \"finish_time\", \"elapsed_seconds\", \"question_label\", \"answer_label\", \"answer_uuid\"]]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Something to keep in mind!"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "**In Specialist csvs, one \"quiz_task_uuid\" can have multiple contributors. To uniquely identify each input, use \"quiz_taskrun_uuid**"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Helper functions"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "# check if a certain column contains any duplicate values \n",
    "# return true if all values in the column are unique\n",
    "def check_duplicate(df, col):\n",
    "    # df - any dataframe \n",
    "    # col - column label \n",
    "    return df[col].is_unique"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [],
   "source": [
    "# count how many Nan rows are there in a df \n",
    "# Nan rows are rows that have all missing values\n",
    "def count_nan_rows(df):\n",
    "    return df.shape[0] - df.dropna(axis=0, how='all').shape[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [],
   "source": [
    "# only select that value, useful for constructing pivot_table\n",
    "def select(x):\n",
    "    return x"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "# return the unique values sorted in a df's column\n",
    "def unique(df, x):\n",
    "    # df - dataframe \n",
    "    # x - the label of the column \n",
    "    a = sorted(df[x].unique())\n",
    "    return a"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "# convert all strings in the array to int \n",
    "# e.g. \"T1.Q1.A5\" -> 5 \n",
    "def to_int(array):\n",
    "    # array - e.g. 2-d array where some entries are \"T1.Q1.A3\"\n",
    "    for i in range(array.shape[0]):\n",
    "        for j in range(array.shape[1]):\n",
    "            cur = array[i][j]\n",
    "            # if the current value is a string\n",
    "            if not pd.isnull(cur):\n",
    "                array[i][j] = int(array[i][j][-1])\n",
    "    return array"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [],
   "source": [
    "# construct a reliability matrix from Specialist csv \n",
    "def to_reliability(input_directory, fileName, question_label):\n",
    "    # input_directory - e.g. \"Specialists data\"\n",
    "    # fileName - e.g. \"Covid_SourceRelevancev1-2020-10-04T1838-DataHunt.csv\"\n",
    "    # question_label - e.g. \"Q2\"\n",
    "    name = os.path.join(input_directory, fileName)\n",
    "    # read in csv \n",
    "    df = pd.read_csv(name)\n",
    "    # select only that question_label \n",
    "    df = df[df[\"question_label\"] == question_label].copy()\n",
    "    # slice \n",
    "    cols = [\"quiz_taskrun_uuid\", \"contributor_uuid\", \"question_label\", \"answer_label\"]\n",
    "    df = df.reindex(columns=cols)\n",
    "    # construct a pivot_table \n",
    "    df = pd.pivot_table(df, values='answer_label', index='contributor_uuid', columns='quiz_taskrun_uuid', aggfunc=select)\n",
    "    # to numpy array\n",
    "    array = df.to_numpy()\n",
    "    # convert string to int \n",
    "    array = to_int(array)\n",
    "    # return the array \n",
    "    return array"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Dictionary for Q - quetion type"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "# dictionaries to distinguish between mc-multiple choice and cl-checklist type of questions\n",
    "source_relevance = {\"mc\":[1, 2, 3, 8, 10, 11], \"cl\":[4, 5, 6, 7], \"tx\":[12]}\n",
    "reasoning = {\"mc\":[7, 10, 11], \"cl\":[1, 2, 3, 6], \"tx\":[8]}\n",
    "probability = {\"mc\":[1, 2, ]}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "# dictionaries to distinguish between orginal and nominal(categorical) type of questions. \n",
    "source_relevance_mc = {\"ordinal\":[], \"nominal\":[]}\n",
    "reasoning_mc = {\"ordinal\":[], \"nominal\":[]}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## One-hot-encode checklist type of questions to 1s and 0s. "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "# e.g. if checklist question Q has 5 different options A-E, we one-hot-encode each one to be 1s and 0s, and do Krippendorff \n",
    "# on each one of them. "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Specialist"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 58,
   "metadata": {},
   "outputs": [],
   "source": [
    "# %load krippendorff.py\n",
    "\"\"\"\n",
    "This module provides a function to compute the Krippendorff's alpha statistical measure of the agreement achieved\n",
    "when coding a set of units based on the values of a variable.\n",
    "\n",
    "For more information, see: https://en.wikipedia.org/wiki/Krippendorff%27s_alpha\n",
    "\n",
    "The module naming follows the one from the Wikipedia link.\n",
    "\"\"\"\n",
    "from typing import Any, Callable, Iterable, Optional, Sequence, Union\n",
    "\n",
    "import numpy as np\n",
    "\n",
    "\n",
    "def _nominal_metric(v1: np.ndarray, v2: np.ndarray, dtype: Any = np.float64, **kwargs) -> np.ndarray:  # noqa\n",
    "    \"\"\"Metric for nominal data.\"\"\"\n",
    "    return (v1 != v2).astype(dtype)\n",
    "\n",
    "\n",
    "def _ordinal_metric(v1: np.ndarray, v2: np.ndarray, i1: np.ndarray, i2: np.ndarray,  # noqa\n",
    "                    n_v: np.ndarray, dtype: Any = np.float64, **kwargs) -> np.ndarray:  # noqa\n",
    "    \"\"\"Metric for ordinal data.\"\"\"\n",
    "    i1, i2 = np.minimum(i1, i2), np.maximum(i1, i2)\n",
    "\n",
    "    ranges = np.dstack((i1, i2 + 1))\n",
    "    sums_between_indices = np.add.reduceat(np.append(n_v, 0), ranges.reshape(-1))[::2].reshape(*i1.shape)\n",
    "\n",
    "    return (sums_between_indices - np.divide(n_v[i1] + n_v[i2], 2, dtype=dtype)) ** 2\n",
    "\n",
    "\n",
    "def _interval_metric(v1: np.ndarray, v2: np.ndarray, dtype: Any = np.float64, **kwargs) -> np.ndarray:  # noqa\n",
    "    \"\"\"Metric for interval data.\"\"\"\n",
    "    return (v1 - v2).astype(dtype) ** 2\n",
    "\n",
    "\n",
    "def _ratio_metric(v1: np.ndarray, v2: np.ndarray, dtype: Any = np.float64, **kwargs) -> np.ndarray:  # noqa\n",
    "    \"\"\"Metric for ratio data.\"\"\"\n",
    "    v1_plus_v2 = v1 + v2\n",
    "    return np.divide(v1 - v2, v1_plus_v2, out=np.zeros(np.broadcast(v1, v2).shape), where=v1_plus_v2 != 0,\n",
    "                     dtype=dtype) ** 2\n",
    "\n",
    "\n",
    "def _coincidences(value_counts: np.ndarray, dtype: Any = np.float64) -> np.ndarray:\n",
    "    \"\"\"Coincidence matrix.\n",
    "\n",
    "    Parameters\n",
    "    ----------\n",
    "    value_counts : ndarray, with shape (N, V)\n",
    "        Number of coders that assigned a certain value to a determined unit, where N is the number of units\n",
    "        and V is the value count.\n",
    "\n",
    "    dtype : data-type\n",
    "        Result and computation data-type.\n",
    "\n",
    "    Returns\n",
    "    -------\n",
    "    o : ndarray, with shape (V, V)\n",
    "        Coincidence matrix.\n",
    "    \"\"\"\n",
    "    N, V = value_counts.shape\n",
    "    pairable = np.maximum(value_counts.sum(axis=1), 2)\n",
    "    diagonals = value_counts[:, np.newaxis, :] * np.eye(V)[np.newaxis, ...]\n",
    "    unnormalized_coincidences = value_counts[..., np.newaxis] * value_counts[:, np.newaxis, :] - diagonals\n",
    "    return np.divide(unnormalized_coincidences, (pairable - 1).reshape((-1, 1, 1)), dtype=dtype).sum(axis=0)\n",
    "\n",
    "\n",
    "def _random_coincidences(n_v: np.ndarray, dtype: Any = np.float64) -> np.ndarray:\n",
    "    \"\"\"Random coincidence matrix.\n",
    "\n",
    "    Parameters\n",
    "    n_v : ndarray, with shape (V,)\n",
    "        Number of pairable elements for each value.\n",
    "\n",
    "    dtype : data-type\n",
    "        Result and computation data-type.\n",
    "\n",
    "    Returns\n",
    "    -------\n",
    "    e : ndarray, with shape (V, V)\n",
    "        Random coincidence matrix.\n",
    "    \"\"\"\n",
    "    return np.divide(np.outer(n_v, n_v) - np.diagflat(n_v), n_v.sum() - 1, dtype=dtype)\n",
    "\n",
    "\n",
    "def _distances(value_domain: np.ndarray, distance_metric: Callable[..., np.ndarray], n_v: np.ndarray,\n",
    "               dtype: Any = np.float64) -> np.ndarray:\n",
    "    \"\"\"Distances of the different possible values.\n",
    "\n",
    "    Parameters\n",
    "    ----------\n",
    "    value_domain : ndarray, with shape (V,)\n",
    "        Possible values V the units can take.\n",
    "        If the level of measurement is not nominal, it must be ordered.\n",
    "\n",
    "    distance_metric : callable\n",
    "        Callable that return the distance of two given values.\n",
    "\n",
    "    n_v : ndarray, with shape (V,)\n",
    "        Number of pairable elements for each value.\n",
    "\n",
    "    dtype : data-type\n",
    "        Result and computation data-type.\n",
    "\n",
    "    Returns\n",
    "    -------\n",
    "    d : ndarray, with shape (V, V)\n",
    "        Distance matrix for each value pair.\n",
    "    \"\"\"\n",
    "    indices = np.arange(len(value_domain))\n",
    "    return distance_metric(value_domain[:, np.newaxis], value_domain[np.newaxis, :], i1=indices[:, np.newaxis],\n",
    "                           i2=indices[np.newaxis, :], n_v=n_v, dtype=dtype)\n",
    "\n",
    "\n",
    "def _distance_metric(level_of_measurement: Union[str, Callable[..., np.ndarray]]) -> Callable[..., np.ndarray]:\n",
    "    \"\"\"Distance metric callable of the level of measurement.\n",
    "\n",
    "    Parameters\n",
    "    ----------\n",
    "    level_of_measurement : string or callable\n",
    "        Steven's level of measurement of the variable.\n",
    "        It must be one of 'nominal', 'ordinal', 'interval', 'ratio' or a callable.\n",
    "\n",
    "    Returns\n",
    "    -------\n",
    "    metric : callable\n",
    "        Distance callable.\n",
    "    \"\"\"\n",
    "    return {\n",
    "        'nominal': _nominal_metric,\n",
    "        'ordinal': _ordinal_metric,\n",
    "        'interval': _interval_metric,\n",
    "        'ratio': _ratio_metric,\n",
    "    }.get(level_of_measurement, level_of_measurement)\n",
    "\n",
    "\n",
    "def _reliability_data_to_value_counts(reliability_data: np.ndarray, value_domain: np.ndarray) -> np.ndarray:\n",
    "    \"\"\"Return the value counts given the reliability data.\n",
    "\n",
    "    Parameters\n",
    "    ----------\n",
    "    reliability_data : ndarray, with shape (M, N)\n",
    "        Reliability data matrix which has the rate the i coder gave to the j unit, where M is the number of raters\n",
    "        and N is the unit count.\n",
    "        Missing rates are represented with `np.nan`.\n",
    "\n",
    "    value_domain : ndarray, with shape (V,)\n",
    "        Possible values the units can take.\n",
    "\n",
    "    Returns\n",
    "    -------\n",
    "    value_counts : ndarray, with shape (N, V)\n",
    "        Number of coders that assigned a certain value to a determined unit, where N is the number of units\n",
    "        and V is the value count.\n",
    "    \"\"\"\n",
    "    return (reliability_data.T[..., np.newaxis] == value_domain[np.newaxis, np.newaxis, :]).sum(axis=1)  # noqa\n",
    "\n",
    "\n",
    "def alpha(reliability_data: Optional[Iterable[Any]] = None, value_counts: Optional[np.ndarray] = None,\n",
    "          value_domain: Optional[Sequence[Any]] = None,\n",
    "          level_of_measurement: Union[str, Callable[..., Any]] = 'interval', dtype: Any = np.float64) -> float:\n",
    "    \"\"\"Compute Krippendorff's alpha.\n",
    "\n",
    "    See https://en.wikipedia.org/wiki/Krippendorff%27s_alpha for more information.\n",
    "\n",
    "    Parameters\n",
    "    ----------\n",
    "    reliability_data : array_like, with shape (M, N)\n",
    "        Reliability data matrix which has the rate the i coder gave to the j unit, where M is the number of raters\n",
    "        and N is the unit count.\n",
    "        Missing rates are represented with `np.nan`.\n",
    "        If it's provided then `value_counts` must not be provided.\n",
    "\n",
    "    value_counts : array_like, with shape (N, V)\n",
    "        Number of coders that assigned a certain value to a determined unit, where N is the number of units\n",
    "        and V is the value count.\n",
    "        If it's provided then `reliability_data` must not be provided.\n",
    "\n",
    "    value_domain : array_like, with shape (V,)\n",
    "        Possible values the units can take.\n",
    "        If the level of measurement is not nominal, it must be ordered.\n",
    "        If `reliability_data` is provided, then the default value is the ordered list of unique rates that appear.\n",
    "        Else, the default value is `list(range(V))`.\n",
    "\n",
    "    level_of_measurement : string or callable\n",
    "        Steven's level of measurement of the variable.\n",
    "        It must be one of 'nominal', 'ordinal', 'interval', 'ratio' or a callable.\n",
    "\n",
    "    dtype : data-type\n",
    "        Result and computation data-type.\n",
    "\n",
    "    Returns\n",
    "    -------\n",
    "    alpha : ndarray\n",
    "        Scalar value of Krippendorff's alpha of type `dtype`.\n",
    "\n",
    "    Examples\n",
    "    --------\n",
    "    >>> reliability_data = [[np.nan, np.nan, np.nan, np.nan, np.nan, 3, 4, 1, 2, 1, 1, 3, 3, np.nan, 3],\n",
    "    ...                     [1, np.nan, 2, 1, 3, 3, 4, 3, np.nan, np.nan, np.nan, np.nan, np.nan, np.nan, np.nan],\n",
    "    ...                     [np.nan, np.nan, 2, 1, 3, 4, 4, np.nan, 2, 1, 1, 3, 3, np.nan, 4]]\n",
    "    >>> print(round(alpha(reliability_data=reliability_data, level_of_measurement='nominal'), 6))\n",
    "    0.691358\n",
    "    >>> print(round(alpha(reliability_data=reliability_data, level_of_measurement='interval'), 6))\n",
    "    0.810845\n",
    "    >>> value_counts = np.array([[1, 0, 0, 0],\n",
    "    ...                          [0, 0, 0, 0],\n",
    "    ...                          [0, 2, 0, 0],\n",
    "    ...                          [2, 0, 0, 0],\n",
    "    ...                          [0, 0, 2, 0],\n",
    "    ...                          [0, 0, 2, 1],\n",
    "    ...                          [0, 0, 0, 3],\n",
    "    ...                          [1, 0, 1, 0],\n",
    "    ...                          [0, 2, 0, 0],\n",
    "    ...                          [2, 0, 0, 0],\n",
    "    ...                          [2, 0, 0, 0],\n",
    "    ...                          [0, 0, 2, 0],\n",
    "    ...                          [0, 0, 2, 0],\n",
    "    ...                          [0, 0, 0, 0],\n",
    "    ...                          [0, 0, 1, 1]])\n",
    "    >>> print(round(alpha(value_counts=value_counts, level_of_measurement='nominal'), 6))\n",
    "    0.691358\n",
    "    >>> # The following examples were extracted from\n",
    "    >>> # https://www.statisticshowto.datasciencecentral.com/wp-content/uploads/2016/07/fulltext.pdf, page 8.\n",
    "    >>> reliability_data = [[1, 2, 3, 3, 2, 1, 4, 1, 2, np.nan, np.nan, np.nan],\n",
    "    ...                     [1, 2, 3, 3, 2, 2, 4, 1, 2, 5, np.nan, 3],\n",
    "    ...                     [np.nan, 3, 3, 3, 2, 3, 4, 2, 2, 5, 1, np.nan],\n",
    "    ...                     [1, 2, 3, 3, 2, 4, 4, 1, 2, 5, 1, np.nan]]\n",
    "    >>> print(round(alpha(reliability_data, level_of_measurement='ordinal'), 3))\n",
    "    0.815\n",
    "    >>> print(round(alpha(reliability_data, level_of_measurement='ratio'), 3))\n",
    "    0.797\n",
    "    >>> reliability_data = [[\"very low\", \"low\", \"mid\", \"mid\", \"low\", \"very low\", \"high\", \"very low\", \"low\", np.nan,\n",
    "    ...                      np.nan, np.nan],\n",
    "    ...                     [\"very low\", \"low\", \"mid\", \"mid\", \"low\", \"low\", \"high\", \"very low\", \"low\", \"very high\",\n",
    "    ...                      np.nan, \"mid\"],\n",
    "    ...                     [np.nan, \"mid\", \"mid\", \"mid\", \"low\", \"mid\", \"high\", \"low\", \"low\", \"very high\", \"very low\",\n",
    "    ...                      np.nan],\n",
    "    ...                     [\"very low\", \"low\", \"mid\", \"mid\", \"low\", \"high\", \"high\", \"very low\", \"low\", \"very high\",\n",
    "    ...                      \"very low\", np.nan]]\n",
    "    >>> print(round(alpha(reliability_data, level_of_measurement='ordinal',\n",
    "    ...                   value_domain=[\"very low\", \"low\", \"mid\", \"high\", \"very high\"]), 3))\n",
    "    0.815\n",
    "    \"\"\"\n",
    "    if (reliability_data is None) == (value_counts is None):\n",
    "        raise ValueError(\"Either reliability_data or value_counts must be provided, but not both.\")\n",
    "\n",
    "    # Don't know if it's a list or numpy array. If it's the latter, the truth value is ambiguous. So, ask for None.\n",
    "    if value_counts is None:\n",
    "        reliability_data = np.asarray(reliability_data)\n",
    "\n",
    "        if value_domain is None:\n",
    "#             value_domain = np.unique(reliability_data[~np.isnan(reliability_data)])\n",
    "            value_domain = np.unique(reliability_data[~pd.isnull(reliability_data)])\n",
    "        else:\n",
    "            value_domain = np.asarray(value_domain)\n",
    "            assert np.isin(reliability_data, np.append(value_domain, np.nan)).all(), \\\n",
    "                \"The reliability data contains out-of-domain values.\"\n",
    "\n",
    "        value_counts = _reliability_data_to_value_counts(reliability_data, value_domain)\n",
    "    else:  # elif reliability_data is None\n",
    "        value_counts = np.asarray(value_counts)\n",
    "\n",
    "        if value_domain is None:\n",
    "            value_domain = np.arange(value_counts.shape[1])\n",
    "        else:\n",
    "            value_domain = np.asarray(value_domain)\n",
    "            assert value_counts.shape[1] == len(value_domain), \\\n",
    "                \"The value domain should be equal to the number of columns of value_counts.\"\n",
    "\n",
    "    assert len(value_domain) > 1, \"There has to be more than one value in the domain.\"\n",
    "\n",
    "    distance_metric = _distance_metric(level_of_measurement)\n",
    "\n",
    "    o = _coincidences(value_counts, dtype=dtype)\n",
    "    n_v = o.sum(axis=0)\n",
    "    e = _random_coincidences(n_v, dtype=dtype)\n",
    "    d = _distances(value_domain, distance_metric, n_v, dtype=dtype)\n",
    "    return 1 - (o * d).sum() / (e * d).sum()\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Demo: to_reliability()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([nan, nan, nan, 5, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan,\n",
       "       nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan,\n",
       "       nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan,\n",
       "       nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, 2, nan,\n",
       "       nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan,\n",
       "       nan, nan, nan, nan, nan, nan, nan, nan, 2, nan, nan, nan, nan, nan,\n",
       "       nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan,\n",
       "       nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan,\n",
       "       nan, 5, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan,\n",
       "       nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan,\n",
       "       nan, nan, 5, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan,\n",
       "       nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, 5, nan, nan,\n",
       "       1, 5, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan,\n",
       "       nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan,\n",
       "       nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan,\n",
       "       nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan,\n",
       "       nan, nan, nan, nan, nan, nan, 3, nan, 3, nan, nan, nan, nan, nan,\n",
       "       nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, 1, nan, nan,\n",
       "       nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan,\n",
       "       nan, nan], dtype=object)"
      ]
     },
     "execution_count": 22,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "array = to_reliability(\"Specialists data\", \"Covid_SourceRelevancev1-2020-10-04T1838-DataHunt.csv\", \"T1.Q2\")\n",
    "array[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "# load the example data \n",
    "relevance = pd.read_csv(\"Specialists data/Covid_SourceRelevancev1-2020-10-04T1838-DataHunt.csv\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Index(['namespace', 'schema_sha256', 'quiz_task_uuid', 'task_url', 'tua_uuid',\n",
       "       'article_batch_name', 'article_number', 'article_filename',\n",
       "       'article_sha256', 'article_text_length', 'destination',\n",
       "       'task_redundancy', 'taskrun_count', 'quiz_taskrun_uuid',\n",
       "       'contributor_uuid', 'created', 'finish_time', 'elapsed_seconds',\n",
       "       'topic_name', 'question_label', 'question_text', 'answer_label',\n",
       "       'answer_content', 'answer_uuid', 'submitted_tua_uuid', 'answer_text',\n",
       "       'case_number', 'highlight_count', 'start_pos', 'end_pos',\n",
       "       'target_text'],\n",
       "      dtype='object')"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "relevance.columns"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "**Question 2: \"Does the experience/knowledge/position of the quoted source qualify them to speak or write such claims?\" is a categorical question**"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['T1.Q1',\n",
       " 'T1.Q10',\n",
       " 'T1.Q2',\n",
       " 'T1.Q3',\n",
       " 'T1.Q4',\n",
       " 'T1.Q5',\n",
       " 'T1.Q6',\n",
       " 'T1.Q7',\n",
       " 'T1.Q8',\n",
       " 'T1.Q9']"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "unique(relevance, \"question_label\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [],
   "source": [
    "relevance_q2 = relevance[relevance[\"question_label\"] == \"T1.Q2\"]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>quiz_taskrun_uuid</th>\n",
       "      <th>contributor_uuid</th>\n",
       "      <th>question_label</th>\n",
       "      <th>answer_label</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>aebae1f3-707a-4145-9fc3-7c5cf79bf0b8</td>\n",
       "      <td>00f548b7-6b63-4b47-828e-8e416b6ca0e2</td>\n",
       "      <td>T1.Q2</td>\n",
       "      <td>T1.Q2.A1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>18</th>\n",
       "      <td>fee75365-83d0-4c47-a520-73e0954af82c</td>\n",
       "      <td>e44f5799-6915-45fd-874e-060f6afedcb9</td>\n",
       "      <td>T1.Q2</td>\n",
       "      <td>T1.Q2.A2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>40</th>\n",
       "      <td>4ee3886b-6e92-44a5-861f-7ffe2f3ad1df</td>\n",
       "      <td>fed45769-aa61-4b8b-847e-a9c799acce15</td>\n",
       "      <td>T1.Q2</td>\n",
       "      <td>T1.Q2.A2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>53</th>\n",
       "      <td>8b229589-f037-4464-85cd-609de66d0612</td>\n",
       "      <td>fb4ae4cc-234b-484a-b541-a8e3a9254a24</td>\n",
       "      <td>T1.Q2</td>\n",
       "      <td>T1.Q2.A1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>73</th>\n",
       "      <td>fc945f0d-ecb8-49a2-8e87-90a83efe543e</td>\n",
       "      <td>3bb03f58-5f09-4414-b54e-43a814b9b09a</td>\n",
       "      <td>T1.Q2</td>\n",
       "      <td>T1.Q2.A1</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                       quiz_taskrun_uuid  \\\n",
       "1   aebae1f3-707a-4145-9fc3-7c5cf79bf0b8   \n",
       "18  fee75365-83d0-4c47-a520-73e0954af82c   \n",
       "40  4ee3886b-6e92-44a5-861f-7ffe2f3ad1df   \n",
       "53  8b229589-f037-4464-85cd-609de66d0612   \n",
       "73  fc945f0d-ecb8-49a2-8e87-90a83efe543e   \n",
       "\n",
       "                        contributor_uuid question_label answer_label  \n",
       "1   00f548b7-6b63-4b47-828e-8e416b6ca0e2          T1.Q2     T1.Q2.A1  \n",
       "18  e44f5799-6915-45fd-874e-060f6afedcb9          T1.Q2     T1.Q2.A2  \n",
       "40  fed45769-aa61-4b8b-847e-a9c799acce15          T1.Q2     T1.Q2.A2  \n",
       "53  fb4ae4cc-234b-484a-b541-a8e3a9254a24          T1.Q2     T1.Q2.A1  \n",
       "73  3bb03f58-5f09-4414-b54e-43a814b9b09a          T1.Q2     T1.Q2.A1  "
      ]
     },
     "execution_count": 27,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# select certain columns \n",
    "cols = [\"quiz_taskrun_uuid\", \"contributor_uuid\", \"question_label\", \"answer_label\"]\n",
    "relevance_q2 = relevance_q2.reindex(columns=cols)\n",
    "relevance_q2.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "**Construct the reliability matrix, the left column is the contributor_uuid, each unit -> quiz_taskrun_uuid**"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 70,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th>quiz_taskrun_uuid</th>\n",
       "      <th>00279610-b46d-4404-9037-3845373e9b62</th>\n",
       "      <th>0158e65d-c960-4565-96ab-574a5fcb750f</th>\n",
       "      <th>036dcfc5-d989-4d1a-a3ea-dcfe2beafd5c</th>\n",
       "      <th>0408ab23-08c4-44f7-84d0-346001d59e93</th>\n",
       "      <th>0734c61c-0448-461b-8dde-7b6abd8003d8</th>\n",
       "      <th>087e32bc-9701-42ef-835c-5b7ba62ce11d</th>\n",
       "      <th>0945a43f-7780-4628-87c4-67d3b0bd3981</th>\n",
       "      <th>0c0adb65-8241-4890-81eb-7e3bece8f0bc</th>\n",
       "      <th>0d209bb1-1795-44f4-8304-de766e94aade</th>\n",
       "      <th>0f0db5e0-540b-4ab7-9062-35170c03320b</th>\n",
       "      <th>...</th>\n",
       "      <th>fc5b9d46-43e7-4472-a112-23a72620a4e3</th>\n",
       "      <th>fc945f0d-ecb8-49a2-8e87-90a83efe543e</th>\n",
       "      <th>fcfd8244-3230-4ea7-ab49-3e49ab70ade9</th>\n",
       "      <th>fd487eb7-857e-4f14-b42f-ea45e67f7d1c</th>\n",
       "      <th>fd49c6a4-9033-47b1-974a-319ec93be21c</th>\n",
       "      <th>fdfb77fa-196e-4aa4-8d07-949d92a1b25c</th>\n",
       "      <th>fe3278b6-e072-46c2-ada3-3bbd148f13f6</th>\n",
       "      <th>fe552cc8-8090-4538-adf7-d963ff1de5a1</th>\n",
       "      <th>fee75365-83d0-4c47-a520-73e0954af82c</th>\n",
       "      <th>ffd3215a-8fc3-4ac7-a59a-0f9c43be2683</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>contributor_uuid</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>00f548b7-6b63-4b47-828e-8e416b6ca0e2</th>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>T1.Q2.A5</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>06e1eba7-c695-4495-85a7-2fb33ef9412c</th>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>0b2ee377-65f2-4ac1-98da-5bc093fee7aa</th>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>0cdf31d0-1966-4f69-a0c8-c483788b5256</th>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>0e4b8a3e-2714-4b5f-ba10-bb576aaa5cc6</th>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>0e81b339-ba89-48a6-b6eb-3160c5ac61db</th>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>0f369711-c76f-497a-bffe-4d10a3ab3619</th>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>172e9a38-d3e4-453e-8556-18bbd0fcb07e</th>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>23338303-8192-4d70-8e03-508a7120d1c3</th>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>28740234-10c8-4421-b56a-c7150e000d48</th>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>343ecb7a-aeb5-4b21-9e9c-df545dafba99</th>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>T1.Q2.A3</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>38b9cf3a-588d-4137-a975-af770121a925</th>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3bb03f58-5f09-4414-b54e-43a814b9b09a</th>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>T1.Q2.A5</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>T1.Q2.A1</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3c387871-0619-43f5-8aaf-4c1a9849c740</th>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>40a947aa-f2fc-43d8-baee-57a88ab21543</th>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>419f3399-0646-43dd-aef8-ea9a8f901007</th>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>T1.Q2.A8</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4a72a97f-24c4-4ef7-897f-9c78d1c6da96</th>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4e256eb7-841b-4239-b6cf-b782de8e492c</th>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>54221500-8c50-493f-bc91-037d13f705f1</th>\n",
       "      <td>T1.Q2.A5</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>566f0fa0-7d81-4f43-9c3f-dca97642bd5f</th>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>T1.Q2.A5</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>619ec434-0748-43e5-bc62-c79039d75e8d</th>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6a11b727-c5bb-4ae4-87b4-f7f27abda2d5</th>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8cddd693-f0df-441c-9c1c-927902e76099</th>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>...</td>\n",
       "      <td>T1.Q2.A5</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>T1.Q2.A5</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>97e53b7d-fba4-491d-97fc-65693f35ddcc</th>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>a1c98085-1fb3-4d6e-9ba6-46206afcb4a4</th>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>a31c365c-986b-4dbe-b752-bbb5986cc7f2</th>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>T1.Q2.A5</td>\n",
       "      <td>NaN</td>\n",
       "      <td>T1.Q2.A5</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>a3b0a59e-04b0-41ab-b05e-247ad2f20331</th>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>T1.Q2.A5</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>af937372-05a4-4143-97fa-7250c7597180</th>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>T1.Q2.A7</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>b633a457-b6d9-48c0-8b69-87cc725f470b</th>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>b72c51cb-0492-4e72-b6d7-43861be35f0c</th>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>bba6ae56-fae7-4c7f-993a-866ea4c3efc2</th>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>bd786026-bad5-4fa8-9a3a-38ca03a16412</th>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>be6f7700-52dd-477d-ae8c-6dbd9c8eeeec</th>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>c1d62210-4d59-4302-9d39-994919558bb1</th>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>T1.Q2.A4</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>c7387eb6-fbe8-4267-8937-38d7a0c7da48</th>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>cb5b27cd-a67a-429f-8bc3-c61a9c9991be</th>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>ce8a8db8-0d7f-4a11-8329-5a7fc065394d</th>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>T1.Q2.A5</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>d3e893ab-423f-4226-ba99-0638cd603b47</th>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>dd916dd6-c208-481f-9c28-c59b32347a46</th>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>T1.Q2.A5</td>\n",
       "      <td>NaN</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>e0834165-64cc-41e5-994f-9be97a16ef93</th>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>T1.Q2.A5</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>e1ae8875-a398-4dde-8f4e-4b21109784e3</th>\n",
       "      <td>NaN</td>\n",
       "      <td>T1.Q2.A3</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>e44f5799-6915-45fd-874e-060f6afedcb9</th>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>T1.Q2.A2</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>f71afc6b-0691-4bb2-81bf-e1a829ccff43</th>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>f9143626-bfe0-4e69-b652-6d1525ab4eb0</th>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>T1.Q2.A5</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>fb4ae4cc-234b-484a-b541-a8e3a9254a24</th>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>fed45769-aa61-4b8b-847e-a9c799acce15</th>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>ff659873-bfba-4da6-94e2-7b91ac8a2c5e</th>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>47 rows × 258 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "quiz_taskrun_uuid                    00279610-b46d-4404-9037-3845373e9b62  \\\n",
       "contributor_uuid                                                            \n",
       "00f548b7-6b63-4b47-828e-8e416b6ca0e2                                  NaN   \n",
       "06e1eba7-c695-4495-85a7-2fb33ef9412c                                  NaN   \n",
       "0b2ee377-65f2-4ac1-98da-5bc093fee7aa                                  NaN   \n",
       "0cdf31d0-1966-4f69-a0c8-c483788b5256                                  NaN   \n",
       "0e4b8a3e-2714-4b5f-ba10-bb576aaa5cc6                                  NaN   \n",
       "0e81b339-ba89-48a6-b6eb-3160c5ac61db                                  NaN   \n",
       "0f369711-c76f-497a-bffe-4d10a3ab3619                                  NaN   \n",
       "172e9a38-d3e4-453e-8556-18bbd0fcb07e                                  NaN   \n",
       "23338303-8192-4d70-8e03-508a7120d1c3                                  NaN   \n",
       "28740234-10c8-4421-b56a-c7150e000d48                                  NaN   \n",
       "343ecb7a-aeb5-4b21-9e9c-df545dafba99                                  NaN   \n",
       "38b9cf3a-588d-4137-a975-af770121a925                                  NaN   \n",
       "3bb03f58-5f09-4414-b54e-43a814b9b09a                                  NaN   \n",
       "3c387871-0619-43f5-8aaf-4c1a9849c740                                  NaN   \n",
       "40a947aa-f2fc-43d8-baee-57a88ab21543                                  NaN   \n",
       "419f3399-0646-43dd-aef8-ea9a8f901007                                  NaN   \n",
       "4a72a97f-24c4-4ef7-897f-9c78d1c6da96                                  NaN   \n",
       "4e256eb7-841b-4239-b6cf-b782de8e492c                                  NaN   \n",
       "54221500-8c50-493f-bc91-037d13f705f1                             T1.Q2.A5   \n",
       "566f0fa0-7d81-4f43-9c3f-dca97642bd5f                                  NaN   \n",
       "619ec434-0748-43e5-bc62-c79039d75e8d                                  NaN   \n",
       "6a11b727-c5bb-4ae4-87b4-f7f27abda2d5                                  NaN   \n",
       "8cddd693-f0df-441c-9c1c-927902e76099                                  NaN   \n",
       "97e53b7d-fba4-491d-97fc-65693f35ddcc                                  NaN   \n",
       "a1c98085-1fb3-4d6e-9ba6-46206afcb4a4                                  NaN   \n",
       "a31c365c-986b-4dbe-b752-bbb5986cc7f2                                  NaN   \n",
       "a3b0a59e-04b0-41ab-b05e-247ad2f20331                                  NaN   \n",
       "af937372-05a4-4143-97fa-7250c7597180                                  NaN   \n",
       "b633a457-b6d9-48c0-8b69-87cc725f470b                                  NaN   \n",
       "b72c51cb-0492-4e72-b6d7-43861be35f0c                                  NaN   \n",
       "bba6ae56-fae7-4c7f-993a-866ea4c3efc2                                  NaN   \n",
       "bd786026-bad5-4fa8-9a3a-38ca03a16412                                  NaN   \n",
       "be6f7700-52dd-477d-ae8c-6dbd9c8eeeec                                  NaN   \n",
       "c1d62210-4d59-4302-9d39-994919558bb1                                  NaN   \n",
       "c7387eb6-fbe8-4267-8937-38d7a0c7da48                                  NaN   \n",
       "cb5b27cd-a67a-429f-8bc3-c61a9c9991be                                  NaN   \n",
       "ce8a8db8-0d7f-4a11-8329-5a7fc065394d                                  NaN   \n",
       "d3e893ab-423f-4226-ba99-0638cd603b47                                  NaN   \n",
       "dd916dd6-c208-481f-9c28-c59b32347a46                                  NaN   \n",
       "e0834165-64cc-41e5-994f-9be97a16ef93                                  NaN   \n",
       "e1ae8875-a398-4dde-8f4e-4b21109784e3                                  NaN   \n",
       "e44f5799-6915-45fd-874e-060f6afedcb9                                  NaN   \n",
       "f71afc6b-0691-4bb2-81bf-e1a829ccff43                                  NaN   \n",
       "f9143626-bfe0-4e69-b652-6d1525ab4eb0                                  NaN   \n",
       "fb4ae4cc-234b-484a-b541-a8e3a9254a24                                  NaN   \n",
       "fed45769-aa61-4b8b-847e-a9c799acce15                                  NaN   \n",
       "ff659873-bfba-4da6-94e2-7b91ac8a2c5e                                  NaN   \n",
       "\n",
       "quiz_taskrun_uuid                    0158e65d-c960-4565-96ab-574a5fcb750f  \\\n",
       "contributor_uuid                                                            \n",
       "00f548b7-6b63-4b47-828e-8e416b6ca0e2                                  NaN   \n",
       "06e1eba7-c695-4495-85a7-2fb33ef9412c                                  NaN   \n",
       "0b2ee377-65f2-4ac1-98da-5bc093fee7aa                                  NaN   \n",
       "0cdf31d0-1966-4f69-a0c8-c483788b5256                                  NaN   \n",
       "0e4b8a3e-2714-4b5f-ba10-bb576aaa5cc6                                  NaN   \n",
       "0e81b339-ba89-48a6-b6eb-3160c5ac61db                                  NaN   \n",
       "0f369711-c76f-497a-bffe-4d10a3ab3619                                  NaN   \n",
       "172e9a38-d3e4-453e-8556-18bbd0fcb07e                                  NaN   \n",
       "23338303-8192-4d70-8e03-508a7120d1c3                                  NaN   \n",
       "28740234-10c8-4421-b56a-c7150e000d48                                  NaN   \n",
       "343ecb7a-aeb5-4b21-9e9c-df545dafba99                                  NaN   \n",
       "38b9cf3a-588d-4137-a975-af770121a925                                  NaN   \n",
       "3bb03f58-5f09-4414-b54e-43a814b9b09a                                  NaN   \n",
       "3c387871-0619-43f5-8aaf-4c1a9849c740                                  NaN   \n",
       "40a947aa-f2fc-43d8-baee-57a88ab21543                                  NaN   \n",
       "419f3399-0646-43dd-aef8-ea9a8f901007                                  NaN   \n",
       "4a72a97f-24c4-4ef7-897f-9c78d1c6da96                                  NaN   \n",
       "4e256eb7-841b-4239-b6cf-b782de8e492c                                  NaN   \n",
       "54221500-8c50-493f-bc91-037d13f705f1                                  NaN   \n",
       "566f0fa0-7d81-4f43-9c3f-dca97642bd5f                                  NaN   \n",
       "619ec434-0748-43e5-bc62-c79039d75e8d                                  NaN   \n",
       "6a11b727-c5bb-4ae4-87b4-f7f27abda2d5                                  NaN   \n",
       "8cddd693-f0df-441c-9c1c-927902e76099                                  NaN   \n",
       "97e53b7d-fba4-491d-97fc-65693f35ddcc                                  NaN   \n",
       "a1c98085-1fb3-4d6e-9ba6-46206afcb4a4                                  NaN   \n",
       "a31c365c-986b-4dbe-b752-bbb5986cc7f2                                  NaN   \n",
       "a3b0a59e-04b0-41ab-b05e-247ad2f20331                                  NaN   \n",
       "af937372-05a4-4143-97fa-7250c7597180                                  NaN   \n",
       "b633a457-b6d9-48c0-8b69-87cc725f470b                                  NaN   \n",
       "b72c51cb-0492-4e72-b6d7-43861be35f0c                                  NaN   \n",
       "bba6ae56-fae7-4c7f-993a-866ea4c3efc2                                  NaN   \n",
       "bd786026-bad5-4fa8-9a3a-38ca03a16412                                  NaN   \n",
       "be6f7700-52dd-477d-ae8c-6dbd9c8eeeec                                  NaN   \n",
       "c1d62210-4d59-4302-9d39-994919558bb1                                  NaN   \n",
       "c7387eb6-fbe8-4267-8937-38d7a0c7da48                                  NaN   \n",
       "cb5b27cd-a67a-429f-8bc3-c61a9c9991be                                  NaN   \n",
       "ce8a8db8-0d7f-4a11-8329-5a7fc065394d                                  NaN   \n",
       "d3e893ab-423f-4226-ba99-0638cd603b47                                  NaN   \n",
       "dd916dd6-c208-481f-9c28-c59b32347a46                                  NaN   \n",
       "e0834165-64cc-41e5-994f-9be97a16ef93                                  NaN   \n",
       "e1ae8875-a398-4dde-8f4e-4b21109784e3                             T1.Q2.A3   \n",
       "e44f5799-6915-45fd-874e-060f6afedcb9                                  NaN   \n",
       "f71afc6b-0691-4bb2-81bf-e1a829ccff43                                  NaN   \n",
       "f9143626-bfe0-4e69-b652-6d1525ab4eb0                                  NaN   \n",
       "fb4ae4cc-234b-484a-b541-a8e3a9254a24                                  NaN   \n",
       "fed45769-aa61-4b8b-847e-a9c799acce15                                  NaN   \n",
       "ff659873-bfba-4da6-94e2-7b91ac8a2c5e                                  NaN   \n",
       "\n",
       "quiz_taskrun_uuid                    036dcfc5-d989-4d1a-a3ea-dcfe2beafd5c  \\\n",
       "contributor_uuid                                                            \n",
       "00f548b7-6b63-4b47-828e-8e416b6ca0e2                                  NaN   \n",
       "06e1eba7-c695-4495-85a7-2fb33ef9412c                                  NaN   \n",
       "0b2ee377-65f2-4ac1-98da-5bc093fee7aa                                  NaN   \n",
       "0cdf31d0-1966-4f69-a0c8-c483788b5256                                  NaN   \n",
       "0e4b8a3e-2714-4b5f-ba10-bb576aaa5cc6                                  NaN   \n",
       "0e81b339-ba89-48a6-b6eb-3160c5ac61db                                  NaN   \n",
       "0f369711-c76f-497a-bffe-4d10a3ab3619                                  NaN   \n",
       "172e9a38-d3e4-453e-8556-18bbd0fcb07e                                  NaN   \n",
       "23338303-8192-4d70-8e03-508a7120d1c3                                  NaN   \n",
       "28740234-10c8-4421-b56a-c7150e000d48                                  NaN   \n",
       "343ecb7a-aeb5-4b21-9e9c-df545dafba99                                  NaN   \n",
       "38b9cf3a-588d-4137-a975-af770121a925                                  NaN   \n",
       "3bb03f58-5f09-4414-b54e-43a814b9b09a                             T1.Q2.A5   \n",
       "3c387871-0619-43f5-8aaf-4c1a9849c740                                  NaN   \n",
       "40a947aa-f2fc-43d8-baee-57a88ab21543                                  NaN   \n",
       "419f3399-0646-43dd-aef8-ea9a8f901007                                  NaN   \n",
       "4a72a97f-24c4-4ef7-897f-9c78d1c6da96                                  NaN   \n",
       "4e256eb7-841b-4239-b6cf-b782de8e492c                                  NaN   \n",
       "54221500-8c50-493f-bc91-037d13f705f1                                  NaN   \n",
       "566f0fa0-7d81-4f43-9c3f-dca97642bd5f                                  NaN   \n",
       "619ec434-0748-43e5-bc62-c79039d75e8d                                  NaN   \n",
       "6a11b727-c5bb-4ae4-87b4-f7f27abda2d5                                  NaN   \n",
       "8cddd693-f0df-441c-9c1c-927902e76099                                  NaN   \n",
       "97e53b7d-fba4-491d-97fc-65693f35ddcc                                  NaN   \n",
       "a1c98085-1fb3-4d6e-9ba6-46206afcb4a4                                  NaN   \n",
       "a31c365c-986b-4dbe-b752-bbb5986cc7f2                                  NaN   \n",
       "a3b0a59e-04b0-41ab-b05e-247ad2f20331                                  NaN   \n",
       "af937372-05a4-4143-97fa-7250c7597180                                  NaN   \n",
       "b633a457-b6d9-48c0-8b69-87cc725f470b                                  NaN   \n",
       "b72c51cb-0492-4e72-b6d7-43861be35f0c                                  NaN   \n",
       "bba6ae56-fae7-4c7f-993a-866ea4c3efc2                                  NaN   \n",
       "bd786026-bad5-4fa8-9a3a-38ca03a16412                                  NaN   \n",
       "be6f7700-52dd-477d-ae8c-6dbd9c8eeeec                                  NaN   \n",
       "c1d62210-4d59-4302-9d39-994919558bb1                                  NaN   \n",
       "c7387eb6-fbe8-4267-8937-38d7a0c7da48                                  NaN   \n",
       "cb5b27cd-a67a-429f-8bc3-c61a9c9991be                                  NaN   \n",
       "ce8a8db8-0d7f-4a11-8329-5a7fc065394d                                  NaN   \n",
       "d3e893ab-423f-4226-ba99-0638cd603b47                                  NaN   \n",
       "dd916dd6-c208-481f-9c28-c59b32347a46                                  NaN   \n",
       "e0834165-64cc-41e5-994f-9be97a16ef93                                  NaN   \n",
       "e1ae8875-a398-4dde-8f4e-4b21109784e3                                  NaN   \n",
       "e44f5799-6915-45fd-874e-060f6afedcb9                                  NaN   \n",
       "f71afc6b-0691-4bb2-81bf-e1a829ccff43                                  NaN   \n",
       "f9143626-bfe0-4e69-b652-6d1525ab4eb0                                  NaN   \n",
       "fb4ae4cc-234b-484a-b541-a8e3a9254a24                                  NaN   \n",
       "fed45769-aa61-4b8b-847e-a9c799acce15                                  NaN   \n",
       "ff659873-bfba-4da6-94e2-7b91ac8a2c5e                                  NaN   \n",
       "\n",
       "quiz_taskrun_uuid                    0408ab23-08c4-44f7-84d0-346001d59e93  \\\n",
       "contributor_uuid                                                            \n",
       "00f548b7-6b63-4b47-828e-8e416b6ca0e2                             T1.Q2.A5   \n",
       "06e1eba7-c695-4495-85a7-2fb33ef9412c                                  NaN   \n",
       "0b2ee377-65f2-4ac1-98da-5bc093fee7aa                                  NaN   \n",
       "0cdf31d0-1966-4f69-a0c8-c483788b5256                                  NaN   \n",
       "0e4b8a3e-2714-4b5f-ba10-bb576aaa5cc6                                  NaN   \n",
       "0e81b339-ba89-48a6-b6eb-3160c5ac61db                                  NaN   \n",
       "0f369711-c76f-497a-bffe-4d10a3ab3619                                  NaN   \n",
       "172e9a38-d3e4-453e-8556-18bbd0fcb07e                                  NaN   \n",
       "23338303-8192-4d70-8e03-508a7120d1c3                                  NaN   \n",
       "28740234-10c8-4421-b56a-c7150e000d48                                  NaN   \n",
       "343ecb7a-aeb5-4b21-9e9c-df545dafba99                                  NaN   \n",
       "38b9cf3a-588d-4137-a975-af770121a925                                  NaN   \n",
       "3bb03f58-5f09-4414-b54e-43a814b9b09a                                  NaN   \n",
       "3c387871-0619-43f5-8aaf-4c1a9849c740                                  NaN   \n",
       "40a947aa-f2fc-43d8-baee-57a88ab21543                                  NaN   \n",
       "419f3399-0646-43dd-aef8-ea9a8f901007                                  NaN   \n",
       "4a72a97f-24c4-4ef7-897f-9c78d1c6da96                                  NaN   \n",
       "4e256eb7-841b-4239-b6cf-b782de8e492c                                  NaN   \n",
       "54221500-8c50-493f-bc91-037d13f705f1                                  NaN   \n",
       "566f0fa0-7d81-4f43-9c3f-dca97642bd5f                                  NaN   \n",
       "619ec434-0748-43e5-bc62-c79039d75e8d                                  NaN   \n",
       "6a11b727-c5bb-4ae4-87b4-f7f27abda2d5                                  NaN   \n",
       "8cddd693-f0df-441c-9c1c-927902e76099                                  NaN   \n",
       "97e53b7d-fba4-491d-97fc-65693f35ddcc                                  NaN   \n",
       "a1c98085-1fb3-4d6e-9ba6-46206afcb4a4                                  NaN   \n",
       "a31c365c-986b-4dbe-b752-bbb5986cc7f2                                  NaN   \n",
       "a3b0a59e-04b0-41ab-b05e-247ad2f20331                                  NaN   \n",
       "af937372-05a4-4143-97fa-7250c7597180                                  NaN   \n",
       "b633a457-b6d9-48c0-8b69-87cc725f470b                                  NaN   \n",
       "b72c51cb-0492-4e72-b6d7-43861be35f0c                                  NaN   \n",
       "bba6ae56-fae7-4c7f-993a-866ea4c3efc2                                  NaN   \n",
       "bd786026-bad5-4fa8-9a3a-38ca03a16412                                  NaN   \n",
       "be6f7700-52dd-477d-ae8c-6dbd9c8eeeec                                  NaN   \n",
       "c1d62210-4d59-4302-9d39-994919558bb1                                  NaN   \n",
       "c7387eb6-fbe8-4267-8937-38d7a0c7da48                                  NaN   \n",
       "cb5b27cd-a67a-429f-8bc3-c61a9c9991be                                  NaN   \n",
       "ce8a8db8-0d7f-4a11-8329-5a7fc065394d                                  NaN   \n",
       "d3e893ab-423f-4226-ba99-0638cd603b47                                  NaN   \n",
       "dd916dd6-c208-481f-9c28-c59b32347a46                                  NaN   \n",
       "e0834165-64cc-41e5-994f-9be97a16ef93                                  NaN   \n",
       "e1ae8875-a398-4dde-8f4e-4b21109784e3                                  NaN   \n",
       "e44f5799-6915-45fd-874e-060f6afedcb9                                  NaN   \n",
       "f71afc6b-0691-4bb2-81bf-e1a829ccff43                                  NaN   \n",
       "f9143626-bfe0-4e69-b652-6d1525ab4eb0                                  NaN   \n",
       "fb4ae4cc-234b-484a-b541-a8e3a9254a24                                  NaN   \n",
       "fed45769-aa61-4b8b-847e-a9c799acce15                                  NaN   \n",
       "ff659873-bfba-4da6-94e2-7b91ac8a2c5e                                  NaN   \n",
       "\n",
       "quiz_taskrun_uuid                    0734c61c-0448-461b-8dde-7b6abd8003d8  \\\n",
       "contributor_uuid                                                            \n",
       "00f548b7-6b63-4b47-828e-8e416b6ca0e2                                  NaN   \n",
       "06e1eba7-c695-4495-85a7-2fb33ef9412c                                  NaN   \n",
       "0b2ee377-65f2-4ac1-98da-5bc093fee7aa                                  NaN   \n",
       "0cdf31d0-1966-4f69-a0c8-c483788b5256                                  NaN   \n",
       "0e4b8a3e-2714-4b5f-ba10-bb576aaa5cc6                                  NaN   \n",
       "0e81b339-ba89-48a6-b6eb-3160c5ac61db                                  NaN   \n",
       "0f369711-c76f-497a-bffe-4d10a3ab3619                                  NaN   \n",
       "172e9a38-d3e4-453e-8556-18bbd0fcb07e                                  NaN   \n",
       "23338303-8192-4d70-8e03-508a7120d1c3                                  NaN   \n",
       "28740234-10c8-4421-b56a-c7150e000d48                                  NaN   \n",
       "343ecb7a-aeb5-4b21-9e9c-df545dafba99                                  NaN   \n",
       "38b9cf3a-588d-4137-a975-af770121a925                                  NaN   \n",
       "3bb03f58-5f09-4414-b54e-43a814b9b09a                                  NaN   \n",
       "3c387871-0619-43f5-8aaf-4c1a9849c740                                  NaN   \n",
       "40a947aa-f2fc-43d8-baee-57a88ab21543                                  NaN   \n",
       "419f3399-0646-43dd-aef8-ea9a8f901007                                  NaN   \n",
       "4a72a97f-24c4-4ef7-897f-9c78d1c6da96                                  NaN   \n",
       "4e256eb7-841b-4239-b6cf-b782de8e492c                                  NaN   \n",
       "54221500-8c50-493f-bc91-037d13f705f1                                  NaN   \n",
       "566f0fa0-7d81-4f43-9c3f-dca97642bd5f                                  NaN   \n",
       "619ec434-0748-43e5-bc62-c79039d75e8d                                  NaN   \n",
       "6a11b727-c5bb-4ae4-87b4-f7f27abda2d5                                  NaN   \n",
       "8cddd693-f0df-441c-9c1c-927902e76099                                  NaN   \n",
       "97e53b7d-fba4-491d-97fc-65693f35ddcc                                  NaN   \n",
       "a1c98085-1fb3-4d6e-9ba6-46206afcb4a4                                  NaN   \n",
       "a31c365c-986b-4dbe-b752-bbb5986cc7f2                             T1.Q2.A5   \n",
       "a3b0a59e-04b0-41ab-b05e-247ad2f20331                                  NaN   \n",
       "af937372-05a4-4143-97fa-7250c7597180                                  NaN   \n",
       "b633a457-b6d9-48c0-8b69-87cc725f470b                                  NaN   \n",
       "b72c51cb-0492-4e72-b6d7-43861be35f0c                                  NaN   \n",
       "bba6ae56-fae7-4c7f-993a-866ea4c3efc2                                  NaN   \n",
       "bd786026-bad5-4fa8-9a3a-38ca03a16412                                  NaN   \n",
       "be6f7700-52dd-477d-ae8c-6dbd9c8eeeec                                  NaN   \n",
       "c1d62210-4d59-4302-9d39-994919558bb1                                  NaN   \n",
       "c7387eb6-fbe8-4267-8937-38d7a0c7da48                                  NaN   \n",
       "cb5b27cd-a67a-429f-8bc3-c61a9c9991be                                  NaN   \n",
       "ce8a8db8-0d7f-4a11-8329-5a7fc065394d                                  NaN   \n",
       "d3e893ab-423f-4226-ba99-0638cd603b47                                  NaN   \n",
       "dd916dd6-c208-481f-9c28-c59b32347a46                                  NaN   \n",
       "e0834165-64cc-41e5-994f-9be97a16ef93                                  NaN   \n",
       "e1ae8875-a398-4dde-8f4e-4b21109784e3                                  NaN   \n",
       "e44f5799-6915-45fd-874e-060f6afedcb9                                  NaN   \n",
       "f71afc6b-0691-4bb2-81bf-e1a829ccff43                                  NaN   \n",
       "f9143626-bfe0-4e69-b652-6d1525ab4eb0                                  NaN   \n",
       "fb4ae4cc-234b-484a-b541-a8e3a9254a24                                  NaN   \n",
       "fed45769-aa61-4b8b-847e-a9c799acce15                                  NaN   \n",
       "ff659873-bfba-4da6-94e2-7b91ac8a2c5e                                  NaN   \n",
       "\n",
       "quiz_taskrun_uuid                    087e32bc-9701-42ef-835c-5b7ba62ce11d  \\\n",
       "contributor_uuid                                                            \n",
       "00f548b7-6b63-4b47-828e-8e416b6ca0e2                                  NaN   \n",
       "06e1eba7-c695-4495-85a7-2fb33ef9412c                                  NaN   \n",
       "0b2ee377-65f2-4ac1-98da-5bc093fee7aa                                  NaN   \n",
       "0cdf31d0-1966-4f69-a0c8-c483788b5256                                  NaN   \n",
       "0e4b8a3e-2714-4b5f-ba10-bb576aaa5cc6                                  NaN   \n",
       "0e81b339-ba89-48a6-b6eb-3160c5ac61db                                  NaN   \n",
       "0f369711-c76f-497a-bffe-4d10a3ab3619                                  NaN   \n",
       "172e9a38-d3e4-453e-8556-18bbd0fcb07e                                  NaN   \n",
       "23338303-8192-4d70-8e03-508a7120d1c3                                  NaN   \n",
       "28740234-10c8-4421-b56a-c7150e000d48                                  NaN   \n",
       "343ecb7a-aeb5-4b21-9e9c-df545dafba99                                  NaN   \n",
       "38b9cf3a-588d-4137-a975-af770121a925                                  NaN   \n",
       "3bb03f58-5f09-4414-b54e-43a814b9b09a                                  NaN   \n",
       "3c387871-0619-43f5-8aaf-4c1a9849c740                                  NaN   \n",
       "40a947aa-f2fc-43d8-baee-57a88ab21543                                  NaN   \n",
       "419f3399-0646-43dd-aef8-ea9a8f901007                                  NaN   \n",
       "4a72a97f-24c4-4ef7-897f-9c78d1c6da96                                  NaN   \n",
       "4e256eb7-841b-4239-b6cf-b782de8e492c                                  NaN   \n",
       "54221500-8c50-493f-bc91-037d13f705f1                                  NaN   \n",
       "566f0fa0-7d81-4f43-9c3f-dca97642bd5f                                  NaN   \n",
       "619ec434-0748-43e5-bc62-c79039d75e8d                                  NaN   \n",
       "6a11b727-c5bb-4ae4-87b4-f7f27abda2d5                                  NaN   \n",
       "8cddd693-f0df-441c-9c1c-927902e76099                                  NaN   \n",
       "97e53b7d-fba4-491d-97fc-65693f35ddcc                                  NaN   \n",
       "a1c98085-1fb3-4d6e-9ba6-46206afcb4a4                                  NaN   \n",
       "a31c365c-986b-4dbe-b752-bbb5986cc7f2                                  NaN   \n",
       "a3b0a59e-04b0-41ab-b05e-247ad2f20331                                  NaN   \n",
       "af937372-05a4-4143-97fa-7250c7597180                             T1.Q2.A7   \n",
       "b633a457-b6d9-48c0-8b69-87cc725f470b                                  NaN   \n",
       "b72c51cb-0492-4e72-b6d7-43861be35f0c                                  NaN   \n",
       "bba6ae56-fae7-4c7f-993a-866ea4c3efc2                                  NaN   \n",
       "bd786026-bad5-4fa8-9a3a-38ca03a16412                                  NaN   \n",
       "be6f7700-52dd-477d-ae8c-6dbd9c8eeeec                                  NaN   \n",
       "c1d62210-4d59-4302-9d39-994919558bb1                                  NaN   \n",
       "c7387eb6-fbe8-4267-8937-38d7a0c7da48                                  NaN   \n",
       "cb5b27cd-a67a-429f-8bc3-c61a9c9991be                                  NaN   \n",
       "ce8a8db8-0d7f-4a11-8329-5a7fc065394d                                  NaN   \n",
       "d3e893ab-423f-4226-ba99-0638cd603b47                                  NaN   \n",
       "dd916dd6-c208-481f-9c28-c59b32347a46                                  NaN   \n",
       "e0834165-64cc-41e5-994f-9be97a16ef93                                  NaN   \n",
       "e1ae8875-a398-4dde-8f4e-4b21109784e3                                  NaN   \n",
       "e44f5799-6915-45fd-874e-060f6afedcb9                                  NaN   \n",
       "f71afc6b-0691-4bb2-81bf-e1a829ccff43                                  NaN   \n",
       "f9143626-bfe0-4e69-b652-6d1525ab4eb0                                  NaN   \n",
       "fb4ae4cc-234b-484a-b541-a8e3a9254a24                                  NaN   \n",
       "fed45769-aa61-4b8b-847e-a9c799acce15                                  NaN   \n",
       "ff659873-bfba-4da6-94e2-7b91ac8a2c5e                                  NaN   \n",
       "\n",
       "quiz_taskrun_uuid                    0945a43f-7780-4628-87c4-67d3b0bd3981  \\\n",
       "contributor_uuid                                                            \n",
       "00f548b7-6b63-4b47-828e-8e416b6ca0e2                                  NaN   \n",
       "06e1eba7-c695-4495-85a7-2fb33ef9412c                                  NaN   \n",
       "0b2ee377-65f2-4ac1-98da-5bc093fee7aa                                  NaN   \n",
       "0cdf31d0-1966-4f69-a0c8-c483788b5256                                  NaN   \n",
       "0e4b8a3e-2714-4b5f-ba10-bb576aaa5cc6                                  NaN   \n",
       "0e81b339-ba89-48a6-b6eb-3160c5ac61db                                  NaN   \n",
       "0f369711-c76f-497a-bffe-4d10a3ab3619                                  NaN   \n",
       "172e9a38-d3e4-453e-8556-18bbd0fcb07e                                  NaN   \n",
       "23338303-8192-4d70-8e03-508a7120d1c3                                  NaN   \n",
       "28740234-10c8-4421-b56a-c7150e000d48                                  NaN   \n",
       "343ecb7a-aeb5-4b21-9e9c-df545dafba99                                  NaN   \n",
       "38b9cf3a-588d-4137-a975-af770121a925                                  NaN   \n",
       "3bb03f58-5f09-4414-b54e-43a814b9b09a                                  NaN   \n",
       "3c387871-0619-43f5-8aaf-4c1a9849c740                                  NaN   \n",
       "40a947aa-f2fc-43d8-baee-57a88ab21543                                  NaN   \n",
       "419f3399-0646-43dd-aef8-ea9a8f901007                                  NaN   \n",
       "4a72a97f-24c4-4ef7-897f-9c78d1c6da96                                  NaN   \n",
       "4e256eb7-841b-4239-b6cf-b782de8e492c                                  NaN   \n",
       "54221500-8c50-493f-bc91-037d13f705f1                                  NaN   \n",
       "566f0fa0-7d81-4f43-9c3f-dca97642bd5f                                  NaN   \n",
       "619ec434-0748-43e5-bc62-c79039d75e8d                                  NaN   \n",
       "6a11b727-c5bb-4ae4-87b4-f7f27abda2d5                                  NaN   \n",
       "8cddd693-f0df-441c-9c1c-927902e76099                                  NaN   \n",
       "97e53b7d-fba4-491d-97fc-65693f35ddcc                                  NaN   \n",
       "a1c98085-1fb3-4d6e-9ba6-46206afcb4a4                                  NaN   \n",
       "a31c365c-986b-4dbe-b752-bbb5986cc7f2                             T1.Q2.A5   \n",
       "a3b0a59e-04b0-41ab-b05e-247ad2f20331                                  NaN   \n",
       "af937372-05a4-4143-97fa-7250c7597180                                  NaN   \n",
       "b633a457-b6d9-48c0-8b69-87cc725f470b                                  NaN   \n",
       "b72c51cb-0492-4e72-b6d7-43861be35f0c                                  NaN   \n",
       "bba6ae56-fae7-4c7f-993a-866ea4c3efc2                                  NaN   \n",
       "bd786026-bad5-4fa8-9a3a-38ca03a16412                                  NaN   \n",
       "be6f7700-52dd-477d-ae8c-6dbd9c8eeeec                                  NaN   \n",
       "c1d62210-4d59-4302-9d39-994919558bb1                                  NaN   \n",
       "c7387eb6-fbe8-4267-8937-38d7a0c7da48                                  NaN   \n",
       "cb5b27cd-a67a-429f-8bc3-c61a9c9991be                                  NaN   \n",
       "ce8a8db8-0d7f-4a11-8329-5a7fc065394d                                  NaN   \n",
       "d3e893ab-423f-4226-ba99-0638cd603b47                                  NaN   \n",
       "dd916dd6-c208-481f-9c28-c59b32347a46                                  NaN   \n",
       "e0834165-64cc-41e5-994f-9be97a16ef93                                  NaN   \n",
       "e1ae8875-a398-4dde-8f4e-4b21109784e3                                  NaN   \n",
       "e44f5799-6915-45fd-874e-060f6afedcb9                                  NaN   \n",
       "f71afc6b-0691-4bb2-81bf-e1a829ccff43                                  NaN   \n",
       "f9143626-bfe0-4e69-b652-6d1525ab4eb0                                  NaN   \n",
       "fb4ae4cc-234b-484a-b541-a8e3a9254a24                                  NaN   \n",
       "fed45769-aa61-4b8b-847e-a9c799acce15                                  NaN   \n",
       "ff659873-bfba-4da6-94e2-7b91ac8a2c5e                                  NaN   \n",
       "\n",
       "quiz_taskrun_uuid                    0c0adb65-8241-4890-81eb-7e3bece8f0bc  \\\n",
       "contributor_uuid                                                            \n",
       "00f548b7-6b63-4b47-828e-8e416b6ca0e2                                  NaN   \n",
       "06e1eba7-c695-4495-85a7-2fb33ef9412c                                  NaN   \n",
       "0b2ee377-65f2-4ac1-98da-5bc093fee7aa                                  NaN   \n",
       "0cdf31d0-1966-4f69-a0c8-c483788b5256                                  NaN   \n",
       "0e4b8a3e-2714-4b5f-ba10-bb576aaa5cc6                                  NaN   \n",
       "0e81b339-ba89-48a6-b6eb-3160c5ac61db                                  NaN   \n",
       "0f369711-c76f-497a-bffe-4d10a3ab3619                                  NaN   \n",
       "172e9a38-d3e4-453e-8556-18bbd0fcb07e                                  NaN   \n",
       "23338303-8192-4d70-8e03-508a7120d1c3                                  NaN   \n",
       "28740234-10c8-4421-b56a-c7150e000d48                                  NaN   \n",
       "343ecb7a-aeb5-4b21-9e9c-df545dafba99                                  NaN   \n",
       "38b9cf3a-588d-4137-a975-af770121a925                                  NaN   \n",
       "3bb03f58-5f09-4414-b54e-43a814b9b09a                                  NaN   \n",
       "3c387871-0619-43f5-8aaf-4c1a9849c740                                  NaN   \n",
       "40a947aa-f2fc-43d8-baee-57a88ab21543                                  NaN   \n",
       "419f3399-0646-43dd-aef8-ea9a8f901007                                  NaN   \n",
       "4a72a97f-24c4-4ef7-897f-9c78d1c6da96                                  NaN   \n",
       "4e256eb7-841b-4239-b6cf-b782de8e492c                                  NaN   \n",
       "54221500-8c50-493f-bc91-037d13f705f1                                  NaN   \n",
       "566f0fa0-7d81-4f43-9c3f-dca97642bd5f                                  NaN   \n",
       "619ec434-0748-43e5-bc62-c79039d75e8d                                  NaN   \n",
       "6a11b727-c5bb-4ae4-87b4-f7f27abda2d5                                  NaN   \n",
       "8cddd693-f0df-441c-9c1c-927902e76099                                  NaN   \n",
       "97e53b7d-fba4-491d-97fc-65693f35ddcc                                  NaN   \n",
       "a1c98085-1fb3-4d6e-9ba6-46206afcb4a4                                  NaN   \n",
       "a31c365c-986b-4dbe-b752-bbb5986cc7f2                                  NaN   \n",
       "a3b0a59e-04b0-41ab-b05e-247ad2f20331                                  NaN   \n",
       "af937372-05a4-4143-97fa-7250c7597180                                  NaN   \n",
       "b633a457-b6d9-48c0-8b69-87cc725f470b                                  NaN   \n",
       "b72c51cb-0492-4e72-b6d7-43861be35f0c                                  NaN   \n",
       "bba6ae56-fae7-4c7f-993a-866ea4c3efc2                                  NaN   \n",
       "bd786026-bad5-4fa8-9a3a-38ca03a16412                                  NaN   \n",
       "be6f7700-52dd-477d-ae8c-6dbd9c8eeeec                                  NaN   \n",
       "c1d62210-4d59-4302-9d39-994919558bb1                                  NaN   \n",
       "c7387eb6-fbe8-4267-8937-38d7a0c7da48                                  NaN   \n",
       "cb5b27cd-a67a-429f-8bc3-c61a9c9991be                                  NaN   \n",
       "ce8a8db8-0d7f-4a11-8329-5a7fc065394d                             T1.Q2.A5   \n",
       "d3e893ab-423f-4226-ba99-0638cd603b47                                  NaN   \n",
       "dd916dd6-c208-481f-9c28-c59b32347a46                                  NaN   \n",
       "e0834165-64cc-41e5-994f-9be97a16ef93                                  NaN   \n",
       "e1ae8875-a398-4dde-8f4e-4b21109784e3                                  NaN   \n",
       "e44f5799-6915-45fd-874e-060f6afedcb9                                  NaN   \n",
       "f71afc6b-0691-4bb2-81bf-e1a829ccff43                                  NaN   \n",
       "f9143626-bfe0-4e69-b652-6d1525ab4eb0                                  NaN   \n",
       "fb4ae4cc-234b-484a-b541-a8e3a9254a24                                  NaN   \n",
       "fed45769-aa61-4b8b-847e-a9c799acce15                                  NaN   \n",
       "ff659873-bfba-4da6-94e2-7b91ac8a2c5e                                  NaN   \n",
       "\n",
       "quiz_taskrun_uuid                    0d209bb1-1795-44f4-8304-de766e94aade  \\\n",
       "contributor_uuid                                                            \n",
       "00f548b7-6b63-4b47-828e-8e416b6ca0e2                                  NaN   \n",
       "06e1eba7-c695-4495-85a7-2fb33ef9412c                                  NaN   \n",
       "0b2ee377-65f2-4ac1-98da-5bc093fee7aa                                  NaN   \n",
       "0cdf31d0-1966-4f69-a0c8-c483788b5256                                  NaN   \n",
       "0e4b8a3e-2714-4b5f-ba10-bb576aaa5cc6                                  NaN   \n",
       "0e81b339-ba89-48a6-b6eb-3160c5ac61db                                  NaN   \n",
       "0f369711-c76f-497a-bffe-4d10a3ab3619                                  NaN   \n",
       "172e9a38-d3e4-453e-8556-18bbd0fcb07e                                  NaN   \n",
       "23338303-8192-4d70-8e03-508a7120d1c3                                  NaN   \n",
       "28740234-10c8-4421-b56a-c7150e000d48                                  NaN   \n",
       "343ecb7a-aeb5-4b21-9e9c-df545dafba99                                  NaN   \n",
       "38b9cf3a-588d-4137-a975-af770121a925                                  NaN   \n",
       "3bb03f58-5f09-4414-b54e-43a814b9b09a                                  NaN   \n",
       "3c387871-0619-43f5-8aaf-4c1a9849c740                                  NaN   \n",
       "40a947aa-f2fc-43d8-baee-57a88ab21543                                  NaN   \n",
       "419f3399-0646-43dd-aef8-ea9a8f901007                                  NaN   \n",
       "4a72a97f-24c4-4ef7-897f-9c78d1c6da96                                  NaN   \n",
       "4e256eb7-841b-4239-b6cf-b782de8e492c                                  NaN   \n",
       "54221500-8c50-493f-bc91-037d13f705f1                                  NaN   \n",
       "566f0fa0-7d81-4f43-9c3f-dca97642bd5f                                  NaN   \n",
       "619ec434-0748-43e5-bc62-c79039d75e8d                                  NaN   \n",
       "6a11b727-c5bb-4ae4-87b4-f7f27abda2d5                                  NaN   \n",
       "8cddd693-f0df-441c-9c1c-927902e76099                                  NaN   \n",
       "97e53b7d-fba4-491d-97fc-65693f35ddcc                                  NaN   \n",
       "a1c98085-1fb3-4d6e-9ba6-46206afcb4a4                                  NaN   \n",
       "a31c365c-986b-4dbe-b752-bbb5986cc7f2                                  NaN   \n",
       "a3b0a59e-04b0-41ab-b05e-247ad2f20331                                  NaN   \n",
       "af937372-05a4-4143-97fa-7250c7597180                                  NaN   \n",
       "b633a457-b6d9-48c0-8b69-87cc725f470b                                  NaN   \n",
       "b72c51cb-0492-4e72-b6d7-43861be35f0c                                  NaN   \n",
       "bba6ae56-fae7-4c7f-993a-866ea4c3efc2                                  NaN   \n",
       "bd786026-bad5-4fa8-9a3a-38ca03a16412                                  NaN   \n",
       "be6f7700-52dd-477d-ae8c-6dbd9c8eeeec                                  NaN   \n",
       "c1d62210-4d59-4302-9d39-994919558bb1                                  NaN   \n",
       "c7387eb6-fbe8-4267-8937-38d7a0c7da48                                  NaN   \n",
       "cb5b27cd-a67a-429f-8bc3-c61a9c9991be                                  NaN   \n",
       "ce8a8db8-0d7f-4a11-8329-5a7fc065394d                                  NaN   \n",
       "d3e893ab-423f-4226-ba99-0638cd603b47                                  NaN   \n",
       "dd916dd6-c208-481f-9c28-c59b32347a46                             T1.Q2.A5   \n",
       "e0834165-64cc-41e5-994f-9be97a16ef93                                  NaN   \n",
       "e1ae8875-a398-4dde-8f4e-4b21109784e3                                  NaN   \n",
       "e44f5799-6915-45fd-874e-060f6afedcb9                                  NaN   \n",
       "f71afc6b-0691-4bb2-81bf-e1a829ccff43                                  NaN   \n",
       "f9143626-bfe0-4e69-b652-6d1525ab4eb0                                  NaN   \n",
       "fb4ae4cc-234b-484a-b541-a8e3a9254a24                                  NaN   \n",
       "fed45769-aa61-4b8b-847e-a9c799acce15                                  NaN   \n",
       "ff659873-bfba-4da6-94e2-7b91ac8a2c5e                                  NaN   \n",
       "\n",
       "quiz_taskrun_uuid                    0f0db5e0-540b-4ab7-9062-35170c03320b  \\\n",
       "contributor_uuid                                                            \n",
       "00f548b7-6b63-4b47-828e-8e416b6ca0e2                                  NaN   \n",
       "06e1eba7-c695-4495-85a7-2fb33ef9412c                                  NaN   \n",
       "0b2ee377-65f2-4ac1-98da-5bc093fee7aa                                  NaN   \n",
       "0cdf31d0-1966-4f69-a0c8-c483788b5256                                  NaN   \n",
       "0e4b8a3e-2714-4b5f-ba10-bb576aaa5cc6                                  NaN   \n",
       "0e81b339-ba89-48a6-b6eb-3160c5ac61db                                  NaN   \n",
       "0f369711-c76f-497a-bffe-4d10a3ab3619                                  NaN   \n",
       "172e9a38-d3e4-453e-8556-18bbd0fcb07e                                  NaN   \n",
       "23338303-8192-4d70-8e03-508a7120d1c3                                  NaN   \n",
       "28740234-10c8-4421-b56a-c7150e000d48                                  NaN   \n",
       "343ecb7a-aeb5-4b21-9e9c-df545dafba99                                  NaN   \n",
       "38b9cf3a-588d-4137-a975-af770121a925                                  NaN   \n",
       "3bb03f58-5f09-4414-b54e-43a814b9b09a                                  NaN   \n",
       "3c387871-0619-43f5-8aaf-4c1a9849c740                                  NaN   \n",
       "40a947aa-f2fc-43d8-baee-57a88ab21543                                  NaN   \n",
       "419f3399-0646-43dd-aef8-ea9a8f901007                             T1.Q2.A8   \n",
       "4a72a97f-24c4-4ef7-897f-9c78d1c6da96                                  NaN   \n",
       "4e256eb7-841b-4239-b6cf-b782de8e492c                                  NaN   \n",
       "54221500-8c50-493f-bc91-037d13f705f1                                  NaN   \n",
       "566f0fa0-7d81-4f43-9c3f-dca97642bd5f                                  NaN   \n",
       "619ec434-0748-43e5-bc62-c79039d75e8d                                  NaN   \n",
       "6a11b727-c5bb-4ae4-87b4-f7f27abda2d5                                  NaN   \n",
       "8cddd693-f0df-441c-9c1c-927902e76099                                  NaN   \n",
       "97e53b7d-fba4-491d-97fc-65693f35ddcc                                  NaN   \n",
       "a1c98085-1fb3-4d6e-9ba6-46206afcb4a4                                  NaN   \n",
       "a31c365c-986b-4dbe-b752-bbb5986cc7f2                                  NaN   \n",
       "a3b0a59e-04b0-41ab-b05e-247ad2f20331                                  NaN   \n",
       "af937372-05a4-4143-97fa-7250c7597180                                  NaN   \n",
       "b633a457-b6d9-48c0-8b69-87cc725f470b                                  NaN   \n",
       "b72c51cb-0492-4e72-b6d7-43861be35f0c                                  NaN   \n",
       "bba6ae56-fae7-4c7f-993a-866ea4c3efc2                                  NaN   \n",
       "bd786026-bad5-4fa8-9a3a-38ca03a16412                                  NaN   \n",
       "be6f7700-52dd-477d-ae8c-6dbd9c8eeeec                                  NaN   \n",
       "c1d62210-4d59-4302-9d39-994919558bb1                                  NaN   \n",
       "c7387eb6-fbe8-4267-8937-38d7a0c7da48                                  NaN   \n",
       "cb5b27cd-a67a-429f-8bc3-c61a9c9991be                                  NaN   \n",
       "ce8a8db8-0d7f-4a11-8329-5a7fc065394d                                  NaN   \n",
       "d3e893ab-423f-4226-ba99-0638cd603b47                                  NaN   \n",
       "dd916dd6-c208-481f-9c28-c59b32347a46                                  NaN   \n",
       "e0834165-64cc-41e5-994f-9be97a16ef93                                  NaN   \n",
       "e1ae8875-a398-4dde-8f4e-4b21109784e3                                  NaN   \n",
       "e44f5799-6915-45fd-874e-060f6afedcb9                                  NaN   \n",
       "f71afc6b-0691-4bb2-81bf-e1a829ccff43                                  NaN   \n",
       "f9143626-bfe0-4e69-b652-6d1525ab4eb0                                  NaN   \n",
       "fb4ae4cc-234b-484a-b541-a8e3a9254a24                                  NaN   \n",
       "fed45769-aa61-4b8b-847e-a9c799acce15                                  NaN   \n",
       "ff659873-bfba-4da6-94e2-7b91ac8a2c5e                                  NaN   \n",
       "\n",
       "quiz_taskrun_uuid                     ...  \\\n",
       "contributor_uuid                      ...   \n",
       "00f548b7-6b63-4b47-828e-8e416b6ca0e2  ...   \n",
       "06e1eba7-c695-4495-85a7-2fb33ef9412c  ...   \n",
       "0b2ee377-65f2-4ac1-98da-5bc093fee7aa  ...   \n",
       "0cdf31d0-1966-4f69-a0c8-c483788b5256  ...   \n",
       "0e4b8a3e-2714-4b5f-ba10-bb576aaa5cc6  ...   \n",
       "0e81b339-ba89-48a6-b6eb-3160c5ac61db  ...   \n",
       "0f369711-c76f-497a-bffe-4d10a3ab3619  ...   \n",
       "172e9a38-d3e4-453e-8556-18bbd0fcb07e  ...   \n",
       "23338303-8192-4d70-8e03-508a7120d1c3  ...   \n",
       "28740234-10c8-4421-b56a-c7150e000d48  ...   \n",
       "343ecb7a-aeb5-4b21-9e9c-df545dafba99  ...   \n",
       "38b9cf3a-588d-4137-a975-af770121a925  ...   \n",
       "3bb03f58-5f09-4414-b54e-43a814b9b09a  ...   \n",
       "3c387871-0619-43f5-8aaf-4c1a9849c740  ...   \n",
       "40a947aa-f2fc-43d8-baee-57a88ab21543  ...   \n",
       "419f3399-0646-43dd-aef8-ea9a8f901007  ...   \n",
       "4a72a97f-24c4-4ef7-897f-9c78d1c6da96  ...   \n",
       "4e256eb7-841b-4239-b6cf-b782de8e492c  ...   \n",
       "54221500-8c50-493f-bc91-037d13f705f1  ...   \n",
       "566f0fa0-7d81-4f43-9c3f-dca97642bd5f  ...   \n",
       "619ec434-0748-43e5-bc62-c79039d75e8d  ...   \n",
       "6a11b727-c5bb-4ae4-87b4-f7f27abda2d5  ...   \n",
       "8cddd693-f0df-441c-9c1c-927902e76099  ...   \n",
       "97e53b7d-fba4-491d-97fc-65693f35ddcc  ...   \n",
       "a1c98085-1fb3-4d6e-9ba6-46206afcb4a4  ...   \n",
       "a31c365c-986b-4dbe-b752-bbb5986cc7f2  ...   \n",
       "a3b0a59e-04b0-41ab-b05e-247ad2f20331  ...   \n",
       "af937372-05a4-4143-97fa-7250c7597180  ...   \n",
       "b633a457-b6d9-48c0-8b69-87cc725f470b  ...   \n",
       "b72c51cb-0492-4e72-b6d7-43861be35f0c  ...   \n",
       "bba6ae56-fae7-4c7f-993a-866ea4c3efc2  ...   \n",
       "bd786026-bad5-4fa8-9a3a-38ca03a16412  ...   \n",
       "be6f7700-52dd-477d-ae8c-6dbd9c8eeeec  ...   \n",
       "c1d62210-4d59-4302-9d39-994919558bb1  ...   \n",
       "c7387eb6-fbe8-4267-8937-38d7a0c7da48  ...   \n",
       "cb5b27cd-a67a-429f-8bc3-c61a9c9991be  ...   \n",
       "ce8a8db8-0d7f-4a11-8329-5a7fc065394d  ...   \n",
       "d3e893ab-423f-4226-ba99-0638cd603b47  ...   \n",
       "dd916dd6-c208-481f-9c28-c59b32347a46  ...   \n",
       "e0834165-64cc-41e5-994f-9be97a16ef93  ...   \n",
       "e1ae8875-a398-4dde-8f4e-4b21109784e3  ...   \n",
       "e44f5799-6915-45fd-874e-060f6afedcb9  ...   \n",
       "f71afc6b-0691-4bb2-81bf-e1a829ccff43  ...   \n",
       "f9143626-bfe0-4e69-b652-6d1525ab4eb0  ...   \n",
       "fb4ae4cc-234b-484a-b541-a8e3a9254a24  ...   \n",
       "fed45769-aa61-4b8b-847e-a9c799acce15  ...   \n",
       "ff659873-bfba-4da6-94e2-7b91ac8a2c5e  ...   \n",
       "\n",
       "quiz_taskrun_uuid                    fc5b9d46-43e7-4472-a112-23a72620a4e3  \\\n",
       "contributor_uuid                                                            \n",
       "00f548b7-6b63-4b47-828e-8e416b6ca0e2                                  NaN   \n",
       "06e1eba7-c695-4495-85a7-2fb33ef9412c                                  NaN   \n",
       "0b2ee377-65f2-4ac1-98da-5bc093fee7aa                                  NaN   \n",
       "0cdf31d0-1966-4f69-a0c8-c483788b5256                                  NaN   \n",
       "0e4b8a3e-2714-4b5f-ba10-bb576aaa5cc6                                  NaN   \n",
       "0e81b339-ba89-48a6-b6eb-3160c5ac61db                                  NaN   \n",
       "0f369711-c76f-497a-bffe-4d10a3ab3619                                  NaN   \n",
       "172e9a38-d3e4-453e-8556-18bbd0fcb07e                                  NaN   \n",
       "23338303-8192-4d70-8e03-508a7120d1c3                                  NaN   \n",
       "28740234-10c8-4421-b56a-c7150e000d48                                  NaN   \n",
       "343ecb7a-aeb5-4b21-9e9c-df545dafba99                                  NaN   \n",
       "38b9cf3a-588d-4137-a975-af770121a925                                  NaN   \n",
       "3bb03f58-5f09-4414-b54e-43a814b9b09a                                  NaN   \n",
       "3c387871-0619-43f5-8aaf-4c1a9849c740                                  NaN   \n",
       "40a947aa-f2fc-43d8-baee-57a88ab21543                                  NaN   \n",
       "419f3399-0646-43dd-aef8-ea9a8f901007                                  NaN   \n",
       "4a72a97f-24c4-4ef7-897f-9c78d1c6da96                                  NaN   \n",
       "4e256eb7-841b-4239-b6cf-b782de8e492c                                  NaN   \n",
       "54221500-8c50-493f-bc91-037d13f705f1                                  NaN   \n",
       "566f0fa0-7d81-4f43-9c3f-dca97642bd5f                                  NaN   \n",
       "619ec434-0748-43e5-bc62-c79039d75e8d                                  NaN   \n",
       "6a11b727-c5bb-4ae4-87b4-f7f27abda2d5                                  NaN   \n",
       "8cddd693-f0df-441c-9c1c-927902e76099                             T1.Q2.A5   \n",
       "97e53b7d-fba4-491d-97fc-65693f35ddcc                                  NaN   \n",
       "a1c98085-1fb3-4d6e-9ba6-46206afcb4a4                                  NaN   \n",
       "a31c365c-986b-4dbe-b752-bbb5986cc7f2                                  NaN   \n",
       "a3b0a59e-04b0-41ab-b05e-247ad2f20331                                  NaN   \n",
       "af937372-05a4-4143-97fa-7250c7597180                                  NaN   \n",
       "b633a457-b6d9-48c0-8b69-87cc725f470b                                  NaN   \n",
       "b72c51cb-0492-4e72-b6d7-43861be35f0c                                  NaN   \n",
       "bba6ae56-fae7-4c7f-993a-866ea4c3efc2                                  NaN   \n",
       "bd786026-bad5-4fa8-9a3a-38ca03a16412                                  NaN   \n",
       "be6f7700-52dd-477d-ae8c-6dbd9c8eeeec                                  NaN   \n",
       "c1d62210-4d59-4302-9d39-994919558bb1                                  NaN   \n",
       "c7387eb6-fbe8-4267-8937-38d7a0c7da48                                  NaN   \n",
       "cb5b27cd-a67a-429f-8bc3-c61a9c9991be                                  NaN   \n",
       "ce8a8db8-0d7f-4a11-8329-5a7fc065394d                                  NaN   \n",
       "d3e893ab-423f-4226-ba99-0638cd603b47                                  NaN   \n",
       "dd916dd6-c208-481f-9c28-c59b32347a46                                  NaN   \n",
       "e0834165-64cc-41e5-994f-9be97a16ef93                                  NaN   \n",
       "e1ae8875-a398-4dde-8f4e-4b21109784e3                                  NaN   \n",
       "e44f5799-6915-45fd-874e-060f6afedcb9                                  NaN   \n",
       "f71afc6b-0691-4bb2-81bf-e1a829ccff43                                  NaN   \n",
       "f9143626-bfe0-4e69-b652-6d1525ab4eb0                                  NaN   \n",
       "fb4ae4cc-234b-484a-b541-a8e3a9254a24                                  NaN   \n",
       "fed45769-aa61-4b8b-847e-a9c799acce15                                  NaN   \n",
       "ff659873-bfba-4da6-94e2-7b91ac8a2c5e                                  NaN   \n",
       "\n",
       "quiz_taskrun_uuid                    fc945f0d-ecb8-49a2-8e87-90a83efe543e  \\\n",
       "contributor_uuid                                                            \n",
       "00f548b7-6b63-4b47-828e-8e416b6ca0e2                                  NaN   \n",
       "06e1eba7-c695-4495-85a7-2fb33ef9412c                                  NaN   \n",
       "0b2ee377-65f2-4ac1-98da-5bc093fee7aa                                  NaN   \n",
       "0cdf31d0-1966-4f69-a0c8-c483788b5256                                  NaN   \n",
       "0e4b8a3e-2714-4b5f-ba10-bb576aaa5cc6                                  NaN   \n",
       "0e81b339-ba89-48a6-b6eb-3160c5ac61db                                  NaN   \n",
       "0f369711-c76f-497a-bffe-4d10a3ab3619                                  NaN   \n",
       "172e9a38-d3e4-453e-8556-18bbd0fcb07e                                  NaN   \n",
       "23338303-8192-4d70-8e03-508a7120d1c3                                  NaN   \n",
       "28740234-10c8-4421-b56a-c7150e000d48                                  NaN   \n",
       "343ecb7a-aeb5-4b21-9e9c-df545dafba99                                  NaN   \n",
       "38b9cf3a-588d-4137-a975-af770121a925                                  NaN   \n",
       "3bb03f58-5f09-4414-b54e-43a814b9b09a                             T1.Q2.A1   \n",
       "3c387871-0619-43f5-8aaf-4c1a9849c740                                  NaN   \n",
       "40a947aa-f2fc-43d8-baee-57a88ab21543                                  NaN   \n",
       "419f3399-0646-43dd-aef8-ea9a8f901007                                  NaN   \n",
       "4a72a97f-24c4-4ef7-897f-9c78d1c6da96                                  NaN   \n",
       "4e256eb7-841b-4239-b6cf-b782de8e492c                                  NaN   \n",
       "54221500-8c50-493f-bc91-037d13f705f1                                  NaN   \n",
       "566f0fa0-7d81-4f43-9c3f-dca97642bd5f                                  NaN   \n",
       "619ec434-0748-43e5-bc62-c79039d75e8d                                  NaN   \n",
       "6a11b727-c5bb-4ae4-87b4-f7f27abda2d5                                  NaN   \n",
       "8cddd693-f0df-441c-9c1c-927902e76099                                  NaN   \n",
       "97e53b7d-fba4-491d-97fc-65693f35ddcc                                  NaN   \n",
       "a1c98085-1fb3-4d6e-9ba6-46206afcb4a4                                  NaN   \n",
       "a31c365c-986b-4dbe-b752-bbb5986cc7f2                                  NaN   \n",
       "a3b0a59e-04b0-41ab-b05e-247ad2f20331                                  NaN   \n",
       "af937372-05a4-4143-97fa-7250c7597180                                  NaN   \n",
       "b633a457-b6d9-48c0-8b69-87cc725f470b                                  NaN   \n",
       "b72c51cb-0492-4e72-b6d7-43861be35f0c                                  NaN   \n",
       "bba6ae56-fae7-4c7f-993a-866ea4c3efc2                                  NaN   \n",
       "bd786026-bad5-4fa8-9a3a-38ca03a16412                                  NaN   \n",
       "be6f7700-52dd-477d-ae8c-6dbd9c8eeeec                                  NaN   \n",
       "c1d62210-4d59-4302-9d39-994919558bb1                                  NaN   \n",
       "c7387eb6-fbe8-4267-8937-38d7a0c7da48                                  NaN   \n",
       "cb5b27cd-a67a-429f-8bc3-c61a9c9991be                                  NaN   \n",
       "ce8a8db8-0d7f-4a11-8329-5a7fc065394d                                  NaN   \n",
       "d3e893ab-423f-4226-ba99-0638cd603b47                                  NaN   \n",
       "dd916dd6-c208-481f-9c28-c59b32347a46                                  NaN   \n",
       "e0834165-64cc-41e5-994f-9be97a16ef93                                  NaN   \n",
       "e1ae8875-a398-4dde-8f4e-4b21109784e3                                  NaN   \n",
       "e44f5799-6915-45fd-874e-060f6afedcb9                                  NaN   \n",
       "f71afc6b-0691-4bb2-81bf-e1a829ccff43                                  NaN   \n",
       "f9143626-bfe0-4e69-b652-6d1525ab4eb0                                  NaN   \n",
       "fb4ae4cc-234b-484a-b541-a8e3a9254a24                                  NaN   \n",
       "fed45769-aa61-4b8b-847e-a9c799acce15                                  NaN   \n",
       "ff659873-bfba-4da6-94e2-7b91ac8a2c5e                                  NaN   \n",
       "\n",
       "quiz_taskrun_uuid                    fcfd8244-3230-4ea7-ab49-3e49ab70ade9  \\\n",
       "contributor_uuid                                                            \n",
       "00f548b7-6b63-4b47-828e-8e416b6ca0e2                                  NaN   \n",
       "06e1eba7-c695-4495-85a7-2fb33ef9412c                                  NaN   \n",
       "0b2ee377-65f2-4ac1-98da-5bc093fee7aa                                  NaN   \n",
       "0cdf31d0-1966-4f69-a0c8-c483788b5256                                  NaN   \n",
       "0e4b8a3e-2714-4b5f-ba10-bb576aaa5cc6                                  NaN   \n",
       "0e81b339-ba89-48a6-b6eb-3160c5ac61db                                  NaN   \n",
       "0f369711-c76f-497a-bffe-4d10a3ab3619                                  NaN   \n",
       "172e9a38-d3e4-453e-8556-18bbd0fcb07e                                  NaN   \n",
       "23338303-8192-4d70-8e03-508a7120d1c3                                  NaN   \n",
       "28740234-10c8-4421-b56a-c7150e000d48                                  NaN   \n",
       "343ecb7a-aeb5-4b21-9e9c-df545dafba99                                  NaN   \n",
       "38b9cf3a-588d-4137-a975-af770121a925                                  NaN   \n",
       "3bb03f58-5f09-4414-b54e-43a814b9b09a                                  NaN   \n",
       "3c387871-0619-43f5-8aaf-4c1a9849c740                                  NaN   \n",
       "40a947aa-f2fc-43d8-baee-57a88ab21543                                  NaN   \n",
       "419f3399-0646-43dd-aef8-ea9a8f901007                                  NaN   \n",
       "4a72a97f-24c4-4ef7-897f-9c78d1c6da96                                  NaN   \n",
       "4e256eb7-841b-4239-b6cf-b782de8e492c                                  NaN   \n",
       "54221500-8c50-493f-bc91-037d13f705f1                                  NaN   \n",
       "566f0fa0-7d81-4f43-9c3f-dca97642bd5f                             T1.Q2.A5   \n",
       "619ec434-0748-43e5-bc62-c79039d75e8d                                  NaN   \n",
       "6a11b727-c5bb-4ae4-87b4-f7f27abda2d5                                  NaN   \n",
       "8cddd693-f0df-441c-9c1c-927902e76099                                  NaN   \n",
       "97e53b7d-fba4-491d-97fc-65693f35ddcc                                  NaN   \n",
       "a1c98085-1fb3-4d6e-9ba6-46206afcb4a4                                  NaN   \n",
       "a31c365c-986b-4dbe-b752-bbb5986cc7f2                                  NaN   \n",
       "a3b0a59e-04b0-41ab-b05e-247ad2f20331                                  NaN   \n",
       "af937372-05a4-4143-97fa-7250c7597180                                  NaN   \n",
       "b633a457-b6d9-48c0-8b69-87cc725f470b                                  NaN   \n",
       "b72c51cb-0492-4e72-b6d7-43861be35f0c                                  NaN   \n",
       "bba6ae56-fae7-4c7f-993a-866ea4c3efc2                                  NaN   \n",
       "bd786026-bad5-4fa8-9a3a-38ca03a16412                                  NaN   \n",
       "be6f7700-52dd-477d-ae8c-6dbd9c8eeeec                                  NaN   \n",
       "c1d62210-4d59-4302-9d39-994919558bb1                                  NaN   \n",
       "c7387eb6-fbe8-4267-8937-38d7a0c7da48                                  NaN   \n",
       "cb5b27cd-a67a-429f-8bc3-c61a9c9991be                                  NaN   \n",
       "ce8a8db8-0d7f-4a11-8329-5a7fc065394d                                  NaN   \n",
       "d3e893ab-423f-4226-ba99-0638cd603b47                                  NaN   \n",
       "dd916dd6-c208-481f-9c28-c59b32347a46                                  NaN   \n",
       "e0834165-64cc-41e5-994f-9be97a16ef93                                  NaN   \n",
       "e1ae8875-a398-4dde-8f4e-4b21109784e3                                  NaN   \n",
       "e44f5799-6915-45fd-874e-060f6afedcb9                                  NaN   \n",
       "f71afc6b-0691-4bb2-81bf-e1a829ccff43                                  NaN   \n",
       "f9143626-bfe0-4e69-b652-6d1525ab4eb0                                  NaN   \n",
       "fb4ae4cc-234b-484a-b541-a8e3a9254a24                                  NaN   \n",
       "fed45769-aa61-4b8b-847e-a9c799acce15                                  NaN   \n",
       "ff659873-bfba-4da6-94e2-7b91ac8a2c5e                                  NaN   \n",
       "\n",
       "quiz_taskrun_uuid                    fd487eb7-857e-4f14-b42f-ea45e67f7d1c  \\\n",
       "contributor_uuid                                                            \n",
       "00f548b7-6b63-4b47-828e-8e416b6ca0e2                                  NaN   \n",
       "06e1eba7-c695-4495-85a7-2fb33ef9412c                                  NaN   \n",
       "0b2ee377-65f2-4ac1-98da-5bc093fee7aa                                  NaN   \n",
       "0cdf31d0-1966-4f69-a0c8-c483788b5256                                  NaN   \n",
       "0e4b8a3e-2714-4b5f-ba10-bb576aaa5cc6                                  NaN   \n",
       "0e81b339-ba89-48a6-b6eb-3160c5ac61db                                  NaN   \n",
       "0f369711-c76f-497a-bffe-4d10a3ab3619                                  NaN   \n",
       "172e9a38-d3e4-453e-8556-18bbd0fcb07e                                  NaN   \n",
       "23338303-8192-4d70-8e03-508a7120d1c3                                  NaN   \n",
       "28740234-10c8-4421-b56a-c7150e000d48                                  NaN   \n",
       "343ecb7a-aeb5-4b21-9e9c-df545dafba99                             T1.Q2.A3   \n",
       "38b9cf3a-588d-4137-a975-af770121a925                                  NaN   \n",
       "3bb03f58-5f09-4414-b54e-43a814b9b09a                                  NaN   \n",
       "3c387871-0619-43f5-8aaf-4c1a9849c740                                  NaN   \n",
       "40a947aa-f2fc-43d8-baee-57a88ab21543                                  NaN   \n",
       "419f3399-0646-43dd-aef8-ea9a8f901007                                  NaN   \n",
       "4a72a97f-24c4-4ef7-897f-9c78d1c6da96                                  NaN   \n",
       "4e256eb7-841b-4239-b6cf-b782de8e492c                                  NaN   \n",
       "54221500-8c50-493f-bc91-037d13f705f1                                  NaN   \n",
       "566f0fa0-7d81-4f43-9c3f-dca97642bd5f                                  NaN   \n",
       "619ec434-0748-43e5-bc62-c79039d75e8d                                  NaN   \n",
       "6a11b727-c5bb-4ae4-87b4-f7f27abda2d5                                  NaN   \n",
       "8cddd693-f0df-441c-9c1c-927902e76099                                  NaN   \n",
       "97e53b7d-fba4-491d-97fc-65693f35ddcc                                  NaN   \n",
       "a1c98085-1fb3-4d6e-9ba6-46206afcb4a4                                  NaN   \n",
       "a31c365c-986b-4dbe-b752-bbb5986cc7f2                                  NaN   \n",
       "a3b0a59e-04b0-41ab-b05e-247ad2f20331                                  NaN   \n",
       "af937372-05a4-4143-97fa-7250c7597180                                  NaN   \n",
       "b633a457-b6d9-48c0-8b69-87cc725f470b                                  NaN   \n",
       "b72c51cb-0492-4e72-b6d7-43861be35f0c                                  NaN   \n",
       "bba6ae56-fae7-4c7f-993a-866ea4c3efc2                                  NaN   \n",
       "bd786026-bad5-4fa8-9a3a-38ca03a16412                                  NaN   \n",
       "be6f7700-52dd-477d-ae8c-6dbd9c8eeeec                                  NaN   \n",
       "c1d62210-4d59-4302-9d39-994919558bb1                                  NaN   \n",
       "c7387eb6-fbe8-4267-8937-38d7a0c7da48                                  NaN   \n",
       "cb5b27cd-a67a-429f-8bc3-c61a9c9991be                                  NaN   \n",
       "ce8a8db8-0d7f-4a11-8329-5a7fc065394d                                  NaN   \n",
       "d3e893ab-423f-4226-ba99-0638cd603b47                                  NaN   \n",
       "dd916dd6-c208-481f-9c28-c59b32347a46                                  NaN   \n",
       "e0834165-64cc-41e5-994f-9be97a16ef93                                  NaN   \n",
       "e1ae8875-a398-4dde-8f4e-4b21109784e3                                  NaN   \n",
       "e44f5799-6915-45fd-874e-060f6afedcb9                                  NaN   \n",
       "f71afc6b-0691-4bb2-81bf-e1a829ccff43                                  NaN   \n",
       "f9143626-bfe0-4e69-b652-6d1525ab4eb0                                  NaN   \n",
       "fb4ae4cc-234b-484a-b541-a8e3a9254a24                                  NaN   \n",
       "fed45769-aa61-4b8b-847e-a9c799acce15                                  NaN   \n",
       "ff659873-bfba-4da6-94e2-7b91ac8a2c5e                                  NaN   \n",
       "\n",
       "quiz_taskrun_uuid                    fd49c6a4-9033-47b1-974a-319ec93be21c  \\\n",
       "contributor_uuid                                                            \n",
       "00f548b7-6b63-4b47-828e-8e416b6ca0e2                                  NaN   \n",
       "06e1eba7-c695-4495-85a7-2fb33ef9412c                                  NaN   \n",
       "0b2ee377-65f2-4ac1-98da-5bc093fee7aa                                  NaN   \n",
       "0cdf31d0-1966-4f69-a0c8-c483788b5256                                  NaN   \n",
       "0e4b8a3e-2714-4b5f-ba10-bb576aaa5cc6                                  NaN   \n",
       "0e81b339-ba89-48a6-b6eb-3160c5ac61db                                  NaN   \n",
       "0f369711-c76f-497a-bffe-4d10a3ab3619                                  NaN   \n",
       "172e9a38-d3e4-453e-8556-18bbd0fcb07e                                  NaN   \n",
       "23338303-8192-4d70-8e03-508a7120d1c3                                  NaN   \n",
       "28740234-10c8-4421-b56a-c7150e000d48                                  NaN   \n",
       "343ecb7a-aeb5-4b21-9e9c-df545dafba99                                  NaN   \n",
       "38b9cf3a-588d-4137-a975-af770121a925                                  NaN   \n",
       "3bb03f58-5f09-4414-b54e-43a814b9b09a                                  NaN   \n",
       "3c387871-0619-43f5-8aaf-4c1a9849c740                                  NaN   \n",
       "40a947aa-f2fc-43d8-baee-57a88ab21543                                  NaN   \n",
       "419f3399-0646-43dd-aef8-ea9a8f901007                                  NaN   \n",
       "4a72a97f-24c4-4ef7-897f-9c78d1c6da96                                  NaN   \n",
       "4e256eb7-841b-4239-b6cf-b782de8e492c                                  NaN   \n",
       "54221500-8c50-493f-bc91-037d13f705f1                                  NaN   \n",
       "566f0fa0-7d81-4f43-9c3f-dca97642bd5f                                  NaN   \n",
       "619ec434-0748-43e5-bc62-c79039d75e8d                                  NaN   \n",
       "6a11b727-c5bb-4ae4-87b4-f7f27abda2d5                                  NaN   \n",
       "8cddd693-f0df-441c-9c1c-927902e76099                             T1.Q2.A5   \n",
       "97e53b7d-fba4-491d-97fc-65693f35ddcc                                  NaN   \n",
       "a1c98085-1fb3-4d6e-9ba6-46206afcb4a4                                  NaN   \n",
       "a31c365c-986b-4dbe-b752-bbb5986cc7f2                                  NaN   \n",
       "a3b0a59e-04b0-41ab-b05e-247ad2f20331                                  NaN   \n",
       "af937372-05a4-4143-97fa-7250c7597180                                  NaN   \n",
       "b633a457-b6d9-48c0-8b69-87cc725f470b                                  NaN   \n",
       "b72c51cb-0492-4e72-b6d7-43861be35f0c                                  NaN   \n",
       "bba6ae56-fae7-4c7f-993a-866ea4c3efc2                                  NaN   \n",
       "bd786026-bad5-4fa8-9a3a-38ca03a16412                                  NaN   \n",
       "be6f7700-52dd-477d-ae8c-6dbd9c8eeeec                                  NaN   \n",
       "c1d62210-4d59-4302-9d39-994919558bb1                                  NaN   \n",
       "c7387eb6-fbe8-4267-8937-38d7a0c7da48                                  NaN   \n",
       "cb5b27cd-a67a-429f-8bc3-c61a9c9991be                                  NaN   \n",
       "ce8a8db8-0d7f-4a11-8329-5a7fc065394d                                  NaN   \n",
       "d3e893ab-423f-4226-ba99-0638cd603b47                                  NaN   \n",
       "dd916dd6-c208-481f-9c28-c59b32347a46                                  NaN   \n",
       "e0834165-64cc-41e5-994f-9be97a16ef93                                  NaN   \n",
       "e1ae8875-a398-4dde-8f4e-4b21109784e3                                  NaN   \n",
       "e44f5799-6915-45fd-874e-060f6afedcb9                                  NaN   \n",
       "f71afc6b-0691-4bb2-81bf-e1a829ccff43                                  NaN   \n",
       "f9143626-bfe0-4e69-b652-6d1525ab4eb0                                  NaN   \n",
       "fb4ae4cc-234b-484a-b541-a8e3a9254a24                                  NaN   \n",
       "fed45769-aa61-4b8b-847e-a9c799acce15                                  NaN   \n",
       "ff659873-bfba-4da6-94e2-7b91ac8a2c5e                                  NaN   \n",
       "\n",
       "quiz_taskrun_uuid                    fdfb77fa-196e-4aa4-8d07-949d92a1b25c  \\\n",
       "contributor_uuid                                                            \n",
       "00f548b7-6b63-4b47-828e-8e416b6ca0e2                                  NaN   \n",
       "06e1eba7-c695-4495-85a7-2fb33ef9412c                                  NaN   \n",
       "0b2ee377-65f2-4ac1-98da-5bc093fee7aa                                  NaN   \n",
       "0cdf31d0-1966-4f69-a0c8-c483788b5256                                  NaN   \n",
       "0e4b8a3e-2714-4b5f-ba10-bb576aaa5cc6                                  NaN   \n",
       "0e81b339-ba89-48a6-b6eb-3160c5ac61db                                  NaN   \n",
       "0f369711-c76f-497a-bffe-4d10a3ab3619                                  NaN   \n",
       "172e9a38-d3e4-453e-8556-18bbd0fcb07e                                  NaN   \n",
       "23338303-8192-4d70-8e03-508a7120d1c3                                  NaN   \n",
       "28740234-10c8-4421-b56a-c7150e000d48                                  NaN   \n",
       "343ecb7a-aeb5-4b21-9e9c-df545dafba99                                  NaN   \n",
       "38b9cf3a-588d-4137-a975-af770121a925                                  NaN   \n",
       "3bb03f58-5f09-4414-b54e-43a814b9b09a                                  NaN   \n",
       "3c387871-0619-43f5-8aaf-4c1a9849c740                                  NaN   \n",
       "40a947aa-f2fc-43d8-baee-57a88ab21543                                  NaN   \n",
       "419f3399-0646-43dd-aef8-ea9a8f901007                                  NaN   \n",
       "4a72a97f-24c4-4ef7-897f-9c78d1c6da96                                  NaN   \n",
       "4e256eb7-841b-4239-b6cf-b782de8e492c                                  NaN   \n",
       "54221500-8c50-493f-bc91-037d13f705f1                                  NaN   \n",
       "566f0fa0-7d81-4f43-9c3f-dca97642bd5f                                  NaN   \n",
       "619ec434-0748-43e5-bc62-c79039d75e8d                                  NaN   \n",
       "6a11b727-c5bb-4ae4-87b4-f7f27abda2d5                                  NaN   \n",
       "8cddd693-f0df-441c-9c1c-927902e76099                                  NaN   \n",
       "97e53b7d-fba4-491d-97fc-65693f35ddcc                                  NaN   \n",
       "a1c98085-1fb3-4d6e-9ba6-46206afcb4a4                                  NaN   \n",
       "a31c365c-986b-4dbe-b752-bbb5986cc7f2                                  NaN   \n",
       "a3b0a59e-04b0-41ab-b05e-247ad2f20331                                  NaN   \n",
       "af937372-05a4-4143-97fa-7250c7597180                                  NaN   \n",
       "b633a457-b6d9-48c0-8b69-87cc725f470b                                  NaN   \n",
       "b72c51cb-0492-4e72-b6d7-43861be35f0c                                  NaN   \n",
       "bba6ae56-fae7-4c7f-993a-866ea4c3efc2                                  NaN   \n",
       "bd786026-bad5-4fa8-9a3a-38ca03a16412                                  NaN   \n",
       "be6f7700-52dd-477d-ae8c-6dbd9c8eeeec                                  NaN   \n",
       "c1d62210-4d59-4302-9d39-994919558bb1                                  NaN   \n",
       "c7387eb6-fbe8-4267-8937-38d7a0c7da48                                  NaN   \n",
       "cb5b27cd-a67a-429f-8bc3-c61a9c9991be                                  NaN   \n",
       "ce8a8db8-0d7f-4a11-8329-5a7fc065394d                                  NaN   \n",
       "d3e893ab-423f-4226-ba99-0638cd603b47                                  NaN   \n",
       "dd916dd6-c208-481f-9c28-c59b32347a46                                  NaN   \n",
       "e0834165-64cc-41e5-994f-9be97a16ef93                             T1.Q2.A5   \n",
       "e1ae8875-a398-4dde-8f4e-4b21109784e3                                  NaN   \n",
       "e44f5799-6915-45fd-874e-060f6afedcb9                                  NaN   \n",
       "f71afc6b-0691-4bb2-81bf-e1a829ccff43                                  NaN   \n",
       "f9143626-bfe0-4e69-b652-6d1525ab4eb0                                  NaN   \n",
       "fb4ae4cc-234b-484a-b541-a8e3a9254a24                                  NaN   \n",
       "fed45769-aa61-4b8b-847e-a9c799acce15                                  NaN   \n",
       "ff659873-bfba-4da6-94e2-7b91ac8a2c5e                                  NaN   \n",
       "\n",
       "quiz_taskrun_uuid                    fe3278b6-e072-46c2-ada3-3bbd148f13f6  \\\n",
       "contributor_uuid                                                            \n",
       "00f548b7-6b63-4b47-828e-8e416b6ca0e2                                  NaN   \n",
       "06e1eba7-c695-4495-85a7-2fb33ef9412c                                  NaN   \n",
       "0b2ee377-65f2-4ac1-98da-5bc093fee7aa                                  NaN   \n",
       "0cdf31d0-1966-4f69-a0c8-c483788b5256                                  NaN   \n",
       "0e4b8a3e-2714-4b5f-ba10-bb576aaa5cc6                                  NaN   \n",
       "0e81b339-ba89-48a6-b6eb-3160c5ac61db                                  NaN   \n",
       "0f369711-c76f-497a-bffe-4d10a3ab3619                                  NaN   \n",
       "172e9a38-d3e4-453e-8556-18bbd0fcb07e                                  NaN   \n",
       "23338303-8192-4d70-8e03-508a7120d1c3                                  NaN   \n",
       "28740234-10c8-4421-b56a-c7150e000d48                                  NaN   \n",
       "343ecb7a-aeb5-4b21-9e9c-df545dafba99                                  NaN   \n",
       "38b9cf3a-588d-4137-a975-af770121a925                                  NaN   \n",
       "3bb03f58-5f09-4414-b54e-43a814b9b09a                                  NaN   \n",
       "3c387871-0619-43f5-8aaf-4c1a9849c740                                  NaN   \n",
       "40a947aa-f2fc-43d8-baee-57a88ab21543                                  NaN   \n",
       "419f3399-0646-43dd-aef8-ea9a8f901007                                  NaN   \n",
       "4a72a97f-24c4-4ef7-897f-9c78d1c6da96                                  NaN   \n",
       "4e256eb7-841b-4239-b6cf-b782de8e492c                                  NaN   \n",
       "54221500-8c50-493f-bc91-037d13f705f1                                  NaN   \n",
       "566f0fa0-7d81-4f43-9c3f-dca97642bd5f                                  NaN   \n",
       "619ec434-0748-43e5-bc62-c79039d75e8d                                  NaN   \n",
       "6a11b727-c5bb-4ae4-87b4-f7f27abda2d5                                  NaN   \n",
       "8cddd693-f0df-441c-9c1c-927902e76099                                  NaN   \n",
       "97e53b7d-fba4-491d-97fc-65693f35ddcc                                  NaN   \n",
       "a1c98085-1fb3-4d6e-9ba6-46206afcb4a4                                  NaN   \n",
       "a31c365c-986b-4dbe-b752-bbb5986cc7f2                                  NaN   \n",
       "a3b0a59e-04b0-41ab-b05e-247ad2f20331                                  NaN   \n",
       "af937372-05a4-4143-97fa-7250c7597180                                  NaN   \n",
       "b633a457-b6d9-48c0-8b69-87cc725f470b                                  NaN   \n",
       "b72c51cb-0492-4e72-b6d7-43861be35f0c                                  NaN   \n",
       "bba6ae56-fae7-4c7f-993a-866ea4c3efc2                                  NaN   \n",
       "bd786026-bad5-4fa8-9a3a-38ca03a16412                                  NaN   \n",
       "be6f7700-52dd-477d-ae8c-6dbd9c8eeeec                                  NaN   \n",
       "c1d62210-4d59-4302-9d39-994919558bb1                             T1.Q2.A4   \n",
       "c7387eb6-fbe8-4267-8937-38d7a0c7da48                                  NaN   \n",
       "cb5b27cd-a67a-429f-8bc3-c61a9c9991be                                  NaN   \n",
       "ce8a8db8-0d7f-4a11-8329-5a7fc065394d                                  NaN   \n",
       "d3e893ab-423f-4226-ba99-0638cd603b47                                  NaN   \n",
       "dd916dd6-c208-481f-9c28-c59b32347a46                                  NaN   \n",
       "e0834165-64cc-41e5-994f-9be97a16ef93                                  NaN   \n",
       "e1ae8875-a398-4dde-8f4e-4b21109784e3                                  NaN   \n",
       "e44f5799-6915-45fd-874e-060f6afedcb9                                  NaN   \n",
       "f71afc6b-0691-4bb2-81bf-e1a829ccff43                                  NaN   \n",
       "f9143626-bfe0-4e69-b652-6d1525ab4eb0                                  NaN   \n",
       "fb4ae4cc-234b-484a-b541-a8e3a9254a24                                  NaN   \n",
       "fed45769-aa61-4b8b-847e-a9c799acce15                                  NaN   \n",
       "ff659873-bfba-4da6-94e2-7b91ac8a2c5e                                  NaN   \n",
       "\n",
       "quiz_taskrun_uuid                    fe552cc8-8090-4538-adf7-d963ff1de5a1  \\\n",
       "contributor_uuid                                                            \n",
       "00f548b7-6b63-4b47-828e-8e416b6ca0e2                                  NaN   \n",
       "06e1eba7-c695-4495-85a7-2fb33ef9412c                                  NaN   \n",
       "0b2ee377-65f2-4ac1-98da-5bc093fee7aa                                  NaN   \n",
       "0cdf31d0-1966-4f69-a0c8-c483788b5256                                  NaN   \n",
       "0e4b8a3e-2714-4b5f-ba10-bb576aaa5cc6                                  NaN   \n",
       "0e81b339-ba89-48a6-b6eb-3160c5ac61db                                  NaN   \n",
       "0f369711-c76f-497a-bffe-4d10a3ab3619                                  NaN   \n",
       "172e9a38-d3e4-453e-8556-18bbd0fcb07e                                  NaN   \n",
       "23338303-8192-4d70-8e03-508a7120d1c3                                  NaN   \n",
       "28740234-10c8-4421-b56a-c7150e000d48                                  NaN   \n",
       "343ecb7a-aeb5-4b21-9e9c-df545dafba99                                  NaN   \n",
       "38b9cf3a-588d-4137-a975-af770121a925                                  NaN   \n",
       "3bb03f58-5f09-4414-b54e-43a814b9b09a                                  NaN   \n",
       "3c387871-0619-43f5-8aaf-4c1a9849c740                                  NaN   \n",
       "40a947aa-f2fc-43d8-baee-57a88ab21543                                  NaN   \n",
       "419f3399-0646-43dd-aef8-ea9a8f901007                                  NaN   \n",
       "4a72a97f-24c4-4ef7-897f-9c78d1c6da96                                  NaN   \n",
       "4e256eb7-841b-4239-b6cf-b782de8e492c                                  NaN   \n",
       "54221500-8c50-493f-bc91-037d13f705f1                                  NaN   \n",
       "566f0fa0-7d81-4f43-9c3f-dca97642bd5f                                  NaN   \n",
       "619ec434-0748-43e5-bc62-c79039d75e8d                                  NaN   \n",
       "6a11b727-c5bb-4ae4-87b4-f7f27abda2d5                                  NaN   \n",
       "8cddd693-f0df-441c-9c1c-927902e76099                                  NaN   \n",
       "97e53b7d-fba4-491d-97fc-65693f35ddcc                                  NaN   \n",
       "a1c98085-1fb3-4d6e-9ba6-46206afcb4a4                                  NaN   \n",
       "a31c365c-986b-4dbe-b752-bbb5986cc7f2                                  NaN   \n",
       "a3b0a59e-04b0-41ab-b05e-247ad2f20331                             T1.Q2.A5   \n",
       "af937372-05a4-4143-97fa-7250c7597180                                  NaN   \n",
       "b633a457-b6d9-48c0-8b69-87cc725f470b                                  NaN   \n",
       "b72c51cb-0492-4e72-b6d7-43861be35f0c                                  NaN   \n",
       "bba6ae56-fae7-4c7f-993a-866ea4c3efc2                                  NaN   \n",
       "bd786026-bad5-4fa8-9a3a-38ca03a16412                                  NaN   \n",
       "be6f7700-52dd-477d-ae8c-6dbd9c8eeeec                                  NaN   \n",
       "c1d62210-4d59-4302-9d39-994919558bb1                                  NaN   \n",
       "c7387eb6-fbe8-4267-8937-38d7a0c7da48                                  NaN   \n",
       "cb5b27cd-a67a-429f-8bc3-c61a9c9991be                                  NaN   \n",
       "ce8a8db8-0d7f-4a11-8329-5a7fc065394d                                  NaN   \n",
       "d3e893ab-423f-4226-ba99-0638cd603b47                                  NaN   \n",
       "dd916dd6-c208-481f-9c28-c59b32347a46                                  NaN   \n",
       "e0834165-64cc-41e5-994f-9be97a16ef93                                  NaN   \n",
       "e1ae8875-a398-4dde-8f4e-4b21109784e3                                  NaN   \n",
       "e44f5799-6915-45fd-874e-060f6afedcb9                                  NaN   \n",
       "f71afc6b-0691-4bb2-81bf-e1a829ccff43                                  NaN   \n",
       "f9143626-bfe0-4e69-b652-6d1525ab4eb0                                  NaN   \n",
       "fb4ae4cc-234b-484a-b541-a8e3a9254a24                                  NaN   \n",
       "fed45769-aa61-4b8b-847e-a9c799acce15                                  NaN   \n",
       "ff659873-bfba-4da6-94e2-7b91ac8a2c5e                                  NaN   \n",
       "\n",
       "quiz_taskrun_uuid                    fee75365-83d0-4c47-a520-73e0954af82c  \\\n",
       "contributor_uuid                                                            \n",
       "00f548b7-6b63-4b47-828e-8e416b6ca0e2                                  NaN   \n",
       "06e1eba7-c695-4495-85a7-2fb33ef9412c                                  NaN   \n",
       "0b2ee377-65f2-4ac1-98da-5bc093fee7aa                                  NaN   \n",
       "0cdf31d0-1966-4f69-a0c8-c483788b5256                                  NaN   \n",
       "0e4b8a3e-2714-4b5f-ba10-bb576aaa5cc6                                  NaN   \n",
       "0e81b339-ba89-48a6-b6eb-3160c5ac61db                                  NaN   \n",
       "0f369711-c76f-497a-bffe-4d10a3ab3619                                  NaN   \n",
       "172e9a38-d3e4-453e-8556-18bbd0fcb07e                                  NaN   \n",
       "23338303-8192-4d70-8e03-508a7120d1c3                                  NaN   \n",
       "28740234-10c8-4421-b56a-c7150e000d48                                  NaN   \n",
       "343ecb7a-aeb5-4b21-9e9c-df545dafba99                                  NaN   \n",
       "38b9cf3a-588d-4137-a975-af770121a925                                  NaN   \n",
       "3bb03f58-5f09-4414-b54e-43a814b9b09a                                  NaN   \n",
       "3c387871-0619-43f5-8aaf-4c1a9849c740                                  NaN   \n",
       "40a947aa-f2fc-43d8-baee-57a88ab21543                                  NaN   \n",
       "419f3399-0646-43dd-aef8-ea9a8f901007                                  NaN   \n",
       "4a72a97f-24c4-4ef7-897f-9c78d1c6da96                                  NaN   \n",
       "4e256eb7-841b-4239-b6cf-b782de8e492c                                  NaN   \n",
       "54221500-8c50-493f-bc91-037d13f705f1                                  NaN   \n",
       "566f0fa0-7d81-4f43-9c3f-dca97642bd5f                                  NaN   \n",
       "619ec434-0748-43e5-bc62-c79039d75e8d                                  NaN   \n",
       "6a11b727-c5bb-4ae4-87b4-f7f27abda2d5                                  NaN   \n",
       "8cddd693-f0df-441c-9c1c-927902e76099                                  NaN   \n",
       "97e53b7d-fba4-491d-97fc-65693f35ddcc                                  NaN   \n",
       "a1c98085-1fb3-4d6e-9ba6-46206afcb4a4                                  NaN   \n",
       "a31c365c-986b-4dbe-b752-bbb5986cc7f2                                  NaN   \n",
       "a3b0a59e-04b0-41ab-b05e-247ad2f20331                                  NaN   \n",
       "af937372-05a4-4143-97fa-7250c7597180                                  NaN   \n",
       "b633a457-b6d9-48c0-8b69-87cc725f470b                                  NaN   \n",
       "b72c51cb-0492-4e72-b6d7-43861be35f0c                                  NaN   \n",
       "bba6ae56-fae7-4c7f-993a-866ea4c3efc2                                  NaN   \n",
       "bd786026-bad5-4fa8-9a3a-38ca03a16412                                  NaN   \n",
       "be6f7700-52dd-477d-ae8c-6dbd9c8eeeec                                  NaN   \n",
       "c1d62210-4d59-4302-9d39-994919558bb1                                  NaN   \n",
       "c7387eb6-fbe8-4267-8937-38d7a0c7da48                                  NaN   \n",
       "cb5b27cd-a67a-429f-8bc3-c61a9c9991be                                  NaN   \n",
       "ce8a8db8-0d7f-4a11-8329-5a7fc065394d                                  NaN   \n",
       "d3e893ab-423f-4226-ba99-0638cd603b47                                  NaN   \n",
       "dd916dd6-c208-481f-9c28-c59b32347a46                                  NaN   \n",
       "e0834165-64cc-41e5-994f-9be97a16ef93                                  NaN   \n",
       "e1ae8875-a398-4dde-8f4e-4b21109784e3                                  NaN   \n",
       "e44f5799-6915-45fd-874e-060f6afedcb9                             T1.Q2.A2   \n",
       "f71afc6b-0691-4bb2-81bf-e1a829ccff43                                  NaN   \n",
       "f9143626-bfe0-4e69-b652-6d1525ab4eb0                                  NaN   \n",
       "fb4ae4cc-234b-484a-b541-a8e3a9254a24                                  NaN   \n",
       "fed45769-aa61-4b8b-847e-a9c799acce15                                  NaN   \n",
       "ff659873-bfba-4da6-94e2-7b91ac8a2c5e                                  NaN   \n",
       "\n",
       "quiz_taskrun_uuid                    ffd3215a-8fc3-4ac7-a59a-0f9c43be2683  \n",
       "contributor_uuid                                                           \n",
       "00f548b7-6b63-4b47-828e-8e416b6ca0e2                                  NaN  \n",
       "06e1eba7-c695-4495-85a7-2fb33ef9412c                                  NaN  \n",
       "0b2ee377-65f2-4ac1-98da-5bc093fee7aa                                  NaN  \n",
       "0cdf31d0-1966-4f69-a0c8-c483788b5256                                  NaN  \n",
       "0e4b8a3e-2714-4b5f-ba10-bb576aaa5cc6                                  NaN  \n",
       "0e81b339-ba89-48a6-b6eb-3160c5ac61db                                  NaN  \n",
       "0f369711-c76f-497a-bffe-4d10a3ab3619                                  NaN  \n",
       "172e9a38-d3e4-453e-8556-18bbd0fcb07e                                  NaN  \n",
       "23338303-8192-4d70-8e03-508a7120d1c3                                  NaN  \n",
       "28740234-10c8-4421-b56a-c7150e000d48                                  NaN  \n",
       "343ecb7a-aeb5-4b21-9e9c-df545dafba99                                  NaN  \n",
       "38b9cf3a-588d-4137-a975-af770121a925                                  NaN  \n",
       "3bb03f58-5f09-4414-b54e-43a814b9b09a                                  NaN  \n",
       "3c387871-0619-43f5-8aaf-4c1a9849c740                                  NaN  \n",
       "40a947aa-f2fc-43d8-baee-57a88ab21543                                  NaN  \n",
       "419f3399-0646-43dd-aef8-ea9a8f901007                                  NaN  \n",
       "4a72a97f-24c4-4ef7-897f-9c78d1c6da96                                  NaN  \n",
       "4e256eb7-841b-4239-b6cf-b782de8e492c                                  NaN  \n",
       "54221500-8c50-493f-bc91-037d13f705f1                                  NaN  \n",
       "566f0fa0-7d81-4f43-9c3f-dca97642bd5f                                  NaN  \n",
       "619ec434-0748-43e5-bc62-c79039d75e8d                                  NaN  \n",
       "6a11b727-c5bb-4ae4-87b4-f7f27abda2d5                                  NaN  \n",
       "8cddd693-f0df-441c-9c1c-927902e76099                                  NaN  \n",
       "97e53b7d-fba4-491d-97fc-65693f35ddcc                                  NaN  \n",
       "a1c98085-1fb3-4d6e-9ba6-46206afcb4a4                                  NaN  \n",
       "a31c365c-986b-4dbe-b752-bbb5986cc7f2                                  NaN  \n",
       "a3b0a59e-04b0-41ab-b05e-247ad2f20331                                  NaN  \n",
       "af937372-05a4-4143-97fa-7250c7597180                                  NaN  \n",
       "b633a457-b6d9-48c0-8b69-87cc725f470b                                  NaN  \n",
       "b72c51cb-0492-4e72-b6d7-43861be35f0c                                  NaN  \n",
       "bba6ae56-fae7-4c7f-993a-866ea4c3efc2                                  NaN  \n",
       "bd786026-bad5-4fa8-9a3a-38ca03a16412                                  NaN  \n",
       "be6f7700-52dd-477d-ae8c-6dbd9c8eeeec                                  NaN  \n",
       "c1d62210-4d59-4302-9d39-994919558bb1                                  NaN  \n",
       "c7387eb6-fbe8-4267-8937-38d7a0c7da48                                  NaN  \n",
       "cb5b27cd-a67a-429f-8bc3-c61a9c9991be                                  NaN  \n",
       "ce8a8db8-0d7f-4a11-8329-5a7fc065394d                                  NaN  \n",
       "d3e893ab-423f-4226-ba99-0638cd603b47                                  NaN  \n",
       "dd916dd6-c208-481f-9c28-c59b32347a46                                  NaN  \n",
       "e0834165-64cc-41e5-994f-9be97a16ef93                                  NaN  \n",
       "e1ae8875-a398-4dde-8f4e-4b21109784e3                                  NaN  \n",
       "e44f5799-6915-45fd-874e-060f6afedcb9                                  NaN  \n",
       "f71afc6b-0691-4bb2-81bf-e1a829ccff43                                  NaN  \n",
       "f9143626-bfe0-4e69-b652-6d1525ab4eb0                             T1.Q2.A5  \n",
       "fb4ae4cc-234b-484a-b541-a8e3a9254a24                                  NaN  \n",
       "fed45769-aa61-4b8b-847e-a9c799acce15                                  NaN  \n",
       "ff659873-bfba-4da6-94e2-7b91ac8a2c5e                                  NaN  \n",
       "\n",
       "[47 rows x 258 columns]"
      ]
     },
     "execution_count": 70,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df = pd.pivot_table(relevance_q2, values='answer_label', index='contributor_uuid', columns='quiz_taskrun_uuid', aggfunc=select)\n",
    "# drop nan rows \n",
    "df = df.dropna(axis=1, how='all')\n",
    "df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 71,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([nan, nan, nan, 'T1.Q2.A5', nan, nan, nan, nan, nan, nan, nan, nan,\n",
       "       nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan,\n",
       "       nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan,\n",
       "       nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan,\n",
       "       nan, 'T1.Q2.A2', nan, nan, nan, nan, nan, nan, nan, nan, nan, nan,\n",
       "       nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan,\n",
       "       'T1.Q2.A2', nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan,\n",
       "       nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan,\n",
       "       nan, nan, nan, nan, nan, nan, nan, nan, 'T1.Q2.A5', nan, nan, nan,\n",
       "       nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan,\n",
       "       nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, 'T1.Q2.A5',\n",
       "       nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan,\n",
       "       nan, nan, nan, nan, nan, nan, nan, nan, nan, 'T1.Q2.A5', nan, nan,\n",
       "       'T1.Q2.A1', 'T1.Q2.A5', nan, nan, nan, nan, nan, nan, nan, nan,\n",
       "       nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan,\n",
       "       nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan,\n",
       "       nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan,\n",
       "       nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, 'T1.Q2.A3', nan,\n",
       "       'T1.Q2.A3', nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan,\n",
       "       nan, nan, nan, nan, nan, 'T1.Q2.A1', nan, nan, nan, nan, nan, nan,\n",
       "       nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan],\n",
       "      dtype=object)"
      ]
     },
     "execution_count": 71,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "array = df.to_numpy()\n",
    "array[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 76,
   "metadata": {},
   "outputs": [],
   "source": [
    "# iterate through all values in array and convert string to int \n",
    "for i in range(array.shape[0]):\n",
    "    for j in range(array.shape[1]):\n",
    "        cur = array[i][j]\n",
    "        # if the current value is a string\n",
    "        if not pd.isnull(cur):\n",
    "            array[i][j] = int(array[i][j][-1])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 77,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([nan, nan, nan, 5, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan,\n",
       "       nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan,\n",
       "       nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan,\n",
       "       nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, 2, nan,\n",
       "       nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan,\n",
       "       nan, nan, nan, nan, nan, nan, nan, nan, 2, nan, nan, nan, nan, nan,\n",
       "       nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan,\n",
       "       nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan,\n",
       "       nan, 5, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan,\n",
       "       nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan,\n",
       "       nan, nan, 5, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan,\n",
       "       nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, 5, nan, nan,\n",
       "       1, 5, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan,\n",
       "       nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan,\n",
       "       nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan,\n",
       "       nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan,\n",
       "       nan, nan, nan, nan, nan, nan, 3, nan, 3, nan, nan, nan, nan, nan,\n",
       "       nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, 1, nan, nan,\n",
       "       nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan,\n",
       "       nan, nan], dtype=object)"
      ]
     },
     "execution_count": 77,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "array[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 78,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "nan\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\thefi\\Anaconda3\\lib\\site-packages\\ipykernel_launcher.py:278: RuntimeWarning: invalid value encountered in double_scalars\n"
     ]
    }
   ],
   "source": [
    "print(round(alpha(array, level_of_measurement='nominal'), 3))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Triager to uAlpha"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "# create mapping for topic_name\n",
    "dic = {\"Quoted Sources\":1,\n",
    "           \"Arguments\":2,\n",
    "           \"Assertions\":3, \n",
    "           \"Needs Fact-Check\":4, \n",
    "           \"Language\":5, \n",
    "           \"Reasoning\":6, \n",
    "           \"Probability\":7, \n",
    "           \"Evidence\":8}\n",
    "\n",
    "    \n",
    "# mapping topic_name to integers \n",
    "def mapping(df, dic):\n",
    "    # df - dataframe to be mapped \n",
    "    # dic - dictionary of mapping \n",
    "    df[\"topic_name\"] = df[\"topic_name\"].map(dic)\n",
    "    return df \n",
    "\n",
    "\n",
    "# filter df and select only certain cols as input\n",
    "def slice_input(df):\n",
    "    cols = [\"contributor_uuid\", \"topic_name\", \"start_pos\",\n",
    "            \"end_pos\", \"article_text_length\", \"article_number\", \"created\"]\n",
    "    return df.reindex(columns=cols)\n",
    "\n",
    "\n",
    "# filter df and select only certain cols as output\n",
    "def slice_output(df):\n",
    "    cols = [\"contributor_uuid\", \"topic_name\", \"blank\",  \n",
    "            \"start_pos_adjusted\", \"end_pos_adjusted\"]\n",
    "    return df.reindex(columns=cols)\n",
    "    \n",
    "\n",
    "\n",
    "# read in a directory that contains Triager csvs \n",
    "# split each Triager csv to 4 other csvs based on topic_name (argument, reasoning, etc)\n",
    "# and write it as csvs in the output directory\n",
    "def triager_split(input_directory, output_directory):\n",
    "    # input_directory - e.g. \"Triager data\"\n",
    "    # output_directory - e.g. \"Triager output\"\n",
    "    for file in os.listdir(input_directory):\n",
    "        triager_split_help(input_directory, file, output_directory)\n",
    "    print(\"Triager Tranformation Done! Ready to be imported to uAlpha!\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "# resolve overlapping issues by merging overlapping rows!!!\n",
    "def resolve_overlapping(df):\n",
    "    # df - a dataframe with 5 cols {0: index, 1: user_id, 2:topic_name(as int), 3:blank, 4:start_pos, 5:end_pos}\n",
    "    # this function only works when start_pos are sorted for each user!!!\n",
    "    created = df[\"created\"].unique()\n",
    "    # iterate through all unique user \n",
    "    for time in created:\n",
    "        # start and end index of that user \n",
    "        start_i = df[df[\"created\"] == time].index.values[0]\n",
    "        end_i = df[df[\"created\"] == time].index.values[-1]\n",
    "        # iterate through the rows of that user \n",
    "        i = start_i\n",
    "        while i < end_i:\n",
    "            # if the start_pos of next row > end_pos of this row\n",
    "            # no overlap\n",
    "            # 5 - the col of end_pos\n",
    "            # the 10th column is end_pos_adjusted\n",
    "            end_pos = df.iat[i, 10]\n",
    "            # the 9th column is start_pos_adjusted\n",
    "            start_pos = df.iat[i+1, 9]\n",
    "            if start_pos > end_pos:\n",
    "                i += 1 \n",
    "            else:\n",
    "                # if the article_number is the same, doesn't really matter \n",
    "                # modify the end_pos of this row, merge \n",
    "                if df.iat[i, 10] < df.iat[i+1 , 10]:\n",
    "                    df.iat[i, 10] = df.iat[i+1, 10].copy()\n",
    "                # drop the next row \n",
    "                df.drop(i+1, inplace=True)\n",
    "                # reindex\n",
    "                df.reset_index(drop=True, inplace=True)\n",
    "                # decrement end_i \n",
    "                end_i -= 1\n",
    "    return df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "# add adjusted start_pos and adjusted end_pos for df \n",
    "def adjust(df):\n",
    "    # df - a dataframe with [\"article_number\", \"article_text_length\", \"start_pos\", \"end_pos\"] cols\n",
    "    article_number = df[\"article_number\"]\n",
    "    article_text_length = df[\"article_text_length\"]\n",
    "    start_pos = df[\"start_pos\"]\n",
    "    end_pos = df[\"end_pos\"]\n",
    "    # the cumulative sum of article_text_length\n",
    "    cumulative = []\n",
    "    start_pos_adjusted = []\n",
    "    end_pos_adjusted = []\n",
    "    pre_total = 0\n",
    "    total = article_text_length[0]\n",
    "    cur_num = article_number[0]\n",
    "    for i in np.arange(len(article_number)):\n",
    "        if article_number[i] == cur_num:\n",
    "            total = total + 0\n",
    "            cumulative.append(total)\n",
    "        else:\n",
    "            pre_total = total\n",
    "            cur_num = article_number[i]\n",
    "            total = total + article_text_length[i]\n",
    "            cumulative.append(total)\n",
    "        start_pos_adjusted.append(start_pos[i] + pre_total)\n",
    "        end_pos_adjusted.append(end_pos[i] + pre_total)\n",
    "    df[\"cumulative\"] = cumulative\n",
    "    df[\"start_pos_adjusted\"] = start_pos_adjusted\n",
    "    df[\"end_pos_adjusted\"] = end_pos_adjusted\n",
    "    return df "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "# take in a Triager csv and split it to 4 different csvs \n",
    "# based on topic_name \n",
    "def triager_split_help(input_directory, fileName, output_directory):\n",
    "    # input_directory - e.g. \"Triager data\"\n",
    "    # fileName - e.g. \"Covid_Form1.0.adjudicated-2020-10-04T2314-Tags.csv\"\n",
    "    # output_directory - e.g. \"Triager output\"\n",
    "    name = os.path.join(input_directory, fileName)\n",
    "    df = pd.read_csv(name)\n",
    "    # different topic_names \n",
    "    topic_names = df[\"topic_name\"].unique()\n",
    "    for topic in topic_names: \n",
    "        # select all rows for that specific topic name \n",
    "        df_topic = df.loc[df[\"topic_name\"] == topic]\n",
    "        # select only these columns \n",
    "        filtered = slice_input(df_topic)\n",
    "        # slice contributor_uuid with the first 6 chars \n",
    "        filtered[\"contributor_uuid\"] = filtered[\"contributor_uuid\"].str[:6]\n",
    "        # map topic_name to integers\n",
    "        mapping(filtered, dic)\n",
    "        # add a blank column \n",
    "        filtered.insert(loc=3, column=\"blank\", value=\"\")\n",
    "        # sort\n",
    "        filtered = filtered.sort_values(by=[\"article_number\", \"contributor_uuid\", \"start_pos\"])\n",
    "        # re-index such that its index starts from 0 again\n",
    "        filtered.reset_index(drop=True, inplace=True)\n",
    "        # add modified columns\n",
    "        filtered = adjust(filtered)\n",
    "        # resolve overlapping issues \n",
    "        filtered = resolve_overlapping(filtered)\n",
    "        # filter and select only certain columns \n",
    "        output = slice_output(filtered)\n",
    "        # sort again by contributor\n",
    "        output = output.sort_values(by=[\"contributor_uuid\", \"start_pos_adjusted\"])\n",
    "        # add one column of u + str(row) and set it as index \n",
    "        index = 'u' + pd.Series(filtered.index).astype(str)\n",
    "        output.set_index(keys=index, inplace=True)\n",
    "        # the batch_name, e.g. \"Covid\"\n",
    "        batch_name = re.split(r'_', fileName)[0]\n",
    "        name = '{0}-Triager-{1}.csv'.format(batch_name, topic)\n",
    "        path = os.path.join(output_directory, name)\n",
    "        # write to csvs\n",
    "        output.to_csv(path, header=False)\n",
    "    return None"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Triager Tranformation Done! Ready to be imported to uAlpha!\n"
     ]
    }
   ],
   "source": [
    "triager_split(\"Triager data\", \"Triager output\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "**Example of resolving overlap**"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>contributor_uuid</th>\n",
       "      <th>topic_name</th>\n",
       "      <th>start_pos</th>\n",
       "      <th>blank</th>\n",
       "      <th>end_pos</th>\n",
       "      <th>article_text_length</th>\n",
       "      <th>article_number</th>\n",
       "      <th>created</th>\n",
       "      <th>cumulative</th>\n",
       "      <th>start_pos_adjusted</th>\n",
       "      <th>end_pos_adjusted</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>aac18e</td>\n",
       "      <td>2</td>\n",
       "      <td>136</td>\n",
       "      <td>NaN</td>\n",
       "      <td>287</td>\n",
       "      <td>1870</td>\n",
       "      <td>100054</td>\n",
       "      <td>2020-03-18 00:15:42.364545</td>\n",
       "      <td>5755</td>\n",
       "      <td>4021</td>\n",
       "      <td>4172</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>aac18e</td>\n",
       "      <td>2</td>\n",
       "      <td>563</td>\n",
       "      <td>NaN</td>\n",
       "      <td>776</td>\n",
       "      <td>1870</td>\n",
       "      <td>100054</td>\n",
       "      <td>2020-03-18 00:15:42.364545</td>\n",
       "      <td>5755</td>\n",
       "      <td>4448</td>\n",
       "      <td>4661</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>aac18e</td>\n",
       "      <td>2</td>\n",
       "      <td>651</td>\n",
       "      <td>NaN</td>\n",
       "      <td>667</td>\n",
       "      <td>1870</td>\n",
       "      <td>100054</td>\n",
       "      <td>2020-03-18 00:15:42.364545</td>\n",
       "      <td>5755</td>\n",
       "      <td>4536</td>\n",
       "      <td>4552</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "  contributor_uuid  topic_name  start_pos  blank  end_pos  \\\n",
       "0           aac18e           2        136    NaN      287   \n",
       "1           aac18e           2        563    NaN      776   \n",
       "2           aac18e           2        651    NaN      667   \n",
       "\n",
       "   article_text_length  article_number                     created  \\\n",
       "0                 1870          100054  2020-03-18 00:15:42.364545   \n",
       "1                 1870          100054  2020-03-18 00:15:42.364545   \n",
       "2                 1870          100054  2020-03-18 00:15:42.364545   \n",
       "\n",
       "   cumulative  start_pos_adjusted  end_pos_adjusted  \n",
       "0        5755                4021              4172  \n",
       "1        5755                4448              4661  \n",
       "2        5755                4536              4552  "
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "a = pd.read_csv(\"example.csv\")\n",
    "a.drop(a.columns[0], axis=1, inplace=True)\n",
    "a"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>contributor_uuid</th>\n",
       "      <th>topic_name</th>\n",
       "      <th>start_pos</th>\n",
       "      <th>blank</th>\n",
       "      <th>end_pos</th>\n",
       "      <th>article_text_length</th>\n",
       "      <th>article_number</th>\n",
       "      <th>created</th>\n",
       "      <th>cumulative</th>\n",
       "      <th>start_pos_adjusted</th>\n",
       "      <th>end_pos_adjusted</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>aac18e</td>\n",
       "      <td>2</td>\n",
       "      <td>136</td>\n",
       "      <td>NaN</td>\n",
       "      <td>287</td>\n",
       "      <td>1870</td>\n",
       "      <td>100054</td>\n",
       "      <td>2020-03-18 00:15:42.364545</td>\n",
       "      <td>5755</td>\n",
       "      <td>4021</td>\n",
       "      <td>4172</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>aac18e</td>\n",
       "      <td>2</td>\n",
       "      <td>563</td>\n",
       "      <td>NaN</td>\n",
       "      <td>776</td>\n",
       "      <td>1870</td>\n",
       "      <td>100054</td>\n",
       "      <td>2020-03-18 00:15:42.364545</td>\n",
       "      <td>5755</td>\n",
       "      <td>4448</td>\n",
       "      <td>4661</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "  contributor_uuid  topic_name  start_pos  blank  end_pos  \\\n",
       "0           aac18e           2        136    NaN      287   \n",
       "1           aac18e           2        563    NaN      776   \n",
       "\n",
       "   article_text_length  article_number                     created  \\\n",
       "0                 1870          100054  2020-03-18 00:15:42.364545   \n",
       "1                 1870          100054  2020-03-18 00:15:42.364545   \n",
       "\n",
       "   cumulative  start_pos_adjusted  end_pos_adjusted  \n",
       "0        5755                4021              4172  \n",
       "1        5755                4448              4661  "
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "a = resolve_overlapping(a)\n",
    "a"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [],
   "source": [
    "# debugging function - find potential overlapping issue \n",
    "def detect_overlapping(input_directory, fileName):\n",
    "    # input directory - e.g. \"Triager output\"\n",
    "    # fileName - e.g. \"Covid-Triager-Arguments.csv\"\n",
    "    name = os.path.join(input_directory, fileName)\n",
    "    df = pd.read_csv(name, header=None)\n",
    "    # all the unique user \n",
    "    users = df.iloc[:, 1].unique()\n",
    "    # iterate through all unique user \n",
    "    for user in users:\n",
    "        # first only select the rows of that user \n",
    "        # use copy to deal with the wanring\n",
    "        piece = df.loc[df[1] == user].copy()\n",
    "        # sort the 4th column\n",
    "        piece.sort_values(by=4, inplace=True)\n",
    "        # find or not\n",
    "        found = False \n",
    "        start_pos = np.array(piece[4])\n",
    "        end_pos = np.array(piece[5])\n",
    "        for i in np.arange(1, len(start_pos)):\n",
    "            if start_pos[i] > end_pos[i-1]:\n",
    "                continue\n",
    "            else:\n",
    "                found = True\n",
    "                print('user {0}, start_pos {1}, end_pos {2} begin overlapping!'.format(user, start_pos[i], end_pos[i]))\n",
    "                break \n",
    "    return None "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [],
   "source": [
    "detect_overlapping(\"Triager output\", \"Covid-Triager-Arguments.csv\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "**Example of removing a row**"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>first</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>23</th>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>24</th>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>25</th>\n",
       "      <td>3</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>26</th>\n",
       "      <td>4</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>27</th>\n",
       "      <td>5</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "    first\n",
       "23      1\n",
       "24      2\n",
       "25      3\n",
       "26      4\n",
       "27      5"
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "a = pd.DataFrame(index=np.array([23, 24, 25, 26, 27]), data={\"first\": [1, 2, 3, 4, 5]})\n",
    "a"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>first</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>23</th>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>24</th>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>26</th>\n",
       "      <td>4</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>27</th>\n",
       "      <td>5</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "    first\n",
       "23      1\n",
       "24      2\n",
       "26      4\n",
       "27      5"
      ]
     },
     "execution_count": 19,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "a.drop(25, inplace=True)\n",
    "a"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}