01-activity-data-extraction.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import sys\n",
    "sys.path.append(\"./dbaasp_api_helper_libraries/python\")\n",
    "sys.path.append(\"./dbaasp_api_helper_libraries/python/request\")\n",
    "%load_ext autoreload\n",
    "%autoreload 2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "INFO: Pandarallel will run on 8 workers.\n",
      "INFO: Pandarallel will use Memory file system to transfer data between the main process and workers.\n"
     ]
    }
   ],
   "source": [
    "import json\n",
    "import pandas as pd\n",
    "from pandas import Series, DataFrame\n",
    "from pandarallel import pandarallel\n",
    "pandarallel.initialize(progress_bar=True)\n",
    "import APICaller,Complexity, FormatType, LookupType, MathOperationTypes\n",
    "import random\n",
    "import numpy as np\n",
    "import os\n",
    "folder = \"/data/AIpep-clean/\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "SEED = 12456789\n",
    "np.random.seed(SEED)\n",
    "random.seed(SEED)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "def print_terminus(T_list, T_acc):\n",
    "    for t in T_list:\n",
    "        if t not in T_acc:\n",
    "            print(t)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "aminoacids = [\"A\",\"C\",\"D\",\"E\",\"F\",\"G\",\"H\",\"I\",\"L\",\"M\",\"N\",\"P\",\"K\",\"Q\",\"R\",\"S\",\"T\",\"V\",\"W\",\"Y\"]\n",
    "def is_natural(seq):\n",
    "    try:\n",
    "        seq = seq.upper()\n",
    "        for aa in seq:\n",
    "            if aa not in aminoacids:\n",
    "                return False\n",
    "        return True\n",
    "    except:\n",
    "        return False"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "def float_ignore_plus_minus(mynumber):\n",
    "    try:\n",
    "        return sum(map(float,mynumber.split(\"±\")))\n",
    "    except:\n",
    "        return float(\"inf\")\n",
    "\n",
    "def is_active(identifier):\n",
    "    results = []\n",
    "    try:\n",
    "        peptideCardRequest = APICaller.PeptideCardRequest()\n",
    "        peptideCardRequest.peptide_id = identifier;\n",
    "        peptideCardRequest.format = FormatType.FormatType.JSON;\n",
    "        dbaasp_peptide = json.loads(peptideCardRequest.request())\n",
    "        NAs = [\"NA\", \"na\", \"Na\", \"nA\", \"N/A\", \"n/a\"]\n",
    "\n",
    "        if 'errorCode' in dbaasp_peptide:\n",
    "#            print(\"db error\", identifier)\n",
    "            return []\n",
    "        if \"targetActivities\" not in dbaasp_peptide[\"peptideCard\"]:\n",
    "#            print(\"no target\", identifier)#, dbaasp_peptide[\"peptideCard\"])\n",
    "            return []  \n",
    "    \n",
    "        species = dbaasp_peptide[\"peptideCard\"][\"targetActivities\"]\n",
    "        for specie in species:\n",
    "            if not (\"unit\" and \"concentration\" and \"targetSpecies\" and \"activityMeasure\" in specie):\n",
    "                continue\n",
    "            unit = specie[\"unit\"]\n",
    "\n",
    "            #aaaaaaaaaaaaaaaaaaaaaaaaaaaaaAAAaaaaaaaaaargh\n",
    "            concentration_str = specie[\"concentration\"].replace(\" \",\"\")\n",
    "            concentration_str = concentration_str.replace(\"–\",\"-\")\n",
    "            concentration_str = concentration_str.replace(\"->\",\"-\") \n",
    "            concentration_str = concentration_str.replace(\"0,\",\"0.\")\n",
    "            concentration_str = concentration_str.replace(\",\",\"\") \n",
    "\n",
    "\n",
    "            if concentration_str[0] == '<':\n",
    "                if concentration_str[1] == '=':\n",
    "                    concentration_tmp = float_ignore_plus_minus(concentration_str[2:])\n",
    "                else:\n",
    "                    concentration_tmp = float_ignore_plus_minus(concentration_str[1:])\n",
    "                concentration = concentration_tmp\n",
    "            elif concentration_str[0] == '>' or concentration_str in NAs:\n",
    "                concentration = float(\"inf\")\n",
    "            elif \"-\"  in concentration_str:\n",
    "                concentrations = concentration_str.split(\"-\")\n",
    "                concentration =  float_ignore_plus_minus(concentrations[0]) + float_ignore_plus_minus(concentrations[1])\n",
    "                concentration /= 2\n",
    "            else:\n",
    "                concentration = float_ignore_plus_minus(concentration_str)\n",
    "\n",
    "            if (unit == \"µM\" and concentration < 10) or (unit == \"nM\" and concentration < 10000) or (unit == \"µg/ml\" and concentration < 32): \n",
    "                results.append([concentration, unit, specie[\"targetSpecies\"], specie[\"activityMeasure\"]])\n",
    "\n",
    "            elif unit != \"µM\" and unit != \"nM\" and unit != \"µg/ml\" and concentration_str not in NAs:\n",
    "                pass \n",
    "                # print(\"no unit\",unit, identifier)#, species)\n",
    "\n",
    "        return results\n",
    "    except:\n",
    "        return results\n",
    "    \n",
    "def is_inactive(identifier):\n",
    "    results = []\n",
    "    try:\n",
    "        peptideCardRequest = APICaller.PeptideCardRequest()\n",
    "        peptideCardRequest.peptide_id = identifier;\n",
    "        peptideCardRequest.format = FormatType.FormatType.JSON;\n",
    "        dbaasp_peptide = json.loads(peptideCardRequest.request())\n",
    "        NAs = [\"NA\", \"na\", \"Na\", \"nA\", \"N/A\", \"n/a\"]\n",
    "\n",
    "        if 'errorCode' in dbaasp_peptide:\n",
    "#            print(\"db error\", identifier)\n",
    "            return []\n",
    "        if \"targetActivities\" not in dbaasp_peptide[\"peptideCard\"]:\n",
    "#            print(\"no target\", identifier)#, dbaasp_peptide[\"peptideCard\"])\n",
    "            return []  \n",
    "    \n",
    "        species = dbaasp_peptide[\"peptideCard\"][\"targetActivities\"]\n",
    "        for specie in species:\n",
    "            if not (\"unit\" and \"concentration\" and \"targetSpecies\" and \"activityMeasure\" in specie):\n",
    "                continue\n",
    "            unit = specie[\"unit\"]\n",
    "\n",
    "            #aaaaaaaaaaaaaaaaaaaaaaaaaaaaaAAAaaaaaaaaaargh\n",
    "            concentration_str = specie[\"concentration\"].replace(\" \",\"\")\n",
    "            concentration_str = concentration_str.replace(\"–\",\"-\")\n",
    "            concentration_str = concentration_str.replace(\"->\",\"-\") \n",
    "            #concentration_str = concentration_str.replace(\"0,\",\"0.\")\n",
    "            concentration_str = concentration_str.replace(\",\",\".\") \n",
    "\n",
    "\n",
    "            if concentration_str[0] == '<':\n",
    "                if concentration_str[1] == '=':\n",
    "                    concentration_tmp = float_ignore_plus_minus(concentration_str[2:])\n",
    "                else:\n",
    "                    concentration_tmp = float_ignore_plus_minus(concentration_str[1:])\n",
    "                concentration = concentration_tmp\n",
    "            elif concentration_str[0] == '>' or concentration_str in NAs:\n",
    "                if concentration_str[1] == '=':\n",
    "                    concentration_tmp = float_ignore_plus_minus(concentration_str[2:])\n",
    "                else:\n",
    "                    concentration_tmp = float_ignore_plus_minus(concentration_str[1:])\n",
    "                concentration = concentration_tmp\n",
    "            elif \"-\"  in concentration_str:\n",
    "                concentrations = concentration_str.split(\"-\")\n",
    "                concentration =  float_ignore_plus_minus(concentrations[0]) + float_ignore_plus_minus(concentrations[1])\n",
    "                concentration /= 2\n",
    "            else:\n",
    "                concentration = float_ignore_plus_minus(concentration_str)\n",
    "\n",
    "            if (unit == \"µM\" and concentration > 10) or (unit == \"nM\" and concentration > 10000) or (unit == \"µg/ml\" and concentration > 32): \n",
    "                results.append([concentration, unit, specie[\"targetSpecies\"], specie[\"activityMeasure\"]])\n",
    "\n",
    "\n",
    "            elif unit != \"µM\" and unit != \"nM\" and unit != \"µg/ml\" and concentration_str not in NAs:\n",
    "                pass \n",
    "                # print(\"no unit\",unit, identifier)#, species)\n",
    "            else:\n",
    "                return []\n",
    "\n",
    "        return results\n",
    "    except:\n",
    "        return results\n",
    "    "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Actives from DBAASP dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "db_path = folder+\"pickles/daasp_with_activities.pkl\"\n",
    "if not os.path.exists(db_path):\n",
    "    \n",
    "    pandarallel.initialize(nb_workers=64, progress_bar=False)\n",
    "    # read\n",
    "    df_dbaasp = pd.read_csv(\"data/DBAASP_nointrabond.csv\", sep =';')\n",
    "    \n",
    "    # remove seq duplicates\n",
    "    df_dbaasp = df_dbaasp.drop_duplicates(\"Sequence\").copy()\n",
    "\n",
    "    # check that N and C terminus are absent or ACT (actetyl) and AMD (amide) respectively\n",
    "    cols = df_dbaasp.columns\n",
    "    cols = cols.map(lambda x: x.replace(' ', '_'))\n",
    "    df_dbaasp.columns = cols\n",
    "    df_dbaasp[\"C_terminus\"]= df_dbaasp[\"C_terminus\"].map(str)\n",
    "    df_dbaasp[\"N_terminus\"]= df_dbaasp[\"N_terminus\"].map(str)\n",
    "    df_dbaasp = df_dbaasp.query(\" ((N_terminus ==  'nan' or N_terminus ==  'ACT') and (C_terminus ==  'nan' or C_terminus == 'AMD'))\")\n",
    "\n",
    "    # only natural aminoacid sequences are kept\n",
    "    df_dbaasp['isNatural'] = df_dbaasp.Sequence.map(is_natural)\n",
    "    df_dbaasp = df_dbaasp.loc[df_dbaasp['isNatural'] == True].copy()\n",
    "    del df_dbaasp[\"isNatural\"]\n",
    "\n",
    "    # check that entries have an associated target with activity below 10 µM, 10000 nM and 32 µg/ml\n",
    "    df_dbaasp['isActive'] = df_dbaasp.ID.parallel_map(is_active)\n",
    "    s = df_dbaasp[\"isActive\"].apply(Series,1).stack()\n",
    "    s.index = s.index.droplevel(-1)\n",
    "    s.name = \"isActive\"\n",
    "    df_dbaasp = df_dbaasp.copy(deep=True)\n",
    "    del df_dbaasp[\"isActive\"]\n",
    "    df_dbaasp = df_dbaasp.join(s.apply(lambda x: Series(x)))\n",
    "    df_dbaasp.columns = ['ID', 'Name', 'N terminus', 'Sequence', 'C terminus', \"concentration\", \"unit\", \"targetSpecies\", \"activityMeasure\"]\n",
    "    df_dbaasp = df_dbaasp.dropna(subset = ['Sequence',\"concentration\", \"unit\", \"targetSpecies\", \"activityMeasure\"]).copy()\n",
    "\n",
    "    df_dbaasp.to_pickle(db_path)\n",
    "    pandarallel.initialize(progress_bar=True)\n",
    "\n",
    "else:\n",
    "    df_dbaasp = pd.read_pickle(db_path)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_dbaasp = pd.read_pickle(db_path)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "group = df_dbaasp.groupby([\"ID\",\"Name\", \"N terminus\", \"Sequence\", \"C terminus\"], as_index=False)\n",
    "actives = pd.DataFrame(group[\"targetSpecies\"].aggregate(list))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "def kills_bacteria(row, bacteria):\n",
    "    targets = row.targetSpecies\n",
    "    for target in targets:\n",
    "        if bacteria in target:\n",
    "            return True    \n",
    "    return False"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "fa7533afa65847ae89081fa4c0ea3e15",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "VBox(children=(HBox(children=(IntProgress(value=0, description='0.00%', max=597), Label(value='0 / 597'))), HB…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "03f2868d2509433aacf3789301aaeaaa",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "VBox(children=(HBox(children=(IntProgress(value=0, description='0.00%', max=597), Label(value='0 / 597'))), HB…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "3c7b54a4e20f4b4ebce23c13411dd425",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "VBox(children=(HBox(children=(IntProgress(value=0, description='0.00%', max=597), Label(value='0 / 597'))), HB…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "for bacteria in [\"baumannii\", \"aureus\", \"aeruginosa\"]:\n",
    "    actives[bacteria] =  actives.parallel_apply(lambda x: kills_bacteria(x, bacteria), axis=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "actives[\"activity\"] = 1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "actives_list = actives.Sequence.to_list()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Inactives dataset"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Inactives from DBAASP dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "db_path = folder+\"pickles/daasp_inactives_with_activities.pkl\"\n",
    "if not os.path.exists(db_path):\n",
    "    \n",
    "    pandarallel.initialize(nb_workers=64, progress_bar=False)\n",
    "    # read\n",
    "    df_dbaasp = pd.read_csv(\"data/DBAASP_nointrabond.csv\", sep =';')\n",
    "    \n",
    "    # remove seq duplicates\n",
    "    df_dbaasp = df_dbaasp.drop_duplicates(\"Sequence\").copy()\n",
    "\n",
    "    # check that N and C terminus are absent or ACT (actetyl) and AMD (amide) respectively\n",
    "    cols = df_dbaasp.columns\n",
    "    cols = cols.map(lambda x: x.replace(' ', '_'))\n",
    "    df_dbaasp.columns = cols\n",
    "    df_dbaasp[\"C_terminus\"]= df_dbaasp[\"C_terminus\"].map(str)\n",
    "    df_dbaasp[\"N_terminus\"]= df_dbaasp[\"N_terminus\"].map(str)\n",
    "    df_dbaasp = df_dbaasp.query(\" ((N_terminus ==  'nan' or N_terminus ==  'ACT') and (C_terminus ==  'nan' or C_terminus == 'AMD'))\")\n",
    "\n",
    "    # only natural aminoacid sequences are kept\n",
    "    df_dbaasp['isNatural'] = df_dbaasp.Sequence.map(is_natural)\n",
    "    df_dbaasp = df_dbaasp.loc[df_dbaasp['isNatural'] == True].copy()\n",
    "    del df_dbaasp[\"isNatural\"]\n",
    "\n",
    "    # check that entries have an associated target with activity below 10 µM, 10000 nM and 32 µg/ml\n",
    "    df_dbaasp['isInactive'] = df_dbaasp.ID.parallel_map(is_inactive)\n",
    "    s = df_dbaasp[\"isInactive\"].apply(Series,1).stack()\n",
    "    s.index = s.index.droplevel(-1)\n",
    "    s.name = \"isInactive\"\n",
    "    df_dbaasp = df_dbaasp.copy(deep=True)\n",
    "    del df_dbaasp[\"isInactive\"]\n",
    "    df_dbaasp = df_dbaasp.join(s.apply(lambda x: Series(x)))\n",
    "    df_dbaasp.columns = ['ID', 'Name', 'N terminus', 'Sequence', 'C terminus', \"concentration\", \"unit\", \"targetSpecies\", \"activityMeasure\"]\n",
    "    df_dbaasp = df_dbaasp.dropna(subset = ['Sequence',\"concentration\", \"unit\", \"targetSpecies\", \"activityMeasure\"]).copy()\n",
    "\n",
    "    df_dbaasp.to_pickle(db_path)\n",
    "    pandarallel.initialize(progress_bar=True)\n",
    "\n",
    "else:\n",
    "    df_dbaasp = pd.read_pickle(db_path)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "group = df_dbaasp.groupby([\"ID\",\"Name\", \"N terminus\", \"Sequence\", \"C terminus\"], as_index=False)\n",
    "inactives_confirmed = pd.DataFrame(group[\"targetSpecies\"].aggregate(list))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "4c5935d75c184679924e905b6177501a",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "VBox(children=(HBox(children=(IntProgress(value=0, description='0.00%', max=234), Label(value='0 / 234'))), HB…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "07d9fa7d6aea4e51adef269457782806",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "VBox(children=(HBox(children=(IntProgress(value=0, description='0.00%', max=234), Label(value='0 / 234'))), HB…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "1398abece0d64c6ea5a0c69f9d0ff5bf",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "VBox(children=(HBox(children=(IntProgress(value=0, description='0.00%', max=234), Label(value='0 / 234'))), HB…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "for bacteria in [\"baumannii\", \"aureus\", \"aeruginosa\"]:\n",
    "    inactives_confirmed[bacteria] =  inactives_confirmed.parallel_apply(lambda x: kills_bacteria(x, bacteria), axis=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "number_confirmed_actives = len(actives)\n",
    "number_confirmed_inactives = len(inactives_confirmed)\n",
    "number_scrambled_inactives = (number_confirmed_actives - number_confirmed_inactives) // 2\n",
    "number_swissprot_inactives = number_confirmed_actives - (number_confirmed_inactives + number_scrambled_inactives)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Inactives from scrambled actives dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [],
   "source": [
    "def scramble(seq):\n",
    "    new_seq = ''\n",
    "    aas = list(seq)\n",
    "    while len(new_seq) < len(seq):\n",
    "        aa = random.choice(aas)\n",
    "        aas.remove(aa)\n",
    "        new_seq += aa\n",
    "    return new_seq\n",
    "\n",
    "def scramble_less(seq):\n",
    "    lengths = [1,2]\n",
    "    n = random.choice(lengths)\n",
    "    seq_list = [seq[i:i+n] for i in range(0, len(seq), n)]\n",
    "    new_seq = ''\n",
    "    parts = seq_list\n",
    "    while len(new_seq) < len(seq):\n",
    "        part = random.choice(parts)\n",
    "        parts.remove(part)\n",
    "        new_seq += part\n",
    "    return new_seq\n",
    "\n",
    "def new_inactive_scrambled(row, actives_list = actives_list):\n",
    "    seq = scramble_less(row[\"Sequence\"])\n",
    "    while seq in actives_list:\n",
    "        seq = scramble_less(row[\"Sequence\"])\n",
    "    cid = \"scr_{}\".format(row[\"ID\"])\n",
    "    new_row = {\"ID\":[cid], \"Sequence\":[seq]}\n",
    "    return pd.DataFrame(new_row)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [],
   "source": [
    "# sample actives to make scrambled inactives until you have enough unique scrambled sequences\n",
    "\n",
    "def get_scrambled_samples(df_actives, n, SEED=SEED):\n",
    "    actives_subset = df_actives.sample(n, random_state=SEED).reset_index(drop = True)\n",
    "    inactives_scrambled = actives_subset.apply(new_inactive_scrambled, axis = 1)\n",
    "    inactives_scrambled = pd.concat(inactives_scrambled.tolist()).reset_index(drop = True)\n",
    "    return inactives_scrambled\n",
    "\n",
    "SEED_counter = SEED\n",
    "inactives_scrambled = get_scrambled_samples(actives, number_scrambled_inactives).drop_duplicates(\"Sequence\")\n",
    "while len(inactives_scrambled) < number_scrambled_inactives:\n",
    "    SEED_counter += 1\n",
    "    new_samples = get_scrambled_samples(actives, number_scrambled_inactives-len(inactives_scrambled) , SEED=SEED_counter).drop_duplicates(\"Sequence\")\n",
    "    inactives_scrambled = inactives_scrambled.append(new_samples)\n",
    "    inactives_scrambled.drop_duplicates(\"Sequence\").reset_index(drop = True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Inactives from Swissprot dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [],
   "source": [
    "def random_fragment(longseq, length):\n",
    "    index1 = random.randrange(len(longseq))\n",
    "    index2 = index1+length\n",
    "    seq = longseq[index1:index2]\n",
    "    return seq\n",
    "\n",
    "def new_inactive_swissprot(row, df_actives, df_inactives, actives_list = actives_list):\n",
    "    row_act = df_actives.iloc[row]\n",
    "    row_inact = df_inactives.iloc[row]\n",
    "    longseq = row_inact[\"Sequence\"]\n",
    "    length = len(row_act[\"Sequence\"])\n",
    "    seq = random_fragment(longseq, length)\n",
    "    while seq in actives_list:\n",
    "        seq = random_fragment(longseq, length)\n",
    "    old_cid = row_act[\"ID\"]\n",
    "    cid = \"frag_{}_{}\".format(len(longseq), old_cid)\n",
    "    new_row = {\"ID\":[cid], \"Sequence\":[seq]}\n",
    "    return pd.DataFrame(new_row)\n",
    "\n",
    "seqs = []\n",
    "seq = ''\n",
    "newvalues_dictionary = {}\n",
    "first = True\n",
    "\n",
    "with open(folder+\"data/uniprot_sprot.fasta\") as inFile:\n",
    "    for line in inFile:\n",
    "        line = line.strip()\n",
    "        if line[0] == \">\":\n",
    "            if first ==  True:\n",
    "                cid = line.replace(\">\", \"\")\n",
    "                first = False\n",
    "                continue\n",
    "            newvalues_dictionary[cid] = seq\n",
    "            cid = line.replace(\">\", \"\")\n",
    "            seq = \"\"\n",
    "            continue\n",
    "        else:\n",
    "            seq+=line\n",
    "            continue\n",
    "\n",
    "swissprot = pd.DataFrame(newvalues_dictionary.items(), columns=['ID', 'Sequence']) \n",
    "swissprot['length'] = swissprot.Sequence.map(len)\n",
    "swissprot[\"isNatural\"] = swissprot[\"Sequence\"].map(is_natural)\n",
    "swissprot = swissprot[swissprot['isNatural'] == True].reset_index(drop = True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [],
   "source": [
    "# generate inactives picking fragments from uniprot\n",
    "# sample actives to make uniprot inactives until you have enough unique uniprot sequences\n",
    "\n",
    "def get_uniprot_samples(df_actives, n, SEED=SEED):\n",
    "    actives_subset = df_actives.sample(n, random_state=SEED).reset_index(drop = True)\n",
    "    swissprot_subset =  swissprot.sample(n, random_state=SEED).reset_index(drop = True)\n",
    "    inactives_swissprot = [new_inactive_swissprot(i, actives_subset, swissprot_subset) for i in range(n)]\n",
    "    inactives_swissprot = pd.concat(inactives_swissprot).reset_index(drop = True)\n",
    "    return inactives_swissprot\n",
    "\n",
    "SEED_counter = SEED\n",
    "inactives_swissprot = get_uniprot_samples(actives, number_swissprot_inactives).drop_duplicates(\"Sequence\")\n",
    "n_of_samples = (len(actives.append(inactives_scrambled).append(inactives_swissprot).Sequence.unique()) - len(actives) - len(inactives_scrambled))\n",
    "while n_of_samples < number_swissprot_inactives:\n",
    "    SEED_counter += 1\n",
    "    new_samples = get_uniprot_samples(actives, number_swissprot_inactives - n_of_samples, SEED=SEED_counter).drop_duplicates(\"Sequence\")\n",
    "    inactives_swissprot = inactives_swissprot.append(new_samples)\n",
    "    inactives_swissprot.drop_duplicates(\"Sequence\").reset_index(drop = True)\n",
    "    n_of_samples = (len(actives.append(inactives_scrambled).append(inactives_swissprot).Sequence.unique()) - len(actives) - len(inactives_scrambled))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Finalize and save dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [],
   "source": [
    "# all inactives together\n",
    "inactives = pd.concat([inactives_confirmed, inactives_scrambled, inactives_swissprot]).reset_index(drop=True)\n",
    "\n",
    "inactives[\"activity\"]=0\n",
    "\n",
    "# assign to training or test set inactives\n",
    "inactives[\"Set\"] = \"test\"\n",
    "training_inactives = inactives.sample(frac=0.75, random_state=SEED)\n",
    "inactives.loc[training_inactives.index, 'Set'] = \"training\"\n",
    "\n",
    "# assign to training or test set actives\n",
    "actives[\"Set\"] = \"test\"\n",
    "training_actives = actives.sample(frac=0.75, random_state=SEED)\n",
    "actives.loc[training_actives.index, 'Set'] = \"training\"\n",
    "\n",
    "# actives and inactives together and saved\n",
    "dataset_actives_inactives = inactives.append(actives).reset_index(drop=True)\n",
    "dataset_actives_inactives.to_csv(folder + \"data/DAASP_RNN_dataset.csv\", index=False)\n",
    "dataset_actives_inactives.to_pickle(folder + \"pickles/DAASP_RNN_dataset.plk\")"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.10"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}