Ex_06_PySpark_Random_Forest_Regressor.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "Ex-06-PySpark Random Forest Regressor.ipynb",
      "provenance": []
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "language_info": {
      "name": "python"
    }
  },
  "cells": [
    {
      "cell_type": "code",
      "metadata": {
        "id": "wNQvOcLSJC4D"
      },
      "source": [
        "!apt-get install openjdk-8-jdk-headless -qq > /dev/null\n",
        "!wget -q https://downloads.apache.org/spark/spark-2.4.8/spark-2.4.8-bin-hadoop2.7.tgz\n",
        "!tar xf spark-2.4.8-bin-hadoop2.7.tgz\n",
        "!pip install -q findspark"
      ],
      "execution_count": 2,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "FIH7ZgU6JD6M",
        "outputId": "9dbc6714-af5d-4851-a72b-cf007e2fd4f0"
      },
      "source": [
        "!pip install pyspark"
      ],
      "execution_count": 3,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Collecting pyspark\n",
            "  Downloading pyspark-3.1.2.tar.gz (212.4 MB)\n",
            "\u001b[K     |████████████████████████████████| 212.4 MB 60 kB/s \n",
            "\u001b[?25hCollecting py4j==0.10.9\n",
            "  Downloading py4j-0.10.9-py2.py3-none-any.whl (198 kB)\n",
            "\u001b[K     |████████████████████████████████| 198 kB 71.6 MB/s \n",
            "\u001b[?25hBuilding wheels for collected packages: pyspark\n",
            "  Building wheel for pyspark (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
            "  Created wheel for pyspark: filename=pyspark-3.1.2-py2.py3-none-any.whl size=212880768 sha256=f308fd933b1e90c371b7dfb6e188a2348f30864188ff6925afcf8898e7fd532f\n",
            "  Stored in directory: /root/.cache/pip/wheels/a5/0a/c1/9561f6fecb759579a7d863dcd846daaa95f598744e71b02c77\n",
            "Successfully built pyspark\n",
            "Installing collected packages: py4j, pyspark\n",
            "Successfully installed py4j-0.10.9 pyspark-3.1.2\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "VQDp31ePJF07"
      },
      "source": [
        "from pyspark import SparkContext, SparkConf"
      ],
      "execution_count": 4,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Nu0J9qykJZFz"
      },
      "source": [
        "conf = SparkConf().setAppName('Spark Prac').setMaster('local')\n",
        "sc = SparkContext(conf = conf)"
      ],
      "execution_count": 5,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "resources": {
            "http://localhost:8080/nbextensions/google.colab/files.js": {
              "data": "Ly8gQ29weXJpZ2h0IDIwMTcgR29vZ2xlIExMQwovLwovLyBMaWNlbnNlZCB1bmRlciB0aGUgQXBhY2hlIExpY2Vuc2UsIFZlcnNpb24gMi4wICh0aGUgIkxpY2Vuc2UiKTsKLy8geW91IG1heSBub3QgdXNlIHRoaXMgZmlsZSBleGNlcHQgaW4gY29tcGxpYW5jZSB3aXRoIHRoZSBMaWNlbnNlLgovLyBZb3UgbWF5IG9idGFpbiBhIGNvcHkgb2YgdGhlIExpY2Vuc2UgYXQKLy8KLy8gICAgICBodHRwOi8vd3d3LmFwYWNoZS5vcmcvbGljZW5zZXMvTElDRU5TRS0yLjAKLy8KLy8gVW5sZXNzIHJlcXVpcmVkIGJ5IGFwcGxpY2FibGUgbGF3IG9yIGFncmVlZCB0byBpbiB3cml0aW5nLCBzb2Z0d2FyZQovLyBkaXN0cmlidXRlZCB1bmRlciB0aGUgTGljZW5zZSBpcyBkaXN0cmlidXRlZCBvbiBhbiAiQVMgSVMiIEJBU0lTLAovLyBXSVRIT1VUIFdBUlJBTlRJRVMgT1IgQ09ORElUSU9OUyBPRiBBTlkgS0lORCwgZWl0aGVyIGV4cHJlc3Mgb3IgaW1wbGllZC4KLy8gU2VlIHRoZSBMaWNlbnNlIGZvciB0aGUgc3BlY2lmaWMgbGFuZ3VhZ2UgZ292ZXJuaW5nIHBlcm1pc3Npb25zIGFuZAovLyBsaW1pdGF0aW9ucyB1bmRlciB0aGUgTGljZW5zZS4KCi8qKgogKiBAZmlsZW92ZXJ2aWV3IEhlbHBlcnMgZm9yIGdvb2dsZS5jb2xhYiBQeXRob24gbW9kdWxlLgogKi8KKGZ1bmN0aW9uKHNjb3BlKSB7CmZ1bmN0aW9uIHNwYW4odGV4dCwgc3R5bGVBdHRyaWJ1dGVzID0ge30pIHsKICBjb25zdCBlbGVtZW50ID0gZG9jdW1lbnQuY3JlYXRlRWxlbWVudCgnc3BhbicpOwogIGVsZW1lbnQudGV4dENvbnRlbnQgPSB0ZXh0OwogIGZvciAoY29uc3Qga2V5IG9mIE9iamVjdC5rZXlzKHN0eWxlQXR0cmlidXRlcykpIHsKICAgIGVsZW1lbnQuc3R5bGVba2V5XSA9IHN0eWxlQXR0cmlidXRlc1trZXldOwogIH0KICByZXR1cm4gZWxlbWVudDsKfQoKLy8gTWF4IG51bWJlciBvZiBieXRlcyB3aGljaCB3aWxsIGJlIHVwbG9hZGVkIGF0IGEgdGltZS4KY29uc3QgTUFYX1BBWUxPQURfU0laRSA9IDEwMCAqIDEwMjQ7CgpmdW5jdGlvbiBfdXBsb2FkRmlsZXMoaW5wdXRJZCwgb3V0cHV0SWQpIHsKICBjb25zdCBzdGVwcyA9IHVwbG9hZEZpbGVzU3RlcChpbnB1dElkLCBvdXRwdXRJZCk7CiAgY29uc3Qgb3V0cHV0RWxlbWVudCA9IGRvY3VtZW50LmdldEVsZW1lbnRCeUlkKG91dHB1dElkKTsKICAvLyBDYWNoZSBzdGVwcyBvbiB0aGUgb3V0cHV0RWxlbWVudCB0byBtYWtlIGl0IGF2YWlsYWJsZSBmb3IgdGhlIG5leHQgY2FsbAogIC8vIHRvIHVwbG9hZEZpbGVzQ29udGludWUgZnJvbSBQeXRob24uCiAgb3V0cHV0RWxlbWVudC5zdGVwcyA9IHN0ZXBzOwoKICByZXR1cm4gX3VwbG9hZEZpbGVzQ29udGludWUob3V0cHV0SWQpOwp9CgovLyBUaGlzIGlzIHJvdWdobHkgYW4gYXN5bmMgZ2VuZXJhdG9yIChub3Qgc3VwcG9ydGVkIGluIHRoZSBicm93c2VyIHlldCksCi8vIHdoZXJlIHRoZXJlIGFyZSBtdWx0aXBsZSBhc3luY2hyb25vdXMgc3RlcHMgYW5kIHRoZSBQeXRob24gc2lkZSBpcyBnb2luZwovLyB0byBwb2xsIGZvciBjb21wbGV0aW9uIG9mIGVhY2ggc3RlcC4KLy8gVGhpcyB1c2VzIGEgUHJvbWlzZSB0byBibG9jayB0aGUgcHl0aG9uIHNpZGUgb24gY29tcGxldGlvbiBvZiBlYWNoIHN0ZXAsCi8vIHRoZW4gcGFzc2VzIHRoZSByZXN1bHQgb2YgdGhlIHByZXZpb3VzIHN0ZXAgYXMgdGhlIGlucHV0IHRvIHRoZSBuZXh0IHN0ZXAuCmZ1bmN0aW9uIF91cGxvYWRGaWxlc0NvbnRpbnVlKG91dHB1dElkKSB7CiAgY29uc3Qgb3V0cHV0RWxlbWVudCA9IGRvY3VtZW50LmdldEVsZW1lbnRCeUlkKG91dHB1dElkKTsKICBjb25zdCBzdGVwcyA9IG91dHB1dEVsZW1lbnQuc3RlcHM7CgogIGNvbnN0IG5leHQgPSBzdGVwcy5uZXh0KG91dHB1dEVsZW1lbnQubGFzdFByb21pc2VWYWx1ZSk7CiAgcmV0dXJuIFByb21pc2UucmVzb2x2ZShuZXh0LnZhbHVlLnByb21pc2UpLnRoZW4oKHZhbHVlKSA9PiB7CiAgICAvLyBDYWNoZSB0aGUgbGFzdCBwcm9taXNlIHZhbHVlIHRvIG1ha2UgaXQgYXZhaWxhYmxlIHRvIHRoZSBuZXh0CiAgICAvLyBzdGVwIG9mIHRoZSBnZW5lcmF0b3IuCiAgICBvdXRwdXRFbGVtZW50Lmxhc3RQcm9taXNlVmFsdWUgPSB2YWx1ZTsKICAgIHJldHVybiBuZXh0LnZhbHVlLnJlc3BvbnNlOwogIH0pOwp9CgovKioKICogR2VuZXJhdG9yIGZ1bmN0aW9uIHdoaWNoIGlzIGNhbGxlZCBiZXR3ZWVuIGVhY2ggYXN5bmMgc3RlcCBvZiB0aGUgdXBsb2FkCiAqIHByb2Nlc3MuCiAqIEBwYXJhbSB7c3RyaW5nfSBpbnB1dElkIEVsZW1lbnQgSUQgb2YgdGhlIGlucHV0IGZpbGUgcGlja2VyIGVsZW1lbnQuCiAqIEBwYXJhbSB7c3RyaW5nfSBvdXRwdXRJZCBFbGVtZW50IElEIG9mIHRoZSBvdXRwdXQgZGlzcGxheS4KICogQHJldHVybiB7IUl0ZXJhYmxlPCFPYmplY3Q+fSBJdGVyYWJsZSBvZiBuZXh0IHN0ZXBzLgogKi8KZnVuY3Rpb24qIHVwbG9hZEZpbGVzU3RlcChpbnB1dElkLCBvdXRwdXRJZCkgewogIGNvbnN0IGlucHV0RWxlbWVudCA9IGRvY3VtZW50LmdldEVsZW1lbnRCeUlkKGlucHV0SWQpOwogIGlucHV0RWxlbWVudC5kaXNhYmxlZCA9IGZhbHNlOwoKICBjb25zdCBvdXRwdXRFbGVtZW50ID0gZG9jdW1lbnQuZ2V0RWxlbWVudEJ5SWQob3V0cHV0SWQpOwogIG91dHB1dEVsZW1lbnQuaW5uZXJIVE1MID0gJyc7CgogIGNvbnN0IHBpY2tlZFByb21pc2UgPSBuZXcgUHJvbWlzZSgocmVzb2x2ZSkgPT4gewogICAgaW5wdXRFbGVtZW50LmFkZEV2ZW50TGlzdGVuZXIoJ2NoYW5nZScsIChlKSA9PiB7CiAgICAgIHJlc29sdmUoZS50YXJnZXQuZmlsZXMpOwogICAgfSk7CiAgfSk7CgogIGNvbnN0IGNhbmNlbCA9IGRvY3VtZW50LmNyZWF0ZUVsZW1lbnQoJ2J1dHRvbicpOwogIGlucHV0RWxlbWVudC5wYXJlbnRFbGVtZW50LmFwcGVuZENoaWxkKGNhbmNlbCk7CiAgY2FuY2VsLnRleHRDb250ZW50ID0gJ0NhbmNlbCB1cGxvYWQnOwogIGNvbnN0IGNhbmNlbFByb21pc2UgPSBuZXcgUHJvbWlzZSgocmVzb2x2ZSkgPT4gewogICAgY2FuY2VsLm9uY2xpY2sgPSAoKSA9PiB7CiAgICAgIHJlc29sdmUobnVsbCk7CiAgICB9OwogIH0pOwoKICAvLyBXYWl0IGZvciB0aGUgdXNlciB0byBwaWNrIHRoZSBmaWxlcy4KICBjb25zdCBmaWxlcyA9IHlpZWxkIHsKICAgIHByb21pc2U6IFByb21pc2UucmFjZShbcGlja2VkUHJvbWlzZSwgY2FuY2VsUHJvbWlzZV0pLAogICAgcmVzcG9uc2U6IHsKICAgICAgYWN0aW9uOiAnc3RhcnRpbmcnLAogICAgfQogIH07CgogIGNhbmNlbC5yZW1vdmUoKTsKCiAgLy8gRGlzYWJsZSB0aGUgaW5wdXQgZWxlbWVudCBzaW5jZSBmdXJ0aGVyIHBpY2tzIGFyZSBub3QgYWxsb3dlZC4KICBpbnB1dEVsZW1lbnQuZGlzYWJsZWQgPSB0cnVlOwoKICBpZiAoIWZpbGVzKSB7CiAgICByZXR1cm4gewogICAgICByZXNwb25zZTogewogICAgICAgIGFjdGlvbjogJ2NvbXBsZXRlJywKICAgICAgfQogICAgfTsKICB9CgogIGZvciAoY29uc3QgZmlsZSBvZiBmaWxlcykgewogICAgY29uc3QgbGkgPSBkb2N1bWVudC5jcmVhdGVFbGVtZW50KCdsaScpOwogICAgbGkuYXBwZW5kKHNwYW4oZmlsZS5uYW1lLCB7Zm9udFdlaWdodDogJ2JvbGQnfSkpOwogICAgbGkuYXBwZW5kKHNwYW4oCiAgICAgICAgYCgke2ZpbGUudHlwZSB8fCAnbi9hJ30pIC0gJHtmaWxlLnNpemV9IGJ5dGVzLCBgICsKICAgICAgICBgbGFzdCBtb2RpZmllZDogJHsKICAgICAgICAgICAgZmlsZS5sYXN0TW9kaWZpZWREYXRlID8gZmlsZS5sYXN0TW9kaWZpZWREYXRlLnRvTG9jYWxlRGF0ZVN0cmluZygpIDoKICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgJ24vYSd9IC0gYCkpOwogICAgY29uc3QgcGVyY2VudCA9IHNwYW4oJzAlIGRvbmUnKTsKICAgIGxpLmFwcGVuZENoaWxkKHBlcmNlbnQpOwoKICAgIG91dHB1dEVsZW1lbnQuYXBwZW5kQ2hpbGQobGkpOwoKICAgIGNvbnN0IGZpbGVEYXRhUHJvbWlzZSA9IG5ldyBQcm9taXNlKChyZXNvbHZlKSA9PiB7CiAgICAgIGNvbnN0IHJlYWRlciA9IG5ldyBGaWxlUmVhZGVyKCk7CiAgICAgIHJlYWRlci5vbmxvYWQgPSAoZSkgPT4gewogICAgICAgIHJlc29sdmUoZS50YXJnZXQucmVzdWx0KTsKICAgICAgfTsKICAgICAgcmVhZGVyLnJlYWRBc0FycmF5QnVmZmVyKGZpbGUpOwogICAgfSk7CiAgICAvLyBXYWl0IGZvciB0aGUgZGF0YSB0byBiZSByZWFkeS4KICAgIGxldCBmaWxlRGF0YSA9IHlpZWxkIHsKICAgICAgcHJvbWlzZTogZmlsZURhdGFQcm9taXNlLAogICAgICByZXNwb25zZTogewogICAgICAgIGFjdGlvbjogJ2NvbnRpbnVlJywKICAgICAgfQogICAgfTsKCiAgICAvLyBVc2UgYSBjaHVua2VkIHNlbmRpbmcgdG8gYXZvaWQgbWVzc2FnZSBzaXplIGxpbWl0cy4gU2VlIGIvNjIxMTU2NjAuCiAgICBsZXQgcG9zaXRpb24gPSAwOwogICAgZG8gewogICAgICBjb25zdCBsZW5ndGggPSBNYXRoLm1pbihmaWxlRGF0YS5ieXRlTGVuZ3RoIC0gcG9zaXRpb24sIE1BWF9QQVlMT0FEX1NJWkUpOwogICAgICBjb25zdCBjaHVuayA9IG5ldyBVaW50OEFycmF5KGZpbGVEYXRhLCBwb3NpdGlvbiwgbGVuZ3RoKTsKICAgICAgcG9zaXRpb24gKz0gbGVuZ3RoOwoKICAgICAgY29uc3QgYmFzZTY0ID0gYnRvYShTdHJpbmcuZnJvbUNoYXJDb2RlLmFwcGx5KG51bGwsIGNodW5rKSk7CiAgICAgIHlpZWxkIHsKICAgICAgICByZXNwb25zZTogewogICAgICAgICAgYWN0aW9uOiAnYXBwZW5kJywKICAgICAgICAgIGZpbGU6IGZpbGUubmFtZSwKICAgICAgICAgIGRhdGE6IGJhc2U2NCwKICAgICAgICB9LAogICAgICB9OwoKICAgICAgbGV0IHBlcmNlbnREb25lID0gZmlsZURhdGEuYnl0ZUxlbmd0aCA9PT0gMCA/CiAgICAgICAgICAxMDAgOgogICAgICAgICAgTWF0aC5yb3VuZCgocG9zaXRpb24gLyBmaWxlRGF0YS5ieXRlTGVuZ3RoKSAqIDEwMCk7CiAgICAgIHBlcmNlbnQudGV4dENvbnRlbnQgPSBgJHtwZXJjZW50RG9uZX0lIGRvbmVgOwoKICAgIH0gd2hpbGUgKHBvc2l0aW9uIDwgZmlsZURhdGEuYnl0ZUxlbmd0aCk7CiAgfQoKICAvLyBBbGwgZG9uZS4KICB5aWVsZCB7CiAgICByZXNwb25zZTogewogICAgICBhY3Rpb246ICdjb21wbGV0ZScsCiAgICB9CiAgfTsKfQoKc2NvcGUuZ29vZ2xlID0gc2NvcGUuZ29vZ2xlIHx8IHt9OwpzY29wZS5nb29nbGUuY29sYWIgPSBzY29wZS5nb29nbGUuY29sYWIgfHwge307CnNjb3BlLmdvb2dsZS5jb2xhYi5fZmlsZXMgPSB7CiAgX3VwbG9hZEZpbGVzLAogIF91cGxvYWRGaWxlc0NvbnRpbnVlLAp9Owp9KShzZWxmKTsK",
              "ok": true,
              "headers": [
                [
                  "content-type",
                  "application/javascript"
                ]
              ],
              "status": 200,
              "status_text": ""
            }
          },
          "base_uri": "https://localhost:8080/",
          "height": 72
        },
        "id": "CdunAvM1Jejq",
        "outputId": "a4547112-4b84-40a9-86c6-ed997275782c"
      },
      "source": [
        "from google.colab import files\n",
        "uploaded = files.upload()"
      ],
      "execution_count": 6,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "text/html": [
              "\n",
              "     <input type=\"file\" id=\"files-886a260a-d47d-4db8-b3ec-eead14db0755\" name=\"files[]\" multiple disabled\n",
              "        style=\"border:none\" />\n",
              "     <output id=\"result-886a260a-d47d-4db8-b3ec-eead14db0755\">\n",
              "      Upload widget is only available when the cell has been executed in the\n",
              "      current browser session. Please rerun this cell to enable.\n",
              "      </output>\n",
              "      <script src=\"/nbextensions/google.colab/files.js\"></script> "
            ],
            "text/plain": [
              "<IPython.core.display.HTML object>"
            ]
          },
          "metadata": {}
        },
        {
          "output_type": "stream",
          "text": [
            "Saving Admission_Prediction.csv to Admission_Prediction.csv\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "NTA-EgRcKFSb"
      },
      "source": [
        "import io\n",
        "data1 = io.BytesIO(uploaded['Admission_Prediction.csv'])    "
      ],
      "execution_count": 7,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "qLWtEmhuKoMa"
      },
      "source": [
        "data = sc.textFile('Admission_Prediction.csv')"
      ],
      "execution_count": 8,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "yFgucSWId9yy"
      },
      "source": [
        "#### Import PySpark ML Libraries"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "9GiDs0Iib1fR"
      },
      "source": [
        "from __future__ import print_function\n",
        "import pyspark\n",
        "from pyspark.sql import SparkSession\n",
        "from pyspark.ml.feature import VectorAssembler, VectorIndexer\n",
        "from pyspark.ml.feature import VectorAssembler\n",
        "from pyspark.ml.regression import LinearRegression\n",
        "from pyspark.ml.regression import RandomForestRegressor\n",
        "from pyspark.ml.evaluation import RegressionEvaluator"
      ],
      "execution_count": 9,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "QVFPPBfkb1iU"
      },
      "source": [
        "spark = SparkSession.builder.appName(\"Spark ML Algo\").getOrCreate()"
      ],
      "execution_count": 10,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "e1-2JhZ-b1lt"
      },
      "source": [
        "dataframe = spark.read.csv(\"Admission_Prediction.csv\", header=True)"
      ],
      "execution_count": 11,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "AztfoAWpb1qW",
        "outputId": "67565647-1186-4f0a-c5d3-1b3c8ed31c73"
      },
      "source": [
        "type(dataframe)"
      ],
      "execution_count": 12,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "pyspark.sql.dataframe.DataFrame"
            ]
          },
          "metadata": {},
          "execution_count": 12
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "qzabGQgBb1tW",
        "outputId": "7a7cb128-6327-41f5-e827-e5e19181a858"
      },
      "source": [
        "dataframe.show()"
      ],
      "execution_count": 13,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "+---------+-----------+-----------------+----+----+----+--------+---------------+\n",
            "|GRE Score|TOEFL Score|University Rating| SOP| LOR|CGPA|Research|Chance of Admit|\n",
            "+---------+-----------+-----------------+----+----+----+--------+---------------+\n",
            "|   337.00|     118.00|                4|4.50|4.50|9.65|    1.00|           0.92|\n",
            "|   324.00|     107.00|                4|4.00|4.50|8.87|    1.00|           0.76|\n",
            "|     null|     104.00|                3|3.00|3.50|8.00|    1.00|           0.72|\n",
            "|   322.00|     110.00|                3|3.50|2.50|8.67|    1.00|           0.80|\n",
            "|   314.00|     103.00|                2|2.00|3.00|8.21|    0.00|           0.65|\n",
            "|   330.00|     115.00|                5|4.50|3.00|9.34|    1.00|           0.90|\n",
            "|   321.00|     109.00|             null|3.00|4.00|8.20|    1.00|           0.75|\n",
            "|   308.00|     101.00|                2|3.00|4.00|7.90|    0.00|           0.68|\n",
            "|   302.00|     102.00|                1|2.00|1.50|8.00|    0.00|           0.50|\n",
            "|   323.00|     108.00|                3|3.50|3.00|8.60|    0.00|           0.45|\n",
            "|   325.00|     106.00|                3|3.50|4.00|8.40|    1.00|           0.52|\n",
            "|   327.00|     111.00|                4|4.00|4.50|9.00|    1.00|           0.84|\n",
            "|     null|     112.00|                4|4.00|4.50|9.10|    1.00|           0.78|\n",
            "|   307.00|     109.00|                3|4.00|3.00|8.00|    1.00|           0.62|\n",
            "|   311.00|     104.00|                3|3.50|2.00|8.20|    1.00|           0.61|\n",
            "|   314.00|     105.00|                3|3.50|2.50|8.30|    0.00|           0.54|\n",
            "|   317.00|     107.00|                3|4.00|3.00|8.70|    0.00|           0.66|\n",
            "|   319.00|     106.00|                3|4.00|3.00|8.00|    1.00|           0.65|\n",
            "|   318.00|     110.00|                3|4.00|3.00|8.80|    0.00|           0.63|\n",
            "|   303.00|     102.00|                3|3.50|3.00|8.50|    0.00|           0.62|\n",
            "+---------+-----------+-----------------+----+----+----+--------+---------------+\n",
            "only showing top 20 rows\n",
            "\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "6h0a6XO1b1wF",
        "outputId": "eee242e0-f21c-4d69-95cc-46dc9dc83d7b"
      },
      "source": [
        "dataframe.printSchema()"
      ],
      "execution_count": 14,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "root\n",
            " |-- GRE Score: string (nullable = true)\n",
            " |-- TOEFL Score: string (nullable = true)\n",
            " |-- University Rating: string (nullable = true)\n",
            " |-- SOP: string (nullable = true)\n",
            " |-- LOR: string (nullable = true)\n",
            " |-- CGPA: string (nullable = true)\n",
            " |-- Research: string (nullable = true)\n",
            " |-- Chance of Admit: string (nullable = true)\n",
            "\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "KqiZAPIMb1yi"
      },
      "source": [
        "from pyspark.sql.functions import col\n",
        "new_dataframe = dataframe.select(*(col(c).cast(\"float\").alias(c) for c in dataframe.columns))"
      ],
      "execution_count": 15,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "jcTuJC-ikNSm",
        "outputId": "55161044-490b-4920-8b7f-b281f559ac3d"
      },
      "source": [
        "new_dataframe.printSchema()"
      ],
      "execution_count": 16,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "root\n",
            " |-- GRE Score: float (nullable = true)\n",
            " |-- TOEFL Score: float (nullable = true)\n",
            " |-- University Rating: float (nullable = true)\n",
            " |-- SOP: float (nullable = true)\n",
            " |-- LOR: float (nullable = true)\n",
            " |-- CGPA: float (nullable = true)\n",
            " |-- Research: float (nullable = true)\n",
            " |-- Chance of Admit: float (nullable = true)\n",
            "\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "FiEierfbkNWF"
      },
      "source": [
        "from pyspark.sql.functions import col, count, isnan, when"
      ],
      "execution_count": 17,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "YVqXbKK3kNZN",
        "outputId": "09abe500-476a-40dc-d198-18410a9ad8a8"
      },
      "source": [
        "for c in new_dataframe.columns:\n",
        "  print(c)"
      ],
      "execution_count": 18,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "GRE Score\n",
            "TOEFL Score\n",
            "University Rating\n",
            "SOP\n",
            "LOR\n",
            "CGPA\n",
            "Research\n",
            "Chance of Admit\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "oeJrSXQ5b14c",
        "outputId": "71b36d5d-e791-483d-c611-b0f072d918d4"
      },
      "source": [
        "### Checking for null ir nan type values in out columns\n",
        "new_dataframe.select([count(when(col(c).isNull(), c)).alias(c) for c in new_dataframe.columns]).show()"
      ],
      "execution_count": 19,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "+---------+-----------+-----------------+---+---+----+--------+---------------+\n",
            "|GRE Score|TOEFL Score|University Rating|SOP|LOR|CGPA|Research|Chance of Admit|\n",
            "+---------+-----------+-----------------+---+---+----+--------+---------------+\n",
            "|       15|         10|               15|  0|  0|   0|       0|              0|\n",
            "+---------+-----------+-----------------+---+---+----+--------+---------------+\n",
            "\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "I3_MsHiupjuN"
      },
      "source": [
        "from pyspark.ml.feature import Imputer"
      ],
      "execution_count": 20,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "edtWrqmmsMnE"
      },
      "source": [
        "imputer = Imputer(inputCols=[\"GRE Score\", \"TOEFL Score\",\"University Rating\"], \n",
        "                  outputCols=[\"GRE Score\", \"TOEFL Score\",\"University Rating\"])\n",
        "model = imputer.fit(new_dataframe)\n",
        "\n",
        "imputed_data = model.transform(new_dataframe)"
      ],
      "execution_count": 21,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "phzcbc8PsXbG",
        "outputId": "17e505ea-f12b-48e1-ecb6-3e9bd577a0f0"
      },
      "source": [
        "imputed_data"
      ],
      "execution_count": 22,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "DataFrame[GRE Score: float, TOEFL Score: float, University Rating: float, SOP: float, LOR: float, CGPA: float, Research: float, Chance of Admit: float]"
            ]
          },
          "metadata": {},
          "execution_count": 22
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "__U8GAMqsi2t",
        "outputId": "58696e69-2b32-4c89-b89c-c71c75ea4227"
      },
      "source": [
        "#checking for null ir nan type values in our columns\n",
        "imputed_data.select([count(when(col(c).isNull(), c)).alias(c) for c in imputed_data.columns]).show()"
      ],
      "execution_count": 23,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "+---------+-----------+-----------------+---+---+----+--------+---------------+\n",
            "|GRE Score|TOEFL Score|University Rating|SOP|LOR|CGPA|Research|Chance of Admit|\n",
            "+---------+-----------+-----------------+---+---+----+--------+---------------+\n",
            "|        0|          0|                0|  0|  0|   0|       0|              0|\n",
            "+---------+-----------+-----------------+---+---+----+--------+---------------+\n",
            "\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "N7zkak-fsltU"
      },
      "source": [
        "features = imputed_data.drop('Chance of Admit')"
      ],
      "execution_count": 24,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "I925bBLQsnq1",
        "outputId": "5df17ef4-2f77-4a93-8300-dc6a70f4679a"
      },
      "source": [
        "features"
      ],
      "execution_count": 25,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "DataFrame[GRE Score: float, TOEFL Score: float, University Rating: float, SOP: float, LOR: float, CGPA: float, Research: float]"
            ]
          },
          "metadata": {},
          "execution_count": 25
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "CutmTZhTso9c"
      },
      "source": [
        "#let's assemble our features together using vectorAssembler\n",
        "assembler = VectorAssembler( inputCols=features.columns,outputCol=\"features\")"
      ],
      "execution_count": 26,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "D-G5yD1Fsq0s"
      },
      "source": [
        "output = assembler.transform(imputed_data)"
      ],
      "execution_count": 27,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "j3ku1CRazpIg"
      },
      "source": [
        "#### Linear Regressor"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "dtSW13HxzuOs"
      },
      "source": [
        "output= output.select(\"features\", \"Chance of Admit\")"
      ],
      "execution_count": 28,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "1_bAHs6LzyPm"
      },
      "source": [
        "output = assembler.transform(imputed_data)"
      ],
      "execution_count": 29,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "YDlX0iyHz0E1"
      },
      "source": [
        "output= output.select(\"features\", \"Chance of Admit\")"
      ],
      "execution_count": 30,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "X2_MwCrn0Wce"
      },
      "source": [
        "train_df,test_df = output.randomSplit([0.7, 0.3])"
      ],
      "execution_count": 31,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "FpIJaQi40Yq-",
        "outputId": "e6c1fe75-dc82-4d60-ebfe-3f4d14bb224c"
      },
      "source": [
        "train_df.show()\n",
        "test_df.show()"
      ],
      "execution_count": 32,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "+--------------------+---------------+\n",
            "|            features|Chance of Admit|\n",
            "+--------------------+---------------+\n",
            "|[290.0,100.0,1.0,...|           0.47|\n",
            "|[290.0,104.0,4.0,...|           0.45|\n",
            "|[294.0,93.0,1.0,1...|           0.46|\n",
            "|[294.0,95.0,1.0,1...|           0.49|\n",
            "|[295.0,96.0,2.0,1...|           0.47|\n",
            "|[295.0,99.0,1.0,2...|           0.37|\n",
            "|[295.0,99.0,2.0,2...|           0.57|\n",
            "|[295.0,101.0,2.0,...|           0.69|\n",
            "|[296.0,95.0,2.0,3...|           0.44|\n",
            "|[296.0,99.0,2.0,2...|           0.61|\n",
            "|[296.0,99.0,2.0,3...|           0.47|\n",
            "|[296.0,101.0,1.0,...|            0.6|\n",
            "|[297.0,99.0,4.0,3...|           0.54|\n",
            "|[297.0,100.0,1.0,...|           0.52|\n",
            "|[297.0,101.0,3.0,...|           0.57|\n",
            "|[298.0,92.0,1.0,2...|           0.51|\n",
            "|[298.0,97.0,3.121...|           0.45|\n",
            "|[298.0,98.0,2.0,4...|           0.34|\n",
            "|[298.0,99.0,1.0,1...|           0.53|\n",
            "|[298.0,105.0,3.0,...|           0.69|\n",
            "+--------------------+---------------+\n",
            "only showing top 20 rows\n",
            "\n",
            "+--------------------+---------------+\n",
            "|            features|Chance of Admit|\n",
            "+--------------------+---------------+\n",
            "|[293.0,97.0,2.0,2...|           0.64|\n",
            "|[295.0,93.0,1.0,2...|           0.46|\n",
            "|[296.0,97.0,2.0,1...|           0.49|\n",
            "|[297.0,96.0,2.0,2...|           0.43|\n",
            "|[297.0,96.0,2.0,2...|           0.34|\n",
            "|[297.0,98.0,2.0,2...|           0.59|\n",
            "|[298.0,98.0,2.0,1...|           0.44|\n",
            "|[298.0,100.0,3.0,...|           0.58|\n",
            "|[298.0,101.0,2.0,...|           0.54|\n",
            "|[298.0,101.0,4.0,...|           0.53|\n",
            "|[298.0,107.187751...|           0.46|\n",
            "|[299.0,97.0,3.0,5...|           0.38|\n",
            "|[299.0,100.0,2.0,...|           0.51|\n",
            "|[300.0,97.0,2.0,3...|           0.65|\n",
            "|[300.0,99.0,1.0,3...|           0.36|\n",
            "|[300.0,102.0,3.0,...|           0.63|\n",
            "|[300.0,104.0,3.0,...|           0.71|\n",
            "|[301.0,97.0,2.0,3...|           0.44|\n",
            "|[301.0,100.0,3.0,...|           0.67|\n",
            "|[301.0,107.0,3.0,...|           0.62|\n",
            "+--------------------+---------------+\n",
            "only showing top 20 rows\n",
            "\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "9w_Rj1O40a8H"
      },
      "source": [
        "lin_reg = LinearRegression(featuresCol = 'features', labelCol='Chance of Admit')\n",
        "linear_model = lin_reg.fit(train_df)"
      ],
      "execution_count": 33,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "K2eaD4xt0fNf",
        "outputId": "fa7e5f05-515f-4530-cfe7-031c941243be"
      },
      "source": [
        "print(\"Coefficients: \" + str(linear_model.coefficients))\n",
        "print(\"Intercept: \" + str(linear_model.intercept))"
      ],
      "execution_count": 34,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Coefficients: [0.001164748062041676,0.002731724659456793,0.002444087344080545,0.010395272333342713,0.013826716932124027,0.12110420909124693,0.035173941251165235]\n",
            "Intercept: -1.092643840771405\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "up7nM4LE0g0f",
        "outputId": "57fcd9a5-209f-42b6-ffdf-b62646250704"
      },
      "source": [
        "trainSummary = linear_model.summary\n",
        "print(\"RMSE: %f\" % trainSummary.rootMeanSquaredError)\n",
        "print(\"r2: %f\" % trainSummary.r2)"
      ],
      "execution_count": 35,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "RMSE: 0.059702\n",
            "r2: 0.814640\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "U1ZBTuVd0jBm",
        "outputId": "bf9b348d-d111-458d-e578-e05a80822758"
      },
      "source": [
        "# prediction\n",
        "\n",
        "predictions = linear_model.transform(test_df)\n",
        "predictions.select(\"prediction\",\"Chance of Admit\",\"features\").show()"
      ],
      "execution_count": 36,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "+-------------------+---------------+--------------------+\n",
            "|         prediction|Chance of Admit|            features|\n",
            "+-------------------+---------------+--------------------+\n",
            "| 0.5743770157191435|           0.64|[293.0,97.0,2.0,2...|\n",
            "|0.42784557909356846|           0.46|[295.0,93.0,1.0,2...|\n",
            "| 0.5098462486231841|           0.49|[296.0,97.0,2.0,1...|\n",
            "| 0.5226605254433108|           0.43|[297.0,96.0,2.0,2...|\n",
            "| 0.4738659431076402|           0.34|[297.0,96.0,2.0,2...|\n",
            "| 0.5222211495690103|           0.59|[297.0,98.0,2.0,2...|\n",
            "| 0.5206634832977826|           0.44|[298.0,98.0,2.0,1...|\n",
            "| 0.6142032386845719|           0.58|[298.0,100.0,3.0,...|\n",
            "| 0.5303688890009308|           0.54|[298.0,101.0,2.0,...|\n",
            "| 0.5948053448188801|           0.53|[298.0,101.0,4.0,...|\n",
            "| 0.5417731818487697|           0.46|[298.0,107.187751...|\n",
            "| 0.5559534778676702|           0.38|[299.0,97.0,3.0,5...|\n",
            "| 0.5364216304421325|           0.51|[299.0,100.0,2.0,...|\n",
            "| 0.6154300933808228|           0.65|[300.0,97.0,2.0,3...|\n",
            "|0.44718724350611594|           0.36|[300.0,99.0,1.0,3...|\n",
            "| 0.6031203981499476|           0.63|[300.0,102.0,3.0,...|\n",
            "| 0.6142861361254568|           0.71|[300.0,104.0,3.0,...|\n",
            "| 0.5899518831044777|           0.44|[301.0,97.0,2.0,3...|\n",
            "| 0.5899914943179985|           0.67|[301.0,100.0,3.0,...|\n",
            "| 0.6875321524775919|           0.62|[301.0,107.0,3.0,...|\n",
            "+-------------------+---------------+--------------------+\n",
            "only showing top 20 rows\n",
            "\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "aIk5xQ4M0lm4",
        "outputId": "4f79e06d-2d34-4496-f028-c4062d782bbf"
      },
      "source": [
        "from pyspark.ml.evaluation import RegressionEvaluator\n",
        "pred_evaluator = RegressionEvaluator(predictionCol=\"prediction\", \\\n",
        "                 labelCol=\"Chance of Admit\",metricName=\"r2\")\n",
        "print(\"R Squared (R2) on test data =\", pred_evaluator.evaluate(predictions))"
      ],
      "execution_count": 37,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "R Squared (R2) on test data = 0.822919129851789\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "QlJ0_QoJnkO-"
      },
      "source": [
        "#### Random Forest Regressor"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "hKJIDhEHnpJp"
      },
      "source": [
        "featureIndexer =VectorIndexer(inputCol=\"features\", outputCol=\"indexedFeatures\", maxCategories=4).fit(output)"
      ],
      "execution_count": 38,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "ffO6NywJnxxy"
      },
      "source": [
        "featureIndexer = featureIndexer.transform(output)"
      ],
      "execution_count": 39,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "8ooaetyRoq37"
      },
      "source": [
        "new_indexed_data = featureIndexer.select(\"indexedFeatures\", \"Chance of Admit\")"
      ],
      "execution_count": 40,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "q22T41nGoskj"
      },
      "source": [
        "training, test = new_indexed_data.randomSplit([0.7, 0.3])"
      ],
      "execution_count": 41,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "LhSvNCHhouPo",
        "outputId": "22fecfa3-ea61-41a7-b41f-efbc9a785144"
      },
      "source": [
        "training.show()"
      ],
      "execution_count": 42,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "+--------------------+---------------+\n",
            "|     indexedFeatures|Chance of Admit|\n",
            "+--------------------+---------------+\n",
            "|[290.0,104.0,4.0,...|           0.45|\n",
            "|[293.0,97.0,2.0,2...|           0.64|\n",
            "|[294.0,95.0,1.0,1...|           0.49|\n",
            "|[295.0,96.0,2.0,1...|           0.47|\n",
            "|[295.0,99.0,1.0,2...|           0.37|\n",
            "|[295.0,101.0,2.0,...|           0.69|\n",
            "|[296.0,95.0,2.0,3...|           0.44|\n",
            "|[296.0,99.0,2.0,2...|           0.61|\n",
            "|[296.0,99.0,2.0,3...|           0.47|\n",
            "|[297.0,96.0,2.0,2...|           0.43|\n",
            "|[297.0,96.0,2.0,2...|           0.34|\n",
            "|[297.0,98.0,2.0,2...|           0.59|\n",
            "|[297.0,99.0,4.0,3...|           0.54|\n",
            "|[297.0,101.0,3.0,...|           0.57|\n",
            "|[298.0,92.0,1.0,2...|           0.51|\n",
            "|[298.0,98.0,2.0,4...|           0.34|\n",
            "|[298.0,99.0,1.0,1...|           0.53|\n",
            "|[298.0,100.0,3.0,...|           0.58|\n",
            "|[298.0,101.0,2.0,...|           0.54|\n",
            "|[298.0,101.0,4.0,...|           0.53|\n",
            "+--------------------+---------------+\n",
            "only showing top 20 rows\n",
            "\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "8N1tOj06ov6J",
        "outputId": "6899fbed-fa5d-4e9d-a4ab-bb63ff5ff14f"
      },
      "source": [
        "test.show()"
      ],
      "execution_count": 43,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "+--------------------+---------------+\n",
            "|     indexedFeatures|Chance of Admit|\n",
            "+--------------------+---------------+\n",
            "|[290.0,100.0,1.0,...|           0.47|\n",
            "|[294.0,93.0,1.0,1...|           0.46|\n",
            "|[295.0,93.0,1.0,2...|           0.46|\n",
            "|[295.0,99.0,2.0,2...|           0.57|\n",
            "|[296.0,97.0,2.0,1...|           0.49|\n",
            "|[296.0,101.0,1.0,...|            0.6|\n",
            "|[297.0,100.0,1.0,...|           0.52|\n",
            "|[298.0,97.0,3.121...|           0.45|\n",
            "|[298.0,98.0,2.0,1...|           0.44|\n",
            "|[298.0,105.0,3.0,...|           0.69|\n",
            "|[298.0,107.187751...|           0.46|\n",
            "|[299.0,94.0,1.0,1...|           0.42|\n",
            "|[299.0,97.0,3.0,5...|           0.38|\n",
            "|[300.0,97.0,2.0,3...|           0.65|\n",
            "|[300.0,98.0,1.0,2...|           0.61|\n",
            "|[300.0,105.0,1.0,...|           0.58|\n",
            "|[301.0,96.0,1.0,3...|           0.54|\n",
            "|[301.0,98.0,1.0,2...|           0.67|\n",
            "|[301.0,100.0,3.0,...|           0.67|\n",
            "|[301.0,102.0,3.0,...|           0.68|\n",
            "+--------------------+---------------+\n",
            "only showing top 20 rows\n",
            "\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "ewbBugYkox4Y"
      },
      "source": [
        "random_forest_reg = RandomForestRegressor(featuresCol=\"indexedFeatures\",labelCol=\"Chance of Admit\" )"
      ],
      "execution_count": 44,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "6XL2y7m7ozxx"
      },
      "source": [
        "# Train model.  This also runs the indexer.\n",
        "model = random_forest_reg.fit(test)"
      ],
      "execution_count": 50,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "rP_Wu2zSo1Yg"
      },
      "source": [
        "# Make predictions.\n",
        "predictions = model.transform(test)"
      ],
      "execution_count": 51,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "r-mJWsMMo4D5",
        "outputId": "c1d55c7d-41ca-4cc3-d43a-a46f99426382"
      },
      "source": [
        "predictions.show()"
      ],
      "execution_count": 52,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "+--------------------+---------------+-------------------+\n",
            "|     indexedFeatures|Chance of Admit|         prediction|\n",
            "+--------------------+---------------+-------------------+\n",
            "|[290.0,100.0,1.0,...|           0.47| 0.4777091139306625|\n",
            "|[294.0,93.0,1.0,1...|           0.46|0.46320655139784017|\n",
            "|[295.0,93.0,1.0,2...|           0.46| 0.4712184565549805|\n",
            "|[295.0,99.0,2.0,2...|           0.57| 0.5531493390883001|\n",
            "|[296.0,97.0,2.0,1...|           0.49| 0.5342833346057506|\n",
            "|[296.0,101.0,1.0,...|            0.6| 0.5833469651425871|\n",
            "|[297.0,100.0,1.0,...|           0.52| 0.5613966458523401|\n",
            "|[298.0,97.0,3.121...|           0.45| 0.4776309536681289|\n",
            "|[298.0,98.0,2.0,1...|           0.44| 0.4717858990778526|\n",
            "|[298.0,105.0,3.0,...|           0.69| 0.6983823208545262|\n",
            "|[298.0,107.187751...|           0.46| 0.5209569958580408|\n",
            "|[299.0,94.0,1.0,1...|           0.42| 0.4394690415014823|\n",
            "|[299.0,97.0,3.0,5...|           0.38|0.49078205158440474|\n",
            "|[300.0,97.0,2.0,3...|           0.65| 0.6034195112287496|\n",
            "|[300.0,98.0,1.0,2...|           0.61| 0.6075186057061146|\n",
            "|[300.0,105.0,1.0,...|           0.58| 0.5777494277192092|\n",
            "|[301.0,96.0,1.0,3...|           0.54| 0.5736594075875315|\n",
            "|[301.0,98.0,1.0,2...|           0.67| 0.6258148956536717|\n",
            "|[301.0,100.0,3.0,...|           0.67|  0.639644238335121|\n",
            "|[301.0,102.0,3.0,...|           0.68| 0.6491375967615809|\n",
            "+--------------------+---------------+-------------------+\n",
            "only showing top 20 rows\n",
            "\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "541ENvqYqJ2K",
        "outputId": "50e47211-ced0-473f-d1ad-d0f227f00ddc"
      },
      "source": [
        "evaluator = RegressionEvaluator(labelCol=\"Chance of Admit\", predictionCol=\"prediction\", metricName=\"rmse\")\n",
        "print (\"Root Mean Squared Error (RMSE) on test data = \",evaluator.evaluate(predictions))"
      ],
      "execution_count": 53,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Root Mean Squared Error (RMSE) on test data =  0.04024199137120952\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "I0Q5_eaEqRx7",
        "outputId": "942ebba2-bcff-471f-ca7c-53d8deafe82a"
      },
      "source": [
        "evaluator = RegressionEvaluator(labelCol=\"Chance of Admit\", predictionCol=\"prediction\", metricName=\"r2\")\n",
        "print(\"R Squared (R2) on test data =\", evaluator.evaluate(predictions))"
      ],
      "execution_count": 54,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "R Squared (R2) on test data = 0.9153660775567585\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "RxXek97KqTdx"
      },
      "source": [
        ""
      ],
      "execution_count": null,
      "outputs": []
    }
  ]
}