Classification de type d'étoile selon l'apprentissage supervisé.ipynb

{"cells":[{"cell_type":"code","source":["#https://medium.com/data-science-school/practical-apache-spark-in-10-minutes-part-4-mllib-fca02fecf5b8\n#https://towardsdatascience.com/uci-heart-disease-classification-with-pyspark-eadc8e99663f\n#Import Data & Exploratory Data Analysis (EDA)\n#Load the data\nfrom pyspark.sql import SQLContext\nfrom pyspark.sql.types import *\nsqlContext = SQLContext(sc)\nstars_df= sqlContext.read.load('/FileStore/tables/Stars.csv', \n                          format='com.databricks.spark.csv', \n                          header='true', \n                          inferSchema='true')\n#stars_df.show(truncate=False)\nstars_df.show(3)\n\nprint(\"There are\", stars_df.count(), \"rows\" ,len(stars_df.columns),\"columns\" ,\"in the data .\")\n#There are 240 rows 7 columns in the data "],"metadata":{"application/vnd.databricks.v1+cell":{"title":"","showTitle":false,"inputWidgets":{},"nuid":"6517d77b-9bfb-43a6-88a9-a8488fa21adf"}},"outputs":[{"output_type":"display_data","metadata":{"application/vnd.databricks.v1+output":{"datasetInfos":[],"data":"+-----------+------+------+-----+-----+--------------+----+\n|Temperature|     L|     R|  A_M|Color|Spectral_Class|Type|\n+-----------+------+------+-----+-----+--------------+----+\n|       3068|0.0024|  0.17|16.12|  Red|             M|   0|\n|       3042|5.0E-4|0.1542| 16.6|  Red|             M|   0|\n|       2600|3.0E-4| 0.102| 18.7|  Red|             M|   0|\n+-----------+------+------+-----+-----+--------------+----+\nonly showing top 3 rows\n\nThere are 240 rows 7 columns in the data .\n","removedWidgets":[],"addedWidgets":{},"metadata":{},"type":"ansi","arguments":{}}},"output_type":"display_data","data":{"text/plain":["+-----------+------+------+-----+-----+--------------+----+\n|Temperature|     L|     R|  A_M|Color|Spectral_Class|Type|\n+-----------+------+------+-----+-----+--------------+----+\n|       3068|0.0024|  0.17|16.12|  Red|             M|   0|\n|       3042|5.0E-4|0.1542| 16.6|  Red|             M|   0|\n|       2600|3.0E-4| 0.102| 18.7|  Red|             M|   0|\n+-----------+------+------+-----+-----+--------------+----+\nonly showing top 3 rows\n\nThere are 240 rows 7 columns in the data .\n"]}}],"execution_count":0},{"cell_type":"code","source":["#Our classes are perfect balanced.\nimport pandas as pd\npd.DataFrame(stars_df.take(5), columns=stars_df.columns).transpose()"],"metadata":{"application/vnd.databricks.v1+cell":{"title":"","showTitle":false,"inputWidgets":{},"nuid":"d0d1eb12-43a4-41c1-b125-d96b846d5276"}},"outputs":[{"output_type":"display_data","metadata":{"application/vnd.databricks.v1+output":{"datasetInfos":[],"data":"<div>\n<style scoped>\n    .dataframe tbody tr th:only-of-type {\n        vertical-align: middle;\n    }\n\n    .dataframe tbody tr th {\n        vertical-align: top;\n    }\n\n    .dataframe thead th {\n        text-align: right;\n    }\n</style>\n<table border=\"1\" class=\"dataframe\">\n  <thead>\n    <tr style=\"text-align: right;\">\n      <th></th>\n      <th>0</th>\n      <th>1</th>\n      <th>2</th>\n      <th>3</th>\n      <th>4</th>\n    </tr>\n  </thead>\n  <tbody>\n    <tr>\n      <th>Temperature</th>\n      <td>3068</td>\n      <td>3042</td>\n      <td>2600</td>\n      <td>2800</td>\n      <td>1939</td>\n    </tr>\n    <tr>\n      <th>L</th>\n      <td>0.0024</td>\n      <td>0.0005</td>\n      <td>0.0003</td>\n      <td>0.0002</td>\n      <td>0.000138</td>\n    </tr>\n    <tr>\n      <th>R</th>\n      <td>0.17</td>\n      <td>0.1542</td>\n      <td>0.102</td>\n      <td>0.16</td>\n      <td>0.103</td>\n    </tr>\n    <tr>\n      <th>A_M</th>\n      <td>16.12</td>\n      <td>16.6</td>\n      <td>18.7</td>\n      <td>16.65</td>\n      <td>20.06</td>\n    </tr>\n    <tr>\n      <th>Color</th>\n      <td>Red</td>\n      <td>Red</td>\n      <td>Red</td>\n      <td>Red</td>\n      <td>Red</td>\n    </tr>\n    <tr>\n      <th>Spectral_Class</th>\n      <td>M</td>\n      <td>M</td>\n      <td>M</td>\n      <td>M</td>\n      <td>M</td>\n    </tr>\n    <tr>\n      <th>Type</th>\n      <td>0</td>\n      <td>0</td>\n      <td>0</td>\n      <td>0</td>\n      <td>0</td>\n    </tr>\n  </tbody>\n</table>\n</div>","textData":null,"removedWidgets":[],"addedWidgets":{},"metadata":{},"type":"htmlSandbox","arguments":{}}},"output_type":"display_data","data":{"text/html":["<div>\n<style scoped>\n    .dataframe tbody tr th:only-of-type {\n        vertical-align: middle;\n    }\n\n    .dataframe tbody tr th {\n        vertical-align: top;\n    }\n\n    .dataframe thead th {\n        text-align: right;\n    }\n</style>\n<table border=\"1\" class=\"dataframe\">\n  <thead>\n    <tr style=\"text-align: right;\">\n      <th></th>\n      <th>0</th>\n      <th>1</th>\n      <th>2</th>\n      <th>3</th>\n      <th>4</th>\n    </tr>\n  </thead>\n  <tbody>\n    <tr>\n      <th>Temperature</th>\n      <td>3068</td>\n      <td>3042</td>\n      <td>2600</td>\n      <td>2800</td>\n      <td>1939</td>\n    </tr>\n    <tr>\n      <th>L</th>\n      <td>0.0024</td>\n      <td>0.0005</td>\n      <td>0.0003</td>\n      <td>0.0002</td>\n      <td>0.000138</td>\n    </tr>\n    <tr>\n      <th>R</th>\n      <td>0.17</td>\n      <td>0.1542</td>\n      <td>0.102</td>\n      <td>0.16</td>\n      <td>0.103</td>\n    </tr>\n    <tr>\n      <th>A_M</th>\n      <td>16.12</td>\n      <td>16.6</td>\n      <td>18.7</td>\n      <td>16.65</td>\n      <td>20.06</td>\n    </tr>\n    <tr>\n      <th>Color</th>\n      <td>Red</td>\n      <td>Red</td>\n      <td>Red</td>\n      <td>Red</td>\n      <td>Red</td>\n    </tr>\n    <tr>\n      <th>Spectral_Class</th>\n      <td>M</td>\n      <td>M</td>\n      <td>M</td>\n      <td>M</td>\n      <td>M</td>\n    </tr>\n    <tr>\n      <th>Type</th>\n      <td>0</td>\n      <td>0</td>\n      <td>0</td>\n      <td>0</td>\n      <td>0</td>\n    </tr>\n  </tbody>\n</table>\n</div>"]}}],"execution_count":0},{"cell_type":"code","source":["#https://towardsdatascience.com/machine-learning-with-pyspark-and-mllib-solving-a-binary-classification-problem-96396065d2aa\ndf = stars_df.select('Temperature','L','R','A_M','Type','Color','Spectral_Class')\ncols = df.columns\ndf.printSchema()"],"metadata":{"application/vnd.databricks.v1+cell":{"title":"","showTitle":false,"inputWidgets":{},"nuid":"d7100821-5457-451e-8dac-045c04696b79"}},"outputs":[{"output_type":"display_data","metadata":{"application/vnd.databricks.v1+output":{"datasetInfos":[],"data":"root\n |-- Temperature: integer (nullable = true)\n |-- L: double (nullable = true)\n |-- R: double (nullable = true)\n |-- A_M: double (nullable = true)\n |-- Type: integer (nullable = true)\n |-- Color: string (nullable = true)\n |-- Spectral_Class: string (nullable = true)\n\n","removedWidgets":[],"addedWidgets":{},"metadata":{},"type":"ansi","arguments":{}}},"output_type":"display_data","data":{"text/plain":["root\n |-- Temperature: integer (nullable = true)\n |-- L: double (nullable = true)\n |-- R: double (nullable = true)\n |-- A_M: double (nullable = true)\n |-- Type: integer (nullable = true)\n |-- Color: string (nullable = true)\n |-- Spectral_Class: string (nullable = true)\n\n"]}}],"execution_count":0},{"cell_type":"code","source":["from pyspark.ml.feature import OneHotEncoder, StringIndexer, VectorAssembler\ncategoricalColumns = ['Color', 'Spectral_Class']\nstages = []\nfor categoricalCol in categoricalColumns:\n    stringIndexer = StringIndexer(inputCol = categoricalCol, outputCol = categoricalCol + 'Index')\n    encoder = OneHotEncoder(inputCols=[stringIndexer.getOutputCol()], outputCols=[categoricalCol + \"classVec\"])\n    stages += [stringIndexer, encoder]\nlabel_stringIdx = StringIndexer(inputCol = 'Type', outputCol = 'target')\nstages += [label_stringIdx]\n\nnumericCols = ['Temperature','L','R','A_M']\nassemblerInputs = [c + \"classVec\" for c in categoricalColumns] + numericCols\nassembler = VectorAssembler(inputCols=assemblerInputs, outputCol=\"features\")\nstages += [assembler]"],"metadata":{"application/vnd.databricks.v1+cell":{"title":"","showTitle":false,"inputWidgets":{},"nuid":"26b8a06c-7491-42a3-ac8b-381e3f0fecb3"}},"outputs":[],"execution_count":0},{"cell_type":"code","source":["#https://docs.databricks.com/notebooks/visualizations/index.html\n# Run the stages as a Pipeline. This puts the data through all of the feature transformations in a single call.\nfrom pyspark.ml import Pipeline\npipeline = Pipeline().setStages(stages)\np_model = pipeline.fit(df)\nfinal_df = p_model.transform(df)\nfinal_df.printSchema()\n\n#final_df.show(5)"],"metadata":{"application/vnd.databricks.v1+cell":{"title":"","showTitle":false,"inputWidgets":{},"nuid":"f427f41a-f42e-44b8-8c92-15b6de786cb6"}},"outputs":[{"output_type":"display_data","metadata":{"application/vnd.databricks.v1+output":{"datasetInfos":[],"data":"root\n |-- Temperature: integer (nullable = true)\n |-- L: double (nullable = true)\n |-- R: double (nullable = true)\n |-- A_M: double (nullable = true)\n |-- Type: integer (nullable = true)\n |-- Color: string (nullable = true)\n |-- Spectral_Class: string (nullable = true)\n |-- ColorIndex: double (nullable = false)\n |-- ColorclassVec: vector (nullable = true)\n |-- Spectral_ClassIndex: double (nullable = false)\n |-- Spectral_ClassclassVec: vector (nullable = true)\n |-- target: double (nullable = false)\n |-- features: vector (nullable = true)\n\n","removedWidgets":[],"addedWidgets":{},"metadata":{},"type":"ansi","arguments":{}}},"output_type":"display_data","data":{"text/plain":["root\n |-- Temperature: integer (nullable = true)\n |-- L: double (nullable = true)\n |-- R: double (nullable = true)\n |-- A_M: double (nullable = true)\n |-- Type: integer (nullable = true)\n |-- Color: string (nullable = true)\n |-- Spectral_Class: string (nullable = true)\n |-- ColorIndex: double (nullable = false)\n |-- ColorclassVec: vector (nullable = true)\n |-- Spectral_ClassIndex: double (nullable = false)\n |-- Spectral_ClassclassVec: vector (nullable = true)\n |-- target: double (nullable = false)\n |-- features: vector (nullable = true)\n\n"]}}],"execution_count":0},{"cell_type":"code","source":["training_df=final_df.select('target','features')\ntraining_df.show(5)\n#split the Data Frame randomly into train and test sets.\n#train, test = training_df.randomSplit([0.6, 0.4],seed=1234)\n#train, test = training_df.randomSplit([0.7, 0.3],seed=1234)\n#train, test = training_df.randomSplit([0.8, 0.2],seed=1234)\ntrain, test = training_df.randomSplit([0.9, 0.1],seed=1234)\nprint(\"Training Dataset Count: \" + str(train.count()))\nprint(\"Test Dataset Count: \" + str(test.count()))"],"metadata":{"application/vnd.databricks.v1+cell":{"title":"","showTitle":false,"inputWidgets":{},"nuid":"f4f5125d-a1ab-4486-8063-c007773b9729"}},"outputs":[{"output_type":"display_data","metadata":{"application/vnd.databricks.v1+output":{"datasetInfos":[],"data":"+------+--------------------+\n|target|            features|\n+------+--------------------+\n|   0.0|(26,[0,16,22,23,2...|\n|   0.0|(26,[0,16,22,23,2...|\n|   0.0|(26,[0,16,22,23,2...|\n|   0.0|(26,[0,16,22,23,2...|\n|   0.0|(26,[0,16,22,23,2...|\n+------+--------------------+\nonly showing top 5 rows\n\nTraining Dataset Count: 214\nTest Dataset Count: 26\n","removedWidgets":[],"addedWidgets":{},"metadata":{},"type":"ansi","arguments":{}}},"output_type":"display_data","data":{"text/plain":["+------+--------------------+\n|target|            features|\n+------+--------------------+\n|   0.0|(26,[0,16,22,23,2...|\n|   0.0|(26,[0,16,22,23,2...|\n|   0.0|(26,[0,16,22,23,2...|\n|   0.0|(26,[0,16,22,23,2...|\n|   0.0|(26,[0,16,22,23,2...|\n+------+--------------------+\nonly showing top 5 rows\n\nTraining Dataset Count: 214\nTest Dataset Count: 26\n"]}}],"execution_count":0},{"cell_type":"code","source":["#https://towardsdatascience.com/uci-heart-disease-classification-with-pyspark-eadc8e99663f\nfrom pyspark.ml.classification import LogisticRegression\nfrom pyspark.ml.evaluation import BinaryClassificationEvaluator\nlr = LogisticRegression(labelCol ='target', featuresCol ='features',maxIter=10 ,threshold=0.5)\nmodel=lr.fit(train)\npredict_train = model.transform(train)\npredict_test  = model.transform(test)\n\npredict_test.show(5)\npredict_test.printSchema()\nprint(model)\n\npredict_test.select(\"target\", \"rawPrediction\", \"prediction\", \"probability\").show(5,truncate=False)\nevaluator = BinaryClassificationEvaluator(labelCol=\"target\", rawPredictionCol=\"prediction\", metricName='areaUnderROC')\n### Use ROC \ne_roc = BinaryClassificationEvaluator(labelCol=\"target\", rawPredictionCol=\"prediction\")\nroc = e_roc.evaluate(predict_test)\n\n#print(\"ROC of model at predicting label was: {:.4f}\".format(roc))\nprint(\"Test Area Under ROC for train set: \" + str(evaluator.evaluate(predict_train, {evaluator.metricName: \"areaUnderROC\"})))\nprint(\"Test Area Under ROC for test set: \" + str(evaluator.evaluate(predict_test, {evaluator.metricName: \"areaUnderROC\"})))\n\n#train, test[0.6, 0.4] ROC of model at predicting label was: 0.9767\n#Test Area Under ROC for train set: 1.0\n#Test Area Under ROC for test set: 0.9767441860465116\n\n#train, test[0.7, 0.3] Test Area Under ROC: 1.0\n#train, test[0.8, 0.2] Test Area Under ROC  : 1.0 \n#train, test [0.9, 0.1] Test Area Under ROC: 1.0"],"metadata":{"application/vnd.databricks.v1+cell":{"title":"","showTitle":false,"inputWidgets":{},"nuid":"f3eded75-068b-4e03-a23f-3518ce534d61"}},"outputs":[{"output_type":"display_data","metadata":{"application/vnd.databricks.v1+output":{"datasetInfos":[],"data":"+------+--------------------+--------------------+--------------------+----------+\n|target|            features|       rawPrediction|         probability|prediction|\n+------+--------------------+--------------------+--------------------+----------+\n|   0.0|(26,[0,16,22,23,2...|[13.2249930439033...|[0.82995555296062...|       0.0|\n|   0.0|(26,[0,16,22,23,2...|[13.4870106209072...|[0.86904394100456...|       0.0|\n|   1.0|(26,[0,16,22,23,2...|[11.5697128334377...|[0.42333407406151...|       1.0|\n|   1.0|(26,[0,16,22,23,2...|[9.03337827368953...|[0.03811084430665...|       1.0|\n|   1.0|(26,[0,16,22,23,2...|[11.0628054063353...|[0.29363613379428...|       1.0|\n+------+--------------------+--------------------+--------------------+----------+\nonly showing top 5 rows\n\nroot\n |-- target: double (nullable = false)\n |-- features: vector (nullable = true)\n |-- rawPrediction: vector (nullable = true)\n |-- probability: vector (nullable = true)\n |-- prediction: double (nullable = false)\n\nLogisticRegressionModel: uid=LogisticRegression_a0fddaf3b6ba, numClasses=6, numFeatures=26\n+------+---------------------------------------------------------------------------------------------------------------------+----------+--------------------------------------------------------------------------------------------------------------------------------+\n|target|rawPrediction                                                                                                        |prediction|probability                                                                                                                     |\n+------+---------------------------------------------------------------------------------------------------------------------+----------+--------------------------------------------------------------------------------------------------------------------------------+\n|0.0   |[13.224993043903327,11.639644226125583,-6.7278269217991955,-8.307605217840548,1.4218007771921695,-11.251005907581334]|0.0       |[0.8299555529606208,0.1700382362588015,1.7933094022901115E-9,3.6945898168797313E-10,6.208598344059121E-6,1.9465409803026597E-11]|\n|0.0   |[13.487010620907238,11.594446322258674,-6.6405680558548354,-8.308790338219953,1.258179017299983,-11.390277566391104] |0.0       |[0.8690439410045602,0.13095180966388342,1.5766866216791083E-9,2.9733473761726046E-10,4.24744389013304E-6,1.3644975227488175E-11]|\n|1.0   |[11.569712833437753,11.878751794848409,-7.281679798685386,-7.954383756187494,2.1985184315478756,-10.410919504961159] |1.0       |[0.4233340740615171,0.576629878306932,2.7518669338475435E-9,1.4043514697762902E-9,3.604335493546232E-5,1.2039706302319167E-10]  |\n|1.0   |[9.033378273689532,12.261628168460451,-8.12901542072157,-7.490529025902175,3.4598424801384473,-9.135304475664686]    |1.0       |[0.03811084430665187,0.9617444432588667,1.3412683989853128E-9,2.5398426428504627E-9,1.4470806303868453E-4,4.903316293122344E-10]|\n|1.0   |[11.062805406335336,11.940524950742388,-7.451825585050208,-7.748695242117578,2.3675841826145554,-10.170393712524495] |1.0       |[0.29363613379428033,0.7063147114401912,2.6730520975344E-9,1.9864542592046545E-9,4.914992968322283E-5,1.7633896472884813E-10]   |\n+------+---------------------------------------------------------------------------------------------------------------------+----------+--------------------------------------------------------------------------------------------------------------------------------+\nonly showing top 5 rows\n\nTest Area Under ROC for train set: 1.0\nTest Area Under ROC for test set: 1.0\n","removedWidgets":[],"addedWidgets":{},"metadata":{},"type":"ansi","arguments":{}}},"output_type":"display_data","data":{"text/plain":["+------+--------------------+--------------------+--------------------+----------+\n|target|            features|       rawPrediction|         probability|prediction|\n+------+--------------------+--------------------+--------------------+----------+\n|   0.0|(26,[0,16,22,23,2...|[13.2249930439033...|[0.82995555296062...|       0.0|\n|   0.0|(26,[0,16,22,23,2...|[13.4870106209072...|[0.86904394100456...|       0.0|\n|   1.0|(26,[0,16,22,23,2...|[11.5697128334377...|[0.42333407406151...|       1.0|\n|   1.0|(26,[0,16,22,23,2...|[9.03337827368953...|[0.03811084430665...|       1.0|\n|   1.0|(26,[0,16,22,23,2...|[11.0628054063353...|[0.29363613379428...|       1.0|\n+------+--------------------+--------------------+--------------------+----------+\nonly showing top 5 rows\n\nroot\n |-- target: double (nullable = false)\n |-- features: vector (nullable = true)\n |-- rawPrediction: vector (nullable = true)\n |-- probability: vector (nullable = true)\n |-- prediction: double (nullable = false)\n\nLogisticRegressionModel: uid=LogisticRegression_a0fddaf3b6ba, numClasses=6, numFeatures=26\n+------+---------------------------------------------------------------------------------------------------------------------+----------+--------------------------------------------------------------------------------------------------------------------------------+\n|target|rawPrediction                                                                                                        |prediction|probability                                                                                                                     |\n+------+---------------------------------------------------------------------------------------------------------------------+----------+--------------------------------------------------------------------------------------------------------------------------------+\n|0.0   |[13.224993043903327,11.639644226125583,-6.7278269217991955,-8.307605217840548,1.4218007771921695,-11.251005907581334]|0.0       |[0.8299555529606208,0.1700382362588015,1.7933094022901115E-9,3.6945898168797313E-10,6.208598344059121E-6,1.9465409803026597E-11]|\n|0.0   |[13.487010620907238,11.594446322258674,-6.6405680558548354,-8.308790338219953,1.258179017299983,-11.390277566391104] |0.0       |[0.8690439410045602,0.13095180966388342,1.5766866216791083E-9,2.9733473761726046E-10,4.24744389013304E-6,1.3644975227488175E-11]|\n|1.0   |[11.569712833437753,11.878751794848409,-7.281679798685386,-7.954383756187494,2.1985184315478756,-10.410919504961159] |1.0       |[0.4233340740615171,0.576629878306932,2.7518669338475435E-9,1.4043514697762902E-9,3.604335493546232E-5,1.2039706302319167E-10]  |\n|1.0   |[9.033378273689532,12.261628168460451,-8.12901542072157,-7.490529025902175,3.4598424801384473,-9.135304475664686]    |1.0       |[0.03811084430665187,0.9617444432588667,1.3412683989853128E-9,2.5398426428504627E-9,1.4470806303868453E-4,4.903316293122344E-10]|\n|1.0   |[11.062805406335336,11.940524950742388,-7.451825585050208,-7.748695242117578,2.3675841826145554,-10.170393712524495] |1.0       |[0.29363613379428033,0.7063147114401912,2.6730520975344E-9,1.9864542592046545E-9,4.914992968322283E-5,1.7633896472884813E-10]   |\n+------+---------------------------------------------------------------------------------------------------------------------+----------+--------------------------------------------------------------------------------------------------------------------------------+\nonly showing top 5 rows\n\nTest Area Under ROC for train set: 1.0\nTest Area Under ROC for test set: 1.0\n"]}}],"execution_count":0},{"cell_type":"code","source":["display(model, training_df)"],"metadata":{"application/vnd.databricks.v1+cell":{"title":"","showTitle":false,"inputWidgets":{},"nuid":"6a6d60e8-023c-4076-a470-f5ed0840d346"}},"outputs":[{"output_type":"display_data","metadata":{"application/vnd.databricks.v1+output":{"overflow":false,"datasetInfos":[],"data":[[-1.0975366274292164,-0.2502017407140071],[-1.585356301018644,-0.1700382362588015],[-3.76813436193741,-0.022573766608175578],[-1.6715536803426616,-0.15821714283980026],[-2.4572855503111257,-0.0789074007789438],[-2.5003296369255783,-0.07583507447830999],[4.438281393236198,0.011678235826407923],[3.273098764065551,0.036505678722439705],[3.2105477445716795,0.038770716205372646],[3.9292226966623605,0.019279924555004047],[-25.804453079192214,1.9999999999937874],[-28.65679503583266,1.9999999999996414],[-27.75591350674064,1.9999999999991174],[-28.074988256322246,1.9999999999993585],[-27.1334566982454,1.9999999999983553],[-22.52621989367687,2.9999999998351887],[-23.025060503638688,2.999999999899921],[-16.173394104717698,2.999999905379738],[-31.079865109141902,2.999999999999968],[-29.35692680090278,2.999999999999822],[-29.13407765645514,2.9999999999997775],[-3.2325968051564864,3.962042693152814],[-5.7346449942039115,3.996778382808047],[-2.3010399522154024,3.9089631307622486],[-4.8991348987075645,3.9926021087828145],[-16.27387867594197,3.9999999144255227],[-32.85985130725663,4.999999999999995],[-18.821202677211918,4.999999993300297],[-25.182332094860996,4.999999999988427],[-1.1679482598059319,-0.2372260461659115],[-1.8705854338840013,-0.1334739974871252],[-3.910216693195969,-0.019642596703338143],[-3.354121218498626,-0.03376046850621519],[-2.4215098245234614,-0.08154710240629881],[2.7930004395584866,0.05770359303789585],[4.994771713926521,0.006727698664592996],[3.125748660719427,0.042057554992537605],[3.2244599398767826,0.038255556741133345],[2.1729765389916893,0.10220358856220002],[-28.217928944601713,1.999999999999444],[-27.60024897726727,1.9999999999989688],[-27.56554848894203,1.9999999999989324],[-26.85793369514561,1.9999999999978335],[-23.982764507495602,1.9999999999615923],[-26.54866828637316,1.9999999999970484],[-28.368854483374328,1.999999999999522],[-28.829424778854147,2.9999999999996985],[-18.868289074926114,2.9999999936084505],[-23.650154172801923,2.9999999999464366],[-33.92947016961865,2.9999999999999982],[-21.19261022884948,2.999999999374588],[-17.367122056300104,3.9999999713216137],[-22.66331771865592,3.9999999998563034],[-18.16324070762798,3.9999999870638576],[-22.848139113918712,3.999999999880552],[-21.101006207259793,3.9999999993145914],[-17.2936716123443,3.9999999691358847],[-18.813375981335962,4.999999993247655],[-1.8925691882990945,-0.13095180966388342],[-4.597433290803406,-0.009977122880762367],[4.782167329224703,0.008308217017181985],[3.888714454718731,0.020060965368940198],[0.8775521581555976,0.2936852885598088],[-27.519865153599852,1.9999999999988824],[-30.821228655267145,1.999999999999959],[-28.299826283932745,1.9999999999994877],[-27.572642861617858,1.99999999999894],[-22.40224388602005,2.9999999998134355],[-25.517631362817102,2.999999999991724],[-21.978442668301067,2.9999999997149747],[-20.384814223985043,3.999999998597226],[-25.075470423200287,3.9999999999871214],[-24.56533457971247,3.999999999978551],[-26.068784591171326,4.9999999999952305],[-19.831189221835157,4.999999997559808],[-46.29795102064383,5.0],[-51.03623131645012,5.0],[-17.567977982073,4.999999976540211],[-54.34360395495136,5.0],[-2.30543422507384,-0.09067389973507024],[-2.0085002615388947,-0.1183133339279117],[-4.498090340388912,-0.011007712820516855],[3.6431411317597027,0.025502607738942307],[2.215691785891654,0.09835018510745941],[2.6885185454181846,0.06365425951829606],[2.3066413216320885,0.09057442120029768],[-18.520925423145435,1.9999999909538395],[-17.15156045857821,1.9999999644227873],[-25.53256176084041,2.9999999999918465],[-19.288677007947125,2.9999999958020815],[-20.35432293170424,2.999999998553795],[-19.509666505009914,2.9999999966344233],[-30.26290837894331,2.999999999999928],[-28.84013014291795,3.9999999999997016],[-47.913528838646485,4.0],[-38.3689421581031,5.0],[-61.99633482112417,5.0],[-56.36262692836781,5.0]],"plotOptions":{"displayType":"scatterPlot","customPlotOptions":{},"pivotColumns":null,"pivotAggregation":null,"xColumns":[],"yColumns":["fitted values","residuals"]},"columnCustomDisplayInfos":{},"aggType":"","isJsonSchema":true,"removedWidgets":[],"aggSchema":[],"schema":[{"name":"fitted values","type":"\"double\"","metadata":"{}"},{"name":"residuals","type":"\"double\"","metadata":"{}"}],"aggError":"","aggData":[],"addedWidgets":{},"metadata":{},"dbfsResultPath":null,"type":"table","aggOverflow":false,"aggSeriesLimitReached":false,"arguments":{}}},"output_type":"display_data","data":{"text/html":["<style scoped>\n  .table-result-container {\n    max-height: 300px;\n    overflow: auto;\n  }\n  table, th, td {\n    border: 1px solid black;\n    border-collapse: collapse;\n  }\n  th, td {\n    padding: 5px;\n  }\n  th {\n    text-align: left;\n  }\n</style><div class='table-result-container'><table class='table-result'><thead style='background-color: white'><tr><th>fitted values</th><th>residuals</th></tr></thead><tbody><tr><td>-1.0975366274292164</td><td>-0.2502017407140071</td></tr><tr><td>-1.585356301018644</td><td>-0.1700382362588015</td></tr><tr><td>-3.76813436193741</td><td>-0.022573766608175578</td></tr><tr><td>-1.6715536803426616</td><td>-0.15821714283980026</td></tr><tr><td>-2.4572855503111257</td><td>-0.0789074007789438</td></tr><tr><td>-2.5003296369255783</td><td>-0.07583507447830999</td></tr><tr><td>4.438281393236198</td><td>0.011678235826407923</td></tr><tr><td>3.273098764065551</td><td>0.036505678722439705</td></tr><tr><td>3.2105477445716795</td><td>0.038770716205372646</td></tr><tr><td>3.9292226966623605</td><td>0.019279924555004047</td></tr><tr><td>-25.804453079192214</td><td>1.9999999999937874</td></tr><tr><td>-28.65679503583266</td><td>1.9999999999996414</td></tr><tr><td>-27.75591350674064</td><td>1.9999999999991174</td></tr><tr><td>-28.074988256322246</td><td>1.9999999999993585</td></tr><tr><td>-27.1334566982454</td><td>1.9999999999983553</td></tr><tr><td>-22.52621989367687</td><td>2.9999999998351887</td></tr><tr><td>-23.025060503638688</td><td>2.999999999899921</td></tr><tr><td>-16.173394104717698</td><td>2.999999905379738</td></tr><tr><td>-31.079865109141902</td><td>2.999999999999968</td></tr><tr><td>-29.35692680090278</td><td>2.999999999999822</td></tr><tr><td>-29.13407765645514</td><td>2.9999999999997775</td></tr><tr><td>-3.2325968051564864</td><td>3.962042693152814</td></tr><tr><td>-5.7346449942039115</td><td>3.996778382808047</td></tr><tr><td>-2.3010399522154024</td><td>3.9089631307622486</td></tr><tr><td>-4.8991348987075645</td><td>3.9926021087828145</td></tr><tr><td>-16.27387867594197</td><td>3.9999999144255227</td></tr><tr><td>-32.85985130725663</td><td>4.999999999999995</td></tr><tr><td>-18.821202677211918</td><td>4.999999993300297</td></tr><tr><td>-25.182332094860996</td><td>4.999999999988427</td></tr><tr><td>-1.1679482598059319</td><td>-0.2372260461659115</td></tr><tr><td>-1.8705854338840013</td><td>-0.1334739974871252</td></tr><tr><td>-3.910216693195969</td><td>-0.019642596703338143</td></tr><tr><td>-3.354121218498626</td><td>-0.03376046850621519</td></tr><tr><td>-2.4215098245234614</td><td>-0.08154710240629881</td></tr><tr><td>2.7930004395584866</td><td>0.05770359303789585</td></tr><tr><td>4.994771713926521</td><td>0.006727698664592996</td></tr><tr><td>3.125748660719427</td><td>0.042057554992537605</td></tr><tr><td>3.2244599398767826</td><td>0.038255556741133345</td></tr><tr><td>2.1729765389916893</td><td>0.10220358856220002</td></tr><tr><td>-28.217928944601713</td><td>1.999999999999444</td></tr><tr><td>-27.60024897726727</td><td>1.9999999999989688</td></tr><tr><td>-27.56554848894203</td><td>1.9999999999989324</td></tr><tr><td>-26.85793369514561</td><td>1.9999999999978335</td></tr><tr><td>-23.982764507495602</td><td>1.9999999999615923</td></tr><tr><td>-26.54866828637316</td><td>1.9999999999970484</td></tr><tr><td>-28.368854483374328</td><td>1.999999999999522</td></tr><tr><td>-28.829424778854147</td><td>2.9999999999996985</td></tr><tr><td>-18.868289074926114</td><td>2.9999999936084505</td></tr><tr><td>-23.650154172801923</td><td>2.9999999999464366</td></tr><tr><td>-33.92947016961865</td><td>2.9999999999999982</td></tr><tr><td>-21.19261022884948</td><td>2.999999999374588</td></tr><tr><td>-17.367122056300104</td><td>3.9999999713216137</td></tr><tr><td>-22.66331771865592</td><td>3.9999999998563034</td></tr><tr><td>-18.16324070762798</td><td>3.9999999870638576</td></tr><tr><td>-22.848139113918712</td><td>3.999999999880552</td></tr><tr><td>-21.101006207259793</td><td>3.9999999993145914</td></tr><tr><td>-17.2936716123443</td><td>3.9999999691358847</td></tr><tr><td>-18.813375981335962</td><td>4.999999993247655</td></tr><tr><td>-1.8925691882990945</td><td>-0.13095180966388342</td></tr><tr><td>-4.597433290803406</td><td>-0.009977122880762367</td></tr><tr><td>4.782167329224703</td><td>0.008308217017181985</td></tr><tr><td>3.888714454718731</td><td>0.020060965368940198</td></tr><tr><td>0.8775521581555976</td><td>0.2936852885598088</td></tr><tr><td>-27.519865153599852</td><td>1.9999999999988824</td></tr><tr><td>-30.821228655267145</td><td>1.999999999999959</td></tr><tr><td>-28.299826283932745</td><td>1.9999999999994877</td></tr><tr><td>-27.572642861617858</td><td>1.99999999999894</td></tr><tr><td>-22.40224388602005</td><td>2.9999999998134355</td></tr><tr><td>-25.517631362817102</td><td>2.999999999991724</td></tr><tr><td>-21.978442668301067</td><td>2.9999999997149747</td></tr><tr><td>-20.384814223985043</td><td>3.999999998597226</td></tr><tr><td>-25.075470423200287</td><td>3.9999999999871214</td></tr><tr><td>-24.56533457971247</td><td>3.999999999978551</td></tr><tr><td>-26.068784591171326</td><td>4.9999999999952305</td></tr><tr><td>-19.831189221835157</td><td>4.999999997559808</td></tr><tr><td>-46.29795102064383</td><td>5.0</td></tr><tr><td>-51.03623131645012</td><td>5.0</td></tr><tr><td>-17.567977982073</td><td>4.999999976540211</td></tr><tr><td>-54.34360395495136</td><td>5.0</td></tr><tr><td>-2.30543422507384</td><td>-0.09067389973507024</td></tr><tr><td>-2.0085002615388947</td><td>-0.1183133339279117</td></tr><tr><td>-4.498090340388912</td><td>-0.011007712820516855</td></tr><tr><td>3.6431411317597027</td><td>0.025502607738942307</td></tr><tr><td>2.215691785891654</td><td>0.09835018510745941</td></tr><tr><td>2.6885185454181846</td><td>0.06365425951829606</td></tr><tr><td>2.3066413216320885</td><td>0.09057442120029768</td></tr><tr><td>-18.520925423145435</td><td>1.9999999909538395</td></tr><tr><td>-17.15156045857821</td><td>1.9999999644227873</td></tr><tr><td>-25.53256176084041</td><td>2.9999999999918465</td></tr><tr><td>-19.288677007947125</td><td>2.9999999958020815</td></tr><tr><td>-20.35432293170424</td><td>2.999999998553795</td></tr><tr><td>-19.509666505009914</td><td>2.9999999966344233</td></tr><tr><td>-30.26290837894331</td><td>2.999999999999928</td></tr><tr><td>-28.84013014291795</td><td>3.9999999999997016</td></tr><tr><td>-47.913528838646485</td><td>4.0</td></tr><tr><td>-38.3689421581031</td><td>5.0</td></tr><tr><td>-61.99633482112417</td><td>5.0</td></tr><tr><td>-56.36262692836781</td><td>5.0</td></tr></tbody></table></div>"]}}],"execution_count":0},{"cell_type":"code","source":["# LogisticRegression Evaluation\n#https://stackoverflow.com/questions/64090386/get-all-evaluation-metrics-after-classification-in-pyspark\n##https://github.com/apache/spark/blob/39e2bad6a866d27c3ca594d15e574a1da3ee84cc/python/pyspark/mllib/evaluation.py#L255\nfrom pyspark.ml.evaluation import BinaryClassificationEvaluator, MulticlassClassificationEvaluator\ne_accuracy = MulticlassClassificationEvaluator(labelCol=\"target\", predictionCol=\"prediction\", metricName=\"accuracy\")\ne_precision = MulticlassClassificationEvaluator(labelCol=\"target\", predictionCol=\"prediction\", metricName=\"precisionByLabel\")\ne_recall = MulticlassClassificationEvaluator(labelCol=\"target\", predictionCol=\"prediction\", metricName=\"recallByLabel\")\ne_f1 = MulticlassClassificationEvaluator(labelCol=\"target\", predictionCol=\"prediction\", metricName=\"f1\")\ne_falsePositiveRate = MulticlassClassificationEvaluator(labelCol=\"target\", predictionCol=\"prediction\", metricName=\"falsePositiveRateByLabel\")\n\naccuracy = e_accuracy.evaluate(predict_test)\nprecision = e_precision.evaluate(predict_test)\nrecall = e_recall.evaluate(predict_test)\nf1score = e_f1.evaluate(predict_test)\nfalsePositiveRate = e_falsePositiveRate.evaluate(predict_test)\n\nprint(\"Accuracy of model at predicting label was: {:.4f}\".format(accuracy))\nprint(\"Precision of model at predicting label was: {:.4f}\".format(precision))\nprint(\"TPRate(Recall) of model at predicting label was: {:.4f}\".format(recall))#\nprint(\"F1 score of model at predicting label was: {:.4f}\".format(f1score))\nprint(\"FPRate of model at predicting label was: {:.4f}\".format(falsePositiveRate))\nprint(\"TNRate(Specificity) of model at predicting label was: {:.4f}\".format(1-falsePositiveRate))#\n##train, test[0.6, 0.4] \n#Accuracy of model at predicting label was: 0.9519\n#Precision of model at predicting label was: 0.8182\n#TPRate(Recall) of model at predicting label was: 1.0000\n#F1 score of model at predicting label was: 0.9517\n#FPRate of model at predicting label was: 0.0465\n#TNRate(Specificity) of model at predicting label was: 0.9535\n\n#train, test[0.7, 0.3]\n#Accuracy of model at predicting label was: 0.9512\n#Precision of model at predicting label was: 0.8462\n#TPRate(Recall) of model at predicting label was: 0.9167\n#F1 score of model at predicting label was: 0.9513\n#FPRate of model at predicting label was: 0.0286\n#TNRate(Specificity) of model at predicting label was: 0.9714\n\n#train, test[0.8, 0.2]\n#Accuracy of model at predicting label was: 0.9831\n#Precision of model at predicting label was: 1.0000\n#TPRate(Recall) of model at predicting label was: 1.0000\n#F1 score of model at predicting label was: 0.9826\n#FPRate of model at predicting label was: 0.0000\n#TNRate(Specificity) of model at predicting label was: 1.0000\n\n#train, test [0.9, 0.1]\n#Accuracy of model at predicting label was: 1.0000\n#Precision of model at predicting label was: 1.0000\n#TPRate(Recall) of model at predicting label was: 1.0000\n#F1 score of model at predicting label was: 1.0000\n#FPRate of model at predicting label was: 0.0000\n#TNRate(Specificity) of model at predicting label was: 1.0000\n"],"metadata":{"application/vnd.databricks.v1+cell":{"title":"","showTitle":false,"inputWidgets":{},"nuid":"34b7c65e-c0df-40fa-9a66-10567220b03c"}},"outputs":[{"output_type":"display_data","metadata":{"application/vnd.databricks.v1+output":{"datasetInfos":[],"data":"Accuracy of model at predicting label was: 1.0000\nPrecision of model at predicting label was: 1.0000\nTPRate(Recall) of model at predicting label was: 1.0000\nF1 score of model at predicting label was: 1.0000\nFPRate of model at predicting label was: 0.0000\nTNRate(Specificity) of model at predicting label was: 1.0000\n","removedWidgets":[],"addedWidgets":{},"metadata":{},"type":"ansi","arguments":{}}},"output_type":"display_data","data":{"text/plain":["Accuracy of model at predicting label was: 1.0000\nPrecision of model at predicting label was: 1.0000\nTPRate(Recall) of model at predicting label was: 1.0000\nF1 score of model at predicting label was: 1.0000\nFPRate of model at predicting label was: 0.0000\nTNRate(Specificity) of model at predicting label was: 1.0000\n"]}}],"execution_count":0},{"cell_type":"code","source":["# LogisticRegression Evaluation\nfrom pyspark.mllib.evaluation import MulticlassMetrics\nfrom pyspark.sql.types import FloatType\nimport pyspark.sql.functions as F\nevaluator = MulticlassClassificationEvaluator(labelCol=\"target\", predictionCol=\"prediction\",metricName=\"accuracy\")\naccuracy = evaluator.evaluate(predict_test)\nprint(\"Accuracy of model at predicting label on testing data was: {:.4f}\".format(accuracy))\nprint(\"Test Error of model at predicting label on testing data was: {:.4f}\".format(1-accuracy))\npredict = predict_test.select(['prediction','target'])\nmetrics = MulticlassMetrics(predict.rdd.map(tuple))\nprint(metrics.confusionMatrix().toArray())\n#print(\"Training Dataset Count: \" + str(train.count()))\nprint(\"Test Dataset Count: \" + str(test.count()))\n#According to the confusion matrix, for train, test = [0.6, 0.4]:  99 (18+17+15+16+18+15) items are correctly classified out of 104 test data. 5 items are incorrectly classified/Accuracy of model at predicting label on testing data was: 0.9519/Test Error of model at predicting label on testing data was: 0.0481\n\n#According to the confusion matrix, for train, test = [0.7, 0.3]:  82 (12+14+14+15+14+13) items are correctly classified out of 82 test data. 0 items are incorrectly classified/Accuracy of model at predicting label on testing data was: 1.0000/Test Error of model at predicting label on testing data was: 0.0000\n\n#According to the confusion matrix, for train, test = [0.8, 0.2]:  58 (8+11+12+10+12+5) items are correctly classified out of 59 test data. 1 items are incorrectly classified/Accuracy of model at predicting label on testing data was: 0.9831/Test Error of model at predicting label on testing data was: 0.0169\n\n#According to the confusion matrix, for train, test = [0.9, 0.1]:  26 (2+4+6+4+6+2) items are correctly classified out of 59 test data. 0 items are incorrectly classified/Accuracy of model at predicting label on testing data was: 1.0000/Test Error of model at predicting label on testing data was: 0.0000\n"],"metadata":{"application/vnd.databricks.v1+cell":{"title":"","showTitle":false,"inputWidgets":{},"nuid":"9aac39c0-853b-4557-9102-01ea083dba07"}},"outputs":[{"output_type":"display_data","metadata":{"application/vnd.databricks.v1+output":{"datasetInfos":[],"data":"Accuracy of model at predicting label on testing data was: 1.0000\nTest Error of model at predicting label on testing data was: 0.0000\n[[2. 0. 0. 0. 0. 0.]\n [0. 4. 0. 0. 0. 0.]\n [0. 0. 6. 0. 0. 0.]\n [0. 0. 0. 4. 0. 0.]\n [0. 0. 0. 0. 8. 0.]\n [0. 0. 0. 0. 0. 2.]]\nTest Dataset Count: 26\n","removedWidgets":[],"addedWidgets":{},"metadata":{},"type":"ansi","arguments":{}}},"output_type":"display_data","data":{"text/plain":["Accuracy of model at predicting label on testing data was: 1.0000\nTest Error of model at predicting label on testing data was: 0.0000\n[[2. 0. 0. 0. 0. 0.]\n [0. 4. 0. 0. 0. 0.]\n [0. 0. 6. 0. 0. 0.]\n [0. 0. 0. 4. 0. 0.]\n [0. 0. 0. 0. 8. 0.]\n [0. 0. 0. 0. 0. 2.]]\nTest Dataset Count: 26\n"]}}],"execution_count":0},{"cell_type":"code","source":["# LogisticRegression Evaluation\n#https://stackoverflow.com/questions/60772315/how-to-evaluate-a-classifier-with-pyspark-2-4-5\n# Create both evaluators\ne_multi = MulticlassClassificationEvaluator(labelCol=\"target\", predictionCol=\"prediction\")\ne_bin = BinaryClassificationEvaluator(labelCol=\"target\", rawPredictionCol=\"prediction\", metricName='areaUnderROC')\n\n# Make predicitons\nprediction = predict_test.select(\"target\", \"prediction\")\n\n# Get metrics\naccuracy = e_multi.evaluate(prediction, {e_multi.metricName: \"accuracy\"})\nf1 = e_multi.evaluate(prediction, {e_multi.metricName: \"f1\"})\nweightedPrecision = e_multi.evaluate(prediction, {e_multi.metricName: \"weightedPrecision\"})\nweightedRecall = e_multi.evaluate(prediction, {e_multi.metricName: \"weightedRecall\"})\nfalsePositiveRate = e_multi.evaluate(prediction, {e_multi.metricName: \"weightedFalsePositiveRate\"})\nauc = e_bin.evaluate(prediction)\n\nprint(\"Accuracy of model at predicting label was: {:.4f}\".format(accuracy))\nprint(\"F1 of model at predicting label was: {:.4f}\".format(f1))\nprint(\"Precision_weighted of model at predicting label was: {:.4f}\".format(weightedPrecision))\nprint(\"Recall_weighted of model at predicting label was: {:.4f}\".format(weightedRecall))\nprint(\"FPRate of model at predicting label was: {:.4f}\".format(falsePositiveRate))\nprint(\"TNRate(Specificity) of model at predicting label was: {:.4f}\".format(1-falsePositiveRate))\nprint(\"AUC of model at predicting label was: {:.4f}\".format(auc))\n##train, test[0.6, 0.4]\n#Accuracy of model at predicting label was: 0.9519\n#F1 of model at predicting label was: 0.9517\n#Precision_weighted of model at predicting label was: 0.9594\n#Recall_weighted of model at predicting label was: 0.9519\n#FPRate of model at predicting label was: 0.0101\n#TNRate(Specificity) of model at predicting label was: 0.9899\n#AUC of model at predicting label was: 0.9767\n\n#train, test[0.7, 0.3]\n#Accuracy of model at predicting label was: 0.9512\n#F1 of model at predicting label was: 0.9513\n#Precision_weighted of model at predicting label was: 0.9530\n#Recall_weighted of model at predicting label was: 0.9512\n#FPRate of model at predicting label was: 0.0092\n#TNRate(Specificity) of model at predicting label was: 0.9908\n#AUC of model at predicting label was: 0.9774\n\n#train, test[0.8, 0.2]\n#Accuracy of model at predicting label was: 0.9831\n#F1 of model at predicting label was: 0.9826\n#Precision_weighted of model at predicting label was: 0.9844\n#Recall_weighted of model at predicting label was: 0.9831\n#FPRate of model at predicting label was: 0.0043\n#TNRate(Specificity) of model at predicting label was: 0.9957\n#AUC of model at predicting label was: 1.0000\n\n#train, test[0.9, 0.1]\n#Accuracy of model at predicting label was: 1.0000\n#F1 of model at predicting label was: 1.0000\n#Precision_weighted of model at predicting label was: 1.0000\n#Recall_weighted of model at predicting label was: 1.0000\n#FPRate of model at predicting label was: 0.0000\n#TNRate(Specificity) of model at predicting label was: 1.0000\n#AUC of model at predicting label was: 1.0000"],"metadata":{"application/vnd.databricks.v1+cell":{"title":"","showTitle":false,"inputWidgets":{},"nuid":"b8953ef9-f001-406f-beec-5f7a1fc37f90"}},"outputs":[{"output_type":"display_data","metadata":{"application/vnd.databricks.v1+output":{"datasetInfos":[],"data":"Accuracy of model at predicting label was: 1.0000\nF1 of model at predicting label was: 1.0000\nPrecision_weighted of model at predicting label was: 1.0000\nRecall_weighted of model at predicting label was: 1.0000\nFPRate of model at predicting label was: 0.0000\nTNRate(Specificity) of model at predicting label was: 1.0000\nAUC of model at predicting label was: 1.0000\n","removedWidgets":[],"addedWidgets":{},"metadata":{},"type":"ansi","arguments":{}}},"output_type":"display_data","data":{"text/plain":["Accuracy of model at predicting label was: 1.0000\nF1 of model at predicting label was: 1.0000\nPrecision_weighted of model at predicting label was: 1.0000\nRecall_weighted of model at predicting label was: 1.0000\nFPRate of model at predicting label was: 0.0000\nTNRate(Specificity) of model at predicting label was: 1.0000\nAUC of model at predicting label was: 1.0000\n"]}}],"execution_count":0},{"cell_type":"code","source":["#https://www.guru99.com/pyspark-tutorial.html\n# Print the coefficients and intercept for logistic regression\nprint(\"Coefficients: \" + str(model.coefficientMatrix ))\nprint(\"Intercept: \" + str(model.interceptVector))"],"metadata":{"application/vnd.databricks.v1+cell":{"title":"","showTitle":false,"inputWidgets":{},"nuid":"2f0d17f0-2a41-4409-b6bb-484230b67625"}},"outputs":[{"output_type":"display_data","metadata":{"application/vnd.databricks.v1+output":{"datasetInfos":[],"data":"Coefficients: DenseMatrix([[ 1.10034008e-02, -1.73302204e+00, -1.06785971e+00,\n              -2.22517748e+00, -2.58368526e+00, -3.35529687e+00,\n              -3.09785032e+00, -2.93010252e+00, -2.91034724e+00,\n               9.41333750e-01, -2.16403741e+00, -2.71569003e+00,\n              -1.78301731e+00, -2.43488676e+00, -3.27035690e+00,\n              -3.28361606e+00,  7.99195860e-02, -2.61254393e+00,\n              -3.95802953e-01, -2.72499925e+00, -3.04758072e+00,\n              -3.07511790e+00, -1.74442643e-04, -7.56308422e-06,\n              -2.44006111e-03,  8.75033642e-01],\n             [ 6.53434068e+00, -1.77613332e+00, -1.33465732e+00,\n              -1.79606828e+00, -2.45131734e+00, -2.63639335e+00,\n              -2.42551387e+00, -2.30379518e+00, -2.30124800e+00,\n               7.58302889e-01, -2.19531458e+00, -2.42491074e+00,\n              -1.85267476e+00, -2.25249371e+00, -2.53634026e+00,\n              -2.54592692e+00,  6.62957891e+00, -2.26653801e+00,\n              -8.63229607e-01, -2.42235496e+00, -2.60446810e+00,\n              -2.87836326e+00, -2.66197108e-05, -1.36592956e-05,\n              -3.75163122e-03, -1.33307213e-01],\n             [-5.47244380e+00,  3.14903721e+00, -8.06068641e+00,\n               1.18031682e+01, -7.81990005e+00,  1.41319967e+01,\n               1.48150468e+01,  1.37972264e+01,  1.49401504e+01,\n               9.45775678e-01, -7.65558295e+00, -2.84129313e+00,\n              -4.95930317e+00, -2.68617238e+00,  1.38316097e+01,\n               1.38390333e+01, -5.44783796e+00,  7.54347020e+00,\n              -5.41311174e+00,  3.58916484e+00,  4.72067084e+00,\n              -3.18412590e+00, -6.11837712e-05, -1.19324938e-05,\n              -1.35733287e-03,  2.92309927e-01],\n             [-3.72776672e+00, -2.52723227e+00,  1.06782960e+01,\n              -4.60746927e+00,  1.72573003e+01, -8.16851290e+00,\n              -6.15324755e+00, -6.04088619e+00, -6.91510462e+00,\n              -5.64393615e-02,  1.77077780e+01,  1.31084696e+01,\n               1.35675069e+01,  1.27464622e+01, -5.61224272e+00,\n              -5.59838569e+00, -3.60892841e+00, -4.50319200e+00,\n               4.67980137e+00,  5.72775507e+00,  4.79106183e+00,\n               1.44034440e+01,  4.33145505e-04, -1.16956569e-05,\n              -5.02366559e-03, -1.45433439e-01],\n             [ 4.07955437e+00,  6.01613750e+00, -3.46635423e+00,\n              -1.56474221e+00, -2.29370203e+00, -2.63227858e+00,\n              -1.58938873e+00, -1.52162068e+00, -1.62344680e+00,\n              -1.01217747e+00, -2.48347552e+00, -1.87316046e+00,\n              -2.28169571e+00, -1.94757195e+00, -1.44623611e+00,\n              -1.44447782e+00,  4.18571796e+00,  3.93983745e-01,\n               4.37764060e+00, -2.63148428e+00, -2.03099380e+00,\n              -2.89536625e+00, -2.09377535e-04,  3.04619247e-05,\n              -5.59061562e-03, -4.44335979e-01],\n             [-1.42468794e+00, -3.12878708e+00,  3.25126162e+00,\n              -1.60971096e+00, -2.10869558e+00,  2.66048503e+00,\n              -1.54904637e+00, -1.00082180e+00, -1.19000378e+00,\n              -1.57679549e+00, -3.20936756e+00, -3.25341523e+00,\n              -2.69081598e+00, -3.42533740e+00, -9.66433747e-01,\n              -9.66626828e-01, -1.83845009e+00,  1.44481999e+00,\n              -2.38529767e+00, -1.53808142e+00, -1.82869006e+00,\n              -2.37047073e+00,  3.84781552e-05,  1.43886059e-05,\n               1.81633064e-02, -4.44266938e-01]])\nIntercept: [-0.8604576232278583,0.7701800439663096,-0.4735596177865797,0.12643103549062512,1.170294202592161,-0.7328880410346578]\n","removedWidgets":[],"addedWidgets":{},"metadata":{},"type":"ansi","arguments":{}}},"output_type":"display_data","data":{"text/plain":["Coefficients: DenseMatrix([[ 1.10034008e-02, -1.73302204e+00, -1.06785971e+00,\n              -2.22517748e+00, -2.58368526e+00, -3.35529687e+00,\n              -3.09785032e+00, -2.93010252e+00, -2.91034724e+00,\n               9.41333750e-01, -2.16403741e+00, -2.71569003e+00,\n              -1.78301731e+00, -2.43488676e+00, -3.27035690e+00,\n              -3.28361606e+00,  7.99195860e-02, -2.61254393e+00,\n              -3.95802953e-01, -2.72499925e+00, -3.04758072e+00,\n              -3.07511790e+00, -1.74442643e-04, -7.56308422e-06,\n              -2.44006111e-03,  8.75033642e-01],\n             [ 6.53434068e+00, -1.77613332e+00, -1.33465732e+00,\n              -1.79606828e+00, -2.45131734e+00, -2.63639335e+00,\n              -2.42551387e+00, -2.30379518e+00, -2.30124800e+00,\n               7.58302889e-01, -2.19531458e+00, -2.42491074e+00,\n              -1.85267476e+00, -2.25249371e+00, -2.53634026e+00,\n              -2.54592692e+00,  6.62957891e+00, -2.26653801e+00,\n              -8.63229607e-01, -2.42235496e+00, -2.60446810e+00,\n              -2.87836326e+00, -2.66197108e-05, -1.36592956e-05,\n              -3.75163122e-03, -1.33307213e-01],\n             [-5.47244380e+00,  3.14903721e+00, -8.06068641e+00,\n               1.18031682e+01, -7.81990005e+00,  1.41319967e+01,\n               1.48150468e+01,  1.37972264e+01,  1.49401504e+01,\n               9.45775678e-01, -7.65558295e+00, -2.84129313e+00,\n              -4.95930317e+00, -2.68617238e+00,  1.38316097e+01,\n               1.38390333e+01, -5.44783796e+00,  7.54347020e+00,\n              -5.41311174e+00,  3.58916484e+00,  4.72067084e+00,\n              -3.18412590e+00, -6.11837712e-05, -1.19324938e-05,\n              -1.35733287e-03,  2.92309927e-01],\n             [-3.72776672e+00, -2.52723227e+00,  1.06782960e+01,\n              -4.60746927e+00,  1.72573003e+01, -8.16851290e+00,\n              -6.15324755e+00, -6.04088619e+00, -6.91510462e+00,\n              -5.64393615e-02,  1.77077780e+01,  1.31084696e+01,\n               1.35675069e+01,  1.27464622e+01, -5.61224272e+00,\n              -5.59838569e+00, -3.60892841e+00, -4.50319200e+00,\n               4.67980137e+00,  5.72775507e+00,  4.79106183e+00,\n               1.44034440e+01,  4.33145505e-04, -1.16956569e-05,\n              -5.02366559e-03, -1.45433439e-01],\n             [ 4.07955437e+00,  6.01613750e+00, -3.46635423e+00,\n              -1.56474221e+00, -2.29370203e+00, -2.63227858e+00,\n              -1.58938873e+00, -1.52162068e+00, -1.62344680e+00,\n              -1.01217747e+00, -2.48347552e+00, -1.87316046e+00,\n              -2.28169571e+00, -1.94757195e+00, -1.44623611e+00,\n              -1.44447782e+00,  4.18571796e+00,  3.93983745e-01,\n               4.37764060e+00, -2.63148428e+00, -2.03099380e+00,\n              -2.89536625e+00, -2.09377535e-04,  3.04619247e-05,\n              -5.59061562e-03, -4.44335979e-01],\n             [-1.42468794e+00, -3.12878708e+00,  3.25126162e+00,\n              -1.60971096e+00, -2.10869558e+00,  2.66048503e+00,\n              -1.54904637e+00, -1.00082180e+00, -1.19000378e+00,\n              -1.57679549e+00, -3.20936756e+00, -3.25341523e+00,\n              -2.69081598e+00, -3.42533740e+00, -9.66433747e-01,\n              -9.66626828e-01, -1.83845009e+00,  1.44481999e+00,\n              -2.38529767e+00, -1.53808142e+00, -1.82869006e+00,\n              -2.37047073e+00,  3.84781552e-05,  1.43886059e-05,\n               1.81633064e-02, -4.44266938e-01]])\nIntercept: [-0.8604576232278583,0.7701800439663096,-0.4735596177865797,0.12643103549062512,1.170294202592161,-0.7328880410346578]\n"]}}],"execution_count":0},{"cell_type":"code","source":["#Decision Tree Classifier\nfrom pyspark.ml.classification import DecisionTreeClassifier\n#DecisionTreeClassifier model\ndtc = DecisionTreeClassifier(featuresCol = 'features', labelCol = 'target', maxDepth = 3).fit(train)\npredict_test = dtc.transform(test)\n#predict_test.select('target', 'rawPrediction', 'prediction', 'probability').show(5)\n#print DecisionTreeClassifier model\nprint(dtc)\n\n#Evaluate our Decision Tree model.\n#https://www.datatechnotes.com/2021/06/pyspark-decision-tree-classification.html\n# After training the model, we'll predict test data and check the accuracy metrics. Here, we can use MulticlassClassificationEvaluator to check the accuracy. Confusion matrix can be created by using confusion_matrix function of sklearn.metrics module.\nevaluator = MulticlassClassificationEvaluator(labelCol=\"target\", predictionCol=\"prediction\",metricName=\"accuracy\")\naccuracy = evaluator.evaluate(predict_test)\nprint(\"Accuracy of model at predicting label on testing data was: {}\".format(accuracy))\nprint(\"Test Error of model at predicting label on testing data was: {}\".format(1-accuracy))\n\nfrom pyspark.mllib.evaluation import MulticlassMetrics\nfrom pyspark.sql.types import FloatType\nimport pyspark.sql.functions as F\n\npredict = predict_test.select(['prediction','target'])\nmetrics = MulticlassMetrics(predict.rdd.map(tuple))\nprint(metrics.confusionMatrix().toArray())\n#print(\"Training Dataset Count: \" + str(train.count()))\nprint(\"Test Dataset Count: \" + str(test.count()))\n#According to the confusion matrix, for train, test = [0.6, 0.4]:  65 (1+15+15+18+16) items are correctly classified out of 104 test data. 39 item are incorrectly classified/Accuracy of model at predicting label on testing data was: 0.625/Test Error of model at predicting label on testing data was: 0.375\n\n#According to the confusion matrix, for train, test = [0.7, 0.3]:  81 (12+14+14+14+14+13) items are correctly classified out of 82 test data. 1 item are incorrectly classified/Accuracy of model at predicting label on testing data was: 0.9878048780487805/Test Error of model at predicting label on testing data was: 0.012195121951219523\n\n#According to the confusion matrix, for train, test = [0.8, 0.2]:  34 (8+10+10+6) items are correctly classified out of 59 test data. 25 items are incorrectly classified/Accuracy of model at predicting label on testing data was: 0.576271186440678/Test Error of model at predicting label on testing data was: 0.423728813559322\n\n#According to the confusion matrix, for train, test = [0.9, 0.1]:  14 (2+6+4+2) items are correctly classified out of 26 test data. 12 items are incorrectly classified/Accuracy of model at predicting label on testing data was:  0.5384615384615384/Test Error of model at predicting label on testing data was: 0.46153846153846156"],"metadata":{"application/vnd.databricks.v1+cell":{"title":"","showTitle":false,"inputWidgets":{},"nuid":"5feb0850-7d1a-4f11-92ba-03f71eb3fa45"}},"outputs":[{"output_type":"display_data","metadata":{"application/vnd.databricks.v1+output":{"datasetInfos":[],"data":"DecisionTreeClassificationModel: uid=DecisionTreeClassifier_60070a465288, depth=3, numNodes=11, numClasses=6, numFeatures=26\nAccuracy of model at predicting label on testing data was: 0.5384615384615384\nTest Error of model at predicting label on testing data was: 0.46153846153846156\n[[2. 0. 0. 0. 0. 0.]\n [0. 0. 0. 4. 0. 0.]\n [0. 0. 6. 0. 0. 0.]\n [0. 0. 0. 4. 0. 0.]\n [0. 0. 0. 7. 0. 1.]\n [0. 0. 0. 0. 0. 2.]]\nTest Dataset Count: 26\n","removedWidgets":[],"addedWidgets":{},"metadata":{},"type":"ansi","arguments":{}}},"output_type":"display_data","data":{"text/plain":["DecisionTreeClassificationModel: uid=DecisionTreeClassifier_60070a465288, depth=3, numNodes=11, numClasses=6, numFeatures=26\nAccuracy of model at predicting label on testing data was: 0.5384615384615384\nTest Error of model at predicting label on testing data was: 0.46153846153846156\n[[2. 0. 0. 0. 0. 0.]\n [0. 0. 0. 4. 0. 0.]\n [0. 0. 6. 0. 0. 0.]\n [0. 0. 0. 4. 0. 0.]\n [0. 0. 0. 7. 0. 1.]\n [0. 0. 0. 0. 0. 2.]]\nTest Dataset Count: 26\n"]}}],"execution_count":0},{"cell_type":"code","source":["#Decision Tree Classifier Evaluation\nfrom pyspark.ml.evaluation import BinaryClassificationEvaluator, MulticlassClassificationEvaluator\ne_accuracy = MulticlassClassificationEvaluator(labelCol=\"target\", predictionCol=\"prediction\", metricName=\"accuracy\")\ne_precision = MulticlassClassificationEvaluator(labelCol=\"target\", predictionCol=\"prediction\", metricName=\"precisionByLabel\")\ne_recall = MulticlassClassificationEvaluator(labelCol=\"target\", predictionCol=\"prediction\", metricName=\"recallByLabel\")\ne_f1 = MulticlassClassificationEvaluator(labelCol=\"target\", predictionCol=\"prediction\", metricName=\"f1\")\ne_falsePositiveRate = MulticlassClassificationEvaluator(labelCol=\"target\", predictionCol=\"prediction\", metricName=\"falsePositiveRateByLabel\")\ne_roc = BinaryClassificationEvaluator(labelCol=\"target\", rawPredictionCol=\"prediction\")\n\n\nroc = e_roc.evaluate(predict_test)\naccuracy = e_accuracy.evaluate(predict_test)\nprecision = e_precision.evaluate(predict_test)\nrecall = e_recall.evaluate(predict_test)\nf1score = e_f1.evaluate(predict_test)\nfalsePositiveRate = e_falsePositiveRate.evaluate(predict_test)\n\nprint(\"Accuracy of model at predicting label was: {:.4f}\".format(accuracy))\nprint(\"Precision of model at predicting label was: {:.4f}\".format(precision))\nprint(\"TPRate(Recall) of model at predicting label was: {:.4f}\".format(recall))#\nprint(\"F1 score of model at predicting label was: {:.4f}\".format(f1score))\nprint(\"FPRate of model at predicting label was: {:.4f}\".format(falsePositiveRate))\nprint(\"TNRate(Specificity) of model at predicting label was: {:.4f}\".format(1-falsePositiveRate))#\n#print(\"ROC of model at predicting label was: {:.4f}\".format(roc))\nprint(\"Test Area Under ROC for train set: \" + str(e_roc.evaluate(predict_train, {e_roc.metricName: \"areaUnderROC\"})))\nprint(\"Test Area Under ROC for test set: \" + str(e_roc.evaluate(predict_test, {e_roc.metricName: \"areaUnderROC\"})))"],"metadata":{"application/vnd.databricks.v1+cell":{"title":"","showTitle":false,"inputWidgets":{},"nuid":"6b81c0e4-ffe8-4bef-a104-2b72bfcce3be"}},"outputs":[{"output_type":"display_data","metadata":{"application/vnd.databricks.v1+output":{"datasetInfos":[],"data":"Accuracy of model at predicting label was: 0.5385\nPrecision of model at predicting label was: 1.0000\nTPRate(Recall) of model at predicting label was: 1.0000\nF1 score of model at predicting label was: 0.4340\nFPRate of model at predicting label was: 0.0000\nTNRate(Specificity) of model at predicting label was: 1.0000\nTest Area Under ROC for train set: 1.0\nTest Area Under ROC for test set: 1.0\n","removedWidgets":[],"addedWidgets":{},"metadata":{},"type":"ansi","arguments":{}}},"output_type":"display_data","data":{"text/plain":["Accuracy of model at predicting label was: 0.5385\nPrecision of model at predicting label was: 1.0000\nTPRate(Recall) of model at predicting label was: 1.0000\nF1 score of model at predicting label was: 0.4340\nFPRate of model at predicting label was: 0.0000\nTNRate(Specificity) of model at predicting label was: 1.0000\nTest Area Under ROC for train set: 1.0\nTest Area Under ROC for test set: 1.0\n"]}}],"execution_count":0},{"cell_type":"code","source":["#It comes under supervised learning and mainly used for classification but can be used for regression as well. Random forest classifier is useful because,No overfitting, High accuracy,Estimates missing data\n\nfrom pyspark.ml.classification import RandomForestClassifier\n#RandomForest model\nrfc = RandomForestClassifier(labelCol=\"target\",featuresCol=\"features\", numTrees=10).fit(train)\npredict_test = rfc.transform(test)\npredict_train = rfc.transform(train)\n#predict_test.select('target', 'rawPrediction', 'prediction', 'probability').show(5)\n#print RandomForest model\nprint(rfc)\n#https://towardsdatascience.com/a-guide-to-exploit-random-forest-classifier-in-pyspark-46d6999cb5db\n#We can clearly compare the actual values and predicted values with the output below.\npredict_test.select(\"target\", \"prediction\").show(5)\nevaluator =MulticlassClassificationEvaluator(labelCol=\"target\",predictionCol=\"prediction\", metricName=\"accuracy\") \naccuracy = evaluator.evaluate(predict_test)\nprint(\"Accuracy of model at predicting label on testing data was: {}\".format(accuracy))\nprint(\"Test Error of model at predicting label on testing data was: {}\".format(1-accuracy))\n\n#Now we can see that the accuracy of our model is high and the test error is very low. It means our classifier model is performing well.\n\n#https://towardsdatascience.com/a-guide-to-exploit-random-forest-classifier-in-pyspark-46d6999cb5db\n#We can use a confusion matrix to compare the predicted star types and the actual star types.\nfrom pyspark.mllib.evaluation import MulticlassMetrics\nfrom pyspark.sql.types import FloatType\nimport pyspark.sql.functions as F\n\npredict = predict_test.select(['prediction','target'])\nmetrics = MulticlassMetrics(predict.rdd.map(tuple))\nprint(metrics.confusionMatrix().toArray())\n#print(\"Training Dataset Count: \" + str(train.count()))\nprint(\"Test Dataset Count: \" + str(test.count()))\n#According to the confusion matrix, for train, test = [0.6, 0.4]:  104 (18+17+15+20+18+16) items are correctly classified out of 104 test data. 0 item are incorrectly classified/Accuracy of model at predicting label on testing data was: 1.0\n\n#According to the confusion matrix, for train, test = [0.7, 0.3]:  78 (11+12+14+14+14+13) items are correctly classified out of 82 test data. 4 item are incorrectly classified/Accuracy of model at predicting label on testing data was: 0.9512195121951219/Test Error of model at predicting label on testing data was: 0.04878048780487809\n\n#According to the confusion matrix, for train, test = [0.8, 0.2] :56 (7+11+12+8+12+6) items are correctly classified out of 59 test data. 3 items are incorrectly classified/Accuracy of model at predicting label on testing data was: 0.9491525423728814/Test Error of model at predicting label on testing data was: 0.05084745762711862\n\n#According to the confusion matrix, for train, test = [0.9, 0.1] :26 (2+4+6+4+8+2) items are correctly classified out of 26 test data. 0 item are incorrectly classified/Accuracy of model at predicting label on testing data was: 1.0000/Test Error of model at predicting label on testing data was: 0.0000"],"metadata":{"application/vnd.databricks.v1+cell":{"title":"","showTitle":false,"inputWidgets":{},"nuid":"5fc197c7-f863-4033-82b4-c7dc6b6f6ed5"}},"outputs":[{"output_type":"display_data","metadata":{"application/vnd.databricks.v1+output":{"datasetInfos":[],"data":"RandomForestClassificationModel: uid=RandomForestClassifier_a5fcf55d652b, numTrees=10, numClasses=6, numFeatures=26\n+------+----------+\n|target|prediction|\n+------+----------+\n|   0.0|       0.0|\n|   0.0|       0.0|\n|   1.0|       1.0|\n|   1.0|       1.0|\n|   1.0|       1.0|\n+------+----------+\nonly showing top 5 rows\n\nAccuracy of model at predicting label on testing data was: 1.0\nTest Error of model at predicting label on testing data was: 0.0\n[[2. 0. 0. 0. 0. 0.]\n [0. 4. 0. 0. 0. 0.]\n [0. 0. 6. 0. 0. 0.]\n [0. 0. 0. 4. 0. 0.]\n [0. 0. 0. 0. 8. 0.]\n [0. 0. 0. 0. 0. 2.]]\nTest Dataset Count: 26\n","removedWidgets":[],"addedWidgets":{},"metadata":{},"type":"ansi","arguments":{}}},"output_type":"display_data","data":{"text/plain":["RandomForestClassificationModel: uid=RandomForestClassifier_a5fcf55d652b, numTrees=10, numClasses=6, numFeatures=26\n+------+----------+\n|target|prediction|\n+------+----------+\n|   0.0|       0.0|\n|   0.0|       0.0|\n|   1.0|       1.0|\n|   1.0|       1.0|\n|   1.0|       1.0|\n+------+----------+\nonly showing top 5 rows\n\nAccuracy of model at predicting label on testing data was: 1.0\nTest Error of model at predicting label on testing data was: 0.0\n[[2. 0. 0. 0. 0. 0.]\n [0. 4. 0. 0. 0. 0.]\n [0. 0. 6. 0. 0. 0.]\n [0. 0. 0. 4. 0. 0.]\n [0. 0. 0. 0. 8. 0.]\n [0. 0. 0. 0. 0. 2.]]\nTest Dataset Count: 26\n"]}}],"execution_count":0},{"cell_type":"code","source":["#RandomForestClassifier Evaluation\nfrom pyspark.ml.evaluation import BinaryClassificationEvaluator, MulticlassClassificationEvaluator\ne_accuracy = MulticlassClassificationEvaluator(labelCol=\"target\", predictionCol=\"prediction\", metricName=\"accuracy\")\ne_precision = MulticlassClassificationEvaluator(labelCol=\"target\", predictionCol=\"prediction\", metricName=\"precisionByLabel\")\ne_recall = MulticlassClassificationEvaluator(labelCol=\"target\", predictionCol=\"prediction\", metricName=\"recallByLabel\")\ne_f1 = MulticlassClassificationEvaluator(labelCol=\"target\", predictionCol=\"prediction\", metricName=\"f1\")\ne_falsePositiveRate = MulticlassClassificationEvaluator(labelCol=\"target\", predictionCol=\"prediction\", metricName=\"falsePositiveRateByLabel\")\ne_roc = BinaryClassificationEvaluator(labelCol=\"target\", rawPredictionCol=\"prediction\")\n\nroc = e_roc.evaluate(predict_test)\naccuracy = e_accuracy.evaluate(predict_test)\nprecision = e_precision.evaluate(predict_test)\nrecall = e_recall.evaluate(predict_test)\nf1score = e_f1.evaluate(predict_test)\nfalsePositiveRate = e_falsePositiveRate.evaluate(predict_test)\n\nprint(\"Accuracy of model at predicting label was: {:.4f}\".format(accuracy))\nprint(\"Precision of model at predicting label was: {:.4f}\".format(precision))\nprint(\"TPRate(Recall) of model at predicting label was: {:.4f}\".format(recall))#\nprint(\"F1 score of model at predicting label was: {:.4f}\".format(f1score))\nprint(\"FPRate of model at predicting label was: {:.4f}\".format(falsePositiveRate))\nprint(\"TNRate(Specificity) of model at predicting label was: {:.4f}\".format(1-falsePositiveRate))#\n#print(\"ROC of model at predicting label was: {:.4f}\".format(roc))\nprint(\"Test Area Under ROC for train set: \" + str(e_roc.evaluate(predict_train, {e_roc.metricName: \"areaUnderROC\"})))\nprint(\"Test Area Under ROC for test set: \" + str(e_roc.evaluate(predict_test, {e_roc.metricName: \"areaUnderROC\"})))"],"metadata":{"application/vnd.databricks.v1+cell":{"title":"","showTitle":false,"inputWidgets":{},"nuid":"578fa2cc-e1dc-4d0e-b749-815b206cb300"}},"outputs":[{"output_type":"display_data","metadata":{"application/vnd.databricks.v1+output":{"datasetInfos":[],"data":"Accuracy of model at predicting label was: 1.0000\nPrecision of model at predicting label was: 1.0000\nTPRate(Recall) of model at predicting label was: 1.0000\nF1 score of model at predicting label was: 1.0000\nFPRate of model at predicting label was: 0.0000\nTNRate(Specificity) of model at predicting label was: 1.0000\nTest Area Under ROC for train set: 0.9971590909090908\nTest Area Under ROC for test set: 1.0\n","removedWidgets":[],"addedWidgets":{},"metadata":{},"type":"ansi","arguments":{}}},"output_type":"display_data","data":{"text/plain":["Accuracy of model at predicting label was: 1.0000\nPrecision of model at predicting label was: 1.0000\nTPRate(Recall) of model at predicting label was: 1.0000\nF1 score of model at predicting label was: 1.0000\nFPRate of model at predicting label was: 0.0000\nTNRate(Specificity) of model at predicting label was: 1.0000\nTest Area Under ROC for train set: 0.9971590909090908\nTest Area Under ROC for test set: 1.0\n"]}}],"execution_count":0},{"cell_type":"code","source":["#APPENDIX  LogisticRegression\n#https://stackoverflow.com/questions/64090386/get-all-evaluation-metrics-after-classification-in-pyspark\n##https://github.com/apache/spark/blob/39e2bad6a866d27c3ca594d15e574a1da3ee84cc/python/pyspark/mllib/evaluation.py#L255\nfrom pyspark.ml.evaluation import BinaryClassificationEvaluator, MulticlassClassificationEvaluator\ne_accuracy = MulticlassClassificationEvaluator(labelCol=\"target\", predictionCol=\"prediction\", metricName=\"accuracy\")\ne_precision = MulticlassClassificationEvaluator(labelCol=\"target\", predictionCol=\"prediction\", metricName=\"precisionByLabel\")\ne_recall = MulticlassClassificationEvaluator(labelCol=\"target\", predictionCol=\"prediction\", metricName=\"recallByLabel\")\ne_f1 = MulticlassClassificationEvaluator(labelCol=\"target\", predictionCol=\"prediction\", metricName=\"f1\")\ne_falsePositiveRate = MulticlassClassificationEvaluator(labelCol=\"target\", predictionCol=\"prediction\", metricName=\"falsePositiveRateByLabel\")\n\naccuracy = e_accuracy.evaluate(predict_test)\nprecision = e_precision.evaluate(predict_test)\nrecall = e_recall.evaluate(predict_test)\nf1score = e_f1.evaluate(predict_test)\nfalsePositiveRate = e_falsePositiveRate.evaluate(predict_test)\n\nprint(\"Accuracy of model at predicting label was: {:.4f}\".format(accuracy))\nprint(\"Precision of model at predicting label was: {:.4f}\".format(precision))\nprint(\"TPRate(Recall) of model at predicting label was: {:.4f}\".format(recall))\nprint(\"F1 score of model at predicting label was: {:.4f}\".format(f1score))\nprint(\"FPRate of model at predicting label was: {:.4f}\".format(falsePositiveRate))\nprint(\"TNRate(Specificity) of model at predicting label was: {:.4f}\".format(1-falsePositiveRate))"],"metadata":{"application/vnd.databricks.v1+cell":{"title":"","showTitle":false,"inputWidgets":{},"nuid":"c79078da-701d-458c-b8ef-eff07d92a2fd"}},"outputs":[{"output_type":"display_data","metadata":{"application/vnd.databricks.v1+output":{"datasetInfos":[],"data":"Accuracy of model at predicting label was: 1.0000\nPrecision of model at predicting label was: 1.0000\nTPRate(Recall) of model at predicting label was: 1.0000\nF1 score of model at predicting label was: 1.0000\nFPRate of model at predicting label was: 0.0000\nTNRate(Specificity) of model at predicting label was: 1.0000\n","removedWidgets":[],"addedWidgets":{},"metadata":{},"type":"ansi","arguments":{}}},"output_type":"display_data","data":{"text/plain":["Accuracy of model at predicting label was: 1.0000\nPrecision of model at predicting label was: 1.0000\nTPRate(Recall) of model at predicting label was: 1.0000\nF1 score of model at predicting label was: 1.0000\nFPRate of model at predicting label was: 0.0000\nTNRate(Specificity) of model at predicting label was: 1.0000\n"]}}],"execution_count":0},{"cell_type":"code","source":["#APPENDIX\n#https://towardsdatascience.com/evaluating-machine-learning-classification-problems-in-python-5-1-metrics-that-matter-792c6faddf5\n#https://towardsdatascience.com/machine-learning-with-pyspark-and-mllib-solving-a-binary-classification-problem-96396065d2aa\n#https://towardsdatascience.com/binary-classifier-evaluation-made-easy-with-handyspark-3b1e69c12b4f\nfrom pyspark.ml.evaluation import BinaryClassificationEvaluator, MulticlassClassificationEvaluator\ne_auc = BinaryClassificationEvaluator(labelCol=\"target\", rawPredictionCol=\"prediction\")\ne_auroc=BinaryClassificationEvaluator(labelCol=\"target\", rawPredictionCol=\"prediction\")\ne_auprc=BinaryClassificationEvaluator(labelCol=\"target\", rawPredictionCol=\"prediction\")\n\nauc = e_auc.evaluate(predict_test)\nauroc=e_auroc.evaluate(predict_test, {evaluator.metricName: \"areaUnderROC\"})\nauprc=e_auprc.evaluate(predict_test, {evaluator.metricName: \"areaUnderPR\"})\n\nprint(\"AUC of model at predicting label was: {:.4f}\".format(auc))   \nprint(\"Area under ROC Curve: {:.4f}\".format(auroc))\nprint(\"Area under PR Curve: {:.4f}\".format(auprc))\n#train, test[0.6, 0.4]\n#AUC of model at predicting label was: 0.9767\n#Area under ROC Curve: 0.9767\n#Area under PR Curve: 0.9767\n\n#train, test[0.7, 0.3]\n#AUC of model at predicting label was: 0.9774\n#Area under ROC Curve: 0.9774\n#Area under PR Curve: 0.9774\n#train, test[0.8, 0.2]\n#AUC of model at predicting label was: 1.0000\n#Area under ROC Curve: 1.0000\n#Area under PR Curve: 1.0000\n\n#train, test[0.9, 0.1]\n#AUC of model at predicting label was: 1.0000\n#Area under ROC Curve: 1.0000\n#Area under PR Curve: 1.0000"],"metadata":{"application/vnd.databricks.v1+cell":{"title":"","showTitle":false,"inputWidgets":{},"nuid":"da39be49-8dc9-4bd5-8de2-6c9bc50b04b3"}},"outputs":[{"output_type":"display_data","metadata":{"application/vnd.databricks.v1+output":{"datasetInfos":[],"data":"AUC of model at predicting label was: 1.0000\nArea under ROC Curve: 1.0000\nArea under PR Curve: 1.0000\n","removedWidgets":[],"addedWidgets":{},"metadata":{},"type":"ansi","arguments":{}}},"output_type":"display_data","data":{"text/plain":["AUC of model at predicting label was: 1.0000\nArea under ROC Curve: 1.0000\nArea under PR Curve: 1.0000\n"]}}],"execution_count":0},{"cell_type":"code","source":["#APPENDIX LogisticRegression\n#Using RDD based API\n#https://stackoverflow.com/questions/60772315/how-to-evaluate-a-classifier-with-pyspark-2-4-5\nfrom pyspark.mllib.evaluation import BinaryClassificationMetrics, MulticlassMetrics\n# Make prediction\nprediction = predict_test.select(\"target\", \"prediction\")\n\n# Create both evaluators\nm_bin = BinaryClassificationMetrics(prediction.rdd.map(tuple))\nm_multi = MulticlassMetrics(prediction.rdd.map(tuple))\n\naccuracy = m_multi.accuracy\nf1 = m_multi.fMeasure(1.0)\nprecision = m_multi.precision(1.0)\nrecall = m_multi.recall(1.0)\nauc = m_bin.areaUnderROC\ntpr=m_multi.truePositiveRate(1.0)\nfpr=m_multi.falsePositiveRate(1.0)\nprint(\"Accuracy of model at predicting label was: {:.4f}\".format(accuracy))\nprint(\"F1 of model at predicting label was: {:.4f}\".format(f1))\nprint(\"Precision of model at predicting label was: {:.4f}\".format(precision))\nprint(\"Recall of model at predicting label was: {:.4f}\".format(recall))\nprint(\"AUC of model at predicting label was: {:.4f}\".format(auc))\nprint(\"TruePositiveRate of model at predicting label was: {:.4f}\".format(tpr))\nprint(\"FalsePositiveRate of model at predicting label was: {:.4f}\".format(fpr))"],"metadata":{"application/vnd.databricks.v1+cell":{"title":"","showTitle":false,"inputWidgets":{},"nuid":"b2fa1a4b-fd38-4ff7-9449-1481573c2ce1"}},"outputs":[{"output_type":"display_data","metadata":{"application/vnd.databricks.v1+output":{"datasetInfos":[],"data":"Accuracy of model at predicting label was: 1.0000\nF1 of model at predicting label was: 1.0000\nPrecision of model at predicting label was: 1.0000\nRecall of model at predicting label was: 1.0000\nAUC of model at predicting label was: 1.0000\nTruePositiveRate of model at predicting label was: 1.0000\nFalsePositiveRate of model at predicting label was: 0.0000\n","removedWidgets":[],"addedWidgets":{},"metadata":{},"type":"ansi","arguments":{}}},"output_type":"display_data","data":{"text/plain":["Accuracy of model at predicting label was: 1.0000\nF1 of model at predicting label was: 1.0000\nPrecision of model at predicting label was: 1.0000\nRecall of model at predicting label was: 1.0000\nAUC of model at predicting label was: 1.0000\nTruePositiveRate of model at predicting label was: 1.0000\nFalsePositiveRate of model at predicting label was: 0.0000\n"]}}],"execution_count":0},{"cell_type":"code","source":["# #APPENDIX LogisticRegression\n# #Using RDD based API\n# #https://stackoverflow.com/questions/60772315/how-to-evaluate-a-classifier-with-pyspark-2-4-5\n# from pyspark.mllib.evaluation import BinaryClassificationMetrics, MulticlassMetrics\n# # Make prediction\n# prediction = predict_test.select(\"target\", \"prediction\")\n\n# # Create both evaluators\n# m_bin = BinaryClassificationMetrics(prediction.rdd.map(tuple))\n# m_multi = MulticlassMetrics(prediction.rdd.map(tuple))\n\n# accuracy = m_multi.accuracy\n# f1 = m_multi.fMeasure(1.0)\n# precision = m_multi.precision(1.0)\n# recall = m_multi.recall(1.0)\n# auc = m_bin.areaUnderROC\n# tpr=m_multi.truePositiveRate(1.0)\n# fpr=m_multi.falsePositiveRate(1.0)\n# print(\"Accuracy of model at predicting label was: {:.4f}\".format(accuracy))\n# print(\"F1 of model at predicting label was: {:.4f}\".format(f1))\n# print(\"Precision of model at predicting label was: {:.4f}\".format(precision))\n# print(\"Recall of model at predicting label was: {:.4f}\".format(recall))\n# print(\"AUC of model at predicting label was: {:.4f}\".format(auc))\n# print(\"TruePositiveRate of model at predicting label was: {:.4f}\".format(tpr))\n# print(\"FalsePositiveRate of model at predicting label was: {:.4f}\".format(fpr))\n# df = sqlContext.createDataFrame(rdd, [\"prediction\", \"target_index\"])\n# df.show()\n\n# metricsp = MulticlassMetrics(df.rdd)\n# metricsp.recall(1)\n\n# tp = df[(df.target_index == 1) & (df.prediction == 1)].count()\n# tn = df[(df.target_index == 0) & (df.prediction == 0)].count()\n# fp = df[(df.target_index == 0) & (df.prediction == 1)].count()\n# fn = df[(df.target_index == 1) & (df.prediction == 0)].count()\n# print \"True Positives:\", tp\n# print \"True Negatives:\", tn\n# print \"False Positives:\", fp\n# print \"False Negatives:\", fn\n# print \"Total\", df.count()\n\n# r = float(tp)/(tp + fn)\n# print \"recall\", r\n\n# p = float(tp) / (tp + fp)\n# print \"precision\", p"],"metadata":{"application/vnd.databricks.v1+cell":{"title":"","showTitle":false,"inputWidgets":{},"nuid":"42796413-72d5-4747-8de3-dae1b9f41f18"}},"outputs":[],"execution_count":0},{"cell_type":"code","source":["import numpy as np\nfrom sklearn.ensemble import RandomForestClassifier\nfrom sklearn.metrics import accuracy_score, precision_score, recall_score, roc_auc_score, f1_score\n\n# Make predicitons\nprediction = predict_test.select(\"target\", \"prediction\")\nprediction_np = np.array((prediction.collect()))\n\nprobability=predict_test.select(\"target\", \"probability\")\nprobability_np = np.array((probability.collect()))\n#probability_np=(probability_np,dtype=object)\ntype(prediction_np.dtype)\ntype(probability_np.dtype)"],"metadata":{"application/vnd.databricks.v1+cell":{"title":"","showTitle":false,"inputWidgets":{},"nuid":"59a8f938-92f8-4307-9399-53b543364b84"}},"outputs":[{"output_type":"display_data","metadata":{"application/vnd.databricks.v1+output":{"datasetInfos":[],"data":"<command-4419698111305121>:10: VisibleDeprecationWarning: Creating an ndarray from ragged nested sequences (which is a list-or-tuple of lists-or-tuples-or ndarrays with different lengths or shapes) is deprecated. If you meant to do this, you must specify 'dtype=object' when creating the ndarray\n  probability_np = np.array((probability.collect()))\nOut[127]: numpy.dtype","removedWidgets":[],"addedWidgets":{},"metadata":{},"type":"ansi","arguments":{}}},"output_type":"display_data","data":{"text/plain":["<command-4419698111305121>:10: VisibleDeprecationWarning: Creating an ndarray from ragged nested sequences (which is a list-or-tuple of lists-or-tuples-or ndarrays with different lengths or shapes) is deprecated. If you meant to do this, you must specify 'dtype=object' when creating the ndarray\n  probability_np = np.array((probability.collect()))\nOut[127]: numpy.dtype"]}}],"execution_count":0},{"cell_type":"code","source":["#APPENDIX LogisticRegression\n#https://stackoverflow.com/questions/60772315/how-to-evaluate-a-classifier-with-pyspark-2-4-5\n#https://stackoverflow.com/questions/52269187/facing-valueerror-target-is-multiclass-but-average-binary\nimport numpy as np\nfrom sklearn.ensemble import RandomForestClassifier\nfrom sklearn.metrics import accuracy_score, precision_score, recall_score, roc_auc_score, f1_score\n\n# Make predicitons\nprediction = predict_test.select(\"target\", \"prediction\")\nprediction_np = np.array((prediction.collect()))\n\n#probability=predict_test.select(\"target\", \"probability\")\n#probability_np = np.array((probability.collect()))\n\n\naccuracy=accuracy_score(prediction_np[:,0], prediction_np[:,1])\nf1=f1_score(prediction_np[:,0], prediction_np[:,1],average='weighted')\nprecision=precision_score(prediction_np[:,0], prediction_np[:,1],average='micro')\nrecall=recall_score(prediction_np[:,0], prediction_np[:,1],average='micro')\n#areaUnderRoc=roc_auc_score(prediction_np[:,0], prediction_np[:,1], average = 'macro', multi_class=\"ovr\")\n#areaUnderRoc=roc_auc_score(prediction_np[:,0], prediction_np[:,1],multi_class=\"ovr\",average='weighted')\n#areaUnderRoc=roc_auc_score(prediction_np[:,0], prediction_np[:,1],multi_class=\"ovr\",average=None)\n#areaUnderRoc=roc_auc_score(prediction_np[:,0], prediction_np[:,1],multi_class=\"ovr\",average=None)\n#areaUnderRoc=roc_auc_score(prediction_np[:,0], prediction_np[:,1],multi_class=\"ovr\")\n#auc = roc_auc_score(probability_np[:,0], probability_np[:,1])\nprint(\"Accuracy of model at predicting label was: {:.4f}\".format(accuracy))\nprint(\"F1 of model at predicting label was: {:.4f}\".format(f1))\nprint(\"Precision of model at predicting label was: {:.4f}\".format(precision))\nprint(\"Recall of model at predicting label was: {:.4f}\".format(recall))\n#print(\"AreaUnderRoc of model at predicting label was: {:.4f}\".format(auc))"],"metadata":{"application/vnd.databricks.v1+cell":{"title":"","showTitle":false,"inputWidgets":{},"nuid":"856df85f-7870-4542-9a91-ac0a2f48184a"}},"outputs":[{"output_type":"display_data","metadata":{"application/vnd.databricks.v1+output":{"datasetInfos":[],"data":"Accuracy of model at predicting label was: 1.0000\nF1 of model at predicting label was: 1.0000\nPrecision of model at predicting label was: 1.0000\nRecall of model at predicting label was: 1.0000\n","removedWidgets":[],"addedWidgets":{},"metadata":{},"type":"ansi","arguments":{}}},"output_type":"display_data","data":{"text/plain":["Accuracy of model at predicting label was: 1.0000\nF1 of model at predicting label was: 1.0000\nPrecision of model at predicting label was: 1.0000\nRecall of model at predicting label was: 1.0000\n"]}}],"execution_count":0},{"cell_type":"code","source":["#APPENDIX LogisticRegression\n#https://www.guru99.com/pyspark-tutorial.html\n#model accuracy\naccuracy = predict_test.select(\"target\", \"prediction\")\naccuracy.groupby('target').agg({'target': 'count'}).show(3)\naccuracy.groupby('prediction').agg({'prediction': 'count'}).show(3)\naccuracy.filter(accuracy.target == accuracy.prediction).count() / accuracy.count()"],"metadata":{"application/vnd.databricks.v1+cell":{"title":"","showTitle":false,"inputWidgets":{},"nuid":"5a230d41-a99f-4660-8443-116d4930d0dc"}},"outputs":[{"output_type":"display_data","metadata":{"application/vnd.databricks.v1+output":{"datasetInfos":[],"data":"+------+-------------+\n|target|count(target)|\n+------+-------------+\n|   0.0|            2|\n|   1.0|            4|\n|   4.0|            8|\n+------+-------------+\nonly showing top 3 rows\n\n+----------+-----------------+\n|prediction|count(prediction)|\n+----------+-----------------+\n|       0.0|                2|\n|       1.0|                4|\n|       4.0|                8|\n+----------+-----------------+\nonly showing top 3 rows\n\nOut[129]: 1.0","removedWidgets":[],"addedWidgets":{},"metadata":{},"type":"ansi","arguments":{}}},"output_type":"display_data","data":{"text/plain":["+------+-------------+\n|target|count(target)|\n+------+-------------+\n|   0.0|            2|\n|   1.0|            4|\n|   4.0|            8|\n+------+-------------+\nonly showing top 3 rows\n\n+----------+-----------------+\n|prediction|count(prediction)|\n+----------+-----------------+\n|       0.0|                2|\n|       1.0|                4|\n|       4.0|                8|\n+----------+-----------------+\nonly showing top 3 rows\n\nOut[129]: 1.0"]}}],"execution_count":0},{"cell_type":"code","source":["#APPENDIX LogisticRegression\n#https://www.guru99.com/pyspark-tutorial.html\nselect_test = predict_test.select(\"target\", \"prediction\", \"probability\")\nselect_test.show(3)"],"metadata":{"application/vnd.databricks.v1+cell":{"title":"","showTitle":false,"inputWidgets":{},"nuid":"1f4167c6-480a-4fc4-a512-98baa90eae7e"}},"outputs":[{"output_type":"display_data","metadata":{"application/vnd.databricks.v1+output":{"datasetInfos":[],"data":"+------+----------+--------------------+\n|target|prediction|         probability|\n+------+----------+--------------------+\n|   0.0|       0.0|[0.75701127819548...|\n|   0.0|       0.0|[0.93121762740183...|\n|   1.0|       1.0|[0.17756463343419...|\n+------+----------+--------------------+\nonly showing top 3 rows\n\n","removedWidgets":[],"addedWidgets":{},"metadata":{},"type":"ansi","arguments":{}}},"output_type":"display_data","data":{"text/plain":["+------+----------+--------------------+\n|target|prediction|         probability|\n+------+----------+--------------------+\n|   0.0|       0.0|[0.75701127819548...|\n|   0.0|       0.0|[0.93121762740183...|\n|   1.0|       1.0|[0.17756463343419...|\n+------+----------+--------------------+\nonly showing top 3 rows\n\n"]}}],"execution_count":0},{"cell_type":"code","source":["#APPENDIX\n#https://discuss.itversity.com/t/error-regarding-attributeerror-nonetype-object-has-no-attribute-select/23123/2\n#https://colab.research.google.com/github/goodboychan/goodboychan.github.io/blob/main/_notebooks/2020-08-10-03-Classification-in-PySpark.ipynb#scrollTo=SrkP5B-G5iU8\nfrom pyspark.sql.functions import lit, col\nimport pyspark.sql.functions as f\nfrom pyspark.sql.types import IntegerType, FloatType\n# Create a confusion matrix\nlr_selected_df=predict_test.select(\"target\",\"prediction\", \"probability\").toDF(\"target\",\"prediction\", \"probability\")\nlr_conf_matrix = lr_selected_df.\\\nwithColumn('target', lr_selected_df.target.cast(FloatType())).\\\nwithColumn('prediction', lr_selected_df.prediction.cast(FloatType()))\nlr_conf_matrix.printSchema()\n\nlr_conf_matrix.groupBy('target', 'prediction').count().show()\n# Calculate the elements of the confusion matrix\n# TN = lr_conf_matrix.filter('prediction = 0 AND target = prediction').count()\n# TP = lr_conf_matrix.filter('prediction = 1 AND target = prediction').count()\n# FN = lr_conf_matrix.filter('prediction = 0 AND target = 1').count()\n# FP = lr_conf_matrix.filter('prediction = 1 AND target = 0').count()\n\n# # Accuracy measures the proportion of correct predictions\n# accuracy = (TN + TP) / (TN + TP + FN + FP)\n# print(accuracy)"],"metadata":{"application/vnd.databricks.v1+cell":{"title":"","showTitle":false,"inputWidgets":{},"nuid":"48f8ada1-c5a5-445c-b4c6-08916998d96b"}},"outputs":[{"output_type":"display_data","metadata":{"application/vnd.databricks.v1+output":{"datasetInfos":[],"data":"root\n |-- target: float (nullable = false)\n |-- prediction: float (nullable = false)\n |-- probability: vector (nullable = true)\n\n+------+----------+-----+\n|target|prediction|count|\n+------+----------+-----+\n|   5.0|       5.0|    2|\n|   0.0|       0.0|    2|\n|   2.0|       2.0|    6|\n|   4.0|       4.0|    8|\n|   3.0|       3.0|    4|\n|   1.0|       1.0|    4|\n+------+----------+-----+\n\n","removedWidgets":[],"addedWidgets":{},"metadata":{},"type":"ansi","arguments":{}}},"output_type":"display_data","data":{"text/plain":["root\n |-- target: float (nullable = false)\n |-- prediction: float (nullable = false)\n |-- probability: vector (nullable = true)\n\n+------+----------+-----+\n|target|prediction|count|\n+------+----------+-----+\n|   5.0|       5.0|    2|\n|   0.0|       0.0|    2|\n|   2.0|       2.0|    6|\n|   4.0|       4.0|    8|\n|   3.0|       3.0|    4|\n|   1.0|       1.0|    4|\n+------+----------+-----+\n\n"]}}],"execution_count":0},{"cell_type":"code","source":["# #APPENDIX\n# ### Use ROC \n# from pyspark.ml.evaluation import BinaryClassificationEvaluator\n#  predictions = model.transform(test_data)\n# # Evaluate model\n# evaluator = BinaryClassificationEvaluator(rawPredictionCol=\"rawPrediction\")\n# print(evaluator.evaluate(predict_test))\n# print(evaluator.getMetricName())"],"metadata":{"application/vnd.databricks.v1+cell":{"title":"","showTitle":false,"inputWidgets":{},"nuid":"63fafed4-908e-4077-8892-2ed99ab86c13"}},"outputs":[],"execution_count":0}],"metadata":{"application/vnd.databricks.v1+notebook":{"notebookName":"20_02_2022_NASA","dashboards":[],"notebookMetadata":{"pythonIndentUnit":2},"language":"python","widgets":{},"notebookOrigID":716579262882292}},"nbformat":4,"nbformat_minor":0}