建模算法/回归/线性回归/boston.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "96d0444f-57b5-4e54-b96f-d291e1b78648",
   "metadata": {},
   "outputs": [],
   "source": [
    "import os\n",
    "import pandas as pd\n",
    "import numpy as np\n",
    "import matplotlib.pyplot as plt\n",
    "import matplotlib.cm as cm\n",
    "import seaborn as sns"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "4ef9269f-7a9b-4fcf-a2ad-a74ed66af783",
   "metadata": {},
   "source": [
    "## Load Boston dataset from skit-learn"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "129f6ed5-72c6-4429-8c1a-4a06043f7c9a",
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.datasets import load_boston\n",
    "boston = load_boston()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "12b3fc08-d88f-407a-9f42-5b37b93ada35",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "dict_keys(['data', 'target', 'feature_names', 'DESCR', 'filename'])"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "boston.keys()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "7a745bb9-1175-460a-9bcf-c566d10206c5",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(506, 13)"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "boston.data.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "e20d1221-892f-4ace-b8f4-15f412b128c7",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(13,)"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "boston.feature_names.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "3dc694c6-11d0-4553-8a9e-fecb7c575674",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(506,)"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "boston.target.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "3ac87e5c-5c0d-45b0-ad5a-535839eefcbc",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "\".. _boston_dataset:\\n\\nBoston house prices dataset\\n---------------------------\\n\\n**Data Set Characteristics:**  \\n\\n    :Number of Instances: 506 \\n\\n    :Number of Attributes: 13 numeric/categorical predictive. Median Value (attribute 14) is usually the target.\\n\\n    :Attribute Information (in order):\\n        - CRIM     per capita crime rate by town\\n        - ZN       proportion of residential land zoned for lots over 25,000 sq.ft.\\n        - INDUS    proportion of non-retail business acres per town\\n        - CHAS     Charles River dummy variable (= 1 if tract bounds river; 0 otherwise)\\n        - NOX      nitric oxides concentration (parts per 10 million)\\n        - RM       average number of rooms per dwelling\\n        - AGE      proportion of owner-occupied units built prior to 1940\\n        - DIS      weighted distances to five Boston employment centres\\n        - RAD      index of accessibility to radial highways\\n        - TAX      full-value property-tax rate per $10,000\\n        - PTRATIO  pupil-teacher ratio by town\\n        - B        1000(Bk - 0.63)^2 where Bk is the proportion of black people by town\\n        - LSTAT    % lower status of the population\\n        - MEDV     Median value of owner-occupied homes in $1000's\\n\\n    :Missing Attribute Values: None\\n\\n    :Creator: Harrison, D. and Rubinfeld, D.L.\\n\\nThis is a copy of UCI ML housing dataset.\\nhttps://archive.ics.uci.edu/ml/machine-learning-databases/housing/\\n\\n\\nThis dataset was taken from the StatLib library which is maintained at Carnegie Mellon University.\\n\\nThe Boston house-price data of Harrison, D. and Rubinfeld, D.L. 'Hedonic\\nprices and the demand for clean air', J. Environ. Economics & Management,\\nvol.5, 81-102, 1978.   Used in Belsley, Kuh & Welsch, 'Regression diagnostics\\n...', Wiley, 1980.   N.B. Various transformations are used in the table on\\npages 244-261 of the latter.\\n\\nThe Boston house-price data has been used in many machine learning papers that address regression\\nproblems.   \\n     \\n.. topic:: References\\n\\n   - Belsley, Kuh & Welsch, 'Regression diagnostics: Identifying Influential Data and Sources of Collinearity', Wiley, 1980. 244-261.\\n   - Quinlan,R. (1993). Combining Instance-Based and Model-Based Learning. In Proceedings on the Tenth International Conference of Machine Learning, 236-243, University of Massachusetts, Amherst. Morgan Kaufmann.\\n\""
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "boston.DESCR"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "bc59ef48-381c-4256-aadc-b5405ea82e33",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Convert Numpy array to Pandas Data Frame\n",
    "dataset = pd.DataFrame(boston.data)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "a3ca76e6-99a9-4d7c-a9a1-31c8b4d80929",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "      CRIM    ZN  INDUS  CHAS    NOX     RM    AGE     DIS  RAD    TAX  \\\n",
      "0  0.00632  18.0   2.31   0.0  0.538  6.575   65.2  4.0900  1.0  296.0   \n",
      "1  0.02731   0.0   7.07   0.0  0.469  6.421   78.9  4.9671  2.0  242.0   \n",
      "2  0.02729   0.0   7.07   0.0  0.469  7.185   61.1  4.9671  2.0  242.0   \n",
      "3  0.03237   0.0   2.18   0.0  0.458  6.998   45.8  6.0622  3.0  222.0   \n",
      "4  0.06905   0.0   2.18   0.0  0.458  7.147   54.2  6.0622  3.0  222.0   \n",
      "5  0.02985   0.0   2.18   0.0  0.458  6.430   58.7  6.0622  3.0  222.0   \n",
      "6  0.08829  12.5   7.87   0.0  0.524  6.012   66.6  5.5605  5.0  311.0   \n",
      "7  0.14455  12.5   7.87   0.0  0.524  6.172   96.1  5.9505  5.0  311.0   \n",
      "8  0.21124  12.5   7.87   0.0  0.524  5.631  100.0  6.0821  5.0  311.0   \n",
      "9  0.17004  12.5   7.87   0.0  0.524  6.004   85.9  6.5921  5.0  311.0   \n",
      "\n",
      "   PTRATIO       B  LSTAT  \n",
      "0     15.3  396.90   4.98  \n",
      "1     17.8  396.90   9.14  \n",
      "2     17.8  392.83   4.03  \n",
      "3     18.7  394.63   2.94  \n",
      "4     18.7  396.90   5.33  \n",
      "5     18.7  394.12   5.21  \n",
      "6     15.2  395.60  12.43  \n",
      "7     15.2  396.90  19.15  \n",
      "8     15.2  386.63  29.93  \n",
      "9     15.2  386.71  17.10  \n"
     ]
    }
   ],
   "source": [
    "# Add Column Names in the Dataset\n",
    "dataset.columns = boston.feature_names\n",
    "print(dataset.head(10))"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "fd5353b5-ed3e-486e-9595-d60d87517278",
   "metadata": {},
   "source": [
    "CRIM：城镇人均犯罪率。\n",
    "\n",
    "ZN：城镇超过25000平方英尺的住宅区域的占地比例。\n",
    "\n",
    "INDUS：城镇非零售用地占地比例。\n",
    "\n",
    "CHAS：是否靠近河边，1为靠近，0为远离。\n",
    "\n",
    "NOX：一氧化氮浓度\n",
    "\n",
    "RM：每套房产的平均房间个数。\n",
    "\n",
    "AGE：在1940年之前就盖好，且业主自住的房子的比例。\n",
    "\n",
    "DIS：与波士顿市中心的距离。\n",
    "\n",
    "RAD：周边高速公路的便利性指数。\n",
    "\n",
    "TAX：每10000美元的财产税率。\n",
    "\n",
    "PTRATIO：小学老师的比例。\n",
    "\n",
    "B：城镇黑人的比例。\n",
    "\n",
    "LSTAT：地位较低的人口比例"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "1fc17d6d-b6db-4aac-98a7-f8b8abe0d520",
   "metadata": {},
   "outputs": [],
   "source": [
    "dataset['MEDV'] = boston.target"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "39750979-8b59-493c-859f-4de373a0212a",
   "metadata": {},
   "source": [
    "# 1. 数据分析与可视化"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "bf09bad8-f7ea-40da-825c-d5258db00caa",
   "metadata": {},
   "source": [
    "## 皮尔森Pearson相关性"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "67d8d329-8802-46b1-b5c8-57a466ad7a9a",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "e1259bc2a16840c0a10f533efdf18573",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Canvas(toolbar=Toolbar(toolitems=[('Home', 'Reset original view', 'home', 'home'), ('Back', 'Back to previous …"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "[Text(0.5, 0, 'CRIM'),\n",
       " Text(1.5, 0, 'ZN'),\n",
       " Text(2.5, 0, 'INDUS'),\n",
       " Text(3.5, 0, 'CHAS'),\n",
       " Text(4.5, 0, 'NOX'),\n",
       " Text(5.5, 0, 'RM'),\n",
       " Text(6.5, 0, 'AGE'),\n",
       " Text(7.5, 0, 'DIS'),\n",
       " Text(8.5, 0, 'RAD'),\n",
       " Text(9.5, 0, 'TAX'),\n",
       " Text(10.5, 0, 'PTRATIO'),\n",
       " Text(11.5, 0, 'B'),\n",
       " Text(12.5, 0, 'LSTAT'),\n",
       " Text(13.5, 0, 'MEDV')]"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "sns.set(style=\"whitegrid\")\n",
    "%matplotlib widget\n",
    "import palettable\n",
    "fig = plt.figure(figsize=(10,7.5))\n",
    "ax = sns.heatmap(dataset.corr(method = \"pearson\"), \n",
    "                annot=True, \n",
    "                cmap=palettable.cmocean.diverging.Curl_10.mpl_colors,\n",
    "                linewidths=0.5,\n",
    "                linecolor='w')\n",
    "ax.set_title('Pearson Heat Map', fontsize=20)\n",
    "ax.set_xticklabels(ax.get_xticklabels(), rotation=45)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "4d97b618-119d-4387-9f19-b30f85b6d620",
   "metadata": {},
   "source": [
    "## Spearman相关性"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "64bdb106-bb64-41f7-a0ab-370070fa3f10",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "948fd8e9c8164e0e9869de3adad05959",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Canvas(toolbar=Toolbar(toolitems=[('Home', 'Reset original view', 'home', 'home'), ('Back', 'Back to previous …"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "[Text(0.5, 0, 'CRIM'),\n",
       " Text(1.5, 0, 'ZN'),\n",
       " Text(2.5, 0, 'INDUS'),\n",
       " Text(3.5, 0, 'CHAS'),\n",
       " Text(4.5, 0, 'NOX'),\n",
       " Text(5.5, 0, 'RM'),\n",
       " Text(6.5, 0, 'AGE'),\n",
       " Text(7.5, 0, 'DIS'),\n",
       " Text(8.5, 0, 'RAD'),\n",
       " Text(9.5, 0, 'TAX'),\n",
       " Text(10.5, 0, 'PTRATIO'),\n",
       " Text(11.5, 0, 'B'),\n",
       " Text(12.5, 0, 'LSTAT'),\n",
       " Text(13.5, 0, 'MEDV')]"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "sns.set(style=\"whitegrid\")\n",
    "%matplotlib widget\n",
    "fig = plt.figure(figsize=(10,7.5))\n",
    "ax = sns.heatmap(dataset.corr(method = \"spearman\"), \n",
    "                annot=True, \n",
    "                 cmap='Blues',\n",
    "                linewidths=0.5,\n",
    "                linecolor='w')\n",
    "ax.set_title('Spearman Heat Map', fontsize=20)\n",
    "ax.set_xticklabels(ax.get_xticklabels(), rotation=45)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "44bac94b-855f-4168-8192-2e8a4496dd26",
   "metadata": {},
   "source": [
    "## Kendall相关性"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "id": "2f008eb6-cc9f-47fd-bcde-1ae7fbf6f327",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "a6cba1e83a5247f5a3b5aab51b1fe75a",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Canvas(toolbar=Toolbar(toolitems=[('Home', 'Reset original view', 'home', 'home'), ('Back', 'Back to previous …"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "[Text(0.5, 0, 'CRIM'),\n",
       " Text(1.5, 0, 'ZN'),\n",
       " Text(2.5, 0, 'INDUS'),\n",
       " Text(3.5, 0, 'CHAS'),\n",
       " Text(4.5, 0, 'NOX'),\n",
       " Text(5.5, 0, 'RM'),\n",
       " Text(6.5, 0, 'AGE'),\n",
       " Text(7.5, 0, 'DIS'),\n",
       " Text(8.5, 0, 'RAD'),\n",
       " Text(9.5, 0, 'TAX'),\n",
       " Text(10.5, 0, 'PTRATIO'),\n",
       " Text(11.5, 0, 'B'),\n",
       " Text(12.5, 0, 'LSTAT'),\n",
       " Text(13.5, 0, 'MEDV')]"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "sns.set(style=\"whitegrid\")\n",
    "%matplotlib widget\n",
    "fig = plt.figure(figsize=(10,8))\n",
    "ax = sns.heatmap(dataset.corr(method = \"kendall\"), \n",
    "                annot=True, \n",
    "                linewidths=0.5,\n",
    "                linecolor='w')\n",
    "ax.set_title('Kendall Heat Map', fontsize=20)\n",
    "ax.set_xticklabels(ax.get_xticklabels(), rotation=45)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "c80d2021-49a5-4562-93be-a3365993da7f",
   "metadata": {},
   "source": [
    "- Pearson相关系数是在原始数据的方差和协方差基础上计算得到，所以对离群值比较敏感，它度量的是线性相关。因此，即使Pearson相关系数为0，也只能说明变量之间不存在线性相关，但仍有可能存在曲线相关。\n",
    "- Spearman相关系数和Kendall相关系数都是建立在秩和观测值的相对大小的基础上得到，是一种更为一般性的非参数方法，对离群值的敏感度较低，因而也更具有耐受性，度量的主要是变量之间的联系。"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "id": "b18b26ac-67a1-419e-b295-fd6527d5a803",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "8757c92f67e9465b87d7686f159a3657",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Canvas(toolbar=Toolbar(toolitems=[('Home', 'Reset original view', 'home', 'home'), ('Back', 'Back to previous …"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "D:\\Anaconda3\\envs\\mathmodel\\lib\\site-packages\\seaborn\\distributions.py:2619: FutureWarning: `distplot` is a deprecated function and will be removed in a future version. Please adapt your code to use either `displot` (a figure-level function with similar flexibility) or `histplot` (an axes-level function for histograms).\n",
      "  warnings.warn(msg, FutureWarning)\n",
      "D:\\Anaconda3\\envs\\mathmodel\\lib\\site-packages\\seaborn\\distributions.py:2619: FutureWarning: `distplot` is a deprecated function and will be removed in a future version. Please adapt your code to use either `displot` (a figure-level function with similar flexibility) or `histplot` (an axes-level function for histograms).\n",
      "  warnings.warn(msg, FutureWarning)\n",
      "D:\\Anaconda3\\envs\\mathmodel\\lib\\site-packages\\seaborn\\distributions.py:2619: FutureWarning: `distplot` is a deprecated function and will be removed in a future version. Please adapt your code to use either `displot` (a figure-level function with similar flexibility) or `histplot` (an axes-level function for histograms).\n",
      "  warnings.warn(msg, FutureWarning)\n",
      "D:\\Anaconda3\\envs\\mathmodel\\lib\\site-packages\\seaborn\\distributions.py:2619: FutureWarning: `distplot` is a deprecated function and will be removed in a future version. Please adapt your code to use either `displot` (a figure-level function with similar flexibility) or `histplot` (an axes-level function for histograms).\n",
      "  warnings.warn(msg, FutureWarning)\n",
      "D:\\Anaconda3\\envs\\mathmodel\\lib\\site-packages\\seaborn\\distributions.py:2619: FutureWarning: `distplot` is a deprecated function and will be removed in a future version. Please adapt your code to use either `displot` (a figure-level function with similar flexibility) or `histplot` (an axes-level function for histograms).\n",
      "  warnings.warn(msg, FutureWarning)\n",
      "D:\\Anaconda3\\envs\\mathmodel\\lib\\site-packages\\seaborn\\distributions.py:2619: FutureWarning: `distplot` is a deprecated function and will be removed in a future version. Please adapt your code to use either `displot` (a figure-level function with similar flexibility) or `histplot` (an axes-level function for histograms).\n",
      "  warnings.warn(msg, FutureWarning)\n",
      "D:\\Anaconda3\\envs\\mathmodel\\lib\\site-packages\\seaborn\\distributions.py:2619: FutureWarning: `distplot` is a deprecated function and will be removed in a future version. Please adapt your code to use either `displot` (a figure-level function with similar flexibility) or `histplot` (an axes-level function for histograms).\n",
      "  warnings.warn(msg, FutureWarning)\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "<seaborn.axisgrid.PairGrid at 0x238e9950d30>"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "sns.set(style=\"whitegrid\")\n",
    "%matplotlib widget\n",
    "g = sns.PairGrid(dataset.iloc[:,::2])\n",
    "g.map_diag(sns.distplot)\n",
    "g.map_upper(plt.scatter)\n",
    "g.map_lower(sns.kdeplot)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "da974307-373e-4816-9738-d4439b980558",
   "metadata": {},
   "source": [
    "## 房价MEDV与各变量的关系"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "id": "61bc1703-6cc1-47c3-b3b1-d56b7b5983f3",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "87adff7af47d4212b6dfcd2177bed9ab",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Canvas(toolbar=Toolbar(toolitems=[('Home', 'Reset original view', 'home', 'home'), ('Back', 'Back to previous …"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "Text(0.5, 1.0, 'Correlations between MEDV and variables')"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "sns.set(style=\"whitegrid\")\n",
    "%matplotlib widget\n",
    "fig = plt.figure(figsize=(10,7.5))\n",
    "dataset.corr()['MEDV'].sort_values(ascending = False).plot(kind='bar')\n",
    "plt.title(\"Correlations between MEDV and variables\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "id": "31e7b85e-9d97-429e-ae77-0f516100f935",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "eae9b0891ea146b6af34afab4b65e7a9",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Canvas(toolbar=Toolbar(toolitems=[('Home', 'Reset original view', 'home', 'home'), ('Back', 'Back to previous …"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "Text(0.5, 1, '各属性与房价的关系')"
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "sns.set(style=\"whitegrid\")\n",
    "%matplotlib widget\n",
    "plt.rcParams[\"font.family\"] = 'SimHei'  # 将字体改为中文\n",
    "plt.rcParams['axes.unicode_minus'] = False  # 设置了中文字体默认后，坐标的\"-\"号无法显示，设置这个参数就可以避免\n",
    "plt.figure(figsize=(16, 10))  # 设置绘图尺寸\n",
    "column = len(dataset.columns)\n",
    "\n",
    "for i in range(column-1):\n",
    "    plt.subplot(4, 4, (i + 1))\n",
    "    plt.scatter(dataset.iloc[:,i], dataset['MEDV'], edgecolors='w', alpha=0.7)\n",
    "    plt.xlabel('{}'.format(dataset.columns[i]), fontsize=10)  # 设置x轴标签文本\n",
    "    plt.ylabel('房价', fontsize=10)\n",
    "    plt.title('波士顿房价与{}关系'.format(dataset.columns[i]), fontsize=10)  # 设置图标题\n",
    "\n",
    "plt.tight_layout(rect=[0,0,1,0.9])                                   #   优化子图与总标题的位置，防止重叠\n",
    "plt.suptitle('各属性与房价的关系', x=0.5, y=1, fontsize=20)          #   设置总标题\n",
    "\n"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "24575ecc-7e7b-4cd9-80a6-68d79e9f7b12",
   "metadata": {},
   "source": [
    "## 单变量的分布"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "id": "11f9d664-e20d-4d7a-bcd1-743ef0c3ec45",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "911207f00a2e4a3999dff6695165c68b",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Canvas(toolbar=Toolbar(toolitems=[('Home', 'Reset original view', 'home', 'home'), ('Back', 'Back to previous …"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "D:\\Anaconda3\\envs\\mathmodel\\lib\\site-packages\\seaborn\\distributions.py:2619: FutureWarning: `distplot` is a deprecated function and will be removed in a future version. Please adapt your code to use either `displot` (a figure-level function with similar flexibility) or `histplot` (an axes-level function for histograms).\n",
      "  warnings.warn(msg, FutureWarning)\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "Text(0, 0.5, 'Frequency')"
      ]
     },
     "execution_count": 17,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "sns.set(style=\"whitegrid\")\n",
    "%matplotlib widget\n",
    "fig = plt.figure(figsize=(10,7.5))\n",
    "plt.grid(axis='y', alpha=0.5)\n",
    "ax = sns.distplot(dataset['MEDV'], bins=30, hist_kws=dict(edgecolor=\"w\", linewidth=2))\n",
    "ax.set_title('Histogram', fontsize=20)\n",
    "ax.set_xlabel('MEDV or Price', fontsize=20)\n",
    "ax.set_ylabel('Frequency', fontsize=20)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "96d985cc-e8bd-4c54-b3bd-8b7ec90332e5",
   "metadata": {},
   "source": [
    "# 2. 准备数据集"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "id": "20c3ac8c-4f15-4e56-b790-49dd1cab2f74",
   "metadata": {},
   "outputs": [],
   "source": [
    "X=dataset.iloc[:,dataset.keys()!='MEDV']\n",
    "# Z-SCORE标准化\n",
    "X=(X-X.mean())/(X.std())  \n",
    "Y = dataset['MEDV'] "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "id": "9f960415-7c92-4e46-bb47-aa271f396280",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "X Train:  (404, 13)\n",
      "Y Train:  (404,)\n",
      "X Test:  (102, 13)\n",
      "Y Test:  (102,)\n"
     ]
    }
   ],
   "source": [
    "# 划分数据集\n",
    "from sklearn.model_selection import train_test_split\n",
    "\n",
    "X_train, X_test, Y_train, Y_test = train_test_split(X, Y, test_size = 0.20, random_state = 0)\n",
    "print(\"X Train: \", X_train.shape)\n",
    "print(\"Y Train: \", Y_train.shape)\n",
    "print(\"X Test: \", X_test.shape)\n",
    "print(\"Y Test: \", Y_test.shape)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "a24c3c67-d0d4-441d-9871-826a1e672226",
   "metadata": {},
   "source": [
    "# 3. 建立模型"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "7bb0bafd-6da5-4292-b5a8-dc939a46bf4b",
   "metadata": {},
   "source": [
    "## 3.1 多变量线性回归 LinearRegression"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "id": "0dd0a3d2-9966-43f6-bffa-d6844d4640d2",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "LinearRegression()"
      ]
     },
     "execution_count": 20,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from sklearn.linear_model import LinearRegression\n",
    "lr = LinearRegression()\n",
    "lr.fit(X_train, Y_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "id": "45f44d10-b037-4ac4-a44a-92feac3f45a0",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[-1.0273982   1.0443783   0.03763083  0.59455017 -1.86836579  2.60580253\n",
      " -0.0878549  -2.91935098  2.12612403 -1.85216165 -2.2643624   0.74041111\n",
      " -3.51906316]\n",
      "22.480352884751227\n"
     ]
    }
   ],
   "source": [
    "# coef_存放回归系数，intercept_则存放截距\n",
    "print(lr.coef_)\n",
    "print(lr.intercept_)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "afaf063f-8901-42c5-a090-1daa0abf6bc9",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "id": "20c8474a-0504-4bc5-994c-9b2cd7d1d9fc",
   "metadata": {},
   "outputs": [],
   "source": [
    "# 预测\n",
    "Y_pred = lr.predict(X_test) "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "id": "ef39f121-9d67-4e10-b674-e77c294cd278",
   "metadata": {},
   "outputs": [],
   "source": [
    "model_lr = pd.DataFrame(X_test)\n",
    "model_lr['MEDV'] = Y_test\n",
    "model_lr['Predicted MEDV'] = Y_pred"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "id": "3fb712bc-7978-4cda-aef3-d00d3c717d03",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>CRIM</th>\n",
       "      <th>ZN</th>\n",
       "      <th>INDUS</th>\n",
       "      <th>CHAS</th>\n",
       "      <th>NOX</th>\n",
       "      <th>RM</th>\n",
       "      <th>AGE</th>\n",
       "      <th>DIS</th>\n",
       "      <th>RAD</th>\n",
       "      <th>TAX</th>\n",
       "      <th>PTRATIO</th>\n",
       "      <th>B</th>\n",
       "      <th>LSTAT</th>\n",
       "      <th>MEDV</th>\n",
       "      <th>Predicted MEDV</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>329</th>\n",
       "      <td>-0.412284</td>\n",
       "      <td>-0.487240</td>\n",
       "      <td>-1.151075</td>\n",
       "      <td>-0.272329</td>\n",
       "      <td>-0.817198</td>\n",
       "      <td>0.068836</td>\n",
       "      <td>-1.825115</td>\n",
       "      <td>0.674147</td>\n",
       "      <td>-0.637331</td>\n",
       "      <td>0.129128</td>\n",
       "      <td>-0.718509</td>\n",
       "      <td>0.203034</td>\n",
       "      <td>-0.744016</td>\n",
       "      <td>22.6</td>\n",
       "      <td>24.889638</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>371</th>\n",
       "      <td>0.653229</td>\n",
       "      <td>-0.487240</td>\n",
       "      <td>1.014995</td>\n",
       "      <td>-0.272329</td>\n",
       "      <td>0.658496</td>\n",
       "      <td>-0.097684</td>\n",
       "      <td>1.116390</td>\n",
       "      <td>-1.247058</td>\n",
       "      <td>1.659603</td>\n",
       "      <td>1.529413</td>\n",
       "      <td>0.805778</td>\n",
       "      <td>0.103795</td>\n",
       "      <td>-0.437339</td>\n",
       "      <td>50.0</td>\n",
       "      <td>23.721411</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>219</th>\n",
       "      <td>-0.406819</td>\n",
       "      <td>-0.487240</td>\n",
       "      <td>0.401324</td>\n",
       "      <td>3.664771</td>\n",
       "      <td>-0.040517</td>\n",
       "      <td>0.125766</td>\n",
       "      <td>0.846397</td>\n",
       "      <td>-0.205034</td>\n",
       "      <td>-0.522484</td>\n",
       "      <td>-0.784617</td>\n",
       "      <td>-0.949462</td>\n",
       "      <td>0.406003</td>\n",
       "      <td>-0.301505</td>\n",
       "      <td>23.0</td>\n",
       "      <td>29.364999</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>403</th>\n",
       "      <td>2.463299</td>\n",
       "      <td>-0.487240</td>\n",
       "      <td>1.014995</td>\n",
       "      <td>-0.272329</td>\n",
       "      <td>1.193543</td>\n",
       "      <td>-1.331642</td>\n",
       "      <td>0.974288</td>\n",
       "      <td>-0.993604</td>\n",
       "      <td>1.659603</td>\n",
       "      <td>1.529413</td>\n",
       "      <td>0.805778</td>\n",
       "      <td>0.440616</td>\n",
       "      <td>0.996622</td>\n",
       "      <td>8.3</td>\n",
       "      <td>12.122386</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>78</th>\n",
       "      <td>-0.413538</td>\n",
       "      <td>-0.487240</td>\n",
       "      <td>0.246813</td>\n",
       "      <td>-0.272329</td>\n",
       "      <td>-1.015684</td>\n",
       "      <td>-0.074912</td>\n",
       "      <td>-0.528437</td>\n",
       "      <td>0.578929</td>\n",
       "      <td>-0.522484</td>\n",
       "      <td>-0.060741</td>\n",
       "      <td>0.112920</td>\n",
       "      <td>0.325604</td>\n",
       "      <td>-0.043840</td>\n",
       "      <td>21.2</td>\n",
       "      <td>21.443823</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>56</th>\n",
       "      <td>-0.417713</td>\n",
       "      <td>3.157316</td>\n",
       "      <td>-1.515487</td>\n",
       "      <td>-0.272329</td>\n",
       "      <td>-1.248688</td>\n",
       "      <td>0.139999</td>\n",
       "      <td>-1.167895</td>\n",
       "      <td>2.560921</td>\n",
       "      <td>-0.867024</td>\n",
       "      <td>-0.565081</td>\n",
       "      <td>-0.533747</td>\n",
       "      <td>0.440616</td>\n",
       "      <td>-0.963871</td>\n",
       "      <td>24.7</td>\n",
       "      <td>25.442171</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>455</th>\n",
       "      <td>0.132400</td>\n",
       "      <td>-0.487240</td>\n",
       "      <td>1.014995</td>\n",
       "      <td>-0.272329</td>\n",
       "      <td>1.366138</td>\n",
       "      <td>0.342100</td>\n",
       "      <td>0.636797</td>\n",
       "      <td>-0.645503</td>\n",
       "      <td>1.659603</td>\n",
       "      <td>1.529413</td>\n",
       "      <td>0.805778</td>\n",
       "      <td>-3.349082</td>\n",
       "      <td>0.766964</td>\n",
       "      <td>14.1</td>\n",
       "      <td>15.571783</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>60</th>\n",
       "      <td>-0.402742</td>\n",
       "      <td>0.584688</td>\n",
       "      <td>-0.875579</td>\n",
       "      <td>-0.272329</td>\n",
       "      <td>-0.877607</td>\n",
       "      <td>-0.773728</td>\n",
       "      <td>-0.084369</td>\n",
       "      <td>1.629074</td>\n",
       "      <td>-0.177944</td>\n",
       "      <td>-0.737150</td>\n",
       "      <td>0.574826</td>\n",
       "      <td>0.421009</td>\n",
       "      <td>0.069589</td>\n",
       "      <td>18.7</td>\n",
       "      <td>17.937195</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>213</th>\n",
       "      <td>-0.403765</td>\n",
       "      <td>-0.487240</td>\n",
       "      <td>-0.079701</td>\n",
       "      <td>-0.272329</td>\n",
       "      <td>-0.566935</td>\n",
       "      <td>0.128613</td>\n",
       "      <td>-1.288681</td>\n",
       "      <td>0.071405</td>\n",
       "      <td>-0.637331</td>\n",
       "      <td>-0.778684</td>\n",
       "      <td>0.066730</td>\n",
       "      <td>0.319141</td>\n",
       "      <td>-0.458344</td>\n",
       "      <td>28.1</td>\n",
       "      <td>25.305888</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>108</th>\n",
       "      <td>-0.405218</td>\n",
       "      <td>-0.487240</td>\n",
       "      <td>-0.375604</td>\n",
       "      <td>-0.272329</td>\n",
       "      <td>-0.299411</td>\n",
       "      <td>0.269515</td>\n",
       "      <td>1.013366</td>\n",
       "      <td>-0.646880</td>\n",
       "      <td>-0.522484</td>\n",
       "      <td>-0.143809</td>\n",
       "      <td>1.129112</td>\n",
       "      <td>0.422433</td>\n",
       "      <td>-0.053642</td>\n",
       "      <td>19.8</td>\n",
       "      <td>22.373233</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>102 rows × 15 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "         CRIM        ZN     INDUS      CHAS       NOX        RM       AGE  \\\n",
       "329 -0.412284 -0.487240 -1.151075 -0.272329 -0.817198  0.068836 -1.825115   \n",
       "371  0.653229 -0.487240  1.014995 -0.272329  0.658496 -0.097684  1.116390   \n",
       "219 -0.406819 -0.487240  0.401324  3.664771 -0.040517  0.125766  0.846397   \n",
       "403  2.463299 -0.487240  1.014995 -0.272329  1.193543 -1.331642  0.974288   \n",
       "78  -0.413538 -0.487240  0.246813 -0.272329 -1.015684 -0.074912 -0.528437   \n",
       "..        ...       ...       ...       ...       ...       ...       ...   \n",
       "56  -0.417713  3.157316 -1.515487 -0.272329 -1.248688  0.139999 -1.167895   \n",
       "455  0.132400 -0.487240  1.014995 -0.272329  1.366138  0.342100  0.636797   \n",
       "60  -0.402742  0.584688 -0.875579 -0.272329 -0.877607 -0.773728 -0.084369   \n",
       "213 -0.403765 -0.487240 -0.079701 -0.272329 -0.566935  0.128613 -1.288681   \n",
       "108 -0.405218 -0.487240 -0.375604 -0.272329 -0.299411  0.269515  1.013366   \n",
       "\n",
       "          DIS       RAD       TAX   PTRATIO         B     LSTAT  MEDV  \\\n",
       "329  0.674147 -0.637331  0.129128 -0.718509  0.203034 -0.744016  22.6   \n",
       "371 -1.247058  1.659603  1.529413  0.805778  0.103795 -0.437339  50.0   \n",
       "219 -0.205034 -0.522484 -0.784617 -0.949462  0.406003 -0.301505  23.0   \n",
       "403 -0.993604  1.659603  1.529413  0.805778  0.440616  0.996622   8.3   \n",
       "78   0.578929 -0.522484 -0.060741  0.112920  0.325604 -0.043840  21.2   \n",
       "..        ...       ...       ...       ...       ...       ...   ...   \n",
       "56   2.560921 -0.867024 -0.565081 -0.533747  0.440616 -0.963871  24.7   \n",
       "455 -0.645503  1.659603  1.529413  0.805778 -3.349082  0.766964  14.1   \n",
       "60   1.629074 -0.177944 -0.737150  0.574826  0.421009  0.069589  18.7   \n",
       "213  0.071405 -0.637331 -0.778684  0.066730  0.319141 -0.458344  28.1   \n",
       "108 -0.646880 -0.522484 -0.143809  1.129112  0.422433 -0.053642  19.8   \n",
       "\n",
       "     Predicted MEDV  \n",
       "329       24.889638  \n",
       "371       23.721411  \n",
       "219       29.364999  \n",
       "403       12.122386  \n",
       "78        21.443823  \n",
       "..              ...  \n",
       "56        25.442171  \n",
       "455       15.571783  \n",
       "60        17.937195  \n",
       "213       25.305888  \n",
       "108       22.373233  \n",
       "\n",
       "[102 rows x 15 columns]"
      ]
     },
     "execution_count": 24,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "model_lr"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "e37dd0e4-479f-4d8b-b6f8-8d23735396b7",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "id": "76396985-ebcd-4eaf-aeda-578c8179899c",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "mse:33.448979997676545,mae:3.8429092204444983\n"
     ]
    }
   ],
   "source": [
    "# Get Mean Squared Error (MSE)\n",
    "from sklearn.metrics import mean_squared_error\n",
    "mse = mean_squared_error(Y_test, Y_pred)\n",
    "# Get Mean Absolute Error (MAE)\n",
    "from sklearn.metrics import mean_absolute_error\n",
    "mae = mean_absolute_error(Y_test, Y_pred)\n",
    "print(f\"mse:{mse},mae:{mae}\")"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "9cd3f500-f5ba-4d92-9b4e-364f801817f3",
   "metadata": {},
   "source": [
    "线性回归模型常用的优化方法，包括增加多项式特征以及数据归一化处理等\n",
    "## 3.2 多项式回归-非线性 y=wx+b -> y=wx+w1x^2 +w2x^3+…… "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "id": "629103d1-cf57-494b-a731-a67699648bc5",
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.linear_model import LinearRegression\n",
    "from sklearn.preprocessing import PolynomialFeatures\n",
    "from sklearn.pipeline import Pipeline\n",
    "def polynomial_model(degree=1):\n",
    "    polynomial_features = PolynomialFeatures(degree=degree,include_bias=False)\n",
    "    linear_regression = LinearRegression(normalize=True)\n",
    "    # 这是一个流水线，先增加多项式阶数，然后再用线性回归算法来拟合数据\n",
    "    pipeline = Pipeline([(\"polynomial_features\", polynomial_features),\n",
    "                         (\"linear_regression\", linear_regression)])\n",
    "    return pipeline"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "id": "c2984092-11e6-41bb-9d28-7bd0dbf4bb3f",
   "metadata": {},
   "outputs": [],
   "source": [
    "X_train, X_test, Y_train, Y_test = train_test_split(X, Y, test_size = 0.20, random_state = 0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "id": "3a353549-b5ee-416e-ba9f-33fdc9ebaa2c",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Pipeline(steps=[('polynomial_features', PolynomialFeatures(include_bias=False)),\n",
       "                ('linear_regression', LinearRegression(normalize=True))])"
      ]
     },
     "execution_count": 28,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "poly_model = polynomial_model(degree=2)\n",
    "poly_model.fit(X_train, Y_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "id": "a8449d29-a257-46bd-9ec9-62f1bd34338d",
   "metadata": {},
   "outputs": [],
   "source": [
    "# 预测\n",
    "Y_pred = poly_model.predict(X_test) "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "id": "4adfccde-4b13-4df3-9877-d2443bbbdc16",
   "metadata": {},
   "outputs": [],
   "source": [
    "model_poly = pd.DataFrame(X_test)\n",
    "model_poly['MEDV'] = Y_test\n",
    "model_poly['Predicted MEDV'] = Y_pred"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "id": "eeff6c4f-da10-47dd-acb5-e5e629819967",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "mse:31.277814971447977,mae:3.32776330010088\n"
     ]
    }
   ],
   "source": [
    "# Get Mean Squared Error (MSE)\n",
    "from sklearn.metrics import mean_squared_error\n",
    "mse = mean_squared_error(Y_test, Y_pred)\n",
    "# Get Mean Absolute Error (MAE)\n",
    "from sklearn.metrics import mean_absolute_error\n",
    "mae = mean_absolute_error(Y_test, Y_pred)\n",
    "print(f\"mse:{mse},mae:{mae}\")"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "67083c58-59b2-4ecf-862e-d16f51e96ffd",
   "metadata": {},
   "source": [
    "## 3.3 神经网络+ensemble\n",
    "https://ensemble-pytorch.readthedocs.io/en/stable/introduction.html"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "id": "ad92576d-cfd5-4046-af59-dd2e60e77f4a",
   "metadata": {},
   "outputs": [],
   "source": [
    "import torch\n",
    "import torch.nn as nn\n",
    "from torch.nn import functional as F\n",
    "from torch.utils.data import TensorDataset, DataLoader\n",
    "from torchensemble import GradientBoostingRegressor,BaggingRegressor,FusionRegressor,VotingRegressor,SnapshotEnsembleRegressor"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "id": "533dbb28-5ad1-40ea-8915-74b2dd27c1cc",
   "metadata": {},
   "outputs": [],
   "source": [
    "# 1.定义一个model\n",
    "class MLP(nn.Module):\n",
    "\n",
    "    def __init__(self):\n",
    "        super(MLP, self).__init__()\n",
    "        self.linear1 = nn.Linear(13, 128)\n",
    "        self.linear2 = nn.Linear(128, 128)\n",
    "        self.linear3 = nn.Linear(128, 1)\n",
    "\n",
    "    def forward(self, x):\n",
    "        x = x.view(x.size()[0], -1)\n",
    "        x = F.relu(self.linear1(x))\n",
    "        x = F.relu(self.linear2(x))\n",
    "        x = self.linear3(x)\n",
    "        return x\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "id": "8bb4b05f-089d-4c3d-a3b7-f9b9e7013002",
   "metadata": {},
   "outputs": [],
   "source": [
    "# 2. Define the ensemble\n",
    "mlp_model = BaggingRegressor(\n",
    "    estimator=MLP,\n",
    "    n_estimators=10,\n",
    "    cuda=True,\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "id": "f7598049-578d-4276-ad54-d695593fbff9",
   "metadata": {},
   "outputs": [],
   "source": [
    "# 3.准备数据\n",
    "X_train, X_test, Y_train, Y_test = train_test_split(X, Y, test_size = 0.20, random_state = 0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "id": "262441ad-3f21-40e6-a705-f207dc367fbc",
   "metadata": {},
   "outputs": [],
   "source": [
    "X_train=torch.FloatTensor(np.array(X_train))\n",
    "X_test=torch.FloatTensor(np.array(X_test))\n",
    "Y_train=torch.FloatTensor(np.array(Y_train)).reshape(-1, 1)\n",
    "Y_test=torch.FloatTensor(np.array(Y_test)).reshape(-1, 1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "id": "915a394c-ac30-4a10-8b05-a122c415392d",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "torch.Size([404, 13])"
      ]
     },
     "execution_count": 37,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "X_train.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "id": "d4e86f9e-a4f7-4f0b-846a-d6ba221bb7fe",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "torch.Size([404, 1])"
      ]
     },
     "execution_count": 38,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "Y_train.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "id": "c1565332-ebcd-4b3b-bcf6-73d2e142c535",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Tensor -> Data loader\n",
    "train_data = TensorDataset(X_train, Y_train)\n",
    "train_loader = DataLoader(train_data, batch_size=10, shuffle=True)\n",
    "\n",
    "test_data = TensorDataset(X_test, Y_test)\n",
    "test_loader = DataLoader(test_data, batch_size=10, shuffle=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "id": "ccfd53cd-41d5-4f51-a9a3-c9a63c15630e",
   "metadata": {},
   "outputs": [],
   "source": [
    "# 4. 设置优化器optimizer\n",
    "lr = 1e-3\n",
    "weight_decay = 5e-4\n",
    "mlp_model.set_optimizer(\"Adam\", lr=lr, weight_decay=weight_decay)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "id": "94fbbd44-8573-430a-aa64-05f7ae4159c0",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Estimator: 000 | Epoch: 000 | Batch: 000 | Loss: 734.86200\n",
      "Estimator: 001 | Epoch: 000 | Batch: 000 | Loss: 548.19379\n",
      "Estimator: 002 | Epoch: 000 | Batch: 000 | Loss: 901.56519\n",
      "Estimator: 003 | Epoch: 000 | Batch: 000 | Loss: 491.26514\n",
      "Estimator: 004 | Epoch: 000 | Batch: 000 | Loss: 778.43250\n",
      "Estimator: 005 | Epoch: 000 | Batch: 000 | Loss: 825.58850\n",
      "Estimator: 006 | Epoch: 000 | Batch: 000 | Loss: 668.14386\n",
      "Estimator: 007 | Epoch: 000 | Batch: 000 | Loss: 1020.64124\n",
      "Estimator: 008 | Epoch: 000 | Batch: 000 | Loss: 414.30469\n",
      "Estimator: 009 | Epoch: 000 | Batch: 000 | Loss: 666.83954\n",
      "Estimator: 000 | Epoch: 001 | Batch: 000 | Loss: 556.80688\n",
      "Estimator: 001 | Epoch: 001 | Batch: 000 | Loss: 304.08466\n",
      "Estimator: 002 | Epoch: 001 | Batch: 000 | Loss: 108.85747\n",
      "Estimator: 003 | Epoch: 001 | Batch: 000 | Loss: 279.30270\n",
      "Estimator: 004 | Epoch: 001 | Batch: 000 | Loss: 151.60258\n",
      "Estimator: 005 | Epoch: 001 | Batch: 000 | Loss: 371.02634\n",
      "Estimator: 006 | Epoch: 001 | Batch: 000 | Loss: 302.62329\n",
      "Estimator: 007 | Epoch: 001 | Batch: 000 | Loss: 262.92953\n",
      "Estimator: 008 | Epoch: 001 | Batch: 000 | Loss: 84.46468\n",
      "Estimator: 009 | Epoch: 001 | Batch: 000 | Loss: 160.67094\n",
      "Estimator: 000 | Epoch: 002 | Batch: 000 | Loss: 76.11664\n",
      "Estimator: 001 | Epoch: 002 | Batch: 000 | Loss: 14.33029\n",
      "Estimator: 002 | Epoch: 002 | Batch: 000 | Loss: 95.53429\n",
      "Estimator: 003 | Epoch: 002 | Batch: 000 | Loss: 88.87371\n",
      "Estimator: 004 | Epoch: 002 | Batch: 000 | Loss: 61.03003\n",
      "Estimator: 005 | Epoch: 002 | Batch: 000 | Loss: 27.29717\n",
      "Estimator: 006 | Epoch: 002 | Batch: 000 | Loss: 49.59487\n",
      "Estimator: 007 | Epoch: 002 | Batch: 000 | Loss: 53.88290\n",
      "Estimator: 008 | Epoch: 002 | Batch: 000 | Loss: 41.16265\n",
      "Estimator: 009 | Epoch: 002 | Batch: 000 | Loss: 40.10055\n",
      "Estimator: 000 | Epoch: 003 | Batch: 000 | Loss: 31.58309\n",
      "Estimator: 001 | Epoch: 003 | Batch: 000 | Loss: 19.09933\n",
      "Estimator: 002 | Epoch: 003 | Batch: 000 | Loss: 22.84809\n",
      "Estimator: 003 | Epoch: 003 | Batch: 000 | Loss: 17.53284\n",
      "Estimator: 004 | Epoch: 003 | Batch: 000 | Loss: 10.74783\n",
      "Estimator: 005 | Epoch: 003 | Batch: 000 | Loss: 11.57995\n",
      "Estimator: 006 | Epoch: 003 | Batch: 000 | Loss: 17.73235\n",
      "Estimator: 007 | Epoch: 003 | Batch: 000 | Loss: 23.50197\n",
      "Estimator: 008 | Epoch: 003 | Batch: 000 | Loss: 41.85359\n",
      "Estimator: 009 | Epoch: 003 | Batch: 000 | Loss: 28.08158\n",
      "Estimator: 000 | Epoch: 004 | Batch: 000 | Loss: 3.88176\n",
      "Estimator: 001 | Epoch: 004 | Batch: 000 | Loss: 9.08768\n",
      "Estimator: 002 | Epoch: 004 | Batch: 000 | Loss: 21.99819\n",
      "Estimator: 003 | Epoch: 004 | Batch: 000 | Loss: 18.27651\n",
      "Estimator: 004 | Epoch: 004 | Batch: 000 | Loss: 13.50175\n",
      "Estimator: 005 | Epoch: 004 | Batch: 000 | Loss: 7.85741\n",
      "Estimator: 006 | Epoch: 004 | Batch: 000 | Loss: 28.82802\n",
      "Estimator: 007 | Epoch: 004 | Batch: 000 | Loss: 7.88035\n",
      "Estimator: 008 | Epoch: 004 | Batch: 000 | Loss: 15.73082\n",
      "Estimator: 009 | Epoch: 004 | Batch: 000 | Loss: 7.43794\n",
      "Estimator: 000 | Epoch: 005 | Batch: 000 | Loss: 16.57973\n",
      "Estimator: 001 | Epoch: 005 | Batch: 000 | Loss: 8.83157\n",
      "Estimator: 002 | Epoch: 005 | Batch: 000 | Loss: 5.01763\n",
      "Estimator: 003 | Epoch: 005 | Batch: 000 | Loss: 8.69296\n",
      "Estimator: 004 | Epoch: 005 | Batch: 000 | Loss: 11.26826\n",
      "Estimator: 005 | Epoch: 005 | Batch: 000 | Loss: 6.97332\n",
      "Estimator: 006 | Epoch: 005 | Batch: 000 | Loss: 14.58143\n",
      "Estimator: 007 | Epoch: 005 | Batch: 000 | Loss: 13.83267\n",
      "Estimator: 008 | Epoch: 005 | Batch: 000 | Loss: 11.00585\n",
      "Estimator: 009 | Epoch: 005 | Batch: 000 | Loss: 4.99696\n",
      "Estimator: 000 | Epoch: 006 | Batch: 000 | Loss: 6.88395\n",
      "Estimator: 001 | Epoch: 006 | Batch: 000 | Loss: 4.50396\n",
      "Estimator: 002 | Epoch: 006 | Batch: 000 | Loss: 4.76217\n",
      "Estimator: 003 | Epoch: 006 | Batch: 000 | Loss: 17.15960\n",
      "Estimator: 004 | Epoch: 006 | Batch: 000 | Loss: 13.20639\n",
      "Estimator: 005 | Epoch: 006 | Batch: 000 | Loss: 7.38454\n",
      "Estimator: 006 | Epoch: 006 | Batch: 000 | Loss: 14.13037\n",
      "Estimator: 007 | Epoch: 006 | Batch: 000 | Loss: 7.60095\n",
      "Estimator: 008 | Epoch: 006 | Batch: 000 | Loss: 3.02759\n",
      "Estimator: 009 | Epoch: 006 | Batch: 000 | Loss: 5.79994\n",
      "Estimator: 000 | Epoch: 007 | Batch: 000 | Loss: 8.34326\n",
      "Estimator: 001 | Epoch: 007 | Batch: 000 | Loss: 43.81369\n",
      "Estimator: 002 | Epoch: 007 | Batch: 000 | Loss: 4.21651\n",
      "Estimator: 003 | Epoch: 007 | Batch: 000 | Loss: 6.15753\n",
      "Estimator: 004 | Epoch: 007 | Batch: 000 | Loss: 6.08470\n",
      "Estimator: 005 | Epoch: 007 | Batch: 000 | Loss: 66.54764\n",
      "Estimator: 006 | Epoch: 007 | Batch: 000 | Loss: 12.92324\n",
      "Estimator: 007 | Epoch: 007 | Batch: 000 | Loss: 11.47519\n",
      "Estimator: 008 | Epoch: 007 | Batch: 000 | Loss: 10.21284\n",
      "Estimator: 009 | Epoch: 007 | Batch: 000 | Loss: 13.50311\n",
      "Estimator: 000 | Epoch: 008 | Batch: 000 | Loss: 9.85352\n",
      "Estimator: 001 | Epoch: 008 | Batch: 000 | Loss: 6.33376\n",
      "Estimator: 002 | Epoch: 008 | Batch: 000 | Loss: 12.86076\n",
      "Estimator: 003 | Epoch: 008 | Batch: 000 | Loss: 5.43432\n",
      "Estimator: 004 | Epoch: 008 | Batch: 000 | Loss: 14.15513\n",
      "Estimator: 005 | Epoch: 008 | Batch: 000 | Loss: 6.24753\n",
      "Estimator: 006 | Epoch: 008 | Batch: 000 | Loss: 13.86496\n",
      "Estimator: 007 | Epoch: 008 | Batch: 000 | Loss: 8.28096\n",
      "Estimator: 008 | Epoch: 008 | Batch: 000 | Loss: 66.95647\n",
      "Estimator: 009 | Epoch: 008 | Batch: 000 | Loss: 1.55450\n",
      "Estimator: 000 | Epoch: 009 | Batch: 000 | Loss: 11.32053\n",
      "Estimator: 001 | Epoch: 009 | Batch: 000 | Loss: 14.71975\n",
      "Estimator: 002 | Epoch: 009 | Batch: 000 | Loss: 3.84520\n",
      "Estimator: 003 | Epoch: 009 | Batch: 000 | Loss: 14.68738\n",
      "Estimator: 004 | Epoch: 009 | Batch: 000 | Loss: 2.95386\n",
      "Estimator: 005 | Epoch: 009 | Batch: 000 | Loss: 10.11945\n",
      "Estimator: 006 | Epoch: 009 | Batch: 000 | Loss: 9.11514\n",
      "Estimator: 007 | Epoch: 009 | Batch: 000 | Loss: 2.30307\n",
      "Estimator: 008 | Epoch: 009 | Batch: 000 | Loss: 12.85772\n",
      "Estimator: 009 | Epoch: 009 | Batch: 000 | Loss: 3.76540\n"
     ]
    }
   ],
   "source": [
    "# 5. 开始训练\n",
    "mlp_model.fit(train_loader,epochs=10)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "id": "e67c5bb6-bde7-4813-aac9-b6dbd8f1aad4",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "testing_mse = mlp_model.evaluate(test_loader)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "id": "181eb328-6843-4f56-b6a2-67cbc8af2658",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "26.10006436434659"
      ]
     },
     "execution_count": 41,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "testing_mse"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "id": "e65be3f8-b8bd-434e-a305-053a28486396",
   "metadata": {},
   "outputs": [],
   "source": [
    "# 预测\n",
    "Y_pred = mlp_model.predict(X_test) "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "id": "409f0518-050f-4bab-9ef9-5f1cc4f54b48",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "torch.Size([102, 1])"
      ]
     },
     "execution_count": 43,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "Y_pred.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "id": "8c5d632f-6522-4533-bf6b-8a4aee9e8931",
   "metadata": {},
   "outputs": [],
   "source": [
    "model_MLP_vote = pd.DataFrame(np.array(X_test))\n",
    "model_MLP_vote['MEDV'] = np.array(Y_test)\n",
    "model_MLP_vote['Predicted MEDV'] = np.array(Y_pred)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "id": "22fbb4c5-69a8-4be4-be86-7c257cce8561",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>0</th>\n",
       "      <th>1</th>\n",
       "      <th>2</th>\n",
       "      <th>3</th>\n",
       "      <th>4</th>\n",
       "      <th>5</th>\n",
       "      <th>6</th>\n",
       "      <th>7</th>\n",
       "      <th>8</th>\n",
       "      <th>9</th>\n",
       "      <th>10</th>\n",
       "      <th>11</th>\n",
       "      <th>12</th>\n",
       "      <th>MEDV</th>\n",
       "      <th>Predicted MEDV</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>-0.412284</td>\n",
       "      <td>-0.487240</td>\n",
       "      <td>-1.151075</td>\n",
       "      <td>-0.272329</td>\n",
       "      <td>-0.817198</td>\n",
       "      <td>0.068836</td>\n",
       "      <td>-1.825115</td>\n",
       "      <td>0.674147</td>\n",
       "      <td>-0.637331</td>\n",
       "      <td>0.129128</td>\n",
       "      <td>-0.718509</td>\n",
       "      <td>0.203034</td>\n",
       "      <td>-0.744016</td>\n",
       "      <td>22.600000</td>\n",
       "      <td>25.710272</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>0.653229</td>\n",
       "      <td>-0.487240</td>\n",
       "      <td>1.014995</td>\n",
       "      <td>-0.272329</td>\n",
       "      <td>0.658496</td>\n",
       "      <td>-0.097684</td>\n",
       "      <td>1.116390</td>\n",
       "      <td>-1.247058</td>\n",
       "      <td>1.659603</td>\n",
       "      <td>1.529413</td>\n",
       "      <td>0.805778</td>\n",
       "      <td>0.103795</td>\n",
       "      <td>-0.437339</td>\n",
       "      <td>50.000000</td>\n",
       "      <td>22.662968</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>-0.406819</td>\n",
       "      <td>-0.487240</td>\n",
       "      <td>0.401324</td>\n",
       "      <td>3.664771</td>\n",
       "      <td>-0.040517</td>\n",
       "      <td>0.125766</td>\n",
       "      <td>0.846397</td>\n",
       "      <td>-0.205034</td>\n",
       "      <td>-0.522484</td>\n",
       "      <td>-0.784617</td>\n",
       "      <td>-0.949462</td>\n",
       "      <td>0.406003</td>\n",
       "      <td>-0.301505</td>\n",
       "      <td>23.000000</td>\n",
       "      <td>27.233397</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>2.463299</td>\n",
       "      <td>-0.487240</td>\n",
       "      <td>1.014995</td>\n",
       "      <td>-0.272329</td>\n",
       "      <td>1.193543</td>\n",
       "      <td>-1.331642</td>\n",
       "      <td>0.974288</td>\n",
       "      <td>-0.993604</td>\n",
       "      <td>1.659603</td>\n",
       "      <td>1.529413</td>\n",
       "      <td>0.805778</td>\n",
       "      <td>0.440616</td>\n",
       "      <td>0.996622</td>\n",
       "      <td>8.300000</td>\n",
       "      <td>11.190621</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>-0.413538</td>\n",
       "      <td>-0.487240</td>\n",
       "      <td>0.246813</td>\n",
       "      <td>-0.272329</td>\n",
       "      <td>-1.015684</td>\n",
       "      <td>-0.074912</td>\n",
       "      <td>-0.528437</td>\n",
       "      <td>0.578929</td>\n",
       "      <td>-0.522484</td>\n",
       "      <td>-0.060741</td>\n",
       "      <td>0.112920</td>\n",
       "      <td>0.325604</td>\n",
       "      <td>-0.043840</td>\n",
       "      <td>21.200001</td>\n",
       "      <td>18.511448</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>97</th>\n",
       "      <td>-0.417713</td>\n",
       "      <td>3.157316</td>\n",
       "      <td>-1.515487</td>\n",
       "      <td>-0.272329</td>\n",
       "      <td>-1.248688</td>\n",
       "      <td>0.139999</td>\n",
       "      <td>-1.167894</td>\n",
       "      <td>2.560921</td>\n",
       "      <td>-0.867024</td>\n",
       "      <td>-0.565081</td>\n",
       "      <td>-0.533747</td>\n",
       "      <td>0.440616</td>\n",
       "      <td>-0.963871</td>\n",
       "      <td>24.700001</td>\n",
       "      <td>26.361408</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>98</th>\n",
       "      <td>0.132400</td>\n",
       "      <td>-0.487240</td>\n",
       "      <td>1.014995</td>\n",
       "      <td>-0.272329</td>\n",
       "      <td>1.366138</td>\n",
       "      <td>0.342100</td>\n",
       "      <td>0.636797</td>\n",
       "      <td>-0.645503</td>\n",
       "      <td>1.659603</td>\n",
       "      <td>1.529413</td>\n",
       "      <td>0.805778</td>\n",
       "      <td>-3.349082</td>\n",
       "      <td>0.766964</td>\n",
       "      <td>14.100000</td>\n",
       "      <td>14.395020</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>99</th>\n",
       "      <td>-0.402742</td>\n",
       "      <td>0.584688</td>\n",
       "      <td>-0.875579</td>\n",
       "      <td>-0.272329</td>\n",
       "      <td>-0.877607</td>\n",
       "      <td>-0.773728</td>\n",
       "      <td>-0.084369</td>\n",
       "      <td>1.629074</td>\n",
       "      <td>-0.177944</td>\n",
       "      <td>-0.737150</td>\n",
       "      <td>0.574826</td>\n",
       "      <td>0.421009</td>\n",
       "      <td>0.069589</td>\n",
       "      <td>18.700001</td>\n",
       "      <td>15.549146</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>100</th>\n",
       "      <td>-0.403765</td>\n",
       "      <td>-0.487240</td>\n",
       "      <td>-0.079701</td>\n",
       "      <td>-0.272329</td>\n",
       "      <td>-0.566935</td>\n",
       "      <td>0.128613</td>\n",
       "      <td>-1.288681</td>\n",
       "      <td>0.071405</td>\n",
       "      <td>-0.637331</td>\n",
       "      <td>-0.778684</td>\n",
       "      <td>0.066730</td>\n",
       "      <td>0.319141</td>\n",
       "      <td>-0.458344</td>\n",
       "      <td>28.100000</td>\n",
       "      <td>23.625895</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>101</th>\n",
       "      <td>-0.405218</td>\n",
       "      <td>-0.487240</td>\n",
       "      <td>-0.375604</td>\n",
       "      <td>-0.272329</td>\n",
       "      <td>-0.299411</td>\n",
       "      <td>0.269515</td>\n",
       "      <td>1.013366</td>\n",
       "      <td>-0.646880</td>\n",
       "      <td>-0.522484</td>\n",
       "      <td>-0.143809</td>\n",
       "      <td>1.129112</td>\n",
       "      <td>0.422433</td>\n",
       "      <td>-0.053642</td>\n",
       "      <td>19.799999</td>\n",
       "      <td>21.725019</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>102 rows × 15 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "            0         1         2         3         4         5         6  \\\n",
       "0   -0.412284 -0.487240 -1.151075 -0.272329 -0.817198  0.068836 -1.825115   \n",
       "1    0.653229 -0.487240  1.014995 -0.272329  0.658496 -0.097684  1.116390   \n",
       "2   -0.406819 -0.487240  0.401324  3.664771 -0.040517  0.125766  0.846397   \n",
       "3    2.463299 -0.487240  1.014995 -0.272329  1.193543 -1.331642  0.974288   \n",
       "4   -0.413538 -0.487240  0.246813 -0.272329 -1.015684 -0.074912 -0.528437   \n",
       "..        ...       ...       ...       ...       ...       ...       ...   \n",
       "97  -0.417713  3.157316 -1.515487 -0.272329 -1.248688  0.139999 -1.167894   \n",
       "98   0.132400 -0.487240  1.014995 -0.272329  1.366138  0.342100  0.636797   \n",
       "99  -0.402742  0.584688 -0.875579 -0.272329 -0.877607 -0.773728 -0.084369   \n",
       "100 -0.403765 -0.487240 -0.079701 -0.272329 -0.566935  0.128613 -1.288681   \n",
       "101 -0.405218 -0.487240 -0.375604 -0.272329 -0.299411  0.269515  1.013366   \n",
       "\n",
       "            7         8         9        10        11        12       MEDV  \\\n",
       "0    0.674147 -0.637331  0.129128 -0.718509  0.203034 -0.744016  22.600000   \n",
       "1   -1.247058  1.659603  1.529413  0.805778  0.103795 -0.437339  50.000000   \n",
       "2   -0.205034 -0.522484 -0.784617 -0.949462  0.406003 -0.301505  23.000000   \n",
       "3   -0.993604  1.659603  1.529413  0.805778  0.440616  0.996622   8.300000   \n",
       "4    0.578929 -0.522484 -0.060741  0.112920  0.325604 -0.043840  21.200001   \n",
       "..        ...       ...       ...       ...       ...       ...        ...   \n",
       "97   2.560921 -0.867024 -0.565081 -0.533747  0.440616 -0.963871  24.700001   \n",
       "98  -0.645503  1.659603  1.529413  0.805778 -3.349082  0.766964  14.100000   \n",
       "99   1.629074 -0.177944 -0.737150  0.574826  0.421009  0.069589  18.700001   \n",
       "100  0.071405 -0.637331 -0.778684  0.066730  0.319141 -0.458344  28.100000   \n",
       "101 -0.646880 -0.522484 -0.143809  1.129112  0.422433 -0.053642  19.799999   \n",
       "\n",
       "     Predicted MEDV  \n",
       "0         25.710272  \n",
       "1         22.662968  \n",
       "2         27.233397  \n",
       "3         11.190621  \n",
       "4         18.511448  \n",
       "..              ...  \n",
       "97        26.361408  \n",
       "98        14.395020  \n",
       "99        15.549146  \n",
       "100       23.625895  \n",
       "101       21.725019  \n",
       "\n",
       "[102 rows x 15 columns]"
      ]
     },
     "execution_count": 45,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "model_MLP_vote"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "id": "1a207355-b667-4f27-acb8-9ba1e25da0ee",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "mse:27.216806411743164,mae:3.4257354736328125\n"
     ]
    }
   ],
   "source": [
    "Y_test=np.array(Y_test)\n",
    "Y_pred=np.array(Y_pred)\n",
    "# Get Mean Squared Error (MSE)\n",
    "from sklearn.metrics import mean_squared_error\n",
    "mse = mean_squared_error(Y_test, Y_pred)\n",
    "# Get Mean Absolute Error (MAE)\n",
    "from sklearn.metrics import mean_absolute_error\n",
    "mae = mean_absolute_error(Y_test, Y_pred)\n",
    "print(f\"mse:{mse},mae:{mae}\")"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "01175bbb-01b3-4910-a3a9-57c8a60a3714",
   "metadata": {},
   "source": [
    "## 封装神经网络的这个模型，便于之后的检验"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "id": "6bec0acd-4b0d-4469-b9a3-3d0cfdde6d76",
   "metadata": {},
   "outputs": [],
   "source": [
    "class ensemble_model():\n",
    "    def __init__(self,ensemble,network,estimators=10,lr=1e-3,weight_decay=5e-4,epoch=50,use_cuda=False):\n",
    "        super(ensemble_model, self).__init__()\n",
    "        self.epoch=epoch\n",
    "        self.model=ensemble(\n",
    "                        estimator=network,\n",
    "                        n_estimators=10,cuda=use_cuda)\n",
    "        #设置优化器optimizer\n",
    "        self.model.set_optimizer(\"Adam\", lr=lr, weight_decay=weight_decay)\n",
    "    def fit(self,X_train,Y_train):\n",
    "        X_train=torch.FloatTensor(np.array(X_train))\n",
    "        Y_train=torch.FloatTensor(np.array(Y_train)).reshape(-1,1)\n",
    "        # Tensor -> Data loader\n",
    "        train_data = TensorDataset(X_train, Y_train)\n",
    "        train_loader = DataLoader(train_data, batch_size=10, shuffle=True)  \n",
    "        self.model.fit(train_loader, epochs=self.epoch)\n",
    "    def predict(self,X_test):\n",
    "        X_test=torch.FloatTensor(np.array(X_test))\n",
    "        # 预测\n",
    "        Y_pred = self.model.predict(X_test)\n",
    "        return np.array(Y_pred)\n",
    "    def score(self,X_test,Y_test):\n",
    "        X_test=torch.FloatTensor(np.array(X_test))\n",
    "        Y_test=torch.FloatTensor(np.array(Y_test)).reshape(-1, 1)\n",
    "        test_data = TensorDataset(X_test, Y_test)\n",
    "        test_loader = DataLoader(test_data, batch_size=10, shuffle=False)\n",
    "        testing_mse = self.model.evaluate(test_loader)\n",
    "        return testing_mse\n",
    "        \n",
    "        "
   ]
  },
  {
   "cell_type": "markdown",
   "id": "8b2024ee-717e-4d5a-bb87-2a37932cc33a",
   "metadata": {},
   "source": [
    "# 4. 模型检验 mae/mse/r2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 50,
   "id": "860477f5-2a8b-4107-b432-a4a427c67758",
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.model_selection import cross_val_score\n",
    "from sklearn.model_selection import KFold"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 51,
   "id": "7d6482da-da6e-4fb4-943a-a23c8054aa8d",
   "metadata": {},
   "outputs": [],
   "source": [
    "mlp_model=ensemble_model(VotingRegressor,MLP,epoch=10) #只设置10，演示用"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 52,
   "id": "8b168d53-0472-4f17-a154-3796cacc088b",
   "metadata": {},
   "outputs": [],
   "source": [
    "poly_model = polynomial_model(degree=2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 53,
   "id": "741d1a75-596a-4648-956f-6d66c6480584",
   "metadata": {},
   "outputs": [],
   "source": [
    "lr = LinearRegression()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "5d14a81d-555d-4673-bfdb-0cb8afb9cff6",
   "metadata": {},
   "source": [
    "cross_val_score可以选择的scoring方法：https://scikit-learn.org/stable/modules/model_evaluation.html#scoring-parameter"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 54,
   "id": "3ec62e86-dc17-4b6a-bc64-d9096213b538",
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Kfold  7\n"
     ]
    },
    {
     "ename": "ValueError",
     "evalue": "Setting a random_state has no effect since shuffle is False. You should leave random_state to its default (None), or set shuffle=True.",
     "output_type": "error",
     "traceback": [
      "\u001b[1;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[1;31mValueError\u001b[0m                                Traceback (most recent call last)",
      "\u001b[1;32m<ipython-input-54-07143715b8bd>\u001b[0m in \u001b[0;36m<module>\u001b[1;34m\u001b[0m\n\u001b[0;32m      6\u001b[0m     \u001b[0mprint\u001b[0m \u001b[1;33m(\u001b[0m\u001b[1;34m\"Kfold \"\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mkfold\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m      7\u001b[0m     \u001b[1;32mfor\u001b[0m \u001b[0mregressor\u001b[0m \u001b[1;32min\u001b[0m \u001b[0mregressors\u001b[0m \u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m----> 8\u001b[1;33m         \u001b[0mkf\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0mKFold\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mkfold\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mrandom_state\u001b[0m \u001b[1;33m=\u001b[0m \u001b[1;36m0\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m      9\u001b[0m         \u001b[0mcv_results\u001b[0m\u001b[1;33m=\u001b[0m\u001b[1;33m[\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m     10\u001b[0m         \u001b[1;32mfor\u001b[0m \u001b[0mtrain_index\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mtest_index\u001b[0m \u001b[1;32min\u001b[0m \u001b[0mkf\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0msplit\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mX\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32mD:\\Anaconda3\\envs\\mathmodel\\lib\\site-packages\\sklearn\\utils\\validation.py\u001b[0m in \u001b[0;36minner_f\u001b[1;34m(*args, **kwargs)\u001b[0m\n\u001b[0;32m     61\u001b[0m             \u001b[0mextra_args\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mlen\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0margs\u001b[0m\u001b[1;33m)\u001b[0m \u001b[1;33m-\u001b[0m \u001b[0mlen\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mall_args\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m     62\u001b[0m             \u001b[1;32mif\u001b[0m \u001b[0mextra_args\u001b[0m \u001b[1;33m<=\u001b[0m \u001b[1;36m0\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m---> 63\u001b[1;33m                 \u001b[1;32mreturn\u001b[0m \u001b[0mf\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m*\u001b[0m\u001b[0margs\u001b[0m\u001b[1;33m,\u001b[0m \u001b[1;33m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m     64\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m     65\u001b[0m             \u001b[1;31m# extra_args > 0\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32mD:\\Anaconda3\\envs\\mathmodel\\lib\\site-packages\\sklearn\\model_selection\\_split.py\u001b[0m in \u001b[0;36m__init__\u001b[1;34m(self, n_splits, shuffle, random_state)\u001b[0m\n\u001b[0;32m    427\u001b[0m                  random_state=None):\n\u001b[0;32m    428\u001b[0m         super().__init__(n_splits=n_splits, shuffle=shuffle,\n\u001b[1;32m--> 429\u001b[1;33m                          random_state=random_state)\n\u001b[0m\u001b[0;32m    430\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    431\u001b[0m     \u001b[1;32mdef\u001b[0m \u001b[0m_iter_test_indices\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mself\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mX\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0my\u001b[0m\u001b[1;33m=\u001b[0m\u001b[1;32mNone\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mgroups\u001b[0m\u001b[1;33m=\u001b[0m\u001b[1;32mNone\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32mD:\\Anaconda3\\envs\\mathmodel\\lib\\site-packages\\sklearn\\utils\\validation.py\u001b[0m in \u001b[0;36minner_f\u001b[1;34m(*args, **kwargs)\u001b[0m\n\u001b[0;32m     61\u001b[0m             \u001b[0mextra_args\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mlen\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0margs\u001b[0m\u001b[1;33m)\u001b[0m \u001b[1;33m-\u001b[0m \u001b[0mlen\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mall_args\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m     62\u001b[0m             \u001b[1;32mif\u001b[0m \u001b[0mextra_args\u001b[0m \u001b[1;33m<=\u001b[0m \u001b[1;36m0\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m---> 63\u001b[1;33m                 \u001b[1;32mreturn\u001b[0m \u001b[0mf\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m*\u001b[0m\u001b[0margs\u001b[0m\u001b[1;33m,\u001b[0m \u001b[1;33m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m     64\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m     65\u001b[0m             \u001b[1;31m# extra_args > 0\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32mD:\\Anaconda3\\envs\\mathmodel\\lib\\site-packages\\sklearn\\model_selection\\_split.py\u001b[0m in \u001b[0;36m__init__\u001b[1;34m(self, n_splits, shuffle, random_state)\u001b[0m\n\u001b[0;32m    289\u001b[0m         \u001b[1;32mif\u001b[0m \u001b[1;32mnot\u001b[0m \u001b[0mshuffle\u001b[0m \u001b[1;32mand\u001b[0m \u001b[0mrandom_state\u001b[0m \u001b[1;32mis\u001b[0m \u001b[1;32mnot\u001b[0m \u001b[1;32mNone\u001b[0m\u001b[1;33m:\u001b[0m  \u001b[1;31m# None is the default\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    290\u001b[0m             raise ValueError(\n\u001b[1;32m--> 291\u001b[1;33m                 \u001b[1;34m'Setting a random_state has no effect since shuffle is '\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m    292\u001b[0m                 \u001b[1;34m'False. You should leave '\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    293\u001b[0m                 \u001b[1;34m'random_state to its default (None), or set shuffle=True.'\u001b[0m\u001b[1;33m,\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;31mValueError\u001b[0m: Setting a random_state has no effect since shuffle is False. You should leave random_state to its default (None), or set shuffle=True."
     ]
    }
   ],
   "source": [
    "regressors = [mlp_model,lr, poly_model]\n",
    "kfolds = [7]\n",
    "import warnings\n",
    "warnings.filterwarnings(\"ignore\")\n",
    "for kfold in kfolds :\n",
    "    print (\"Kfold \", kfold)\n",
    "    for regressor in regressors :\n",
    "        kf=KFold(kfold, random_state = 0)\n",
    "        cv_results=[]\n",
    "        for train_index, test_index in kf.split(X):\n",
    "            X_train, X_test = X.iloc[train_index], X.iloc[test_index]\n",
    "            y_train, y_test = Y.iloc[train_index], Y.iloc[test_index]\n",
    "            regressor.fit(X_train,y_train)\n",
    "            y_pred=regressor.predict(X_test)\n",
    "            mse = mean_squared_error(y_test, y_pred)\n",
    "            cv_results.append(mse)\n",
    "        if regressor == lr :\n",
    "                print(\"线性回归, MSE: \", (np.array(cv_results)).mean())\n",
    "        elif regressor == poly_model:\n",
    "            print(\"多项式回归, MSE: \", (np.array(cv_results)).mean())\n",
    "        else:\n",
    "            print(\"神经网络集成学习 , MSE: \", np.array(cv_results).mean())\n",
    "            "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 55,
   "id": "863610b5-bd4d-4952-b0bb-51b04aca88c4",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "神经网梯度提升集成学习 , MSE:18.775064210667427,mae:2.8201569940529616,train_r2:0.9598779732456497,test_r2:0.7694286612028111\n",
      "线性回归, MSE:33.44897999767653,mae:3.842909220444498,train_r2:0.7730135569264233,test_r2:0.5892223849182507\n",
      "多项式回归, MSE:31.277814971447178,mae:3.32776330010084,train_r2:0.9490240966612832,test_r2:0.6158858584078863\n"
     ]
    }
   ],
   "source": [
    "mlp_model=ensemble_model(GradientBoostingRegressor,MLP,epoch=10) #只设置10，演示用\n",
    "poly_model = polynomial_model(degree=2)\n",
    "lr = LinearRegression()\n",
    "regressors = [mlp_model,lr, poly_model]\n",
    "import warnings\n",
    "warnings.filterwarnings(\"ignore\")\n",
    "for regressor in regressors :\n",
    "    X_train, X_test, Y_train, Y_test = train_test_split(X, Y, test_size = 0.20, random_state = 0)\n",
    "    regressor.fit(X_train,Y_train)\n",
    "    Y_pred=regressor.predict(X_test)\n",
    "    from sklearn.metrics import mean_squared_error\n",
    "    mse = mean_squared_error(Y_test, Y_pred)\n",
    "    # Get Mean Absolute Error (MAE)\n",
    "    from sklearn.metrics import mean_absolute_error\n",
    "    mae = mean_absolute_error(Y_test, Y_pred)\n",
    "    from  sklearn.metrics import r2_score\n",
    "    train_r2=r2_score( Y_train,regressor.predict(X_train))\n",
    "    test_r2=r2_score( Y_test,Y_pred)\n",
    "    if regressor == lr:\n",
    "        print(f\"线性回归, MSE:{mse},mae:{mae},train_r2:{train_r2},test_r2:{test_r2}\")\n",
    "    elif regressor == poly_model:\n",
    "        print(f\"多项式回归, MSE:{mse},mae:{mae},train_r2:{train_r2},test_r2:{test_r2}\")\n",
    "    else:\n",
    "        print(f\"神经网梯度提升集成学习 , MSE:{mse},mae:{mae},train_r2:{train_r2},test_r2:{test_r2}\")"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "d10f0aff-8102-4d29-944d-9fb56ba5ae55",
   "metadata": {},
   "source": [
    "通过运算决定系数 R2 来量化模型的表现。模型的决定系数是回归分析中十分常用的统计信息，经常被当作衡量模型预测能力好坏的标准。\n",
    "\n",
    "R2的数值范围从0至1，表示目标变量的预测值和实际值之间的相关程度平方的百分比。一个模型的R2 值为0还不如直接用平均值来预测效果好；而一个R2 值为1的模型则可以对目标变量进行完美的预测。从0至1之间的数值，则表示该模型中目标变量中有百分之多少能够用特征来解释。模型也可能出现负值的R2，这种情况下模型所做预测有时会比直接计算目标变量的平均值差很多。"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "0c29a8b7-ddfa-4df2-9d12-63077eda39c3",
   "metadata": {},
   "source": [
    "# 5. 可视化结果"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "a5e72e36-95dc-4dfe-8938-c56110e32983",
   "metadata": {},
   "source": [
    "## 多项式 阶数拟合图"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "47f99284-c691-4ad9-b398-c98686df1329",
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.model_selection import learning_curve\n",
    "import numpy as np\n",
    "\n",
    "def plot_learning_curve(plt, estimator, title, X, y, ylim=None, cv=None,\n",
    "                        n_jobs=1, train_sizes=np.linspace(.1, 1.0, 5)):\n",
    "    \"\"\"\n",
    "    Generate a simple plot of the test and training learning curve.\n",
    "\n",
    "    Parameters\n",
    "    ----------\n",
    "    estimator : object type that implements the \"fit\" and \"predict\" methods\n",
    "        An object of that type which is cloned for each validation.\n",
    "\n",
    "    title : string\n",
    "        Title for the chart.\n",
    "\n",
    "    X : array-like, shape (n_samples, n_features)\n",
    "        Training vector, where n_samples is the number of samples and\n",
    "        n_features is the number of features.\n",
    "\n",
    "    y : array-like, shape (n_samples) or (n_samples, n_features), optional\n",
    "        Target relative to X for classification or regression;\n",
    "        None for unsupervised learning.\n",
    "\n",
    "    ylim : tuple, shape (ymin, ymax), optional\n",
    "        Defines minimum and maximum yvalues plotted.\n",
    "\n",
    "    cv : int, cross-validation generator or an iterable, optional\n",
    "        Determines the cross-validation splitting strategy.\n",
    "        Possible inputs for cv are:\n",
    "          - None, to use the default 3-fold cross-validation,\n",
    "          - integer, to specify the number of folds.\n",
    "          - An object to be used as a cross-validation generator.\n",
    "          - An iterable yielding train/test splits.\n",
    "\n",
    "        For integer/None inputs, if ``y`` is binary or multiclass,\n",
    "        :class:`StratifiedKFold` used. If the estimator is not a classifier\n",
    "        or if ``y`` is neither binary nor multiclass, :class:`KFold` is used.\n",
    "\n",
    "        Refer :ref:`User Guide <cross_validation>` for the various\n",
    "        cross-validators that can be used here.\n",
    "\n",
    "    n_jobs : integer, optional\n",
    "        Number of jobs to run in parallel (default 1).\n",
    "    \"\"\"\n",
    "    plt.title(title)\n",
    "    if ylim is not None:\n",
    "        plt.ylim(*ylim)\n",
    "    plt.xlabel(\"Training examples\")\n",
    "    plt.ylabel(\"Score\")\n",
    "    train_sizes, train_scores, test_scores = learning_curve(\n",
    "        estimator, X, y, cv=cv, n_jobs=n_jobs, train_sizes=train_sizes)\n",
    "    train_scores_mean = np.mean(train_scores, axis=1)\n",
    "    train_scores_std = np.std(train_scores, axis=1)\n",
    "    test_scores_mean = np.mean(test_scores, axis=1)\n",
    "    test_scores_std = np.std(test_scores, axis=1)\n",
    "    plt.grid()\n",
    "\n",
    "    plt.fill_between(train_sizes, train_scores_mean - train_scores_std,\n",
    "                     train_scores_mean + train_scores_std, alpha=0.1,\n",
    "                     color=\"r\")\n",
    "    plt.fill_between(train_sizes, test_scores_mean - test_scores_std,\n",
    "                     test_scores_mean + test_scores_std, alpha=0.1, color=\"g\")\n",
    "    plt.plot(train_sizes, train_scores_mean, 'o--', color=\"r\",\n",
    "             label=\"Training score\")\n",
    "    plt.plot(train_sizes, test_scores_mean, 'o-', color=\"g\",\n",
    "             label=\"Cross-validation score\")\n",
    "\n",
    "    plt.legend(loc=\"best\")\n",
    "    return plt"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "cf254bec-bed2-4b8b-bdb7-b75060632241",
   "metadata": {},
   "outputs": [],
   "source": [
    "sns.set(style=\"whitegrid\")\n",
    "%matplotlib widget\n",
    "from sklearn.model_selection import ShuffleSplit\n",
    "cv = ShuffleSplit(n_splits=10,test_size=0.2,random_state=0)\n",
    "plt.figure(figsize=(15,3))\n",
    "title = 'Learning Curves (degree={0})'\n",
    "degrees = [1,2,3]\n",
    "for i in range(len(degrees)):\n",
    "    plt.subplot(1,3,i+1)\n",
    "    plot_learning_curve(plt,polynomial_model(degrees[i]),title.format(degrees[i]),\n",
    "                        X,Y,ylim=(0.01,1.01),cv=cv)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "32356a14-a91d-4851-ab30-0293dabfb6ee",
   "metadata": {},
   "source": [
    "## 各种神经网络集成方法拟合图"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "da542a1f-c8c1-4899-8c5b-9af9e78a926f",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "X_train, X_test, Y_train, Y_test = train_test_split(X, Y, test_size = 0.20, random_state = 0)\n",
    "method_name=['GradientBoosting','Bagging','Fusion','Voting','SnapshotEnsemble']\n",
    "method=[ensemble_model(i,MLP,epoch=10) for i in [GradientBoostingRegressor,BaggingRegressor,FusionRegressor,VotingRegressor,SnapshotEnsembleRegressor]]\n",
    "predict={}\n",
    "true_={}\n",
    "for i in range(len(method)):\n",
    "    model=method[i]\n",
    "    model.fit(X_train,Y_train)\n",
    "    mse=model.score(X_test,Y_test)\n",
    "    Y_predict=model.predict(X_test)\n",
    "    predict[method_name[i]]=Y_predict[:,0]\n",
    "    true_[method_name[i]]=np.array(Y_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "d092c983-c689-4e6e-a7c6-79fd767a3f86",
   "metadata": {},
   "outputs": [],
   "source": [
    "predict"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "81ad2b87-3864-48cb-86d0-f7ad44e23f0b",
   "metadata": {},
   "outputs": [],
   "source": [
    "def vis_regressor(ax,y_true,y_predict,title):\n",
    "    res_df = pd.concat([pd.DataFrame({'原始值': y_true}), pd.DataFrame({'预测值': y_predict})], axis=1)\n",
    "    sns.lineplot(x=res_df.index.tolist(), y=res_df['预测值'], linewidth=2, ax=ax)\n",
    "    sns.scatterplot(x=res_df.index.tolist(), y=res_df['原始值'], s=60, color='r', marker='v', ax=ax)\n",
    "    ax.set_ylabel('')\n",
    "    ax.set_title(title)\n",
    "    ax.legend(labels=['预测值', '真实值'],loc='upper left', fontsize=15, frameon=True, fancybox=True, framealpha=1, borderpad=0.3,\n",
    "           ncol=1, markerfirst=True, markerscale=1, numpoints=1, handlelength=3.5)\n",
    "    return ax"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "710cd2de-a8f8-4404-a6d5-ed962096e22f",
   "metadata": {},
   "outputs": [],
   "source": [
    "sns.set(style=\"whitegrid\")\n",
    "%matplotlib widget\n",
    "\n",
    "\n",
    "plt.figure(figsize=(20,5))\n",
    "title = 'method={0}'\n",
    "sns.set(style=\"whitegrid\")\n",
    "for i in range(len(method_name)):\n",
    "    plt.rcParams[\"font.family\"] = 'SimHei'  # 将字体改为中文\n",
    "    plt.rcParams['axes.unicode_minus'] = False  # 设置了中文字体默认后，坐标的\"-\"号无法显示，设置这个参数就可以避免\n",
    "    layout = (1, 5)\n",
    "    ax = plt.subplot2grid(layout,(0,i))\n",
    "    vis_regressor(ax,true_[method_name[i]],predict[method_name[i]],title=title.format(method_name[i]))\n",
    "plt.tight_layout()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "e252be3a-2065-4646-b515-62f58f27c8d1",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "id": "994107f0-af6e-44b7-96c1-8310831523b8",
   "metadata": {},
   "outputs": [],
   "source": [
    "mlp_model=ensemble_model(GradientBoostingRegressor,MLP,epoch=50,use_cuda=True) #只设置10，演"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "id": "c9db35cc-a76f-44fe-ade1-dff73969f074",
   "metadata": {},
   "outputs": [],
   "source": [
    "mlp_model.fit(X,Y)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "id": "c2c5cf7d-c727-4437-ab45-04c2d740d690",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "神经网梯度提升集成学习 , MSE:0.7814571551732074,mae:0.5618853388096504,train_r2:0.990743173847922\n"
     ]
    }
   ],
   "source": [
    "Y_pred=mlp_model.predict(X)\n",
    "from sklearn.metrics import mean_squared_error\n",
    "mse = mean_squared_error(Y, Y_pred)\n",
    "# Get Mean Absolute Error (MAE)\n",
    "from sklearn.metrics import mean_absolute_error\n",
    "mae = mean_absolute_error(Y, Y_pred)\n",
    "from  sklearn.metrics import r2_score\n",
    "train_r2=r2_score(Y,mlp_model.predict(X))\n",
    "print(f\"神经网梯度提升集成学习 , MSE:{mse},mae:{mae},train_r2:{train_r2}\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "ac3aeeb5-c0d0-4c74-b4c6-cf059f84cc0f",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.13"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}