{
"cells": [
{
"cell_type": "code",
"execution_count": 1,
"metadata": {
"_cell_guid": "e137a255-49bf-837d-8511-a49e6c84aef7"
},
"outputs": [
{
"name": "stderr",
"output_type": "stream",
"text": [
"/opt/conda/lib/python3.6/site-packages/sklearn/cross_validation.py:43: DeprecationWarning: This module was deprecated in version 0.18 in favor of the model_selection module into which all the refactored classes and functions are moved. Also note that the interface of the new CV iterators are different from that of this module. This module will be removed in 0.20.\n",
" \"This module will be removed in 0.20.\", DeprecationWarning)\n"
]
}
],
"source": [
"import numpy as np\n",
"import pandas as pd\n",
"import matplotlib.pyplot as plt\n",
"import xgboost\n",
"import math\n",
"from __future__ import division\n",
"from scipy.stats import pearsonr\n",
"from sklearn.linear_model import LinearRegression\n",
"from sklearn import cross_validation, tree, linear_model\n",
"from sklearn.model_selection import train_test_split\n",
"from sklearn.metrics import explained_variance_score"
]
},
{
"cell_type": "markdown",
"metadata": {
"_cell_guid": "fae38191-489d-ba9c-858c-f0b09d028f13"
},
"source": [
"# 1. Exploratory Data Analysis"
]
},
{
"cell_type": "code",
"execution_count": 2,
"metadata": {
"_cell_guid": "fe625774-6184-5cb2-819c-ff840e08e392"
},
"outputs": [],
"source": [
"# Read the data into a data frame\n",
"data = pd.read_csv('../input/kc_house_data.csv')"
]
},
{
"cell_type": "code",
"execution_count": 3,
"metadata": {
"_cell_guid": "f9afd217-a50b-9f37-755d-072af53522ca"
},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"21613\n",
"21\n",
"[dtype('int64') dtype('O') dtype('float64')]\n"
]
}
],
"source": [
"# Check the number of data points in the data set\n",
"print(len(data))\n",
"# Check the number of features in the data set\n",
"print(len(data.columns))\n",
"# Check the data types\n",
"print(data.dtypes.unique())"
]
},
{
"cell_type": "markdown",
"metadata": {
"_cell_guid": "c2021183-8a42-5787-bff7-da3181c08fb7"
},
"source": [
"- Since there are Python objects in the data set, we may have some categorical features. Let's check them. "
]
},
{
"cell_type": "code",
"execution_count": 4,
"metadata": {
"_cell_guid": "608870cd-16d1-0f6f-9594-603c455da8a5"
},
"outputs": [
{
"data": {
"text/plain": [
"['date']"
]
},
"execution_count": 4,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"data.select_dtypes(include=['O']).columns.tolist()"
]
},
{
"cell_type": "markdown",
"metadata": {
"_cell_guid": "a8199e20-8553-da22-4b75-99f6fbc60523"
},
"source": [
"- We only have the date column which is a timestamp that we will ignore."
]
},
{
"cell_type": "code",
"execution_count": 5,
"metadata": {
"_cell_guid": "c9d01805-192d-06f0-5e0a-aff8f7bb75bf"
},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"0 / 21\n",
"0 / 21613\n"
]
}
],
"source": [
"# Check any number of columns with NaN\n",
"print(data.isnull().any().sum(), ' / ', len(data.columns))\n",
"# Check any number of data points with NaN\n",
"print(data.isnull().any(axis=1).sum(), ' / ', len(data))"
]
},
{
"cell_type": "markdown",
"metadata": {
"_cell_guid": "619e716e-efcf-3c50-3b5c-5b8826967619"
},
"source": [
"- The data set is pretty much structured and doesn't have any NaN values. So we can jump into finding correlations between the features and the target variable"
]
},
{
"cell_type": "markdown",
"metadata": {
"_cell_guid": "c041ee9f-937b-5df2-100c-4ef6d4f39c3d"
},
"source": [
"# 2. Correlations between features and target"
]
},
{
"cell_type": "code",
"execution_count": 6,
"metadata": {
"_cell_guid": "71b41bea-27dc-ff5f-d2ba-caa214ed661a"
},
"outputs": [],
"source": [
"features = data.iloc[:,3:].columns.tolist()\n",
"target = data.iloc[:,2].name"
]
},
{
"cell_type": "code",
"execution_count": 7,
"metadata": {
"_cell_guid": "ccdb0159-6285-29e1-7ecc-fdd34442519c"
},
"outputs": [],
"source": [
"correlations = {}\n",
"for f in features:\n",
" data_temp = data[[f,target]]\n",
" x1 = data_temp[f].values\n",
" x2 = data_temp[target].values\n",
" key = f + ' vs ' + target\n",
" correlations[key] = pearsonr(x1,x2)[0]"
]
},
{
"cell_type": "code",
"execution_count": 8,
"metadata": {
"_cell_guid": "a5861b15-d8b3-76bf-9903-31995a68cd41"
},
"outputs": [
{
"data": {
"text/html": [
"\n",
"
\n",
" \n",
" \n",
" | \n",
" Value | \n",
"
\n",
" \n",
" \n",
" \n",
" | sqft_living vs price | \n",
" 0.702035 | \n",
"
\n",
" \n",
" | grade vs price | \n",
" 0.667434 | \n",
"
\n",
" \n",
" | sqft_above vs price | \n",
" 0.605567 | \n",
"
\n",
" \n",
" | sqft_living15 vs price | \n",
" 0.585379 | \n",
"
\n",
" \n",
" | bathrooms vs price | \n",
" 0.525138 | \n",
"
\n",
" \n",
" | view vs price | \n",
" 0.397293 | \n",
"
\n",
" \n",
" | sqft_basement vs price | \n",
" 0.323816 | \n",
"
\n",
" \n",
" | bedrooms vs price | \n",
" 0.308350 | \n",
"
\n",
" \n",
" | lat vs price | \n",
" 0.307003 | \n",
"
\n",
" \n",
" | waterfront vs price | \n",
" 0.266369 | \n",
"
\n",
" \n",
" | floors vs price | \n",
" 0.256794 | \n",
"
\n",
" \n",
" | yr_renovated vs price | \n",
" 0.126434 | \n",
"
\n",
" \n",
" | sqft_lot vs price | \n",
" 0.089661 | \n",
"
\n",
" \n",
" | sqft_lot15 vs price | \n",
" 0.082447 | \n",
"
\n",
" \n",
" | yr_built vs price | \n",
" 0.054012 | \n",
"
\n",
" \n",
" | zipcode vs price | \n",
" -0.053203 | \n",
"
\n",
" \n",
" | condition vs price | \n",
" 0.036362 | \n",
"
\n",
" \n",
" | long vs price | \n",
" 0.021626 | \n",
"
\n",
" \n",
"
\n",
"