{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {
    "hideCode": false,
    "hidePrompt": false
   },
   "source": [
    "# Histograms and Density Plots\n",
    "\n",
    "In this notebook we will look at the [NYCFlights13 dataset](https://cran.r-project.org/web/packages/nycflights13/nycflights13.pdf), which contains information on over 300,000 planes departing NYC airports in 2013. We will focus on visualizing the distribution of arrival delays given in minutes. "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "hideCode": false,
    "hidePrompt": false
   },
   "outputs": [],
   "source": [
    "# Pandas and numpy for data manipulation\n",
    "import pandas as pd\n",
    "import numpy as np\n",
    "\n",
    "# Matplotlib and seaborn for plotting\n",
    "import matplotlib.pyplot as plt\n",
    "import matplotlib\n",
    "%matplotlib inline\n",
    "\n",
    "matplotlib.rcParams['font.size'] = 18\n",
    "matplotlib.rcParams['figure.dpi'] = 200\n",
    "\n",
    "import seaborn as sns\n",
    "\n",
    "from IPython.core.pylabtools import figsize"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "hideCode": false,
    "hidePrompt": false
   },
   "source": [
    "## Data Import "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "hideCode": false,
    "hidePrompt": false
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>arr_delay</th>\n",
       "      <th>name</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>11.0</td>\n",
       "      <td>United Air Lines Inc.</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>20.0</td>\n",
       "      <td>United Air Lines Inc.</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>33.0</td>\n",
       "      <td>American Airlines Inc.</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>-18.0</td>\n",
       "      <td>JetBlue Airways</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>-25.0</td>\n",
       "      <td>Delta Air Lines Inc.</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>12.0</td>\n",
       "      <td>United Air Lines Inc.</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>19.0</td>\n",
       "      <td>JetBlue Airways</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>-14.0</td>\n",
       "      <td>ExpressJet Airlines Inc.</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>-8.0</td>\n",
       "      <td>JetBlue Airways</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>8.0</td>\n",
       "      <td>American Airlines Inc.</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   arr_delay                      name\n",
       "0       11.0     United Air Lines Inc.\n",
       "1       20.0     United Air Lines Inc.\n",
       "2       33.0    American Airlines Inc.\n",
       "3      -18.0           JetBlue Airways\n",
       "4      -25.0      Delta Air Lines Inc.\n",
       "5       12.0     United Air Lines Inc.\n",
       "6       19.0           JetBlue Airways\n",
       "7      -14.0  ExpressJet Airlines Inc.\n",
       "8       -8.0           JetBlue Airways\n",
       "9        8.0    American Airlines Inc."
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    " # Read in data\n",
    "flights = pd.read_csv('data/flights.csv')\n",
    "carriers = pd.read_csv('data/by_carrier.csv')\n",
    "airlines = pd.read_csv('data/airlines.csv')\n",
    "\n",
    "flights = flights.merge(airlines, how = 'left', on = 'carrier')\n",
    "flights = flights[['arr_delay', 'name']] \n",
    "\n",
    "# Filter flight delays between -60 and 120 minutes\n",
    "flights = flights[flights['arr_delay'].between(-60, 120)]\n",
    "flights.head(10)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "flights.to_csv('data/formatted_flights.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "hideCode": false,
    "hidePrompt": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "count    317113.000000\n",
       "mean          1.297143\n",
       "std          29.064401\n",
       "min         -60.000000\n",
       "25%         -17.000000\n",
       "50%          -6.000000\n",
       "75%          11.000000\n",
       "max         120.000000\n",
       "Name: arr_delay, dtype: float64"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "flights = pd.read_csv('data/formatted_flights.csv')\n",
    "flights['arr_delay'].describe()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "hideCode": false,
    "hidePrompt": false
   },
   "source": [
    "# Histogram of All Arrival Delays "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "hideCode": false,
    "hidePrompt": false
   },
   "source": [
    "## Matplotlib"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "hideCode": false,
    "hidePrompt": false
   },
   "outputs": [
    {
     "data": {
      "image/png": "[encoded data, 14076 characters]\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x15151cda908>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "figsize(9, 7)\n",
    "\n",
    "# Make the histogram using matplotlib, bins must be integet\n",
    "plt.hist(flights['arr_delay'], color = 'blue', edgecolor = 'black',\n",
    "         bins = int(180/5))\n",
    "\n",
    "# Add labels\n",
    "plt.title('Histogram of Arrival Delays')\n",
    "plt.xlabel('Delay (min)'); plt.ylabel('Flights');"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "hideCode": false,
    "hidePrompt": false
   },
   "source": [
    "## Seaborn"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "hideCode": false,
    "hidePrompt": false
   },
   "outputs": [
    {
     "data": {
      "image/png": "[encoded data, 14260 characters]\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x15151c84cf8>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "figsize(9, 7)\n",
    "\n",
    "sns.distplot(flights['arr_delay'], hist=True, kde=False, \n",
    "             bins=int(180/5), color = 'blue',\n",
    "             hist_kws={'edgecolor':'black'})\n",
    "\n",
    "plt.title('Histogram of Arrival Delays')\n",
    "plt.xlabel('Delay (min)'); plt.ylabel('Flights');\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Histogram with Varying Binwidths"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "[encoded data, 60804 characters]\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x15151b6fc88>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "figsize(18, 16)\n",
    "\n",
    "# Show 4 different binwidths\n",
    "for i, binwidth in enumerate([1, 5, 10, 15]):\n",
    "    \n",
    "    # Set up the plot\n",
    "    ax = plt.subplot(2, 2, i + 1)\n",
    "    \n",
    "    # Draw the plot\n",
    "    ax.hist(flights['arr_delay'], bins = int(180/binwidth),\n",
    "             color = 'blue', edgecolor = 'black')\n",
    "    \n",
    "    # Title and labels\n",
    "    ax.set_title('Histogram with Binwidth = %d' % binwidth, size = 30)\n",
    "    ax.set_xlabel('Delay (min)', size = 22)\n",
    "    ax.set_ylabel('Flights', size= 22)\n",
    "\n",
    "plt.tight_layout()\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "hideCode": false,
    "hidePrompt": false
   },
   "source": [
    "# Where Histograms Fail: Multiple Categories\n",
    "\n",
    "Histograms work very well for display a single variable from one category (in this case the one category was all the flights). However, for displaying multiple categories, a histogram does not work well because the plots are obscured."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "hideCode": false,
    "hidePrompt": false
   },
   "outputs": [],
   "source": [
    "# Extract top five airlines with most flights\n",
    "carrier_flights = pd.Series(flights.groupby('name')['name'].count())\n",
    "carrier_flights = carrier_flights.sort_values(ascending=False)\n",
    "top_five = carrier_flights.index[:5]\n",
    "\n",
    "# Subset to the top 5 airlines\n",
    "top_five_flights = flights[flights['name'].isin(top_five)]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "hideCode": false,
    "hidePrompt": false
   },
   "outputs": [
    {
     "data": {
      "image/png": "[encoded data, 39276 characters]\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x15151bdabe0>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "figsize(10, 8)\n",
    "\n",
    "# Iterate through each of the five airlines\n",
    "for airline in top_five:\n",
    "    # Subset to the airline\n",
    "    subset = top_five_flights[top_five_flights['name'] == airline]\n",
    "    # Make a histogram with the airline\n",
    "    sns.distplot(subset['arr_delay'], bins = int(180/5), \n",
    "                 label = airline, hist = True, kde = False, \n",
    "                 norm_hist=True, \n",
    "                 hist_kws = {'edgecolor': 'black'})\n",
    "    \n",
    "plt.legend(prop={'size': 16}, title = 'Airline')\n",
    "plt.title('Histogram of Delays by Airline'); plt.xlabel('Delay (min)'); plt.ylabel('Normalized Counts');"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Make a separate list for each airline\n",
    "x1 = list(flights[flights['name'] == 'United Air Lines Inc.']['arr_delay'])\n",
    "x2 = list(flights[flights['name'] == 'JetBlue Airways']['arr_delay'])\n",
    "x3 = list(flights[flights['name'] == 'ExpressJet Airlines Inc.']['arr_delay'])\n",
    "x4 = list(flights[flights['name'] == 'Delta Air Lines Inc.']['arr_delay'])\n",
    "x5 = list(flights[flights['name'] == 'American Airlines Inc.']['arr_delay'])\n",
    "\n",
    "# Assign colors for each airline and the names\n",
    "colors = ['#E69F00', '#56B4E9', '#F0E442', '#009E73', '#D55E00']\n",
    "names = ['United Air Lines Inc.', 'JetBlue Airways', 'ExpressJet Airlines Inc.',\n",
    "         'Delta Air Lines Inc.', 'American Airlines Inc.']"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Solution 1: Side-by-Side Histograms"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "[encoded data, 25568 characters]\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x151520ee5f8>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "figsize(10, 8)\n",
    "\n",
    "# Make the histogram using a list of lists\n",
    "# Normalize the flights and assign colors and names\n",
    "plt.hist([x1, x2, x3, x4, x5], bins = int(180/15), normed=True,\n",
    "         color = colors, label=names)\n",
    "\n",
    "# Plot formatting\n",
    "plt.legend()\n",
    "plt.xlabel('Delay (min)')\n",
    "plt.ylabel('Normalized Flights')\n",
    "plt.title('Side-by-Side Histogram with Multiple Airlines');"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Solution 2: Stacked Histograms"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "[encoded data, 27812 characters]\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x15151bc4e10>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "figsize(10, 8)\n",
    "plt.hist([x1, x2, x3, x4, x5], bins = int(180/15), normed=True,\n",
    "         stacked=True,\n",
    "         color = colors, label=names)\n",
    "\n",
    "plt.legend(); plt.xlabel('Delay (min)'); plt.ylabel('Normed Flights');\n",
    "plt.title('Stacked Histogram with Multiple Airlines');"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "hideCode": false,
    "hidePrompt": false
   },
   "source": [
    "# Solution 3: Density Plots\n",
    "\n",
    "A density plot uses a kernel (by default a Gaussian) at every data point to create a distribution and then sums the distribution at each location to create a single distribution.\n",
    "\n",
    "![](images/KDEWIKI.png)\n",
    "\n",
    "This produces a smoothed version of a histogram where the y-axis is in density. This density can be thought of as a probability of the data value corresponding to the value on the x-axis, but this can create issues because the density may be greater than one. I usually just think of it as _density_, a value used for comparison purposes only. "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "hideCode": false,
    "hidePrompt": false
   },
   "source": [
    "## Density Plot of All Delays Using Seaborn"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {
    "hideCode": false,
    "hidePrompt": false
   },
   "outputs": [
    {
     "data": {
      "image/png": "[encoded data, 30268 characters]\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x1515407d6a0>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "figsize(9, 7)\n",
    "\n",
    "sns.distplot(flights['arr_delay'], hist=True, kde=True, \n",
    "             bins=int(180/5), color = 'darkblue', \n",
    "             hist_kws={'edgecolor':'black'},\n",
    "             kde_kws={'linewidth': 4});\n",
    "\n",
    "plt.title('Density Plot and Histogram of Arrival Delays'); \n",
    "plt.xlabel('Delay (min)'); plt.ylabel('Density');"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "hideCode": false,
    "hidePrompt": false
   },
   "source": [
    "# Density Plot with Varying Bandwidths"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {
    "hideCode": false,
    "hidePrompt": false
   },
   "outputs": [
    {
     "data": {
      "image/png": "[encoded data, 76240 characters]\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x15153fa2eb8>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "figsize(12, 10)\n",
    "\n",
    "for bandwidth in ['scott', 0.2, 0.5, 1, 5]:\n",
    "    sns.distplot(flights['arr_delay'], hist = False, kde = True, \n",
    "                 kde_kws={'bw': bandwidth, 'linewidth': 3},\n",
    "                 label = bandwidth)\n",
    "    \n",
    "plt.legend(prop={'size': 16}, title = 'bandwidth')\n",
    "plt.title('Density Plot with Varying Bandwidths'); \n",
    "plt.xlabel('Delay (min)'); plt.ylabel('Density');"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Different Kernels for the Density Plot"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "for kernel in ['gau', 'cos', 'biw', 'epa', 'tri', 'triw']:\n",
    "    sns.distplot(flights['arr_delay'], hist = False, kde = True, \n",
    "                 kde_kws={'kernel': kernel, 'linewidth': 3},\n",
    "                 label = kernel)\n",
    "    \n",
    "plt.legend(prop={'size': 16}, title = 'Kernel')\n",
    "plt.title('Density Plot with Different Kernels'); \n",
    "plt.xlabel('Delay (min)'); plt.ylabel('Density');"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "hideCode": false,
    "hidePrompt": false
   },
   "source": [
    "# Density Plot of Delays of Top Five"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {
    "hideCode": false,
    "hidePrompt": false
   },
   "outputs": [
    {
     "data": {
      "image/png": "[encoded data, 90664 characters]\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x1515064e048>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "figsize(12, 8)\n",
    "\n",
    "# List of five airlines to plot\n",
    "airlines = ['United Air Lines Inc.', 'JetBlue Airways', 'ExpressJet Airlines Inc.',\n",
    "         'Delta Air Lines Inc.', 'American Airlines Inc.']\n",
    "\n",
    "# Iterate through the top five airlines\n",
    "for airline in airlines:\n",
    "    # Subset to the airline\n",
    "    subset = flights[flights['name'] == airline]\n",
    "    \n",
    "    # Draw the density plot\n",
    "    sns.distplot(subset['arr_delay'], hist = False, kde = True,\n",
    "                 kde_kws = {'linewidth': 3},\n",
    "                 label = airline)\n",
    "    \n",
    "plt.legend(prop={'size': 16}, title = 'Airline')\n",
    "plt.title('Density Plot with Multiple Airlines'); \n",
    "plt.xlabel('Delay (min)')\n",
    "plt.ylabel('Density');"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Shaded Density Plot"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "[encoded data, 54548 characters]\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x15150662320>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "for airline in ['United Air Lines Inc.', 'Alaska Airlines Inc.']:\n",
    "    subset = flights[flights['name'] == airline]\n",
    "    \n",
    "    sns.distplot(subset['arr_delay'], hist = False, kde = True,\n",
    "                 kde_kws = {'shade': True, 'linewidth': 3}, label = airline)\n",
    "    \n",
    "plt.legend(prop={'size': 16}, title = 'Airline')\n",
    "plt.title('Shaded Density Plot of Arrival Delays'); \n",
    "plt.xlabel('Delay (min)'); plt.ylabel('Density');"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Density with Rug Plot"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "[encoded data, 36556 characters]\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x15154a96588>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "# Subset to Alaska Airlines\n",
    "subset = flights[flights['name'] == 'Alaska Airlines Inc.']\n",
    "\n",
    "# Density Plot with Rug Plot\n",
    "sns.distplot(subset['arr_delay'], hist = False, kde = True, rug = True,\n",
    "             color = 'darkblue', \n",
    "             kde_kws={'linewidth': 3},\n",
    "             rug_kws={'color': 'black'})\n",
    "\n",
    "# Plot formatting\n",
    "plt.title('Density Plot with Rug Plot for Alaska Airlines');\n",
    "plt.xlabel('Delay (min)'); plt.ylabel('Density');"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "hide_code_all_hidden": false,
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
