Notebooks/1_Preprocessing/2_Data_Aggregation.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "ImvdTW03GMJO"
   },
   "source": [
    "# 0. Importing data (DO NOT ALTER)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "executionInfo": {
     "elapsed": 1474,
     "status": "ok",
     "timestamp": 1599476915623,
     "user": {
      "displayName": "Janka Gecelovská",
      "photoUrl": "https://lh3.googleusercontent.com/a-/AOh14GjVqlh2PlzTOTBDiLLtofNwHsxobYMwGvAajTuXUa0=s64",
      "userId": "04343601701373244670"
     },
     "user_tz": -120
    },
    "id": "HMaXlSbJbEv2"
   },
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import numpy as np"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "executionInfo": {
     "elapsed": 1336,
     "status": "ok",
     "timestamp": 1599476917596,
     "user": {
      "displayName": "Janka Gecelovská",
      "photoUrl": "https://lh3.googleusercontent.com/a-/AOh14GjVqlh2PlzTOTBDiLLtofNwHsxobYMwGvAajTuXUa0=s64",
      "userId": "04343601701373244670"
     },
     "user_tz": -120
    },
    "id": "dITYoLSQX32E"
   },
   "outputs": [],
   "source": [
    "# Load the dataset\n",
    "data = pd.read_csv('Data/avocado.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "78XPjgBkgRIY"
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Date</th>\n",
       "      <th>AveragePrice</th>\n",
       "      <th>Total Volume</th>\n",
       "      <th>Small Hass Avocado</th>\n",
       "      <th>Large Hass Avocado</th>\n",
       "      <th>Extra Large Hass Avocado</th>\n",
       "      <th>Total Bags</th>\n",
       "      <th>Small Bags</th>\n",
       "      <th>Large Bags</th>\n",
       "      <th>XLarge Bags</th>\n",
       "      <th>type</th>\n",
       "      <th>year</th>\n",
       "      <th>region</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>2015-01-04</td>\n",
       "      <td>1.75</td>\n",
       "      <td>27365.89</td>\n",
       "      <td>9307.34</td>\n",
       "      <td>3844.81</td>\n",
       "      <td>615.28</td>\n",
       "      <td>13598.46</td>\n",
       "      <td>13061.10</td>\n",
       "      <td>537.36</td>\n",
       "      <td>0.00</td>\n",
       "      <td>organic</td>\n",
       "      <td>2015</td>\n",
       "      <td>Southeast</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>2015-01-04</td>\n",
       "      <td>1.49</td>\n",
       "      <td>17723.17</td>\n",
       "      <td>1189.35</td>\n",
       "      <td>15628.27</td>\n",
       "      <td>0.00</td>\n",
       "      <td>905.55</td>\n",
       "      <td>905.55</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>organic</td>\n",
       "      <td>2015</td>\n",
       "      <td>Chicago</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>2015-01-04</td>\n",
       "      <td>1.68</td>\n",
       "      <td>2896.72</td>\n",
       "      <td>161.68</td>\n",
       "      <td>206.96</td>\n",
       "      <td>0.00</td>\n",
       "      <td>2528.08</td>\n",
       "      <td>2528.08</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>organic</td>\n",
       "      <td>2015</td>\n",
       "      <td>HarrisburgScranton</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>2015-01-04</td>\n",
       "      <td>1.52</td>\n",
       "      <td>54956.80</td>\n",
       "      <td>3013.04</td>\n",
       "      <td>35456.88</td>\n",
       "      <td>1561.70</td>\n",
       "      <td>14925.18</td>\n",
       "      <td>11264.80</td>\n",
       "      <td>3660.38</td>\n",
       "      <td>0.00</td>\n",
       "      <td>conventional</td>\n",
       "      <td>2015</td>\n",
       "      <td>Pittsburgh</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>2015-01-04</td>\n",
       "      <td>1.64</td>\n",
       "      <td>1505.12</td>\n",
       "      <td>1.27</td>\n",
       "      <td>1129.50</td>\n",
       "      <td>0.00</td>\n",
       "      <td>374.35</td>\n",
       "      <td>186.67</td>\n",
       "      <td>187.68</td>\n",
       "      <td>0.00</td>\n",
       "      <td>organic</td>\n",
       "      <td>2015</td>\n",
       "      <td>Boise</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>18244</th>\n",
       "      <td>2018-03-25</td>\n",
       "      <td>1.36</td>\n",
       "      <td>908202.13</td>\n",
       "      <td>142681.06</td>\n",
       "      <td>463136.28</td>\n",
       "      <td>174975.75</td>\n",
       "      <td>127409.04</td>\n",
       "      <td>103579.41</td>\n",
       "      <td>22467.04</td>\n",
       "      <td>1362.59</td>\n",
       "      <td>conventional</td>\n",
       "      <td>2018</td>\n",
       "      <td>Chicago</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>18245</th>\n",
       "      <td>2018-03-25</td>\n",
       "      <td>0.70</td>\n",
       "      <td>9010588.32</td>\n",
       "      <td>3999735.71</td>\n",
       "      <td>966589.50</td>\n",
       "      <td>30130.82</td>\n",
       "      <td>4014132.29</td>\n",
       "      <td>3398569.92</td>\n",
       "      <td>546409.74</td>\n",
       "      <td>69152.63</td>\n",
       "      <td>conventional</td>\n",
       "      <td>2018</td>\n",
       "      <td>SouthCentral</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>18246</th>\n",
       "      <td>2018-03-25</td>\n",
       "      <td>1.42</td>\n",
       "      <td>163496.70</td>\n",
       "      <td>29253.30</td>\n",
       "      <td>5080.04</td>\n",
       "      <td>0.00</td>\n",
       "      <td>129163.36</td>\n",
       "      <td>109052.26</td>\n",
       "      <td>20111.10</td>\n",
       "      <td>0.00</td>\n",
       "      <td>organic</td>\n",
       "      <td>2018</td>\n",
       "      <td>SouthCentral</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>18247</th>\n",
       "      <td>2018-03-25</td>\n",
       "      <td>1.70</td>\n",
       "      <td>190257.38</td>\n",
       "      <td>29644.09</td>\n",
       "      <td>70982.10</td>\n",
       "      <td>0.00</td>\n",
       "      <td>89631.19</td>\n",
       "      <td>89424.11</td>\n",
       "      <td>207.08</td>\n",
       "      <td>0.00</td>\n",
       "      <td>organic</td>\n",
       "      <td>2018</td>\n",
       "      <td>California</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>18248</th>\n",
       "      <td>2018-03-25</td>\n",
       "      <td>1.34</td>\n",
       "      <td>1774776.77</td>\n",
       "      <td>63905.98</td>\n",
       "      <td>908653.71</td>\n",
       "      <td>843.45</td>\n",
       "      <td>801373.63</td>\n",
       "      <td>774634.09</td>\n",
       "      <td>23833.93</td>\n",
       "      <td>2905.61</td>\n",
       "      <td>conventional</td>\n",
       "      <td>2018</td>\n",
       "      <td>NewYork</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>18249 rows × 13 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "             Date  AveragePrice  Total Volume  Small Hass Avocado  \\\n",
       "0      2015-01-04          1.75      27365.89             9307.34   \n",
       "1      2015-01-04          1.49      17723.17             1189.35   \n",
       "2      2015-01-04          1.68       2896.72              161.68   \n",
       "3      2015-01-04          1.52      54956.80             3013.04   \n",
       "4      2015-01-04          1.64       1505.12                1.27   \n",
       "...           ...           ...           ...                 ...   \n",
       "18244  2018-03-25          1.36     908202.13           142681.06   \n",
       "18245  2018-03-25          0.70    9010588.32          3999735.71   \n",
       "18246  2018-03-25          1.42     163496.70            29253.30   \n",
       "18247  2018-03-25          1.70     190257.38            29644.09   \n",
       "18248  2018-03-25          1.34    1774776.77            63905.98   \n",
       "\n",
       "       Large Hass Avocado  Extra Large Hass Avocado  Total Bags  Small Bags  \\\n",
       "0                 3844.81                    615.28    13598.46    13061.10   \n",
       "1                15628.27                      0.00      905.55      905.55   \n",
       "2                  206.96                      0.00     2528.08     2528.08   \n",
       "3                35456.88                   1561.70    14925.18    11264.80   \n",
       "4                 1129.50                      0.00      374.35      186.67   \n",
       "...                   ...                       ...         ...         ...   \n",
       "18244           463136.28                 174975.75   127409.04   103579.41   \n",
       "18245           966589.50                  30130.82  4014132.29  3398569.92   \n",
       "18246             5080.04                      0.00   129163.36   109052.26   \n",
       "18247            70982.10                      0.00    89631.19    89424.11   \n",
       "18248           908653.71                    843.45   801373.63   774634.09   \n",
       "\n",
       "       Large Bags  XLarge Bags          type  year              region  \n",
       "0          537.36         0.00       organic  2015           Southeast  \n",
       "1            0.00         0.00       organic  2015             Chicago  \n",
       "2            0.00         0.00       organic  2015  HarrisburgScranton  \n",
       "3         3660.38         0.00  conventional  2015          Pittsburgh  \n",
       "4          187.68         0.00       organic  2015               Boise  \n",
       "...           ...          ...           ...   ...                 ...  \n",
       "18244    22467.04      1362.59  conventional  2018             Chicago  \n",
       "18245   546409.74     69152.63  conventional  2018        SouthCentral  \n",
       "18246    20111.10         0.00       organic  2018        SouthCentral  \n",
       "18247      207.08         0.00       organic  2018          California  \n",
       "18248    23833.93      2905.61  conventional  2018             NewYork  \n",
       "\n",
       "[18249 rows x 13 columns]"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Preview the data\n",
    "data"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "8vK6YEP8jbBF"
   },
   "source": [
    "# 1. 1-D aggregations on Pandas Series\n",
    "\n",
    "Let's recall computing aggregations such as `sum()`, `mean()`, `median()`, `max()` and `min()` using Pandas Series. "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "rx-Ku8_xcjca"
   },
   "outputs": [],
   "source": [
    "# Create Pandas Series using values: [8.45, 3.15, 1.25, 10.55, 2.40]\n",
    "\n",
    "our_series = pd.Series([8.45, 3.15, 1.25, 10.55, 2.40])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "21oSgZtcm8rG"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "The rounded count of the values is: 26.0\n",
      "The average value is: 5.159999999999999\n",
      "The median value is: 3.15\n",
      "The maximum value is: 10.55\n",
      "The minimum value is: 1.25\n"
     ]
    }
   ],
   "source": [
    "# TASK 1 >>>> Print computing aggregations \n",
    "print(f'The rounded count of the values is: {our_series.sum().round()}')\n",
    "\n",
    "# Fill in the empy brackets {} (returns error if not filled)\n",
    "print(f'The average value is: {}')\n",
    "print(f'The median value is: {}')\n",
    "print(f'The maximum value is: {}')\n",
    "print(f'The minimum value is: {}')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "WgQ0ENnOJiEi"
   },
   "source": [
    "# 2. 2-D aggregations on Pandas DataFrame\n",
    "\n",
    "To understand the true power of `groupby()` we can take a look at what is going on under the hood.  \n",
    "Let's say we want to compute the average price of avocados based on their type: conventional and organic. \n",
    "\n",
    "Firstly, we have to split our dataset into 2 different groups based on the type:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "executionInfo": {
     "elapsed": 1740,
     "status": "ok",
     "timestamp": 1599476928983,
     "user": {
      "displayName": "Janka Gecelovská",
      "photoUrl": "https://lh3.googleusercontent.com/a-/AOh14GjVqlh2PlzTOTBDiLLtofNwHsxobYMwGvAajTuXUa0=s64",
      "userId": "04343601701373244670"
     },
     "user_tz": -120
    },
    "id": "cmzctVlW3G_z"
   },
   "outputs": [],
   "source": [
    "# Filter only those records that are organic type and assign it to variable filter_o\n",
    "filter_o = data['type'] == 'organic'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "wbgFZkT2HqVv"
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Date</th>\n",
       "      <th>AveragePrice</th>\n",
       "      <th>Total Volume</th>\n",
       "      <th>Small Hass Avocado</th>\n",
       "      <th>Large Hass Avocado</th>\n",
       "      <th>Extra Large Hass Avocado</th>\n",
       "      <th>Total Bags</th>\n",
       "      <th>Small Bags</th>\n",
       "      <th>Large Bags</th>\n",
       "      <th>XLarge Bags</th>\n",
       "      <th>type</th>\n",
       "      <th>year</th>\n",
       "      <th>region</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>2015-01-04</td>\n",
       "      <td>1.75</td>\n",
       "      <td>27365.89</td>\n",
       "      <td>9307.34</td>\n",
       "      <td>3844.81</td>\n",
       "      <td>615.28</td>\n",
       "      <td>13598.46</td>\n",
       "      <td>13061.10</td>\n",
       "      <td>537.36</td>\n",
       "      <td>0.0</td>\n",
       "      <td>organic</td>\n",
       "      <td>2015</td>\n",
       "      <td>Southeast</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>2015-01-04</td>\n",
       "      <td>1.49</td>\n",
       "      <td>17723.17</td>\n",
       "      <td>1189.35</td>\n",
       "      <td>15628.27</td>\n",
       "      <td>0.00</td>\n",
       "      <td>905.55</td>\n",
       "      <td>905.55</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.0</td>\n",
       "      <td>organic</td>\n",
       "      <td>2015</td>\n",
       "      <td>Chicago</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>2015-01-04</td>\n",
       "      <td>1.68</td>\n",
       "      <td>2896.72</td>\n",
       "      <td>161.68</td>\n",
       "      <td>206.96</td>\n",
       "      <td>0.00</td>\n",
       "      <td>2528.08</td>\n",
       "      <td>2528.08</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.0</td>\n",
       "      <td>organic</td>\n",
       "      <td>2015</td>\n",
       "      <td>HarrisburgScranton</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>2015-01-04</td>\n",
       "      <td>1.64</td>\n",
       "      <td>1505.12</td>\n",
       "      <td>1.27</td>\n",
       "      <td>1129.50</td>\n",
       "      <td>0.00</td>\n",
       "      <td>374.35</td>\n",
       "      <td>186.67</td>\n",
       "      <td>187.68</td>\n",
       "      <td>0.0</td>\n",
       "      <td>organic</td>\n",
       "      <td>2015</td>\n",
       "      <td>Boise</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>2015-01-04</td>\n",
       "      <td>1.50</td>\n",
       "      <td>6329.83</td>\n",
       "      <td>3730.80</td>\n",
       "      <td>2141.91</td>\n",
       "      <td>0.00</td>\n",
       "      <td>457.12</td>\n",
       "      <td>426.67</td>\n",
       "      <td>30.45</td>\n",
       "      <td>0.0</td>\n",
       "      <td>organic</td>\n",
       "      <td>2015</td>\n",
       "      <td>LasVegas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>18240</th>\n",
       "      <td>2018-03-25</td>\n",
       "      <td>1.75</td>\n",
       "      <td>5518.73</td>\n",
       "      <td>171.22</td>\n",
       "      <td>806.65</td>\n",
       "      <td>3.93</td>\n",
       "      <td>4536.93</td>\n",
       "      <td>2944.89</td>\n",
       "      <td>1592.04</td>\n",
       "      <td>0.0</td>\n",
       "      <td>organic</td>\n",
       "      <td>2018</td>\n",
       "      <td>Jacksonville</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>18241</th>\n",
       "      <td>2018-03-25</td>\n",
       "      <td>1.42</td>\n",
       "      <td>17340.49</td>\n",
       "      <td>295.16</td>\n",
       "      <td>3478.97</td>\n",
       "      <td>81.25</td>\n",
       "      <td>13485.11</td>\n",
       "      <td>12149.49</td>\n",
       "      <td>1335.62</td>\n",
       "      <td>0.0</td>\n",
       "      <td>organic</td>\n",
       "      <td>2018</td>\n",
       "      <td>RichmondNorfolk</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>18243</th>\n",
       "      <td>2018-03-25</td>\n",
       "      <td>1.74</td>\n",
       "      <td>38441.23</td>\n",
       "      <td>27.25</td>\n",
       "      <td>2031.54</td>\n",
       "      <td>0.00</td>\n",
       "      <td>36382.44</td>\n",
       "      <td>29466.95</td>\n",
       "      <td>6915.49</td>\n",
       "      <td>0.0</td>\n",
       "      <td>organic</td>\n",
       "      <td>2018</td>\n",
       "      <td>Boston</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>18246</th>\n",
       "      <td>2018-03-25</td>\n",
       "      <td>1.42</td>\n",
       "      <td>163496.70</td>\n",
       "      <td>29253.30</td>\n",
       "      <td>5080.04</td>\n",
       "      <td>0.00</td>\n",
       "      <td>129163.36</td>\n",
       "      <td>109052.26</td>\n",
       "      <td>20111.10</td>\n",
       "      <td>0.0</td>\n",
       "      <td>organic</td>\n",
       "      <td>2018</td>\n",
       "      <td>SouthCentral</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>18247</th>\n",
       "      <td>2018-03-25</td>\n",
       "      <td>1.70</td>\n",
       "      <td>190257.38</td>\n",
       "      <td>29644.09</td>\n",
       "      <td>70982.10</td>\n",
       "      <td>0.00</td>\n",
       "      <td>89631.19</td>\n",
       "      <td>89424.11</td>\n",
       "      <td>207.08</td>\n",
       "      <td>0.0</td>\n",
       "      <td>organic</td>\n",
       "      <td>2018</td>\n",
       "      <td>California</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>9123 rows × 13 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "             Date  AveragePrice  Total Volume  Small Hass Avocado  \\\n",
       "0      2015-01-04          1.75      27365.89             9307.34   \n",
       "1      2015-01-04          1.49      17723.17             1189.35   \n",
       "2      2015-01-04          1.68       2896.72              161.68   \n",
       "4      2015-01-04          1.64       1505.12                1.27   \n",
       "7      2015-01-04          1.50       6329.83             3730.80   \n",
       "...           ...           ...           ...                 ...   \n",
       "18240  2018-03-25          1.75       5518.73              171.22   \n",
       "18241  2018-03-25          1.42      17340.49              295.16   \n",
       "18243  2018-03-25          1.74      38441.23               27.25   \n",
       "18246  2018-03-25          1.42     163496.70            29253.30   \n",
       "18247  2018-03-25          1.70     190257.38            29644.09   \n",
       "\n",
       "       Large Hass Avocado  Extra Large Hass Avocado  Total Bags  Small Bags  \\\n",
       "0                 3844.81                    615.28    13598.46    13061.10   \n",
       "1                15628.27                      0.00      905.55      905.55   \n",
       "2                  206.96                      0.00     2528.08     2528.08   \n",
       "4                 1129.50                      0.00      374.35      186.67   \n",
       "7                 2141.91                      0.00      457.12      426.67   \n",
       "...                   ...                       ...         ...         ...   \n",
       "18240              806.65                      3.93     4536.93     2944.89   \n",
       "18241             3478.97                     81.25    13485.11    12149.49   \n",
       "18243             2031.54                      0.00    36382.44    29466.95   \n",
       "18246             5080.04                      0.00   129163.36   109052.26   \n",
       "18247            70982.10                      0.00    89631.19    89424.11   \n",
       "\n",
       "       Large Bags  XLarge Bags     type  year              region  \n",
       "0          537.36          0.0  organic  2015           Southeast  \n",
       "1            0.00          0.0  organic  2015             Chicago  \n",
       "2            0.00          0.0  organic  2015  HarrisburgScranton  \n",
       "4          187.68          0.0  organic  2015               Boise  \n",
       "7           30.45          0.0  organic  2015            LasVegas  \n",
       "...           ...          ...      ...   ...                 ...  \n",
       "18240     1592.04          0.0  organic  2018        Jacksonville  \n",
       "18241     1335.62          0.0  organic  2018     RichmondNorfolk  \n",
       "18243     6915.49          0.0  organic  2018              Boston  \n",
       "18246    20111.10          0.0  organic  2018        SouthCentral  \n",
       "18247      207.08          0.0  organic  2018          California  \n",
       "\n",
       "[9123 rows x 13 columns]"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Use .loc[] on data to access all columns based on our condition filter_o and assign it to the variable data_organic\n",
    "data_organic = data.loc[filter_o]\n",
    "data_organic"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "BKtcJ4fHHNql"
   },
   "source": [
    "See that only organic type remain."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "executionInfo": {
     "elapsed": 1005,
     "status": "ok",
     "timestamp": 1599476953354,
     "user": {
      "displayName": "Janka Gecelovská",
      "photoUrl": "https://lh3.googleusercontent.com/a-/AOh14GjVqlh2PlzTOTBDiLLtofNwHsxobYMwGvAajTuXUa0=s64",
      "userId": "04343601701373244670"
     },
     "user_tz": -120
    },
    "id": "6jcOyEPTHxKN"
   },
   "outputs": [],
   "source": [
    "# TASK 2 >>>> Filter only those records that are of type conventional and assign it to the variable filter_c"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "z_xrgVE6HySt"
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Date</th>\n",
       "      <th>AveragePrice</th>\n",
       "      <th>Total Volume</th>\n",
       "      <th>Small Hass Avocado</th>\n",
       "      <th>Large Hass Avocado</th>\n",
       "      <th>Extra Large Hass Avocado</th>\n",
       "      <th>Total Bags</th>\n",
       "      <th>Small Bags</th>\n",
       "      <th>Large Bags</th>\n",
       "      <th>XLarge Bags</th>\n",
       "      <th>type</th>\n",
       "      <th>year</th>\n",
       "      <th>region</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>2015-01-04</td>\n",
       "      <td>1.52</td>\n",
       "      <td>54956.80</td>\n",
       "      <td>3013.04</td>\n",
       "      <td>35456.88</td>\n",
       "      <td>1561.70</td>\n",
       "      <td>14925.18</td>\n",
       "      <td>11264.80</td>\n",
       "      <td>3660.38</td>\n",
       "      <td>0.00</td>\n",
       "      <td>conventional</td>\n",
       "      <td>2015</td>\n",
       "      <td>Pittsburgh</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>2015-01-04</td>\n",
       "      <td>0.75</td>\n",
       "      <td>758118.95</td>\n",
       "      <td>426878.87</td>\n",
       "      <td>147958.43</td>\n",
       "      <td>15267.89</td>\n",
       "      <td>168013.76</td>\n",
       "      <td>165202.22</td>\n",
       "      <td>2811.54</td>\n",
       "      <td>0.00</td>\n",
       "      <td>conventional</td>\n",
       "      <td>2015</td>\n",
       "      <td>WestTexNewMexico</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>2015-01-04</td>\n",
       "      <td>0.85</td>\n",
       "      <td>2682159.95</td>\n",
       "      <td>1837999.65</td>\n",
       "      <td>524430.47</td>\n",
       "      <td>64225.78</td>\n",
       "      <td>255504.05</td>\n",
       "      <td>215571.80</td>\n",
       "      <td>36981.72</td>\n",
       "      <td>2950.53</td>\n",
       "      <td>conventional</td>\n",
       "      <td>2015</td>\n",
       "      <td>LosAngeles</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>2015-01-04</td>\n",
       "      <td>1.00</td>\n",
       "      <td>435021.49</td>\n",
       "      <td>364302.39</td>\n",
       "      <td>23821.16</td>\n",
       "      <td>82.15</td>\n",
       "      <td>46815.79</td>\n",
       "      <td>16707.15</td>\n",
       "      <td>30108.64</td>\n",
       "      <td>0.00</td>\n",
       "      <td>conventional</td>\n",
       "      <td>2015</td>\n",
       "      <td>Atlanta</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>2015-01-04</td>\n",
       "      <td>0.80</td>\n",
       "      <td>317861.35</td>\n",
       "      <td>134003.07</td>\n",
       "      <td>120628.37</td>\n",
       "      <td>4591.23</td>\n",
       "      <td>58638.68</td>\n",
       "      <td>58126.59</td>\n",
       "      <td>512.09</td>\n",
       "      <td>0.00</td>\n",
       "      <td>conventional</td>\n",
       "      <td>2015</td>\n",
       "      <td>LasVegas</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>18239</th>\n",
       "      <td>2018-03-25</td>\n",
       "      <td>1.39</td>\n",
       "      <td>641462.43</td>\n",
       "      <td>26019.52</td>\n",
       "      <td>479186.12</td>\n",
       "      <td>1883.17</td>\n",
       "      <td>134373.62</td>\n",
       "      <td>109938.39</td>\n",
       "      <td>22955.23</td>\n",
       "      <td>1480.00</td>\n",
       "      <td>conventional</td>\n",
       "      <td>2018</td>\n",
       "      <td>Boston</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>18242</th>\n",
       "      <td>2018-03-25</td>\n",
       "      <td>1.32</td>\n",
       "      <td>429132.50</td>\n",
       "      <td>258217.83</td>\n",
       "      <td>25970.97</td>\n",
       "      <td>1426.07</td>\n",
       "      <td>143517.63</td>\n",
       "      <td>90870.28</td>\n",
       "      <td>52357.35</td>\n",
       "      <td>290.00</td>\n",
       "      <td>conventional</td>\n",
       "      <td>2018</td>\n",
       "      <td>Orlando</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>18244</th>\n",
       "      <td>2018-03-25</td>\n",
       "      <td>1.36</td>\n",
       "      <td>908202.13</td>\n",
       "      <td>142681.06</td>\n",
       "      <td>463136.28</td>\n",
       "      <td>174975.75</td>\n",
       "      <td>127409.04</td>\n",
       "      <td>103579.41</td>\n",
       "      <td>22467.04</td>\n",
       "      <td>1362.59</td>\n",
       "      <td>conventional</td>\n",
       "      <td>2018</td>\n",
       "      <td>Chicago</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>18245</th>\n",
       "      <td>2018-03-25</td>\n",
       "      <td>0.70</td>\n",
       "      <td>9010588.32</td>\n",
       "      <td>3999735.71</td>\n",
       "      <td>966589.50</td>\n",
       "      <td>30130.82</td>\n",
       "      <td>4014132.29</td>\n",
       "      <td>3398569.92</td>\n",
       "      <td>546409.74</td>\n",
       "      <td>69152.63</td>\n",
       "      <td>conventional</td>\n",
       "      <td>2018</td>\n",
       "      <td>SouthCentral</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>18248</th>\n",
       "      <td>2018-03-25</td>\n",
       "      <td>1.34</td>\n",
       "      <td>1774776.77</td>\n",
       "      <td>63905.98</td>\n",
       "      <td>908653.71</td>\n",
       "      <td>843.45</td>\n",
       "      <td>801373.63</td>\n",
       "      <td>774634.09</td>\n",
       "      <td>23833.93</td>\n",
       "      <td>2905.61</td>\n",
       "      <td>conventional</td>\n",
       "      <td>2018</td>\n",
       "      <td>NewYork</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>9126 rows × 13 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "             Date  AveragePrice  Total Volume  Small Hass Avocado  \\\n",
       "3      2015-01-04          1.52      54956.80             3013.04   \n",
       "5      2015-01-04          0.75     758118.95           426878.87   \n",
       "6      2015-01-04          0.85    2682159.95          1837999.65   \n",
       "8      2015-01-04          1.00     435021.49           364302.39   \n",
       "9      2015-01-04          0.80     317861.35           134003.07   \n",
       "...           ...           ...           ...                 ...   \n",
       "18239  2018-03-25          1.39     641462.43            26019.52   \n",
       "18242  2018-03-25          1.32     429132.50           258217.83   \n",
       "18244  2018-03-25          1.36     908202.13           142681.06   \n",
       "18245  2018-03-25          0.70    9010588.32          3999735.71   \n",
       "18248  2018-03-25          1.34    1774776.77            63905.98   \n",
       "\n",
       "       Large Hass Avocado  Extra Large Hass Avocado  Total Bags  Small Bags  \\\n",
       "3                35456.88                   1561.70    14925.18    11264.80   \n",
       "5               147958.43                  15267.89   168013.76   165202.22   \n",
       "6               524430.47                  64225.78   255504.05   215571.80   \n",
       "8                23821.16                     82.15    46815.79    16707.15   \n",
       "9               120628.37                   4591.23    58638.68    58126.59   \n",
       "...                   ...                       ...         ...         ...   \n",
       "18239           479186.12                   1883.17   134373.62   109938.39   \n",
       "18242            25970.97                   1426.07   143517.63    90870.28   \n",
       "18244           463136.28                 174975.75   127409.04   103579.41   \n",
       "18245           966589.50                  30130.82  4014132.29  3398569.92   \n",
       "18248           908653.71                    843.45   801373.63   774634.09   \n",
       "\n",
       "       Large Bags  XLarge Bags          type  year            region  \n",
       "3         3660.38         0.00  conventional  2015        Pittsburgh  \n",
       "5         2811.54         0.00  conventional  2015  WestTexNewMexico  \n",
       "6        36981.72      2950.53  conventional  2015        LosAngeles  \n",
       "8        30108.64         0.00  conventional  2015           Atlanta  \n",
       "9          512.09         0.00  conventional  2015          LasVegas  \n",
       "...           ...          ...           ...   ...               ...  \n",
       "18239    22955.23      1480.00  conventional  2018            Boston  \n",
       "18242    52357.35       290.00  conventional  2018           Orlando  \n",
       "18244    22467.04      1362.59  conventional  2018           Chicago  \n",
       "18245   546409.74     69152.63  conventional  2018      SouthCentral  \n",
       "18248    23833.93      2905.61  conventional  2018           NewYork  \n",
       "\n",
       "[9126 rows x 13 columns]"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# TASK 2 >>>> Use .loc[] on data to access all columns based on our condition filter_c and assign it to the variable data_conventional"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "aIBTB4C3LFfu"
   },
   "source": [
    "Now compute the average price for both types of avocados using the `.mean()` method applied to the column `AveragePrice`."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "executionInfo": {
     "elapsed": 782,
     "status": "ok",
     "timestamp": 1599478554330,
     "user": {
      "displayName": "Janka Gecelovská",
      "photoUrl": "https://lh3.googleusercontent.com/a-/AOh14GjVqlh2PlzTOTBDiLLtofNwHsxobYMwGvAajTuXUa0=s64",
      "userId": "04343601701373244670"
     },
     "user_tz": -120
    },
    "id": "T6q6IHczIHpO"
   },
   "outputs": [],
   "source": [
    "# Compute the average price for filtered organic avocados and assign it to the variable avg_organic\n",
    "avg_organic = data_organic['AveragePrice'].mean()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "executionInfo": {
     "elapsed": 438,
     "status": "ok",
     "timestamp": 1599478555240,
     "user": {
      "displayName": "Janka Gecelovská",
      "photoUrl": "https://lh3.googleusercontent.com/a-/AOh14GjVqlh2PlzTOTBDiLLtofNwHsxobYMwGvAajTuXUa0=s64",
      "userId": "04343601701373244670"
     },
     "user_tz": -120
    },
    "id": "xa-vceLZIdjX"
   },
   "outputs": [],
   "source": [
    "# TASK 3 >>>> Compute the average price for filtered conventional avocados and assign it to the variable avg_conventional"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "L7f3uzWuIi1G"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "1.6539986846432093 1.1580396668858206\n",
      "\n",
      "\n",
      "<class 'numpy.float64'> <class 'numpy.float64'>\n"
     ]
    }
   ],
   "source": [
    "# Print the outputs and the type of the outputs\n",
    "print(avg_organic, avg_conventional)\n",
    "print('\\n')\n",
    "print(type(avg_organic), type(avg_conventional))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "kBeE35oFLemd"
   },
   "source": [
    "Lastly, combine these results into data structure using `Pandas`  `.DataFrame()`. Create a dictionary, where the first key name will be 'Type' and its values 'organic', 'conventional'. The second key name will be 'Average_price' and its values will be our created `avg_organic` and `avg_conventional`, respectively."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "executionInfo": {
     "elapsed": 754,
     "status": "ok",
     "timestamp": 1599478681099,
     "user": {
      "displayName": "Janka Gecelovská",
      "photoUrl": "https://lh3.googleusercontent.com/a-/AOh14GjVqlh2PlzTOTBDiLLtofNwHsxobYMwGvAajTuXUa0=s64",
      "userId": "04343601701373244670"
     },
     "user_tz": -120
    },
    "id": "uu-CkthjIlSl"
   },
   "outputs": [],
   "source": [
    "# Combine these results into a new DataFrame\n",
    "data_output = pd.DataFrame({'Type':['organic','conventional'], \n",
    "                            'Average_price':[avg_organic, avg_conventional]})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "7bT1M7-NI38d"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Result dataframe :\n",
      "            Type  Average_price\n",
      "0       organic       1.653999\n",
      "1  conventional       1.158040\n"
     ]
    }
   ],
   "source": [
    "# Print the resulting DataFrame\n",
    "print('\\nResult dataframe :\\n',data_output)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "asLaYdfYNOP2"
   },
   "source": [
    "However, we can use `groupby()` to achieve the same result with only 1 line of the code!"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "FPyXkdREJO9s"
   },
   "source": [
    "# 3. 2-D aggregations on Pandas DataFrame (KEY LEARNING)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "WS1EkBKKrr8n"
   },
   "source": [
    " `groupby()` function allows us to quickly and efficiently split the data into separate groups to perform computations. When we pass the desired column or columns within `groupby()`, it will return _DataFrameGroupBy object_. We can think of it as a special view on our DataFrame. No computation will be done until we specify functions such as `mean()`, `sum()` etc. "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "TrkKgFYd8_Wz"
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<pandas.core.groupby.generic.DataFrameGroupBy object at 0x7fd002244090>"
      ]
     },
     "execution_count": 22,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Group the data based on the column 'year'\n",
    "data.groupby('year')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "qWce_PzhO8jB"
   },
   "source": [
    "Now we compute the average price for organic and conventional avocados again but we'll make use of `groupby()`."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "9V8leQF1tLb0"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "type\n",
      "conventional    1.158040\n",
      "organic         1.653999\n",
      "Name: AveragePrice, dtype: float64\n"
     ]
    }
   ],
   "source": [
    "# Group the data based on Avocado type\n",
    "# Compute the average price using .mean()\n",
    "\n",
    "by_type_total = data.groupby('type')['AveragePrice'].mean()\n",
    "print(by_type_total)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "RpEjNtiptrek"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "type          year\n",
      "conventional  2015    1.077963\n",
      "              2016    1.105595\n",
      "              2017    1.294888\n",
      "              2018    1.127886\n",
      "organic       2015    1.673324\n",
      "              2016    1.571684\n",
      "              2017    1.735521\n",
      "              2018    1.567176\n",
      "Name: AveragePrice, dtype: float64\n"
     ]
    }
   ],
   "source": [
    "# Group the data based on columns 'type' and 'region' passed into the list and compute the average price\n",
    "\n",
    "by_type_year = data.groupby(['type','year'])['AveragePrice'].mean()\n",
    "print(by_type_year)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "nn-gd3o_xC21"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "type          year  region             \n",
      "conventional  2015  Albany                  3211741.19\n",
      "                    Atlanta                 1840105.14\n",
      "                    BaltimoreWashington    25345911.00\n",
      "                    Boise                    528537.73\n",
      "                    Boston                 21314866.70\n",
      "                                              ...     \n",
      "organic       2018  Syracuse                   1212.50\n",
      "                    Tampa                      7320.94\n",
      "                    TotalUS                 4325043.54\n",
      "                    West                     638653.03\n",
      "                    WestTexNewMexico          27301.02\n",
      "Name: Large Hass Avocado, Length: 432, dtype: float64\n"
     ]
    }
   ],
   "source": [
    "# TASK 4 >>>> Group the data based on columns 'type', 'year' and 'region' passed into the list\n",
    "# and compute how many kg of Large Hass Avocados have been sold in total. \n",
    "# Assign the result to the variable by_year."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "xPnOHEGOKaXx"
   },
   "source": [
    "When we are using the `.groupby()`, the resulting object will be slightly different from a standard Pandas dataframe. You can see it in the print statement and how \"type\" and \"year\" are nicely printed. \n",
    "\n",
    "If we would like to work with the resulting object further, we should reset its row index by using `reset_index()` and convert it into a regular dataframe."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "S9pnZLMY_TZQ"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "             type  year               region  Large Hass Avocado\n",
      "0    conventional  2015               Albany          3211741.19\n",
      "1    conventional  2015              Atlanta          1840105.14\n",
      "2    conventional  2015  BaltimoreWashington         25345911.00\n",
      "3    conventional  2015                Boise           528537.73\n",
      "4    conventional  2015               Boston         21314866.70\n",
      "..            ...   ...                  ...                 ...\n",
      "427       organic  2018             Syracuse             1212.50\n",
      "428       organic  2018                Tampa             7320.94\n",
      "429       organic  2018              TotalUS          4325043.54\n",
      "430       organic  2018                 West           638653.03\n",
      "431       organic  2018     WestTexNewMexico            27301.02\n",
      "\n",
      "[432 rows x 4 columns]\n"
     ]
    }
   ],
   "source": [
    "# Reset the index using .reset_index() method and create a DataFrame\n",
    "our_df = pd.DataFrame(by_year).reset_index()\n",
    "print(our_df)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "jkSmhSgOJ4r2"
   },
   "source": [
    "# 4. Aggregate function (ADVANCED)\n",
    "\n",
    "![](https://keytodatascience.com/wp-content/uploads/2020/04/image-1.png)\n",
    "\n",
    "[image source](https://keytodatascience.com/groupby-pandas-python/)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "xndRkq37OXm1"
   },
   "source": [
    "The aggregation method `agg()`.\n",
    "\n",
    "- it is an alias for aggregate\n",
    "- it is used to pass a function or list of functions to be applied on a series or even each element of a series separately\n",
    "\n",
    "This can be done by passing the columns and functions within a dictionary like this:\n",
    "\n",
    "`our_dataset.agg({'First_column' : ['max', 'min'], 'Second_column' : ['mean', 'median']})`"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "H309MgFfOXL0"
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Total Volume</th>\n",
       "      <th>Small Bags</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>max</th>\n",
       "      <td>62505646.52</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>mean</th>\n",
       "      <td>NaN</td>\n",
       "      <td>182194.686696</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>min</th>\n",
       "      <td>84.56</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "      Total Volume     Small Bags\n",
       "max    62505646.52            NaN\n",
       "mean           NaN  182194.686696\n",
       "min          84.56       0.000000"
      ]
     },
     "execution_count": 28,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Compute maximum and minimum values for column 'Total Volume' and minimum and mean values for column 'Small Bags' using .agg()\n",
    "data.agg({'Total Volume' : ['max', 'min'], 'Small Bags' : ['min', 'mean']})"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "sFfoFWo_E2Wg"
   },
   "source": [
    "We can pass `.agg()` also to our grouped object and compute statistics for selected column."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "executionInfo": {
     "elapsed": 1159,
     "status": "ok",
     "timestamp": 1599479594259,
     "user": {
      "displayName": "Janka Gecelovská",
      "photoUrl": "https://lh3.googleusercontent.com/a-/AOh14GjVqlh2PlzTOTBDiLLtofNwHsxobYMwGvAajTuXUa0=s64",
      "userId": "04343601701373244670"
     },
     "user_tz": -120
    },
    "id": "aU0jsG31DpNI"
   },
   "outputs": [],
   "source": [
    "# Group the data based on the two columns 'region' and 'type'\n",
    "# Compute aggregations 'min','max' and 'mean' for 'AveragePrice'\n",
    "grouped = data.groupby(['region','type']).agg({'AveragePrice':['min','max','mean']})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "j3rNusHWEFXf"
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead tr th {\n",
       "        text-align: left;\n",
       "    }\n",
       "\n",
       "    .dataframe thead tr:last-of-type th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th colspan=\"3\" halign=\"left\">AveragePrice</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th>min</th>\n",
       "      <th>max</th>\n",
       "      <th>mean</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>region</th>\n",
       "      <th>type</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th rowspan=\"2\" valign=\"top\">Albany</th>\n",
       "      <th>conventional</th>\n",
       "      <td>0.85</td>\n",
       "      <td>1.80</td>\n",
       "      <td>1.348757</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>organic</th>\n",
       "      <td>1.32</td>\n",
       "      <td>2.13</td>\n",
       "      <td>1.773314</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th rowspan=\"2\" valign=\"top\">Atlanta</th>\n",
       "      <th>conventional</th>\n",
       "      <td>0.68</td>\n",
       "      <td>1.64</td>\n",
       "      <td>1.068817</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>organic</th>\n",
       "      <td>0.62</td>\n",
       "      <td>2.75</td>\n",
       "      <td>1.607101</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>BaltimoreWashington</th>\n",
       "      <th>conventional</th>\n",
       "      <td>0.95</td>\n",
       "      <td>1.86</td>\n",
       "      <td>1.344201</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>TotalUS</th>\n",
       "      <th>organic</th>\n",
       "      <td>1.00</td>\n",
       "      <td>2.09</td>\n",
       "      <td>1.546036</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th rowspan=\"2\" valign=\"top\">West</th>\n",
       "      <th>conventional</th>\n",
       "      <td>0.66</td>\n",
       "      <td>1.62</td>\n",
       "      <td>0.985089</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>organic</th>\n",
       "      <td>0.90</td>\n",
       "      <td>2.52</td>\n",
       "      <td>1.559349</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th rowspan=\"2\" valign=\"top\">WestTexNewMexico</th>\n",
       "      <th>conventional</th>\n",
       "      <td>0.52</td>\n",
       "      <td>1.22</td>\n",
       "      <td>0.842130</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>organic</th>\n",
       "      <td>1.04</td>\n",
       "      <td>2.93</td>\n",
       "      <td>1.688855</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>108 rows × 3 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                 AveragePrice                \n",
       "                                          min   max      mean\n",
       "region              type                                     \n",
       "Albany              conventional         0.85  1.80  1.348757\n",
       "                    organic              1.32  2.13  1.773314\n",
       "Atlanta             conventional         0.68  1.64  1.068817\n",
       "                    organic              0.62  2.75  1.607101\n",
       "BaltimoreWashington conventional         0.95  1.86  1.344201\n",
       "...                                       ...   ...       ...\n",
       "TotalUS             organic              1.00  2.09  1.546036\n",
       "West                conventional         0.66  1.62  0.985089\n",
       "                    organic              0.90  2.52  1.559349\n",
       "WestTexNewMexico    conventional         0.52  1.22  0.842130\n",
       "                    organic              1.04  2.93  1.688855\n",
       "\n",
       "[108 rows x 3 columns]"
      ]
     },
     "execution_count": 30,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "grouped"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "idp3VZhkQJzP"
   },
   "source": [
    "- within `agg()` we can have our custom function along with computing aggregation"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "jdQ2mgGZvxEF"
   },
   "outputs": [],
   "source": [
    "# Write a function to compute 95th percentile on desired column using .quantile(0.95)\n",
    "def percentile_95(column):\n",
    "    return column.quantile(0.95)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "q8958j6D26yZ"
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Small Bags</th>\n",
       "      <th>Large Bags</th>\n",
       "      <th>XLarge Bags</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>percentile_95</th>\n",
       "      <td>768147.228000</td>\n",
       "      <td>195699.768000</td>\n",
       "      <td>12058.452000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>mean</th>\n",
       "      <td>182194.686696</td>\n",
       "      <td>54338.088145</td>\n",
       "      <td>3106.426507</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                  Small Bags     Large Bags   XLarge Bags\n",
       "percentile_95  768147.228000  195699.768000  12058.452000\n",
       "mean           182194.686696   54338.088145   3106.426507"
      ]
     },
     "execution_count": 33,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# TASK 5 - HARD >>>> Get 95th percentile and mean values for columns: 'Small Bags','Large Bags','XLarge Bags' from DataFrame data, using .agg()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "zgZ5OqgdSDcG"
   },
   "source": [
    "# 5. Bonus Task (HARD)\n",
    "\n",
    "`groupby()` can be useful when we want to look at the proportion of avocado's type. We would like to see what percentages of conventional and organic avocados have been sold. For example: 97 % and 3%.\n",
    "\n",
    "To reach this result:\n",
    "- Group the data by 'type' and obtain sums on the 'Total Volume' column, assign result to `volume_by_type`\n",
    "- Divide `volume_by_type` by the sum of all avocados. Assign the result to the variable `proportion`.\n",
    "- Print the proportion and optionally multiply it by 100 to obtain a figure in percentage"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "v1YDptzDI4xF"
   },
   "outputs": [],
   "source": [
    "# TASK 6.1 >>>> Group data based on their types and compute count of the Total Volume "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "buKPSIOKP3eX"
   },
   "outputs": [],
   "source": [
    "# TASK 6.2 >>>> Compute the proportion of the avocado's type"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "t0AzJayhP9tT"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "type\n",
      "conventional    97.190167\n",
      "organic          2.809833\n",
      "Name: Total Volume, dtype: float64\n"
     ]
    }
   ],
   "source": [
    "# TASK 6.3 >>>> Print the output multiply by 100"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 6. Appendix\n",
    "\n",
    "Data Source: https://www.kaggle.com/neuromusic/avocado-prices\n",
    "\n",
    "License: Database: Open Database, Contents: © Original Authors\n",
    "\n",
    "Material adapted for RBI internal purposes with full permissions from original authors. Source: https://github.com/zatkopatrik/authentic-data-science"
   ]
  }
 ],
 "metadata": {
  "colab": {
   "authorship_tag": "ABX9TyNU73EaUPh+hh9eiF0PQT5e",
   "collapsed_sections": [],
   "name": "2. Data_Aggregation.ipynb",
   "provenance": [
    {
     "file_id": "1nuzd1PY0JPrquiO8HidmLAG7FEjCkm7d",
     "timestamp": 1599457295022
    }
   ],
   "toc_visible": true
  },
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.1"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}