Notebooks/00_Miscellaneous_Formats.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Introduction to Databases\n",
    "\n",
    "### Miscellaneous Formats"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import os\n",
    "import sys\n",
    "import time\n",
    "import datetime\n",
    "import numpy as np\n",
    "import pandas as pd"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "datapath = \"../SampleDBs/\""
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Saving a txt file"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "with open(os.path.join(datapath, 'newfile.txt'), mode='w', encoding='utf-8') as f:\n",
    "    f.write('Weight\\t\\t72\\n')\n",
    "    f.write('Height\\t\\t183\\n')\n",
    "    f.write('Age\\t\\t44\\n')\n",
    "    f.write('Gender\\t\\tMasculine\\n')\n",
    "    f.write('\\n')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Retrieving a txt file"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "with open(os.path.join(datapath, 'newfile.txt'), mode='r', encoding='utf-8') as f:\n",
    "    fulltext = f.read()\n",
    "    f.seek(0)\n",
    "    line = f.readline()\n",
    "    f.seek(0)\n",
    "    list_of_lines = f.readlines()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Weight\t\t72\n",
      "Height\t\t183\n",
      "Age\t\t44\n",
      "Gender\t\tMasculine\n",
      "\n",
      "\n"
     ]
    }
   ],
   "source": [
    "print(fulltext)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Weight\t\t72\n",
      "\n"
     ]
    }
   ],
   "source": [
    "print(line)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "['Weight\\t\\t72\\n', 'Height\\t\\t183\\n', 'Age\\t\\t44\\n', 'Gender\\t\\tMasculine\\n', '\\n']\n"
     ]
    }
   ],
   "source": [
    "print(list_of_lines)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "#erasing file\n",
    "os.remove(os.path.join(datapath, 'newfile.txt'))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Saving Numpy Structures"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[0.    0.005 0.01  0.015 0.02  0.025 0.03  0.035 0.04  0.045 0.05  0.055\n",
      " 0.06  0.065 0.07  0.075 0.08  0.085 0.09  0.095 0.1   0.105 0.11  0.115\n",
      " 0.12  0.125 0.13  0.135 0.14  0.145 0.15  0.155 0.16  0.165 0.17  0.175\n",
      " 0.18  0.185 0.19  0.195 0.2   0.205 0.21  0.215 0.22  0.225 0.23  0.235\n",
      " 0.24  0.245 0.25  0.255 0.26  0.265 0.27  0.275 0.28  0.285 0.29  0.295\n",
      " 0.3   0.305 0.31  0.315 0.32  0.325 0.33  0.335 0.34  0.345 0.35  0.355\n",
      " 0.36  0.365 0.37  0.375 0.38  0.385 0.39  0.395 0.4   0.405 0.41  0.415\n",
      " 0.42  0.425 0.43  0.435 0.44  0.445 0.45  0.455 0.46  0.465 0.47  0.475\n",
      " 0.48  0.485 0.49  0.495 0.5   0.505 0.51  0.515 0.52  0.525 0.53  0.535\n",
      " 0.54  0.545 0.55  0.555 0.56  0.565 0.57  0.575 0.58  0.585 0.59  0.595\n",
      " 0.6   0.605 0.61  0.615 0.62  0.625 0.63  0.635 0.64  0.645 0.65  0.655\n",
      " 0.66  0.665 0.67  0.675 0.68  0.685 0.69  0.695 0.7   0.705 0.71  0.715\n",
      " 0.72  0.725 0.73  0.735 0.74  0.745 0.75  0.755 0.76  0.765 0.77  0.775\n",
      " 0.78  0.785 0.79  0.795 0.8   0.805 0.81  0.815 0.82  0.825 0.83  0.835\n",
      " 0.84  0.845 0.85  0.855 0.86  0.865 0.87  0.875 0.88  0.885 0.89  0.895\n",
      " 0.9   0.905 0.91  0.915 0.92  0.925 0.93  0.935 0.94  0.945 0.95  0.955\n",
      " 0.96  0.965 0.97  0.975 0.98  0.985 0.99  0.995 1.   ]\n"
     ]
    }
   ],
   "source": [
    "data = np.linspace(0,1,201)\n",
    "print(data)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "numpy.ndarray"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "type(data)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "np.savetxt(os.path.join(datapath, 'data.dat'), data)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[0.    0.005 0.01  0.015 0.02  0.025 0.03  0.035 0.04  0.045 0.05  0.055\n",
      " 0.06  0.065 0.07  0.075 0.08  0.085 0.09  0.095 0.1   0.105 0.11  0.115\n",
      " 0.12  0.125 0.13  0.135 0.14  0.145 0.15  0.155 0.16  0.165 0.17  0.175\n",
      " 0.18  0.185 0.19  0.195 0.2   0.205 0.21  0.215 0.22  0.225 0.23  0.235\n",
      " 0.24  0.245 0.25  0.255 0.26  0.265 0.27  0.275 0.28  0.285 0.29  0.295\n",
      " 0.3   0.305 0.31  0.315 0.32  0.325 0.33  0.335 0.34  0.345 0.35  0.355\n",
      " 0.36  0.365 0.37  0.375 0.38  0.385 0.39  0.395 0.4   0.405 0.41  0.415\n",
      " 0.42  0.425 0.43  0.435 0.44  0.445 0.45  0.455 0.46  0.465 0.47  0.475\n",
      " 0.48  0.485 0.49  0.495 0.5   0.505 0.51  0.515 0.52  0.525 0.53  0.535\n",
      " 0.54  0.545 0.55  0.555 0.56  0.565 0.57  0.575 0.58  0.585 0.59  0.595\n",
      " 0.6   0.605 0.61  0.615 0.62  0.625 0.63  0.635 0.64  0.645 0.65  0.655\n",
      " 0.66  0.665 0.67  0.675 0.68  0.685 0.69  0.695 0.7   0.705 0.71  0.715\n",
      " 0.72  0.725 0.73  0.735 0.74  0.745 0.75  0.755 0.76  0.765 0.77  0.775\n",
      " 0.78  0.785 0.79  0.795 0.8   0.805 0.81  0.815 0.82  0.825 0.83  0.835\n",
      " 0.84  0.845 0.85  0.855 0.86  0.865 0.87  0.875 0.88  0.885 0.89  0.895\n",
      " 0.9   0.905 0.91  0.915 0.92  0.925 0.93  0.935 0.94  0.945 0.95  0.955\n",
      " 0.96  0.965 0.97  0.975 0.98  0.985 0.99  0.995 1.   ]\n"
     ]
    }
   ],
   "source": [
    "new_data = np.loadtxt(os.path.join(datapath, 'data.dat'))\n",
    "print(new_data)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "#erasing file\n",
    "os.remove(os.path.join(datapath, 'data.dat'))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[[0.         0.35995744]]\n",
      "[[0.05263158 0.02429669]]\n",
      "[[0.10526316 0.91456658]]\n",
      "[[0.15789474 0.1015429 ]]\n",
      "[[0.21052632 0.53368453]]\n",
      "[[0.26315789 0.93746956]]\n",
      "[[0.31578947 0.51193851]]\n",
      "[[0.36842105 0.80576975]]\n",
      "[[0.42105263 0.21915724]]\n",
      "[[0.47368421 0.04779344]]\n",
      "[[0.52631579 0.83928441]]\n",
      "[[0.57894737 0.17652798]]\n",
      "[[0.63157895 0.51045553]]\n",
      "[[0.68421053 0.93770703]]\n",
      "[[0.73684211 0.33255968]]\n",
      "[[0.78947368 0.26303772]]\n",
      "[[0.84210526 0.24584096]]\n",
      "[[0.89473684 0.01316609]]\n",
      "[[0.94736842 0.88902065]]\n",
      "[[1.         0.30492523]]\n"
     ]
    }
   ],
   "source": [
    "x = np.linspace(0, 1, 20)\n",
    "y = np.random.random(20)\n",
    "header = \"X-Column, Y-Column\\n\"\n",
    "header += \"This is a second line\"\n",
    "\n",
    "with open(os.path.join(datapath,'data2.dat'), 'wb') as f:\n",
    "#with open(os.path.join(datapath,'data2.dat'), 'w') as f:   #using text formatting and no binary files\n",
    "    np.savetxt(f, [], header=header)\n",
    "    for i in range(20):\n",
    "        data = np.column_stack((x[i], y[i]))\n",
    "        np.savetxt(f, data), print(data)\n",
    "        #f.write('{:4.1f} {:.4f}\\n'.format(x[i], y[i]))    #using text formatting and no binary files\n",
    "        f.flush()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "# X-Column, Y-Column\n",
      "# This is a second line\n",
      "0.000000000000000000e+00 3.599574376349565785e-01\n",
      "5.263157894736841813e-02 2.429668802814788364e-02\n",
      "1.052631578947368363e-01 9.145665849112072365e-01\n",
      "1.578947368421052544e-01 1.015428979218716554e-01\n",
      "2.105263157894736725e-01 5.336845286082815276e-01\n",
      "2.631578947368420907e-01 9.374695629905477334e-01\n",
      "3.157894736842105088e-01 5.119385117671290120e-01\n",
      "3.684210526315789269e-01 8.057697472896294677e-01\n",
      "4.210526315789473450e-01 2.191572353698321152e-01\n",
      "4.736842105263157632e-01 4.779344454176259749e-02\n",
      "5.263157894736841813e-01 8.392844109970266642e-01\n",
      "5.789473684210526550e-01 1.765279789042919489e-01\n",
      "6.315789473684210176e-01 5.104555258024013886e-01\n",
      "6.842105263157893802e-01 9.377070336795269023e-01\n",
      "7.368421052631578538e-01 3.325596754255929488e-01\n",
      "7.894736842105263275e-01 2.630377219757922269e-01\n",
      "8.421052631578946901e-01 2.458409615074957921e-01\n",
      "8.947368421052630527e-01 1.316609068643392888e-02\n",
      "9.473684210526315264e-01 8.890206506795464980e-01\n",
      "1.000000000000000000e+00 3.049252322270733506e-01\n"
     ]
    }
   ],
   "source": [
    "! cat ../SampleDBs/data2.dat"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [],
   "source": [
    "#erasing file\n",
    "os.remove(os.path.join(datapath, 'data2.dat'))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Using Pickle"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pickle\n",
    "\n",
    "data = [1, 1.2, 'a', 'b']\n",
    "\n",
    "with open(os.path.join(datapath, 'data.pkl'), 'wb') as f:\n",
    "    pickle.dump(data, f)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[1, 1.2, 'a', 'b']\n"
     ]
    }
   ],
   "source": [
    "with open(os.path.join(datapath, 'data.pkl'), 'rb') as f:\n",
    "    new_data = pickle.load(f)\n",
    "\n",
    "print(new_data)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Pickle with gzip"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [],
   "source": [
    "import gzip"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [],
   "source": [
    "data2 = [1, 1.2, 'a', 'b']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [],
   "source": [
    "with gzip.open(os.path.join(datapath, 'data.pkl.gz'), 'wb') as f:\n",
    "    pickle.dump(data2, f)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [],
   "source": [
    "with gzip.open(os.path.join(datapath, 'data.pkl.gz'), 'rb') as f:\n",
    "    new_data2 = pickle.load(f)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[1, 1.2, 'a', 'b']"
      ]
     },
     "execution_count": 23,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "new_data2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [],
   "source": [
    "#erasing files\n",
    "os.remove(os.path.join(datapath, 'data.pkl.gz'))\n",
    "os.remove(os.path.join(datapath, 'data.pkl'))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Acessing CSV Files"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [],
   "source": [
    "csvfile = \"fakedata.csv\"\n",
    "datafile = os.path.join(datapath, csvfile)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Using pure Python:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[{'name': 'Fatima',\n",
       "  'email': 'Quisque.varius@Integervitae.org',\n",
       "  'birthdate': '11-25-01'},\n",
       " {'name': 'Katelyn',\n",
       "  'email': 'mi.pede.nonummy@Sedid.ca',\n",
       "  'birthdate': '11-27-03'},\n",
       " {'name': 'Gillian',\n",
       "  'email': 'odio.semper@sodalesMaurisblandit.org',\n",
       "  'birthdate': '02-19-14'},\n",
       " {'name': 'Preston',\n",
       "  'email': 'faucibus.orci@lacusQuisque.edu',\n",
       "  'birthdate': '08-07-09'},\n",
       " {'name': 'Priscilla',\n",
       "  'email': 'semper.auctor@cursusvestibulum.co.uk',\n",
       "  'birthdate': '08-12-01'}]"
      ]
     },
     "execution_count": 26,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data = []\n",
    "with open(datafile, \"r\") as f:\n",
    "    keys = f.readline().split('|')\n",
    "    keys = [k.strip() for k in keys]\n",
    "    for i in range(5):\n",
    "        values = f.readline().split('|')\n",
    "        values = [v.strip() for v in values]\n",
    "        d = dict(zip(keys,values))\n",
    "        data.append(d)\n",
    "data[0:5]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Using [CSV](https://docs.python.org/3/library/csv.html) module:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [],
   "source": [
    "import csv"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "['name', 'email', 'birthdate']\n",
      "['Fatima', 'Quisque.varius@Integervitae.org', '11-25-01']\n",
      "['Katelyn', 'mi.pede.nonummy@Sedid.ca', '11-27-03']\n",
      "['Gillian', 'odio.semper@sodalesMaurisblandit.org', '02-19-14']\n",
      "['Preston', 'faucibus.orci@lacusQuisque.edu', '08-07-09']\n",
      "['Priscilla', 'semper.auctor@cursusvestibulum.co.uk', '08-12-01']\n",
      "['Zena', 'ante@magnaPraesent.com', '11-24-11']\n",
      "['Oren', 'rutrum.eu.ultrices@nec.org', '01-12-07']\n",
      "['Jamalia', 'Phasellus.vitae.mauris@vel.org', '05-17-17']\n",
      "['Libby', 'velit.eu@Maecenasmi.edu', '07-29-17']\n",
      "['Finn', 'natoque.penatibus@lectusa.net', '06-23-10']\n",
      "['Graiden', 'neque.Nullam@hendreritidante.com', '09-18-10']\n",
      "['Kitra', 'quis.tristique@estmollis.com', '11-18-02']\n",
      "['Baxter', 'vitae.mauris@sed.org', '04-12-17']\n",
      "['Margaret', 'ullamcorper@nec.ca', '08-16-04']\n",
      "['Sopoline', 'vitae.diam@velitQuisquevarius.co.uk', '09-03-06']\n",
      "['Flynn', 'eget.varius@magnased.org', '08-28-14']\n",
      "['Beck', 'arcu.et.pede@idblandit.co.uk', '01-14-06']\n",
      "['Sheila', 'natoque.penatibus.et@mattis.com', '09-01-20']\n",
      "['Francis', 'ligula.Aliquam@nullaatsem.ca', '08-25-14']\n",
      "['Harrison', 'nec.urna@loremauctorquis.com', '02-10-16']\n",
      "['Imogene', 'malesuada.id@nibh.com', '04-19-03']\n",
      "['Moana', 'sit@Morbinequetellus.net', '07-05-03']\n",
      "['Kamal', 'pede.Nunc.sed@Proindolor.ca', '01-14-01']\n",
      "['Fiona', 'nulla.at@Sedmalesuadaaugue.net', '02-07-01']\n",
      "['Yuri', 'pharetra@cursuspurusNullam.edu', '12-03-19']\n",
      "['Nichole', 'aliquet@euenim.org', '04-06-03']\n",
      "['Irma', 'elit@pedeCras.net', '04-29-18']\n",
      "['Leilani', 'et.ultrices@semegestasblandit.edu', '10-25-14']\n",
      "['Rashad', 'imperdiet.nec@convallisligulaDonec.co.uk', '02-09-11']\n",
      "['Jerry', 'augue@imperdiet.edu', '08-28-02']\n",
      "['Thomas', 'mauris@utodio.com', '09-30-10']\n",
      "['Yvette', 'ligula@Aliquam.edu', '03-03-04']\n",
      "['Hashim', 'Integer.sem@pede.co.uk', '07-29-07']\n",
      "['Theodore', 'ut.pharetra@egetipsumDonec.org', '08-11-05']\n",
      "['Basil', 'facilisis.vitae@Curae.ca', '08-31-15']\n",
      "['Blythe', 'lacinia@vehiculaPellentesquetincidunt.co.uk', '10-05-02']\n",
      "['Suki', 'Suspendisse.commodo.tincidunt@velit.co.uk', '04-17-07']\n",
      "['Mia', 'vulputate.velit@etmagnisdis.org', '08-03-07']\n",
      "['Chastity', 'venenatis.lacus.Etiam@quisurna.edu', '12-07-05']\n",
      "['Natalie', 'at.libero@feugiatplacerat.org', '06-25-14']\n",
      "['Scarlet', 'ac.fermentum@ullamcorpereu.edu', '09-13-00']\n",
      "['Gage', 'dolor.Fusce@ametlorem.edu', '03-25-20']\n",
      "['Miranda', 'auctor.nunc@magnisdis.ca', '09-20-02']\n",
      "['Nicole', 'elit@egestasSed.co.uk', '09-08-15']\n",
      "['Tatyana', 'nec.urna@sagittis.co.uk', '09-24-12']\n",
      "['Marcia', 'consectetuer@fringilla.co.uk', '05-12-19']\n",
      "['Germaine', 'eget.ipsum@dui.ca', '09-08-04']\n",
      "['Keith', 'Sed.et@nisimagna.net', '12-20-05']\n",
      "['Zephr', 'Praesent.interdum.ligula@eu.edu', '06-17-17']\n",
      "['Preston', 'accumsan.convallis.ante@sociisnatoquepenatibus.ca', '08-30-08']\n",
      "['Stone', 'elit.sed@leoinlobortis.ca', '09-30-02']\n",
      "['Phelan', 'Nam.ligula.elit@Sedpharetra.ca', '03-29-02']\n",
      "['Colton', 'risus.Donec@blanditcongueIn.co.uk', '04-28-11']\n",
      "['Stuart', 'senectus.et.netus@sit.com', '03-24-09']\n",
      "['Brendan', 'enim.mi.tempor@nectempusscelerisque.edu', '09-16-20']\n",
      "['Eden', 'velit.egestas.lacinia@non.com', '07-13-04']\n",
      "['Geoffrey', 'Donec.egestas@ullamcorperDuis.co.uk', '11-02-11']\n",
      "['Xantha', 'libero.at.auctor@mauris.ca', '09-17-00']\n",
      "['Dylan', 'primis.in.faucibus@tortorIntegeraliquam.net', '08-25-20']\n",
      "['Claire', 'pede@sagittissemperNam.ca', '05-23-13']\n",
      "['Hayden', 'sed@Nam.org', '02-24-09']\n",
      "['Russell', 'laoreet@infaucibus.net', '03-06-04']\n",
      "['Ingrid', 'tincidunt.nibh@disparturient.co.uk', '12-23-05']\n",
      "['Michael', 'orci.consectetuer@porttitorscelerisqueneque.com', '06-30-18']\n",
      "['Elton', 'sem.molestie.sodales@vitaesemperegestas.edu', '03-07-01']\n",
      "['Zena', 'sed@malesuadamalesuadaInteger.net', '12-23-14']\n",
      "['Emi', 'dictum.eu@lorem.org', '06-15-02']\n",
      "['Walter', 'non@loremacrisus.edu', '08-27-08']\n",
      "['Jason', 'scelerisque.dui@maurissitamet.net', '08-30-19']\n",
      "['Shannon', 'elit@sempercursus.net', '10-31-13']\n",
      "['Seth', 'Aliquam.nec@eu.ca', '10-30-09']\n",
      "['Magee', 'dui@odiosagittis.com', '11-01-02']\n",
      "['Kasper', 'Praesent@Proinvel.edu', '08-02-18']\n",
      "['Chester', 'et@orciPhasellus.org', '11-04-07']\n",
      "['Cadman', 'a.sollicitudin@enimconsequat.com', '12-03-15']\n",
      "['Erich', 'ligula.eu@sociosquadlitora.org', '10-07-06']\n",
      "['Allegra', 'lorem@Suspendisse.edu', '02-05-08']\n",
      "['Quemby', 'Curae.Donec@imperdietnonvestibulum.co.uk', '09-28-06']\n",
      "['Beverly', 'ullamcorper@temporeratneque.net', '11-17-08']\n",
      "['Kylee', 'luctus.felis.purus@variusultrices.net', '01-16-04']\n",
      "['Aidan', 'Curae@sempereratin.net', '10-15-08']\n",
      "['India', 'luctus@et.com', '08-03-02']\n",
      "['Asher', 'dictum.Proin@in.edu', '07-02-07']\n",
      "['Illana', 'interdum.libero@sitamet.com', '08-05-21']\n",
      "['Maxine', 'enim.commodo.hendrerit@consequat.com', '08-04-02']\n",
      "['Rashad', 'convallis.in@dapibusquam.org', '07-19-20']\n",
      "['Pearl', 'facilisis.Suspendisse@arcuMorbi.co.uk', '05-13-10']\n",
      "['Yael', 'facilisis.eget.ipsum@placerat.edu', '09-19-17']\n",
      "['Michael', 'facilisi.Sed@consequat.co.uk', '07-08-21']\n",
      "['Lawrence', 'luctus.Curabitur@lectusantedictum.ca', '06-02-09']\n",
      "['Raja', 'et.rutrum@orciinconsequat.edu', '02-13-11']\n",
      "['Irene', 'fermentum@maurisut.com', '09-03-02']\n",
      "['Perry', 'velit@Pellentesqueut.co.uk', '09-30-08']\n",
      "['Kellie', 'elit.Curabitur@erat.net', '07-08-02']\n",
      "['Quemby', 'sed@sedest.ca', '07-09-15']\n",
      "['Adrienne', 'Sed@faucibus.edu', '03-06-01']\n",
      "['Quintessa', 'Aliquam.adipiscing@urna.org', '08-21-07']\n",
      "['Baxter', 'blandit.at.nisi@Donec.edu', '01-15-01']\n",
      "['Rylee', 'at@cursus.net', '07-12-16']\n",
      "['Keegan', 'eu.lacus.Quisque@Inscelerisquescelerisque.net', '12-23-05']\n"
     ]
    }
   ],
   "source": [
    "with open(datafile, \"r\") as f:\n",
    "    data2 = csv.reader(f, delimiter='|')\n",
    "    #data2 = csv.DictReader(f, delimiter='|')\n",
    "    for row in data2:\n",
    "        print(row)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Using Pandas:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>name</th>\n",
       "      <th>email</th>\n",
       "      <th>birthdate</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>Fatima</td>\n",
       "      <td>Quisque.varius@Integervitae.org</td>\n",
       "      <td>11-25-01</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>Katelyn</td>\n",
       "      <td>mi.pede.nonummy@Sedid.ca</td>\n",
       "      <td>11-27-03</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>Gillian</td>\n",
       "      <td>odio.semper@sodalesMaurisblandit.org</td>\n",
       "      <td>02-19-14</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>Preston</td>\n",
       "      <td>faucibus.orci@lacusQuisque.edu</td>\n",
       "      <td>08-07-09</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>Priscilla</td>\n",
       "      <td>semper.auctor@cursusvestibulum.co.uk</td>\n",
       "      <td>08-12-01</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "        name                                 email birthdate\n",
       "0     Fatima       Quisque.varius@Integervitae.org  11-25-01\n",
       "1    Katelyn              mi.pede.nonummy@Sedid.ca  11-27-03\n",
       "2    Gillian  odio.semper@sodalesMaurisblandit.org  02-19-14\n",
       "3    Preston        faucibus.orci@lacusQuisque.edu  08-07-09\n",
       "4  Priscilla  semper.auctor@cursusvestibulum.co.uk  08-12-01"
      ]
     },
     "execution_count": 29,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_csv = pd.read_csv(datafile, sep=\"|\")\n",
    "df_csv.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<class 'pandas.core.frame.DataFrame'>\n",
      "RangeIndex: 100 entries, 0 to 99\n",
      "Data columns (total 3 columns):\n",
      " #   Column     Non-Null Count  Dtype \n",
      "---  ------     --------------  ----- \n",
      " 0   name       100 non-null    object\n",
      " 1   email      100 non-null    object\n",
      " 2   birthdate  100 non-null    object\n",
      "dtypes: object(3)\n",
      "memory usage: 2.5+ KB\n"
     ]
    }
   ],
   "source": [
    "df_csv.info()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Manipulating [Zip Files](https://docs.python.org/3/library/zipfile.html)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [],
   "source": [
    "from zipfile import ZipFile"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### CSV Files with Pandas"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {},
   "outputs": [],
   "source": [
    "compressedfile = \"HousingPrices.zip\"\n",
    "datafile = os.path.join(datapath, compressedfile)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>TransactionDate</th>\n",
       "      <th>HouseAge</th>\n",
       "      <th>DistanceToMRT</th>\n",
       "      <th>NumStores</th>\n",
       "      <th>Latitude</th>\n",
       "      <th>Longitude</th>\n",
       "      <th>UnitPrice</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>2012</td>\n",
       "      <td>32.0</td>\n",
       "      <td>84.87882</td>\n",
       "      <td>10</td>\n",
       "      <td>24.98298</td>\n",
       "      <td>121.54024</td>\n",
       "      <td>37.9</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>2012</td>\n",
       "      <td>19.5</td>\n",
       "      <td>306.59470</td>\n",
       "      <td>9</td>\n",
       "      <td>24.98034</td>\n",
       "      <td>121.53951</td>\n",
       "      <td>42.2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>2013</td>\n",
       "      <td>13.3</td>\n",
       "      <td>561.98450</td>\n",
       "      <td>5</td>\n",
       "      <td>24.98746</td>\n",
       "      <td>121.54391</td>\n",
       "      <td>47.3</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>2013</td>\n",
       "      <td>13.3</td>\n",
       "      <td>561.98450</td>\n",
       "      <td>5</td>\n",
       "      <td>24.98746</td>\n",
       "      <td>121.54391</td>\n",
       "      <td>54.8</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>2012</td>\n",
       "      <td>5.0</td>\n",
       "      <td>390.56840</td>\n",
       "      <td>5</td>\n",
       "      <td>24.97937</td>\n",
       "      <td>121.54245</td>\n",
       "      <td>43.1</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   TransactionDate  HouseAge  DistanceToMRT  NumStores  Latitude  Longitude  \\\n",
       "0             2012      32.0       84.87882         10  24.98298  121.54024   \n",
       "1             2012      19.5      306.59470          9  24.98034  121.53951   \n",
       "2             2013      13.3      561.98450          5  24.98746  121.54391   \n",
       "3             2013      13.3      561.98450          5  24.98746  121.54391   \n",
       "4             2012       5.0      390.56840          5  24.97937  121.54245   \n",
       "\n",
       "   UnitPrice  \n",
       "0       37.9  \n",
       "1       42.2  \n",
       "2       47.3  \n",
       "3       54.8  \n",
       "4       43.1  "
      ]
     },
     "execution_count": 45,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "pd_csv_zipped = pd.read_csv(datafile, compression='zip', engine='python')\n",
    "pd_csv_zipped.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### XLS files with Pandas and Zipfile"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 56,
   "metadata": {},
   "outputs": [],
   "source": [
    "compressedfile = \"2013_ERCOT_Hourly_Load_Data.zip\"\n",
    "datafile = os.path.join(datapath, compressedfile)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 58,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "['2013_ERCOT_Hourly_Load_Data.xls']\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Hour_End</th>\n",
       "      <th>COAST</th>\n",
       "      <th>EAST</th>\n",
       "      <th>FAR_WEST</th>\n",
       "      <th>NORTH</th>\n",
       "      <th>NORTH_C</th>\n",
       "      <th>SOUTHERN</th>\n",
       "      <th>SOUTH_C</th>\n",
       "      <th>WEST</th>\n",
       "      <th>ERCOT</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>2013-01-01 01:00:00</td>\n",
       "      <td>7606.263544</td>\n",
       "      <td>1073.892438</td>\n",
       "      <td>1411.750567</td>\n",
       "      <td>784.978166</td>\n",
       "      <td>10369.094390</td>\n",
       "      <td>2206.675077</td>\n",
       "      <td>4368.490945</td>\n",
       "      <td>882.931901</td>\n",
       "      <td>28704.077028</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>2013-01-01 02:00:00</td>\n",
       "      <td>7388.082714</td>\n",
       "      <td>1035.021938</td>\n",
       "      <td>1403.472287</td>\n",
       "      <td>776.307387</td>\n",
       "      <td>10152.358518</td>\n",
       "      <td>2159.733208</td>\n",
       "      <td>4233.587967</td>\n",
       "      <td>872.404750</td>\n",
       "      <td>28020.968769</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>2013-01-01 03:00:00</td>\n",
       "      <td>7178.867878</td>\n",
       "      <td>1036.088697</td>\n",
       "      <td>1395.053150</td>\n",
       "      <td>768.125748</td>\n",
       "      <td>9988.051418</td>\n",
       "      <td>2065.114706</td>\n",
       "      <td>4082.862860</td>\n",
       "      <td>868.853938</td>\n",
       "      <td>27383.018395</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>2013-01-01 04:00:00</td>\n",
       "      <td>7038.822581</td>\n",
       "      <td>1032.648841</td>\n",
       "      <td>1395.508820</td>\n",
       "      <td>770.937969</td>\n",
       "      <td>9946.658655</td>\n",
       "      <td>1990.903699</td>\n",
       "      <td>4010.489608</td>\n",
       "      <td>865.701201</td>\n",
       "      <td>27051.671374</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>2013-01-01 05:00:00</td>\n",
       "      <td>6990.857940</td>\n",
       "      <td>1042.823044</td>\n",
       "      <td>1401.216842</td>\n",
       "      <td>779.089313</td>\n",
       "      <td>10096.664190</td>\n",
       "      <td>1954.807585</td>\n",
       "      <td>4038.655997</td>\n",
       "      <td>879.924249</td>\n",
       "      <td>27184.039160</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "             Hour_End        COAST         EAST     FAR_WEST       NORTH  \\\n",
       "0 2013-01-01 01:00:00  7606.263544  1073.892438  1411.750567  784.978166   \n",
       "1 2013-01-01 02:00:00  7388.082714  1035.021938  1403.472287  776.307387   \n",
       "2 2013-01-01 03:00:00  7178.867878  1036.088697  1395.053150  768.125748   \n",
       "3 2013-01-01 04:00:00  7038.822581  1032.648841  1395.508820  770.937969   \n",
       "4 2013-01-01 05:00:00  6990.857940  1042.823044  1401.216842  779.089313   \n",
       "\n",
       "        NORTH_C     SOUTHERN      SOUTH_C        WEST         ERCOT  \n",
       "0  10369.094390  2206.675077  4368.490945  882.931901  28704.077028  \n",
       "1  10152.358518  2159.733208  4233.587967  872.404750  28020.968769  \n",
       "2   9988.051418  2065.114706  4082.862860  868.853938  27383.018395  \n",
       "3   9946.658655  1990.903699  4010.489608  865.701201  27051.671374  \n",
       "4  10096.664190  1954.807585  4038.655997  879.924249  27184.039160  "
      ]
     },
     "execution_count": 58,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "with ZipFile(datafile, 'r') as myzip:\n",
    "    print(myzip.namelist())\n",
    "    unzipped = myzip.namelist()[0]\n",
    "    with myzip.open(unzipped, 'r') as myfile:\n",
    "        pd_excel = pd.read_excel(myfile)\n",
    "        \n",
    "pd_excel.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## [Acessing](https://pypi.python.org/pypi/xlrd) and [Writing](https://pypi.python.org/pypi/xlwt) [Excel Files](https://github.com/python-excel/tutorial/blob/master/python-excel.pdf)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 59,
   "metadata": {},
   "outputs": [],
   "source": [
    "#!sudo pip install -U xlrd xlwt"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Accessing a workbook"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 60,
   "metadata": {},
   "outputs": [],
   "source": [
    "import xlrd"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 61,
   "metadata": {},
   "outputs": [],
   "source": [
    "with ZipFile(datafile, 'r') as myzip:\n",
    "    unzipped = myzip.namelist()[0]\n",
    "    myzip.extractall(datapath)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 62,
   "metadata": {},
   "outputs": [],
   "source": [
    "workbook = xlrd.open_workbook(os.path.join(datapath, unzipped))\n",
    "sheet = workbook.sheet_by_index(0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 63,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[['Hour_End',\n",
       "  'COAST',\n",
       "  'EAST',\n",
       "  'FAR_WEST',\n",
       "  'NORTH',\n",
       "  'NORTH_C',\n",
       "  'SOUTHERN',\n",
       "  'SOUTH_C',\n",
       "  'WEST',\n",
       "  'ERCOT'],\n",
       " [41275.041666666664,\n",
       "  7606.263544000012,\n",
       "  1073.892438,\n",
       "  1411.7505669999982,\n",
       "  784.9781659999992,\n",
       "  10369.094390000051,\n",
       "  2206.6750770000012,\n",
       "  4368.490945000006,\n",
       "  882.9319009999975,\n",
       "  28704.077028000065]]"
      ]
     },
     "execution_count": 63,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "sheet_data = [[sheet.cell_value(r, col) for col in range(sheet.ncols)] for r in range(sheet.nrows)]\n",
    "sheet_data[:2]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 64,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Number of rows in the sheet:\n",
      "7296\n"
     ]
    }
   ],
   "source": [
    "print(\"Number of rows in the sheet:\"),\n",
    "print(sheet.nrows)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 65,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Type of data in cell (row 3, col 2):\n",
      "2\n"
     ]
    }
   ],
   "source": [
    "print(\"Type of data in cell (row 3, col 2):\"), \n",
    "print(sheet.cell_type(3, 2))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 66,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Value in cell (row 3, col 2):\n",
      "1036.0886969999988\n"
     ]
    }
   ],
   "source": [
    "print(\"Value in cell (row 3, col 2):\", )\n",
    "print(sheet.cell_value(3, 2))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 67,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Get a slice of values in column 3, from rows 1-3:\n",
      "[1411.7505669999982, 1403.4722870000019, 1395.053150000001]\n"
     ]
    }
   ],
   "source": [
    "print(\"Get a slice of values in column 3, from rows 1-3:\",)\n",
    "print(sheet.col_values(3, start_rowx=1, end_rowx=4))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 68,
   "metadata": {},
   "outputs": [],
   "source": [
    "coast = sheet.col_values(1, start_rowx=1)\n",
    "data3 = {}    \n",
    "data3['maxvalue'] = max(coast)\n",
    "data3['minvalue'] = min(coast)\n",
    "data3['avgcoast'] = np.mean(coast)\n",
    "\n",
    "rowmax = coast.index(max(coast))+1\n",
    "rowmin = coast.index(min(coast))+1\n",
    "\n",
    "data3['maxtime'] = xlrd.xldate_as_tuple(sheet.cell_value(rowmax,0), 0)\n",
    "data3['mintime'] = xlrd.xldate_as_tuple(sheet.cell_value(rowmin,0), 0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 69,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'maxvalue': 18779.025510000003,\n",
       " 'minvalue': 6602.113898999982,\n",
       " 'avgcoast': 10976.933460679784,\n",
       " 'maxtime': (2013, 8, 13, 17, 0, 0),\n",
       " 'mintime': (2013, 2, 3, 4, 0, 0)}"
      ]
     },
     "execution_count": 69,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data3"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Using Pandas"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "http://pandas.pydata.org/pandas-docs/stable/generated/pandas.read_excel.html  "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 70,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Hour_End</th>\n",
       "      <th>COAST</th>\n",
       "      <th>EAST</th>\n",
       "      <th>FAR_WEST</th>\n",
       "      <th>NORTH</th>\n",
       "      <th>NORTH_C</th>\n",
       "      <th>SOUTHERN</th>\n",
       "      <th>SOUTH_C</th>\n",
       "      <th>WEST</th>\n",
       "      <th>ERCOT</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>2013-01-01 01:00:00</td>\n",
       "      <td>7606.263544</td>\n",
       "      <td>1073.892438</td>\n",
       "      <td>1411.750567</td>\n",
       "      <td>784.978166</td>\n",
       "      <td>10369.094390</td>\n",
       "      <td>2206.675077</td>\n",
       "      <td>4368.490945</td>\n",
       "      <td>882.931901</td>\n",
       "      <td>28704.077028</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>2013-01-01 02:00:00</td>\n",
       "      <td>7388.082714</td>\n",
       "      <td>1035.021938</td>\n",
       "      <td>1403.472287</td>\n",
       "      <td>776.307387</td>\n",
       "      <td>10152.358518</td>\n",
       "      <td>2159.733208</td>\n",
       "      <td>4233.587967</td>\n",
       "      <td>872.404750</td>\n",
       "      <td>28020.968769</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>2013-01-01 03:00:00</td>\n",
       "      <td>7178.867878</td>\n",
       "      <td>1036.088697</td>\n",
       "      <td>1395.053150</td>\n",
       "      <td>768.125748</td>\n",
       "      <td>9988.051418</td>\n",
       "      <td>2065.114706</td>\n",
       "      <td>4082.862860</td>\n",
       "      <td>868.853938</td>\n",
       "      <td>27383.018395</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>2013-01-01 04:00:00</td>\n",
       "      <td>7038.822581</td>\n",
       "      <td>1032.648841</td>\n",
       "      <td>1395.508820</td>\n",
       "      <td>770.937969</td>\n",
       "      <td>9946.658655</td>\n",
       "      <td>1990.903699</td>\n",
       "      <td>4010.489608</td>\n",
       "      <td>865.701201</td>\n",
       "      <td>27051.671374</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>2013-01-01 05:00:00</td>\n",
       "      <td>6990.857940</td>\n",
       "      <td>1042.823044</td>\n",
       "      <td>1401.216842</td>\n",
       "      <td>779.089313</td>\n",
       "      <td>10096.664190</td>\n",
       "      <td>1954.807585</td>\n",
       "      <td>4038.655997</td>\n",
       "      <td>879.924249</td>\n",
       "      <td>27184.039160</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "             Hour_End        COAST         EAST     FAR_WEST       NORTH  \\\n",
       "0 2013-01-01 01:00:00  7606.263544  1073.892438  1411.750567  784.978166   \n",
       "1 2013-01-01 02:00:00  7388.082714  1035.021938  1403.472287  776.307387   \n",
       "2 2013-01-01 03:00:00  7178.867878  1036.088697  1395.053150  768.125748   \n",
       "3 2013-01-01 04:00:00  7038.822581  1032.648841  1395.508820  770.937969   \n",
       "4 2013-01-01 05:00:00  6990.857940  1042.823044  1401.216842  779.089313   \n",
       "\n",
       "        NORTH_C     SOUTHERN      SOUTH_C        WEST         ERCOT  \n",
       "0  10369.094390  2206.675077  4368.490945  882.931901  28704.077028  \n",
       "1  10152.358518  2159.733208  4233.587967  872.404750  28020.968769  \n",
       "2   9988.051418  2065.114706  4082.862860  868.853938  27383.018395  \n",
       "3   9946.658655  1990.903699  4010.489608  865.701201  27051.671374  \n",
       "4  10096.664190  1954.807585  4038.655997  879.924249  27184.039160  "
      ]
     },
     "execution_count": 70,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "pd_excel = pd.read_excel(os.path.join(datapath, unzipped))\n",
    "pd_excel.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 71,
   "metadata": {},
   "outputs": [],
   "source": [
    "#erasing file\n",
    "os.remove(os.path.join(datapath, unzipped))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Reading and Writing a Workbook"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 72,
   "metadata": {},
   "outputs": [],
   "source": [
    "import xlwt"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 73,
   "metadata": {},
   "outputs": [],
   "source": [
    "style0 = xlwt.easyxf('font: name Times New Roman, color-index red, bold on', num_format_str='#,##0.00')\n",
    "style1 = xlwt.easyxf(num_format_str='D-MMM-YY')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 74,
   "metadata": {},
   "outputs": [],
   "source": [
    "workbook = xlwt.Workbook()\n",
    "workbook_name = 'my_workbook.xls'\n",
    "\n",
    "worksheet1 = workbook.add_sheet('my_first_sheet')\n",
    "worksheet1.write(0, 0, 'Column 1 title', style0)\n",
    "worksheet1.write(0, 1, 'Column 2 title', style0)\n",
    "\n",
    "worksheet2 = workbook.add_sheet('my_second_sheet')\n",
    "worksheet2.write(0, 0, 'Itens')\n",
    "worksheet2.write(0, 1, 'Price')\n",
    "worksheet2.write(1, 0, 'Rice')\n",
    "worksheet2.write(1, 1, 100)\n",
    "worksheet2.write(2, 0, 'Beans')\n",
    "worksheet2.write(2, 1, 200)\n",
    "worksheet2.write(3, 0, 'Pasta')\n",
    "worksheet2.write(3, 1, 500)\n",
    "worksheet2.write(5, 0, 'Total')\n",
    "\n",
    "worksheet2.write(5, 1, xlwt.Formula(\"B2+B3+B4\"))\n",
    "\n",
    "\n",
    "workbook.save(os.path.join(datapath, workbook_name))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 75,
   "metadata": {},
   "outputs": [],
   "source": [
    "#erasing file\n",
    "os.remove(os.path.join(datapath, workbook_name))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Reading PDF Files  \n",
    "\n",
    "More resources [here](https://textract.readthedocs.io/en/stable/) and [here](https://towardsdatascience.com/python-for-pdf-ef0fac2808b0?)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 76,
   "metadata": {},
   "outputs": [],
   "source": [
    "#!sudo pip install -U textract"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 77,
   "metadata": {},
   "outputs": [],
   "source": [
    "import textract"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 78,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Files in the folder:\n",
      "0 -- 1369576221wpdm_DVD Rental ER Diagram.pdf\n",
      "1 -- RobertFrost_Poetry.pdf\n"
     ]
    }
   ],
   "source": [
    "onlyfiles = [f for f in os.listdir(datapath) if os.path.isfile(os.path.join(datapath, f)) and f.endswith('.pdf')]\n",
    "onlyfiles.sort()\n",
    "\n",
    "print('Files in the folder:')\n",
    "for i, w in enumerate(onlyfiles):\n",
    "    print(i, '--' ,w)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 79,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "POSTGRESQL\n",
      "TUTORIAL.COM\n",
      "\n",
      "DVD Rental ER Diagram\n",
      "\n",
      "\f",
      "\n"
     ]
    }
   ],
   "source": [
    "text0 = textract.process(os.path.join(datapath, onlyfiles[0])).decode('utf-8')\n",
    "print(text0[0:1000])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Reading Text in Image Files (OCR)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 80,
   "metadata": {},
   "outputs": [],
   "source": [
    "# !sudo apt install -y tesseract-ocr"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 81,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Files in the folder:\n",
      "0 -- 1 HgrHDlU85dKtyLonNgu2zA.png\n",
      "1 -- SQLAlchemyORM.png\n",
      "2 -- db_schema.png\n",
      "3 -- textimage.png\n"
     ]
    }
   ],
   "source": [
    "onlyfiles = [f for f in os.listdir(datapath) if os.path.isfile(os.path.join(datapath, f)) and f.endswith('.png')]\n",
    "onlyfiles.sort()\n",
    "\n",
    "print('Files in the folder:')\n",
    "for i, w in enumerate(onlyfiles):\n",
    "    print(i, '--' ,w)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 82,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Adobe, the Adobe logo, Acrobat, the Acrobat logo, Acrobat Capture, Adobe Garamond, Adobe\n",
      "Intelligent Document Platform, Adobe PDF, Adobe Reader, Adobe Solutions Network, Aldus, Dis-\n",
      "tiller, ePaper, Extreme, FrameMaker, Illustrator, InDesign, Minion, Myriad, PageMaker, Photo-\n",
      "shop, Poetica, PostScript, and XMP are either registered trademarks or trademarks of Adobe\n",
      "‘Systems Incorporated in the United States and/or other countries. Microsoft and Windows are\n",
      "either registered trademarks or trademarks of Microsoft Corporation in the United States and/or\n",
      "other countries. Apple, Mac, Macintosh, and Power Macintosh are trademarks of Apple Computer,\n",
      "Inc,, registered in the United States and other countries. IBM is a registered trademark of IBM\n",
      "Corporation in the United States. Sun is a trademark or registered trademark of Sun Microsys-\n",
      "tems, Inc. in the United States and other countries. UNIX is\n"
     ]
    }
   ],
   "source": [
    "text1 = textract.process(os.path.join(datapath, onlyfiles[3])).decode('utf-8')\n",
    "\n",
    "print(text1[0:900])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Accessing HTML"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "https://docs.python.org/3/library/urllib.html  \n",
    "http://www.crummy.com/software/BeautifulSoup/  \n",
    "http://docs.python-requests.org/en/latest/  \n",
    "http://lxml.de/  \n",
    "https://docs.python.org/3/library/getpass.html  "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 83,
   "metadata": {},
   "outputs": [],
   "source": [
    "import urllib\n",
    "import requests\n",
    "\n",
    "from bs4 import BeautifulSoup as bs\n",
    "import lxml.html\n",
    "\n",
    "import getpass"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 84,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "\n",
      "\n",
      "\n",
      "Guimarães Rosa – Wikipédia, a enciclopédia livre\n",
      "document.documentElement.className=\"client-js\";RLCONF={\"wgBreakFrames\":!1,\"wgSeparatorTransformTable\":[\",\\t.\",\" \\t,\"],\"wgDigitTransformTable\":[\"\",\"\"],\"wgDefaultDateFormat\":\"dmy\",\"wgMonthNames\":[\"\",\"janeiro\",\"fevereiro\",\"março\",\"abril\",\"maio\",\"junh\n"
     ]
    }
   ],
   "source": [
    "url = \"https://pt.wikipedia.org/wiki/Guimar%C3%A3es_Rosa\"\n",
    "html_txt = urllib.request.urlopen(url).read()\n",
    "soup = bs(html_txt, \"html.parser\")\n",
    " \n",
    "print(soup.text[0:300])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 85,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[None,\n",
       " '#mw-head',\n",
       " '#searchInput',\n",
       " '/wiki/Ficheiro:Guimar%C3%A3es_Rosa,_anos_1960.tif',\n",
       " '/wiki/Arquivo_Nacional_(Brasil)',\n",
       " '/wiki/27_de_junho#Nascimentos',\n",
       " '/wiki/1908',\n",
       " '/wiki/Cordisburgo',\n",
       " '/wiki/Minas_Gerais',\n",
       " '/wiki/19_de_novembro']"
      ]
     },
     "execution_count": 85,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "[line.get('href') for line in soup.find_all('a')][0:10] "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Another approach: [lxml](http://lxml.de/tutorial.html) instead of beautiful soup\n",
    " "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 86,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['#mw-head',\n",
       " '#searchInput',\n",
       " '/wiki/Ficheiro:Guimar%C3%A3es_Rosa,_anos_1960.tif',\n",
       " '/wiki/Arquivo_Nacional_(Brasil)',\n",
       " '/wiki/27_de_junho#Nascimentos',\n",
       " '/wiki/1908',\n",
       " '/wiki/Cordisburgo',\n",
       " '/wiki/Minas_Gerais',\n",
       " '/wiki/19_de_novembro',\n",
       " '/wiki/1967']"
      ]
     },
     "execution_count": 86,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "url = \"https://pt.wikipedia.org/wiki/Guimar%C3%A3es_Rosa\"\n",
    "html_txt = urllib.request.urlopen(url).read()\n",
    "dom =  lxml.html.fromstring(html_txt)\n",
    "    \n",
    "[line for line in dom.xpath('//a/@href')][0:10]   "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Accessing [JSON](http://json.org/)\n",
    "\n",
    "https://docs.python.org/3/library/json.html  \n",
    "https://docs.python.org/3/library/urllib.html  "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 95,
   "metadata": {},
   "outputs": [],
   "source": [
    "import json"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 96,
   "metadata": {},
   "outputs": [],
   "source": [
    "base_url = \"http://musicbrainz.org/ws/2/\"\n",
    "artist_url =  urllib.parse.urljoin(base_url, 'artist/')\n",
    "query_type = {\"simple\": {},\n",
    "              \"atr\": {\"inc\": \"aliases+tags+ratings\"},\n",
    "              \"aliases\": {\"inc\": \"aliases\"},\n",
    "              \"releases\": {\"inc\": \"releases\"}}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 97,
   "metadata": {},
   "outputs": [],
   "source": [
    "def query_site(url, params, uid=\"\", fmt=\"json\"):\n",
    "    params[\"fmt\"] = fmt\n",
    "    r = requests.get(url + uid, params=params)\n",
    "    print(\"requesting\", r.url)\n",
    "\n",
    "    if r.status_code == requests.codes.ok:\n",
    "        return r.json()\n",
    "    else:\n",
    "        r.raise_for_status()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 98,
   "metadata": {},
   "outputs": [],
   "source": [
    "def query_by_name(url, params, name):\n",
    "    params[\"query\"] = \"artist:\" + name\n",
    "    return query_site(url, params)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 99,
   "metadata": {},
   "outputs": [],
   "source": [
    "def pretty_print(data, indent=4):\n",
    "    if type(data) == dict:\n",
    "        print(json.dumps(data, indent=indent, sort_keys=True))\n",
    "    else:\n",
    "        print(data)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 100,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "requesting http://musicbrainz.org/ws/2/artist/?query=artist%3ANirvana&fmt=json\n",
      "[{'id': '5b11f4ce-a62d-471e-81fc-a69a8278c7da', 'type': 'Group', 'type-id': 'e431f5f6-b5d2-343d-8b36-72607fffb74b', 'score': 100, 'name': 'Nirvana', 'sort-name': 'Nirvana', 'country': 'US', 'area': {'id': '489ce91b-6658-3307-9877-795b68554c98', 'type': 'Country', 'type-id': '06dd0ae4-8c74-30bb-b43d-95dcedf961de', 'name': 'United States', 'sort-name': 'United States', 'life-span': {'ended': None}}, 'begin-area': {'id': 'a640b45c-c173-49b1-8030-973603e895b5', 'type': 'City', 'type-id': '6fd8f29a-3d0a-32fc-980d-ea697b69da78', 'name': 'Aberdeen', 'sort-name': 'Aberdeen', 'life-span': {'ended': None}}, 'disambiguation': '90s US grunge band', 'isnis': ['0000000123486830', '0000000123487390'], 'life-span': {'begin': '1988-01', 'end': '1994-04-05', 'ended': True}, 'aliases': [{'sort-name': 'ニルヴァーナ', 'type-id': '894afba6-2816-3c24-8072-eadb66bd04bc', 'name': 'ニルヴァーナ', 'locale': 'ja', 'type': 'Artist name', 'primary': True, 'begin-date': None, 'end-date': None}, {'sort-name': 'Nirvana US', 'name': 'Nirvana US', 'locale': None, 'type': None, 'primary': None, 'begin-date': None, 'end-date': None}, {'sort-name': 'Nirvana', 'type-id': '894afba6-2816-3c24-8072-eadb66bd04bc', 'name': 'Nirvana', 'locale': 'en', 'type': 'Artist name', 'primary': True, 'begin-date': None, 'end-date': None}], 'tags': [{'count': 2, 'name': 'usa'}, {'count': 0, 'name': 'united states'}, {'count': 0, 'name': 'northwest'}, {'count': 27, 'name': 'grunge'}, {'count': 11, 'name': 'alternative rock'}, {'count': 0, 'name': '90'}, {'count': 8, 'name': 'american'}, {'count': 0, 'name': 'américain'}, {'count': 0, 'name': 'alternative'}, {'count': 2, 'name': '90s'}, {'count': 1, 'name': 'acoustic rock'}, {'count': 0, 'name': 'nirvana'}, {'count': 2, 'name': 'noise rock'}, {'count': 17, 'name': 'rock'}, {'count': 0, 'name': 'band'}, {'count': 0, 'name': 'legendary'}, {'count': 0, 'name': 'kurt cobain'}, {'count': 0, 'name': 'rock and indie'}, {'count': 2, 'name': 'punk'}, {'count': 3, 'name': 'seattle'}, {'count': 2, 'name': 'experimental'}]}, {'id': '9282c8b4-ca0b-4c6b-b7e3-4f7762dfc4d6', 'type': 'Group', 'type-id': 'e431f5f6-b5d2-343d-8b36-72607fffb74b', 'score': 79, 'name': 'Nirvana', 'sort-name': 'Nirvana', 'country': 'GB', 'area': {'id': '8a754a16-0027-3a29-b6d7-2b40ea0481ed', 'type': 'Country', 'type-id': '06dd0ae4-8c74-30bb-b43d-95dcedf961de', 'name': 'United Kingdom', 'sort-name': 'United Kingdom', 'life-span': {'ended': None}}, 'begin-area': {'id': 'f03d09b3-39dc-4083-afd6-159e3f0d462f', 'type': 'City', 'type-id': '6fd8f29a-3d0a-32fc-980d-ea697b69da78', 'name': 'London', 'sort-name': 'London', 'life-span': {'ended': None}}, 'disambiguation': '60s band from the UK', 'life-span': {'begin': '1967', 'ended': None}, 'tags': [{'count': 1, 'name': 'english'}, {'count': 1, 'name': 'british'}, {'count': 2, 'name': 'progressive rock'}, {'count': 2, 'name': 'rock'}, {'count': 1, 'name': 'power pop'}, {'count': 1, 'name': 'soft rock'}, {'count': 1, 'name': 'pop'}, {'count': 1, 'name': 'psychedelic rock'}, {'count': 1, 'name': 'pop rock'}, {'count': 1, 'name': 'symphonic rock'}, {'count': 0, 'name': 'orchestral'}, {'count': 1, 'name': 'psychedelic pop'}, {'count': 1, 'name': 'baroque pop'}]}, {'id': 'c3a64a25-251b-4d03-afba-1471440245b8', 'type': 'Group', 'type-id': 'e431f5f6-b5d2-343d-8b36-72607fffb74b', 'score': 74, 'name': 'Approaching Nirvana', 'sort-name': 'Approaching Nirvana', 'country': 'US', 'area': {'id': '489ce91b-6658-3307-9877-795b68554c98', 'type': 'Country', 'type-id': '06dd0ae4-8c74-30bb-b43d-95dcedf961de', 'name': 'United States', 'sort-name': 'United States', 'life-span': {'ended': None}}, 'life-span': {'begin': '2009', 'ended': None}}, {'id': 'c49d69dc-e008-47cf-b5ff-160fafb1fe1f', 'type': 'Group', 'type-id': 'e431f5f6-b5d2-343d-8b36-72607fffb74b', 'score': 71, 'name': 'Nirvana', 'sort-name': 'Nirvana', 'country': 'FR', 'area': {'id': '08310658-51eb-3801-80de-5a0739207115', 'type': 'Country', 'type-id': '06dd0ae4-8c74-30bb-b43d-95dcedf961de', 'name': 'France', 'sort-name': 'France', 'life-span': {'ended': None}}, 'disambiguation': '’70s French band from Martigues', 'life-span': {'ended': None}}, {'id': '85af0709-95db-4fbc-801a-120e9f4766d0', 'type': 'Group', 'type-id': 'e431f5f6-b5d2-343d-8b36-72607fffb74b', 'score': 71, 'name': 'Nirvana', 'sort-name': 'Nirvana', 'country': 'FI', 'area': {'id': '6a264f94-6ff1-30b1-9a81-41f7bfabd616', 'type': 'Country', 'type-id': '06dd0ae4-8c74-30bb-b43d-95dcedf961de', 'name': 'Finland', 'sort-name': 'Finland', 'life-span': {'ended': None}}, 'disambiguation': \"Early 1980's Finnish punk band\", 'life-span': {'ended': None}, 'tags': [{'count': 1, 'name': 'finland'}, {'count': 1, 'name': 'punk'}]}, {'id': '28a4618c-38e4-4027-ad2c-db66a14a2d85', 'type': 'Group', 'type-id': 'e431f5f6-b5d2-343d-8b36-72607fffb74b', 'score': 71, 'name': 'Nirvana', 'sort-name': 'Nirvana', 'country': 'YU', 'area': {'id': '885dce63-c211-3033-8cf7-46cb82d440c7', 'type': 'Country', 'type-id': '06dd0ae4-8c74-30bb-b43d-95dcedf961de', 'name': 'Yugoslavia', 'sort-name': 'Yugoslavia', 'life-span': {'begin': '1918', 'end': '2003', 'ended': True}}, 'disambiguation': 'Croatian prog-rock band active in first half of 70s in former Yugoslavia.', 'life-span': {'ended': None}}, {'id': '3aa878c0-224b-41e5-abd1-63be359d2bca', 'score': 71, 'name': 'Nirvana', 'sort-name': 'Nirvana', 'disambiguation': 'founded in 1987 by a Michael Jackson double/imitator', 'life-span': {'begin': '1987', 'ended': None}}, {'id': 'f2dfdff9-3862-4be0-bf85-9c833fa3059e', 'type': 'Group', 'type-id': 'e431f5f6-b5d2-343d-8b36-72607fffb74b', 'score': 63, 'name': 'Nirvana 2002', 'sort-name': 'Nirvana 2002', 'country': 'SE', 'area': {'id': '23d10872-f5ae-3f0c-bf55-332788a16ecb', 'type': 'Country', 'type-id': '06dd0ae4-8c74-30bb-b43d-95dcedf961de', 'name': 'Sweden', 'sort-name': 'Sweden', 'life-span': {'ended': None}}, 'disambiguation': 'Swedish death metal band', 'life-span': {'begin': '1988', 'end': '2012', 'ended': True}, 'aliases': [{'sort-name': 'Prophet 2002', 'name': 'Prophet 2002', 'locale': None, 'type': None, 'primary': None, 'begin-date': '1988', 'end-date': '1988'}, {'sort-name': 'Nirvana', 'name': 'Nirvana', 'locale': None, 'type': None, 'primary': None, 'begin-date': '1988', 'end-date': '1988'}, {'sort-name': 'N2K2', 'name': 'N2K2', 'locale': None, 'type': None, 'primary': None, 'begin-date': None, 'end-date': None}]}, {'id': '329c04ae-3b73-4ca3-996f-75608ab1befb', 'type': 'Person', 'type-id': 'b6e035f4-3ce9-331c-97df-83397230b0df', 'score': 62, 'name': 'Nirvana Singh', 'sort-name': 'Singh, Nirvana', 'life-span': {'ended': None}}, {'id': '89e9ed9d-9b45-42ad-bce2-54ef36053a1a', 'type': 'Group', 'type-id': 'e431f5f6-b5d2-343d-8b36-72607fffb74b', 'score': 62, 'name': 'Nirvana Seekerz', 'sort-name': 'Nirvana Seekerz', 'life-span': {'ended': None}}, {'id': '86f9ae24-ba2a-4d55-9275-0b89b85f6e3a', 'score': 62, 'name': 'Weed Nirvana', 'sort-name': 'Weed Nirvana', 'life-span': {'ended': None}}, {'id': 'b305320e-c158-43f4-b5be-4450e2f99a32', 'score': 62, 'name': 'El Nirvana', 'sort-name': 'Nirvana, El', 'life-span': {'ended': None}}, {'id': '206419e0-3a7a-49ce-8437-4e757767d02b', 'type': 'Person', 'type-id': 'b6e035f4-3ce9-331c-97df-83397230b0df', 'score': 62, 'name': 'Nirvana Savoury', 'sort-name': 'Savoury, Nirvana', 'gender': 'female', 'country': 'US', 'area': {'id': '489ce91b-6658-3307-9877-795b68554c98', 'type': 'Country', 'type-id': '06dd0ae4-8c74-30bb-b43d-95dcedf961de', 'name': 'United States', 'sort-name': 'United States', 'life-span': {'ended': None}}, 'life-span': {'ended': None}}, {'id': '7c525bf4-abf0-42e3-a1bd-81cbcb55c1f4', 'score': 62, 'name': 'Genta Nirvana', 'sort-name': 'Genta Nirvana', 'country': 'ID', 'area': {'id': 'd3a68bd0-7419-3f99-a5bd-204d6e057089', 'type': 'Country', 'type-id': '06dd0ae4-8c74-30bb-b43d-95dcedf961de', 'name': 'Indonesia', 'sort-name': 'Indonesia', 'life-span': {'ended': None}}, 'disambiguation': 'experimental', 'life-span': {'ended': None}}, {'id': '74ba0d66-e029-4b2e-9bd2-627b0febdc1d', 'score': 62, 'name': 'Cheick Nirvana', 'sort-name': 'Cheick Nirvana', 'life-span': {'ended': None}}, {'id': '8f32371e-4bac-4090-ba13-2c1cd1aeab0d', 'type': 'Group', 'type-id': 'e431f5f6-b5d2-343d-8b36-72607fffb74b', 'score': 62, 'name': 'Nirvana UK', 'sort-name': 'Nirvana UK', 'area': {'id': '07607044-8140-47ba-bb24-7129babe586b', 'type': 'Subdivision', 'type-id': 'fd3d44c5-80a1-3842-9745-2c4972d35afa', 'name': 'West Midlands', 'sort-name': 'West Midlands', 'life-span': {'ended': None}}, 'life-span': {'ended': None}}, {'id': 'da77e424-c473-481e-a2ae-5d966b2ee0b6', 'type': 'Group', 'type-id': 'e431f5f6-b5d2-343d-8b36-72607fffb74b', 'score': 62, 'name': 'Nirvana Undercover', 'sort-name': 'Nirvana Undercover', 'begin-area': {'id': 'ef1b7cc0-cd26-36f4-8ea0-04d9623786c7', 'type': 'Country', 'type-id': '06dd0ae4-8c74-30bb-b43d-95dcedf961de', 'name': 'Netherlands', 'sort-name': 'Netherlands', 'life-span': {'ended': None}}, 'life-span': {'ended': None}}, {'id': 'f58febd3-18de-4371-8d95-4a68d4f79456', 'type': 'Person', 'type-id': 'b6e035f4-3ce9-331c-97df-83397230b0df', 'score': 62, 'name': 'Nirvana Kelly', 'sort-name': 'Kelly, Nirvana', 'life-span': {'ended': None}}, {'id': 'e43ad11b-5d29-45ae-90f7-73ac47fb815d', 'score': 56, 'name': 'smells like nirvana', 'sort-name': 'smells like nirvana', 'life-span': {'ended': None}}, {'id': '1d512080-0ef7-454f-a325-9bf5ff95ce88', 'type': 'Group', 'type-id': 'e431f5f6-b5d2-343d-8b36-72607fffb74b', 'score': 56, 'name': 'Come As Nirvana', 'sort-name': 'Come As Nirvana', 'country': 'BE', 'area': {'id': '5b8a5ee5-0bb3-34cf-9a75-c27c44e341fc', 'type': 'Country', 'type-id': '06dd0ae4-8c74-30bb-b43d-95dcedf961de', 'name': 'Belgium', 'sort-name': 'Belgium', 'life-span': {'ended': None}}, 'disambiguation': 'Belgian', 'life-span': {'ended': None}}, {'id': '02c4e6bb-7b7a-4686-8c23-df01bfd42b0e', 'type': 'Group', 'type-id': 'e431f5f6-b5d2-343d-8b36-72607fffb74b', 'score': 56, 'name': 'Sappy Nirvana Tribute', 'sort-name': 'Sappy Nirvana Tribute', 'area': {'id': 'c621114d-73cc-4832-8afe-f13dc261e5af', 'type': 'City', 'type-id': '6fd8f29a-3d0a-32fc-980d-ea697b69da78', 'name': 'Gatineau', 'sort-name': 'Gatineau', 'life-span': {'ended': None}}, 'begin-area': {'id': 'c621114d-73cc-4832-8afe-f13dc261e5af', 'type': 'City', 'type-id': '6fd8f29a-3d0a-32fc-980d-ea697b69da78', 'name': 'Gatineau', 'sort-name': 'Gatineau', 'life-span': {'ended': None}}, 'life-span': {'begin': '2012-04-05', 'ended': None}}, {'id': '45eacd92-6857-4faa-9283-023e72a1d4b1', 'type': 'Group', 'type-id': 'e431f5f6-b5d2-343d-8b36-72607fffb74b', 'score': 56, 'name': 'The Nirvana Experience', 'sort-name': 'The Nirvana Experience', 'area': {'id': 'c920948b-83e3-40b7-8fe9-9ab5abaac55b', 'type': 'City', 'type-id': '6fd8f29a-3d0a-32fc-980d-ea697b69da78', 'name': 'Houston', 'sort-name': 'Houston', 'life-span': {'ended': None}}, 'begin-area': {'id': 'c920948b-83e3-40b7-8fe9-9ab5abaac55b', 'type': 'City', 'type-id': '6fd8f29a-3d0a-32fc-980d-ea697b69da78', 'name': 'Houston', 'sort-name': 'Houston', 'life-span': {'ended': None}}, 'disambiguation': 'Nirvana Cover Band', 'life-span': {'begin': '2012', 'ended': None}}, {'id': '46d8dae4-abec-438b-9c62-a3dbb2aaa1b7', 'type': 'Group', 'type-id': 'e431f5f6-b5d2-343d-8b36-72607fffb74b', 'score': 56, 'name': 'Nirvana Teen Spirit', 'sort-name': 'Nirvana Teen Spirit', 'area': {'id': 'e8ad73e9-9e7f-41c4-a395-6e29260ff1df', 'type': 'City', 'type-id': '6fd8f29a-3d0a-32fc-980d-ea697b69da78', 'name': 'Graz', 'sort-name': 'Graz', 'life-span': {'ended': None}}, 'begin-area': {'id': 'e8ad73e9-9e7f-41c4-a395-6e29260ff1df', 'type': 'City', 'type-id': '6fd8f29a-3d0a-32fc-980d-ea697b69da78', 'name': 'Graz', 'sort-name': 'Graz', 'life-span': {'ended': None}}, 'disambiguation': 'Nirvana-Coverband', 'life-span': {'begin': '2000', 'ended': None}}, {'id': 'bb94730d-22c2-422d-a0a7-fe16a5b3e429', 'type': 'Group', 'type-id': 'e431f5f6-b5d2-343d-8b36-72607fffb74b', 'score': 52, 'name': 'The Attainment of Nirvana', 'sort-name': 'Attainment of Nirvana, The', 'life-span': {'ended': None}}, {'id': 'e1388435-f80d-434a-9980-f1c9f5aa9b90', 'score': 48, 'name': 'Nirvana Sitar & String Group', 'sort-name': 'Nirvana Sitar & String Group', 'life-span': {'ended': None}}]\n"
     ]
    }
   ],
   "source": [
    "results = query_by_name(artist_url, query_type[\"simple\"], \"Nirvana\")\n",
    "pretty_print(results['artists'])  ##format"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 101,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "dict_keys(['created', 'count', 'offset', 'artists'])"
      ]
     },
     "execution_count": 101,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "results.keys()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 102,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "ARTIST:\n",
      "{\n",
      "    \"aliases\": [\n",
      "        {\n",
      "            \"begin-date\": null,\n",
      "            \"end-date\": null,\n",
      "            \"locale\": \"ja\",\n",
      "            \"name\": \"\\u30cb\\u30eb\\u30f4\\u30a1\\u30fc\\u30ca\",\n",
      "            \"primary\": true,\n",
      "            \"sort-name\": \"\\u30cb\\u30eb\\u30f4\\u30a1\\u30fc\\u30ca\",\n",
      "            \"type\": \"Artist name\",\n",
      "            \"type-id\": \"894afba6-2816-3c24-8072-eadb66bd04bc\"\n",
      "        },\n",
      "        {\n",
      "            \"begin-date\": null,\n",
      "            \"end-date\": null,\n",
      "            \"locale\": null,\n",
      "            \"name\": \"Nirvana US\",\n",
      "            \"primary\": null,\n",
      "            \"sort-name\": \"Nirvana US\",\n",
      "            \"type\": null\n",
      "        },\n",
      "        {\n",
      "            \"begin-date\": null,\n",
      "            \"end-date\": null,\n",
      "            \"locale\": \"en\",\n",
      "            \"name\": \"Nirvana\",\n",
      "            \"primary\": true,\n",
      "            \"sort-name\": \"Nirvana\",\n",
      "            \"type\": \"Artist name\",\n",
      "            \"type-id\": \"894afba6-2816-3c24-8072-eadb66bd04bc\"\n",
      "        }\n",
      "    ],\n",
      "    \"area\": {\n",
      "        \"id\": \"489ce91b-6658-3307-9877-795b68554c98\",\n",
      "        \"life-span\": {\n",
      "            \"ended\": null\n",
      "        },\n",
      "        \"name\": \"United States\",\n",
      "        \"sort-name\": \"United States\",\n",
      "        \"type\": \"Country\",\n",
      "        \"type-id\": \"06dd0ae4-8c74-30bb-b43d-95dcedf961de\"\n",
      "    },\n",
      "    \"begin-area\": {\n",
      "        \"id\": \"a640b45c-c173-49b1-8030-973603e895b5\",\n",
      "        \"life-span\": {\n",
      "            \"ended\": null\n",
      "        },\n",
      "        \"name\": \"Aberdeen\",\n",
      "        \"sort-name\": \"Aberdeen\",\n",
      "        \"type\": \"City\",\n",
      "        \"type-id\": \"6fd8f29a-3d0a-32fc-980d-ea697b69da78\"\n",
      "    },\n",
      "    \"country\": \"US\",\n",
      "    \"disambiguation\": \"90s US grunge band\",\n",
      "    \"id\": \"5b11f4ce-a62d-471e-81fc-a69a8278c7da\",\n",
      "    \"isnis\": [\n",
      "        \"0000000123486830\",\n",
      "        \"0000000123487390\"\n",
      "    ],\n",
      "    \"life-span\": {\n",
      "        \"begin\": \"1988-01\",\n",
      "        \"end\": \"1994-04-05\",\n",
      "        \"ended\": true\n",
      "    },\n",
      "    \"name\": \"Nirvana\",\n",
      "    \"score\": 100,\n",
      "    \"sort-name\": \"Nirvana\",\n",
      "    \"tags\": [\n",
      "        {\n",
      "            \"count\": 2,\n",
      "            \"name\": \"usa\"\n",
      "        },\n",
      "        {\n",
      "            \"count\": 0,\n",
      "            \"name\": \"united states\"\n",
      "        },\n",
      "        {\n",
      "            \"count\": 0,\n",
      "            \"name\": \"northwest\"\n",
      "        },\n",
      "        {\n",
      "            \"count\": 27,\n",
      "            \"name\": \"grunge\"\n",
      "        },\n",
      "        {\n",
      "            \"count\": 11,\n",
      "            \"name\": \"alternative rock\"\n",
      "        },\n",
      "        {\n",
      "            \"count\": 0,\n",
      "            \"name\": \"90\"\n",
      "        },\n",
      "        {\n",
      "            \"count\": 8,\n",
      "            \"name\": \"american\"\n",
      "        },\n",
      "        {\n",
      "            \"count\": 0,\n",
      "            \"name\": \"am\\u00e9ricain\"\n",
      "        },\n",
      "        {\n",
      "            \"count\": 0,\n",
      "            \"name\": \"alternative\"\n",
      "        },\n",
      "        {\n",
      "            \"count\": 2,\n",
      "            \"name\": \"90s\"\n",
      "        },\n",
      "        {\n",
      "            \"count\": 1,\n",
      "            \"name\": \"acoustic rock\"\n",
      "        },\n",
      "        {\n",
      "            \"count\": 0,\n",
      "            \"name\": \"nirvana\"\n",
      "        },\n",
      "        {\n",
      "            \"count\": 2,\n",
      "            \"name\": \"noise rock\"\n",
      "        },\n",
      "        {\n",
      "            \"count\": 17,\n",
      "            \"name\": \"rock\"\n",
      "        },\n",
      "        {\n",
      "            \"count\": 0,\n",
      "            \"name\": \"band\"\n",
      "        },\n",
      "        {\n",
      "            \"count\": 0,\n",
      "            \"name\": \"legendary\"\n",
      "        },\n",
      "        {\n",
      "            \"count\": 0,\n",
      "            \"name\": \"kurt cobain\"\n",
      "        },\n",
      "        {\n",
      "            \"count\": 0,\n",
      "            \"name\": \"rock and indie\"\n",
      "        },\n",
      "        {\n",
      "            \"count\": 2,\n",
      "            \"name\": \"punk\"\n",
      "        },\n",
      "        {\n",
      "            \"count\": 3,\n",
      "            \"name\": \"seattle\"\n",
      "        },\n",
      "        {\n",
      "            \"count\": 2,\n",
      "            \"name\": \"experimental\"\n",
      "        }\n",
      "    ],\n",
      "    \"type\": \"Group\",\n",
      "    \"type-id\": \"e431f5f6-b5d2-343d-8b36-72607fffb74b\"\n",
      "}\n"
     ]
    }
   ],
   "source": [
    "print(\"\\nARTIST:\")\n",
    "pretty_print(results[\"artists\"][0])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 103,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'5b11f4ce-a62d-471e-81fc-a69a8278c7da'"
      ]
     },
     "execution_count": 103,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "artist_id = results[\"artists\"][0][\"id\"]\n",
    "artist_id"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 104,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "requesting http://musicbrainz.org/ws/2/artist/5b11f4ce-a62d-471e-81fc-a69a8278c7da?inc=releases&fmt=json\n",
      "\n",
      "ONE RELEASE:\n",
      "{\n",
      "  \"barcode\": null,\n",
      "  \"country\": \"US\",\n",
      "  \"date\": \"1988-11\",\n",
      "  \"disambiguation\": \"\",\n",
      "  \"id\": \"2c84c9b3-d1d4-48c0-a825-a360f26a5dde\",\n",
      "  \"packaging\": null,\n",
      "  \"packaging-id\": null,\n",
      "  \"quality\": \"normal\",\n",
      "  \"release-events\": [\n",
      "    {\n",
      "      \"area\": {\n",
      "        \"disambiguation\": \"\",\n",
      "        \"id\": \"489ce91b-6658-3307-9877-795b68554c98\",\n",
      "        \"iso-3166-1-codes\": [\n",
      "          \"US\"\n",
      "        ],\n",
      "        \"name\": \"United States\",\n",
      "        \"sort-name\": \"United States\",\n",
      "        \"type\": null,\n",
      "        \"type-id\": null\n",
      "      },\n",
      "      \"date\": \"1988-11\"\n",
      "    }\n",
      "  ],\n",
      "  \"status\": \"Official\",\n",
      "  \"status-id\": \"4e304316-386d-3409-af2e-78857eec5cfe\",\n",
      "  \"text-representation\": {\n",
      "    \"language\": \"eng\",\n",
      "    \"script\": \"Latn\"\n",
      "  },\n",
      "  \"title\": \"Love Buzz / Big Cheese\"\n",
      "}\n"
     ]
    }
   ],
   "source": [
    "artist_data = query_site(artist_url, query_type[\"releases\"], artist_id)\n",
    "releases = artist_data[\"releases\"]\n",
    "print(\"\\nONE RELEASE:\")\n",
    "pretty_print(releases[0], indent=2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 105,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[{'date': '1988-11',\n",
       "  'id': '2c84c9b3-d1d4-48c0-a825-a360f26a5dde',\n",
       "  'packaging': None,\n",
       "  'release-events': [{'date': '1988-11',\n",
       "    'area': {'type-id': None,\n",
       "     'id': '489ce91b-6658-3307-9877-795b68554c98',\n",
       "     'iso-3166-1-codes': ['US'],\n",
       "     'disambiguation': '',\n",
       "     'sort-name': 'United States',\n",
       "     'name': 'United States',\n",
       "     'type': None}}],\n",
       "  'disambiguation': '',\n",
       "  'status': 'Official',\n",
       "  'text-representation': {'language': 'eng', 'script': 'Latn'},\n",
       "  'quality': 'normal',\n",
       "  'packaging-id': None,\n",
       "  'barcode': None,\n",
       "  'status-id': '4e304316-386d-3409-af2e-78857eec5cfe',\n",
       "  'country': 'US',\n",
       "  'title': 'Love Buzz / Big Cheese'},\n",
       " {'release-events': [{'date': '1989-06-01',\n",
       "    'area': {'disambiguation': '',\n",
       "     'id': '489ce91b-6658-3307-9877-795b68554c98',\n",
       "     'type-id': None,\n",
       "     'iso-3166-1-codes': ['US'],\n",
       "     'type': None,\n",
       "     'sort-name': 'United States',\n",
       "     'name': 'United States'}}],\n",
       "  'status': 'Official',\n",
       "  'disambiguation': '',\n",
       "  'date': '1989-06-01',\n",
       "  'id': '7d166a44-cfb5-4b08-aacb-6863bbe677d6',\n",
       "  'packaging': 'Jewel Case',\n",
       "  'quality': 'normal',\n",
       "  'packaging-id': 'ec27701a-4a22-37f4-bfac-6616e0f9750a',\n",
       "  'text-representation': {'script': 'Latn', 'language': 'eng'},\n",
       "  'status-id': '4e304316-386d-3409-af2e-78857eec5cfe',\n",
       "  'country': 'US',\n",
       "  'barcode': '098787003420',\n",
       "  'title': 'Bleach'},\n",
       " {'barcode': None,\n",
       "  'status-id': '4e304316-386d-3409-af2e-78857eec5cfe',\n",
       "  'country': 'US',\n",
       "  'title': 'Bleach',\n",
       "  'date': '1989-08-08',\n",
       "  'id': 'e5fa1cc5-3693-455a-821f-5ace2690aff1',\n",
       "  'packaging': None,\n",
       "  'release-events': [{'date': '1989-08-08',\n",
       "    'area': {'id': '489ce91b-6658-3307-9877-795b68554c98',\n",
       "     'type-id': None,\n",
       "     'iso-3166-1-codes': ['US'],\n",
       "     'disambiguation': '',\n",
       "     'sort-name': 'United States',\n",
       "     'name': 'United States',\n",
       "     'type': None}}],\n",
       "  'disambiguation': '',\n",
       "  'status': 'Official',\n",
       "  'text-representation': {'language': 'eng', 'script': 'Latn'},\n",
       "  'quality': 'normal',\n",
       "  'packaging-id': None},\n",
       " {'barcode': '5018615200611',\n",
       "  'country': 'GB',\n",
       "  'status-id': '4e304316-386d-3409-af2e-78857eec5cfe',\n",
       "  'title': 'Bleach',\n",
       "  'id': 'b7184186-6884-44cd-91ed-62370f880e5d',\n",
       "  'date': '1989-08',\n",
       "  'packaging': None,\n",
       "  'status': 'Official',\n",
       "  'disambiguation': '',\n",
       "  'release-events': [{'area': {'type': None,\n",
       "     'name': 'United Kingdom',\n",
       "     'sort-name': 'United Kingdom',\n",
       "     'disambiguation': '',\n",
       "     'id': '8a754a16-0027-3a29-b6d7-2b40ea0481ed',\n",
       "     'type-id': None,\n",
       "     'iso-3166-1-codes': ['GB']},\n",
       "    'date': '1989-08'}],\n",
       "  'text-representation': {'language': 'eng', 'script': 'Latn'},\n",
       "  'quality': 'normal',\n",
       "  'packaging-id': None},\n",
       " {'barcode': '5018615200628',\n",
       "  'country': 'GB',\n",
       "  'status-id': '4e304316-386d-3409-af2e-78857eec5cfe',\n",
       "  'title': 'Bleach',\n",
       "  'id': '581f7289-93cc-4629-97d6-4c1eae38db17',\n",
       "  'packaging': None,\n",
       "  'date': '1989-08',\n",
       "  'status': 'Official',\n",
       "  'disambiguation': '',\n",
       "  'release-events': [{'area': {'id': '8a754a16-0027-3a29-b6d7-2b40ea0481ed',\n",
       "     'type-id': None,\n",
       "     'iso-3166-1-codes': ['GB'],\n",
       "     'disambiguation': '',\n",
       "     'name': 'United Kingdom',\n",
       "     'sort-name': 'United Kingdom',\n",
       "     'type': None},\n",
       "    'date': '1989-08'}],\n",
       "  'text-representation': {'script': 'Latn', 'language': 'eng'},\n",
       "  'packaging-id': None,\n",
       "  'quality': 'normal'},\n",
       " {'barcode': '5018615200819',\n",
       "  'country': 'GB',\n",
       "  'status-id': '4e304316-386d-3409-af2e-78857eec5cfe',\n",
       "  'title': 'Blew',\n",
       "  'id': 'a5829c6e-ad99-38cf-bf98-2dcbee359c09',\n",
       "  'packaging': None,\n",
       "  'date': '1989-12',\n",
       "  'status': 'Official',\n",
       "  'disambiguation': '',\n",
       "  'release-events': [{'date': '1989-12',\n",
       "    'area': {'type': None,\n",
       "     'sort-name': 'United Kingdom',\n",
       "     'name': 'United Kingdom',\n",
       "     'disambiguation': '',\n",
       "     'type-id': None,\n",
       "     'iso-3166-1-codes': ['GB'],\n",
       "     'id': '8a754a16-0027-3a29-b6d7-2b40ea0481ed'}}],\n",
       "  'text-representation': {'language': 'eng', 'script': 'Latn'},\n",
       "  'packaging-id': None,\n",
       "  'quality': 'normal'},\n",
       " {'title': 'Blew',\n",
       "  'country': 'GB',\n",
       "  'status-id': '4e304316-386d-3409-af2e-78857eec5cfe',\n",
       "  'barcode': '5018615200826',\n",
       "  'quality': 'normal',\n",
       "  'packaging-id': None,\n",
       "  'text-representation': {'language': 'eng', 'script': 'Latn'},\n",
       "  'disambiguation': '',\n",
       "  'status': 'Official',\n",
       "  'release-events': [{'area': {'type-id': None,\n",
       "     'iso-3166-1-codes': ['GB'],\n",
       "     'id': '8a754a16-0027-3a29-b6d7-2b40ea0481ed',\n",
       "     'disambiguation': '',\n",
       "     'sort-name': 'United Kingdom',\n",
       "     'name': 'United Kingdom',\n",
       "     'type': None},\n",
       "    'date': '1989-12'}],\n",
       "  'id': '180abab4-8480-4d1b-9a9c-db3d5e864390',\n",
       "  'date': '1989-12',\n",
       "  'packaging': None},\n",
       " {'text-representation': {'language': 'eng', 'script': 'Latn'},\n",
       "  'quality': 'normal',\n",
       "  'packaging-id': 'ec27701a-4a22-37f4-bfac-6616e0f9750a',\n",
       "  'packaging': 'Jewel Case',\n",
       "  'id': 'a96b59a5-4185-4d87-899a-a0d944385811',\n",
       "  'date': '1989',\n",
       "  'release-events': [{'area': {'type': None,\n",
       "     'sort-name': 'United Kingdom',\n",
       "     'name': 'United Kingdom',\n",
       "     'disambiguation': '',\n",
       "     'iso-3166-1-codes': ['GB'],\n",
       "     'type-id': None,\n",
       "     'id': '8a754a16-0027-3a29-b6d7-2b40ea0481ed'},\n",
       "    'date': '1989'}],\n",
       "  'disambiguation': '',\n",
       "  'status': 'Official',\n",
       "  'title': 'Bleach',\n",
       "  'barcode': '008811929121',\n",
       "  'status-id': '4e304316-386d-3409-af2e-78857eec5cfe',\n",
       "  'country': 'GB'},\n",
       " {'release-events': [{'date': '1989',\n",
       "    'area': {'disambiguation': '',\n",
       "     'type-id': None,\n",
       "     'iso-3166-1-codes': ['GB'],\n",
       "     'id': '8a754a16-0027-3a29-b6d7-2b40ea0481ed',\n",
       "     'type': None,\n",
       "     'name': 'United Kingdom',\n",
       "     'sort-name': 'United Kingdom'}}],\n",
       "  'disambiguation': '',\n",
       "  'status': 'Official',\n",
       "  'date': '1989',\n",
       "  'id': 'd3d44548-b2ed-3ddc-a9e0-3f5ee544c1c9',\n",
       "  'packaging': 'Jewel Case',\n",
       "  'quality': 'normal',\n",
       "  'packaging-id': 'ec27701a-4a22-37f4-bfac-6616e0f9750a',\n",
       "  'text-representation': {'script': 'Latn', 'language': 'eng'},\n",
       "  'status-id': '4e304316-386d-3409-af2e-78857eec5cfe',\n",
       "  'country': 'GB',\n",
       "  'barcode': '5018615200628',\n",
       "  'title': 'Bleach'},\n",
       " {'title': 'Never Mind the Bollocks Here’s Nirvana',\n",
       "  'barcode': '',\n",
       "  'country': 'US',\n",
       "  'status-id': '1156806e-d06a-38bd-83f0-cf2284a808b9',\n",
       "  'text-representation': {'language': 'eng', 'script': 'Latn'},\n",
       "  'packaging-id': 'f7101ce3-0384-39ce-9fde-fbbd0044d35f',\n",
       "  'quality': 'normal',\n",
       "  'id': '0444b84e-15af-47a4-810a-773258eb9395',\n",
       "  'packaging': 'Cardboard/Paper Sleeve',\n",
       "  'date': '1990-01',\n",
       "  'status': 'Bootleg',\n",
       "  'disambiguation': '',\n",
       "  'release-events': [{'date': '1990-01',\n",
       "    'area': {'id': '489ce91b-6658-3307-9877-795b68554c98',\n",
       "     'type-id': None,\n",
       "     'iso-3166-1-codes': ['US'],\n",
       "     'disambiguation': '',\n",
       "     'name': 'United States',\n",
       "     'sort-name': 'United States',\n",
       "     'type': None}}]},\n",
       " {'status-id': '4e304316-386d-3409-af2e-78857eec5cfe',\n",
       "  'country': 'US',\n",
       "  'barcode': '',\n",
       "  'title': 'Sliver',\n",
       "  'release-events': [{'date': '1990-09',\n",
       "    'area': {'disambiguation': '',\n",
       "     'id': '489ce91b-6658-3307-9877-795b68554c98',\n",
       "     'type-id': None,\n",
       "     'iso-3166-1-codes': ['US'],\n",
       "     'type': None,\n",
       "     'sort-name': 'United States',\n",
       "     'name': 'United States'}}],\n",
       "  'disambiguation': '',\n",
       "  'status': 'Official',\n",
       "  'date': '1990-09',\n",
       "  'id': 'eaff5199-b86c-4bca-a4b5-c7c33e10e88d',\n",
       "  'packaging': 'Cardboard/Paper Sleeve',\n",
       "  'quality': 'normal',\n",
       "  'packaging-id': 'f7101ce3-0384-39ce-9fde-fbbd0044d35f',\n",
       "  'text-representation': {'language': 'eng', 'script': 'Latn'}},\n",
       " {'title': 'Sliver',\n",
       "  'barcode': None,\n",
       "  'status-id': '4e304316-386d-3409-af2e-78857eec5cfe',\n",
       "  'country': 'US',\n",
       "  'text-representation': {'language': 'eng', 'script': 'Latn'},\n",
       "  'packaging-id': None,\n",
       "  'quality': 'normal',\n",
       "  'date': '1990-09',\n",
       "  'id': 'acebbf06-a948-46e7-ae3c-730894e08067',\n",
       "  'packaging': None,\n",
       "  'release-events': [{'area': {'type-id': None,\n",
       "     'iso-3166-1-codes': ['US'],\n",
       "     'id': '489ce91b-6658-3307-9877-795b68554c98',\n",
       "     'disambiguation': '',\n",
       "     'sort-name': 'United States',\n",
       "     'name': 'United States',\n",
       "     'type': None},\n",
       "    'date': '1990-09'}],\n",
       "  'disambiguation': '',\n",
       "  'status': 'Official'},\n",
       " {'status-id': '4e304316-386d-3409-af2e-78857eec5cfe',\n",
       "  'country': 'US',\n",
       "  'barcode': None,\n",
       "  'title': 'Sliver',\n",
       "  'release-events': [{'area': {'type-id': None,\n",
       "     'iso-3166-1-codes': ['US'],\n",
       "     'id': '489ce91b-6658-3307-9877-795b68554c98',\n",
       "     'disambiguation': '',\n",
       "     'name': 'United States',\n",
       "     'sort-name': 'United States',\n",
       "     'type': None},\n",
       "    'date': '1990-09'}],\n",
       "  'status': 'Official',\n",
       "  'disambiguation': '',\n",
       "  'id': '21c3ed4e-3961-3fa3-9541-e4f02a980858',\n",
       "  'packaging': None,\n",
       "  'date': '1990-09',\n",
       "  'quality': 'normal',\n",
       "  'packaging-id': None,\n",
       "  'text-representation': {'language': 'eng', 'script': 'Latn'}},\n",
       " {'quality': 'normal',\n",
       "  'packaging-id': None,\n",
       "  'text-representation': {'script': 'Latn', 'language': 'eng'},\n",
       "  'release-events': [{'date': '1990',\n",
       "    'area': {'type-id': None,\n",
       "     'iso-3166-1-codes': ['PL'],\n",
       "     'id': 'dd7f80c8-f017-3d01-8608-2a8c9c32b954',\n",
       "     'disambiguation': '',\n",
       "     'name': 'Poland',\n",
       "     'sort-name': 'Poland',\n",
       "     'type': None}}],\n",
       "  'disambiguation': '',\n",
       "  'status': 'Bootleg',\n",
       "  'id': '202368dc-5497-47a3-953e-bfa631b2a0cb',\n",
       "  'packaging': None,\n",
       "  'date': '1990',\n",
       "  'title': 'Bleach',\n",
       "  'status-id': '1156806e-d06a-38bd-83f0-cf2284a808b9',\n",
       "  'country': 'PL',\n",
       "  'barcode': None},\n",
       " {'text-representation': {'language': 'eng', 'script': None},\n",
       "  'quality': 'normal',\n",
       "  'packaging-id': 'c70b737a-0114-39a9-88f7-82843e54f906',\n",
       "  'packaging': 'Cassette Case',\n",
       "  'id': 'b984300f-2fb5-4147-891b-7740019a6164',\n",
       "  'date': '1990',\n",
       "  'release-events': [{'date': '1990',\n",
       "    'area': {'disambiguation': '',\n",
       "     'type-id': None,\n",
       "     'id': '8a754a16-0027-3a29-b6d7-2b40ea0481ed',\n",
       "     'iso-3166-1-codes': ['GB'],\n",
       "     'type': None,\n",
       "     'sort-name': 'United Kingdom',\n",
       "     'name': 'United Kingdom'}}],\n",
       "  'disambiguation': '',\n",
       "  'status': 'Official',\n",
       "  'title': 'Bleach',\n",
       "  'barcode': '5018615200642',\n",
       "  'status-id': '4e304316-386d-3409-af2e-78857eec5cfe',\n",
       "  'country': 'GB'},\n",
       " {'id': '1a5ad2ee-92e6-47a5-9340-25c529fc39b4',\n",
       "  'date': '1991-01-21',\n",
       "  'packaging': 'Cardboard/Paper Sleeve',\n",
       "  'status': 'Official',\n",
       "  'disambiguation': '',\n",
       "  'release-events': [{'area': {'sort-name': 'United Kingdom',\n",
       "     'name': 'United Kingdom',\n",
       "     'type': None,\n",
       "     'iso-3166-1-codes': ['GB'],\n",
       "     'type-id': None,\n",
       "     'id': '8a754a16-0027-3a29-b6d7-2b40ea0481ed',\n",
       "     'disambiguation': ''},\n",
       "    'date': '1991-01-21'}],\n",
       "  'text-representation': {'script': 'Latn', 'language': 'eng'},\n",
       "  'quality': 'normal',\n",
       "  'packaging-id': 'f7101ce3-0384-39ce-9fde-fbbd0044d35f',\n",
       "  'barcode': '5018615202561',\n",
       "  'country': 'GB',\n",
       "  'status-id': '4e304316-386d-3409-af2e-78857eec5cfe',\n",
       "  'title': 'Sliver'},\n",
       " {'quality': 'normal',\n",
       "  'packaging-id': '36327bc2-f691-3d66-80e5-bd03cec6060a',\n",
       "  'text-representation': {'language': 'eng', 'script': 'Latn'},\n",
       "  'release-events': [{'date': '1991-01',\n",
       "    'area': {'sort-name': 'United Kingdom',\n",
       "     'name': 'United Kingdom',\n",
       "     'type': None,\n",
       "     'id': '8a754a16-0027-3a29-b6d7-2b40ea0481ed',\n",
       "     'type-id': None,\n",
       "     'iso-3166-1-codes': ['GB'],\n",
       "     'disambiguation': ''}}],\n",
       "  'disambiguation': '',\n",
       "  'status': 'Official',\n",
       "  'id': '7f183c32-4a79-4873-ac38-215cde83e0ce',\n",
       "  'date': '1991-01',\n",
       "  'packaging': 'Slim Jewel Case',\n",
       "  'title': 'Sliver',\n",
       "  'status-id': '4e304316-386d-3409-af2e-78857eec5cfe',\n",
       "  'country': 'GB',\n",
       "  'barcode': '5018615202523'},\n",
       " {'quality': 'normal',\n",
       "  'packaging-id': None,\n",
       "  'text-representation': {'script': 'Latn', 'language': 'eng'},\n",
       "  'disambiguation': '',\n",
       "  'status': 'Official',\n",
       "  'release-events': [{'area': {'type': None,\n",
       "     'name': 'United States',\n",
       "     'sort-name': 'United States',\n",
       "     'disambiguation': '',\n",
       "     'iso-3166-1-codes': ['US'],\n",
       "     'type-id': None,\n",
       "     'id': '489ce91b-6658-3307-9877-795b68554c98'},\n",
       "    'date': '1991-01'}],\n",
       "  'date': '1991-01',\n",
       "  'id': 'a8b2b2e8-f366-4903-afe8-d4ca1332c13d',\n",
       "  'packaging': None,\n",
       "  'title': 'Candy / Molly’s Lips',\n",
       "  'country': 'US',\n",
       "  'status-id': '4e304316-386d-3409-af2e-78857eec5cfe',\n",
       "  'barcode': ''},\n",
       " {'title': 'Here She Comes Now / Venus in Furs',\n",
       "  'country': 'US',\n",
       "  'status-id': '4e304316-386d-3409-af2e-78857eec5cfe',\n",
       "  'barcode': None,\n",
       "  'packaging-id': None,\n",
       "  'quality': 'normal',\n",
       "  'text-representation': {'script': 'Latn', 'language': 'eng'},\n",
       "  'status': 'Official',\n",
       "  'disambiguation': '',\n",
       "  'release-events': [{'date': '1991-06',\n",
       "    'area': {'iso-3166-1-codes': ['US'],\n",
       "     'type-id': None,\n",
       "     'id': '489ce91b-6658-3307-9877-795b68554c98',\n",
       "     'disambiguation': '',\n",
       "     'sort-name': 'United States',\n",
       "     'name': 'United States',\n",
       "     'type': None}}],\n",
       "  'packaging': None,\n",
       "  'id': '3fce74e6-2898-478e-81b3-c72a8fb4cb89',\n",
       "  'date': '1991-06'},\n",
       " {'disambiguation': '',\n",
       "  'status': 'Official',\n",
       "  'release-events': [{'area': {'sort-name': 'Europe',\n",
       "     'name': 'Europe',\n",
       "     'type': None,\n",
       "     'type-id': None,\n",
       "     'iso-3166-1-codes': ['XE'],\n",
       "     'id': '89a675c2-3e37-3518-b83c-418bad59a85a',\n",
       "     'disambiguation': ''},\n",
       "    'date': '1991-09-10'}],\n",
       "  'packaging': 'Slim Jewel Case',\n",
       "  'id': '3abad285-fd57-4709-9f82-572b9c1ae051',\n",
       "  'date': '1991-09-10',\n",
       "  'packaging-id': '36327bc2-f691-3d66-80e5-bd03cec6060a',\n",
       "  'quality': 'normal',\n",
       "  'text-representation': {'script': 'Latn', 'language': 'eng'},\n",
       "  'country': 'XE',\n",
       "  'status-id': '4e304316-386d-3409-af2e-78857eec5cfe',\n",
       "  'barcode': '720642167328',\n",
       "  'title': 'Smells Like Teen Spirit'},\n",
       " {'barcode': '720642167328',\n",
       "  'status-id': '4e304316-386d-3409-af2e-78857eec5cfe',\n",
       "  'country': 'DE',\n",
       "  'title': 'Smells Like Teen Spirit',\n",
       "  'date': '1991-09-10',\n",
       "  'id': '68f9fe0a-23c2-3975-8310-7f675b4ec0e6',\n",
       "  'packaging': None,\n",
       "  'release-events': [{'area': {'type': None,\n",
       "     'name': 'Germany',\n",
       "     'sort-name': 'Germany',\n",
       "     'disambiguation': '',\n",
       "     'iso-3166-1-codes': ['DE'],\n",
       "     'type-id': None,\n",
       "     'id': '85752fda-13c4-31a3-bee5-0e5cb1f51dad'},\n",
       "    'date': '1991-09-10'}],\n",
       "  'status': 'Official',\n",
       "  'disambiguation': '',\n",
       "  'text-representation': {'script': 'Latn', 'language': 'eng'},\n",
       "  'quality': 'normal',\n",
       "  'packaging-id': None},\n",
       " {'text-representation': {'script': 'Latn', 'language': 'eng'},\n",
       "  'quality': 'normal',\n",
       "  'packaging-id': '8f931351-d2e2-310f-afc6-37b89ddba246',\n",
       "  'packaging': 'Digipak',\n",
       "  'id': 'b0dd957a-4dd0-40e3-920c-3aac782fad14',\n",
       "  'date': '1991-09-10',\n",
       "  'status': 'Official',\n",
       "  'disambiguation': '',\n",
       "  'release-events': [{'area': {'sort-name': 'United Kingdom',\n",
       "     'name': 'United Kingdom',\n",
       "     'type': None,\n",
       "     'id': '8a754a16-0027-3a29-b6d7-2b40ea0481ed',\n",
       "     'type-id': None,\n",
       "     'iso-3166-1-codes': ['GB'],\n",
       "     'disambiguation': ''},\n",
       "    'date': '1991-09-10'}],\n",
       "  'title': 'Smells Like Teen Spirit',\n",
       "  'barcode': '5021509000525',\n",
       "  'country': 'GB',\n",
       "  'status-id': '4e304316-386d-3409-af2e-78857eec5cfe'},\n",
       " {'disambiguation': '',\n",
       "  'status': 'Official',\n",
       "  'release-events': [{'area': {'type-id': None,\n",
       "     'id': '489ce91b-6658-3307-9877-795b68554c98',\n",
       "     'iso-3166-1-codes': ['US'],\n",
       "     'disambiguation': '',\n",
       "     'sort-name': 'United States',\n",
       "     'name': 'United States',\n",
       "     'type': None},\n",
       "    'date': '1991-09-10'}],\n",
       "  'packaging': 'Digipak',\n",
       "  'id': '6a3e3fe7-3cc5-3a6d-b3aa-6fbb12d5b9b2',\n",
       "  'date': '1991-09-10',\n",
       "  'packaging-id': '8f931351-d2e2-310f-afc6-37b89ddba246',\n",
       "  'quality': 'normal',\n",
       "  'text-representation': {'language': 'eng', 'script': 'Latn'},\n",
       "  'country': 'US',\n",
       "  'status-id': '4e304316-386d-3409-af2e-78857eec5cfe',\n",
       "  'barcode': '720642167328',\n",
       "  'title': 'Smells Like Teen Spirit'},\n",
       " {'text-representation': {'script': 'Latn', 'language': 'eng'},\n",
       "  'quality': 'normal',\n",
       "  'packaging-id': None,\n",
       "  'id': '8d771949-da8f-36df-8d4a-c3cb194cf21d',\n",
       "  'date': '1991-09-10',\n",
       "  'packaging': None,\n",
       "  'disambiguation': '',\n",
       "  'status': 'Official',\n",
       "  'release-events': [{'date': '1991-09-10',\n",
       "    'area': {'type': None,\n",
       "     'name': 'United States',\n",
       "     'sort-name': 'United States',\n",
       "     'disambiguation': '',\n",
       "     'type-id': None,\n",
       "     'iso-3166-1-codes': ['US'],\n",
       "     'id': '489ce91b-6658-3307-9877-795b68554c98'}}],\n",
       "  'title': 'Smells Like Teen Spirit',\n",
       "  'barcode': None,\n",
       "  'country': 'US',\n",
       "  'status-id': '4e304316-386d-3409-af2e-78857eec5cfe'},\n",
       " {'title': 'Smells Like Teen Spirit',\n",
       "  'status-id': '4e304316-386d-3409-af2e-78857eec5cfe',\n",
       "  'country': 'US',\n",
       "  'barcode': None,\n",
       "  'quality': 'normal',\n",
       "  'packaging-id': None,\n",
       "  'text-representation': {'language': 'eng', 'script': 'Latn'},\n",
       "  'release-events': [{'date': '1991-09-10',\n",
       "    'area': {'name': 'United States',\n",
       "     'sort-name': 'United States',\n",
       "     'type': None,\n",
       "     'type-id': None,\n",
       "     'iso-3166-1-codes': ['US'],\n",
       "     'id': '489ce91b-6658-3307-9877-795b68554c98',\n",
       "     'disambiguation': ''}}],\n",
       "  'disambiguation': '',\n",
       "  'status': 'Official',\n",
       "  'packaging': None,\n",
       "  'id': 'e1c5d5fa-b1cb-388b-9027-055aa29a382b',\n",
       "  'date': '1991-09-10'}]"
      ]
     },
     "execution_count": 105,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "releases"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 106,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "ALL TITLES:\n",
      "Love Buzz / Big Cheese\n",
      "Bleach\n",
      "Bleach\n",
      "Bleach\n",
      "Bleach\n",
      "Blew\n",
      "Blew\n",
      "Bleach\n",
      "Bleach\n",
      "Never Mind the Bollocks Here’s Nirvana\n",
      "Sliver\n",
      "Sliver\n",
      "Sliver\n",
      "Bleach\n",
      "Bleach\n",
      "Sliver\n",
      "Sliver\n",
      "Candy / Molly’s Lips\n",
      "Here She Comes Now / Venus in Furs\n",
      "Smells Like Teen Spirit\n",
      "Smells Like Teen Spirit\n",
      "Smells Like Teen Spirit\n",
      "Smells Like Teen Spirit\n",
      "Smells Like Teen Spirit\n",
      "Smells Like Teen Spirit\n"
     ]
    }
   ],
   "source": [
    "release_titles = [r[\"title\"] for r in releases]\n",
    "print(\"\\nALL TITLES:\")\n",
    "for t in release_titles:\n",
    "        print(t)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Using Pandas"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "http://pandas.pydata.org/pandas-docs/stable/io.html#io-json-reader  "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 107,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<class 'pandas.core.frame.DataFrame'>\n",
      "RangeIndex: 5 entries, 0 to 4\n",
      "Data columns (total 26 columns):\n",
      " #   Column                    Non-Null Count  Dtype              \n",
      "---  ------                    --------------  -----              \n",
      " 0   url                       5 non-null      object             \n",
      " 1   repository_url            5 non-null      object             \n",
      " 2   labels_url                5 non-null      object             \n",
      " 3   comments_url              5 non-null      object             \n",
      " 4   events_url                5 non-null      object             \n",
      " 5   html_url                  5 non-null      object             \n",
      " 6   id                        5 non-null      int64              \n",
      " 7   node_id                   5 non-null      object             \n",
      " 8   number                    5 non-null      int64              \n",
      " 9   title                     5 non-null      object             \n",
      " 10  user                      5 non-null      object             \n",
      " 11  labels                    5 non-null      object             \n",
      " 12  state                     5 non-null      object             \n",
      " 13  locked                    5 non-null      bool               \n",
      " 14  assignee                  1 non-null      object             \n",
      " 15  assignees                 5 non-null      object             \n",
      " 16  milestone                 1 non-null      object             \n",
      " 17  comments                  5 non-null      int64              \n",
      " 18  created_at                5 non-null      datetime64[ns, UTC]\n",
      " 19  updated_at                5 non-null      datetime64[ns, UTC]\n",
      " 20  closed_at                 0 non-null      datetime64[ns]     \n",
      " 21  author_association        5 non-null      object             \n",
      " 22  active_lock_reason        0 non-null      float64            \n",
      " 23  body                      5 non-null      object             \n",
      " 24  performed_via_github_app  0 non-null      float64            \n",
      " 25  pull_request              2 non-null      object             \n",
      "dtypes: bool(1), datetime64[ns, UTC](2), datetime64[ns](1), float64(2), int64(3), object(17)\n",
      "memory usage: 1.1+ KB\n"
     ]
    }
   ],
   "source": [
    "df_json = pd.read_json('https://api.github.com/repos/pydata/pandas/issues?per_page=5')\n",
    "df_json.info()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 108,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0    BUG: Setting a MultiIndex through set_index wh...\n",
       "1    Revert \"REF: back IntervalArray by a single nd...\n",
       "2                             QST: Pandas without MKL?\n",
       "3                     STYLE use types_or in pre-commit\n",
       "4      ENH: support 2D in DatetimeArray._from_sequence\n",
       "Name: title, dtype: object"
      ]
     },
     "execution_count": 108,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_json.title.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Accessing YAML"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 109,
   "metadata": {},
   "outputs": [],
   "source": [
    "import yaml"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 110,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[{'martin': {'name': \"Martin D'vloper\", 'job': 'Developer', 'skills': ['python', 'perl', 'pascal']}}, {'tabitha': {'name': 'Tabitha Bitumen', 'job': 'Developer', 'skills': ['lisp', 'fortran', 'erlang']}}]\n"
     ]
    }
   ],
   "source": [
    "with open(os.path.join(datapath, 'simple.yaml'), 'r') as stream:\n",
    "    try:\n",
    "        yamlfile = yaml.load(stream, Loader=yaml.Loader)\n",
    "        print(yamlfile)\n",
    "    except yaml.YAMLError as exc:\n",
    "        print(exc)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 111,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>martin</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>job</th>\n",
       "      <td>Developer</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>name</th>\n",
       "      <td>Martin D'vloper</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>skills</th>\n",
       "      <td>[python, perl, pascal]</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                        martin\n",
       "job                  Developer\n",
       "name           Martin D'vloper\n",
       "skills  [python, perl, pascal]"
      ]
     },
     "execution_count": 111,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "pd.DataFrame(yamlfile[0])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Accessing XML"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "https://docs.python.org/3/library/urllib.html\n",
    "\n",
    "https://docs.python.org/3/library/xml.etree.elementtree.html\n",
    "\n",
    "http://lxml.de/\n",
    "\n",
    "http://www.w3schools.com/xml/xml_examples.asp"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 112,
   "metadata": {},
   "outputs": [],
   "source": [
    "from xml.dom import minidom\n",
    "#from xml.etree import ElementTree as ET\n",
    "from lxml import etree as ET #Supports xpath syntax\n",
    "from collections import defaultdict\n",
    "from pprint import pprint"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 113,
   "metadata": {},
   "outputs": [],
   "source": [
    "url = 'http://www.w3schools.com/xml/cd_catalog.xml'\n",
    "xml_page = urllib.request.urlopen(url).read()\n",
    "e = ET.XML(xml_page)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 114,
   "metadata": {},
   "outputs": [],
   "source": [
    "def etree_to_dict(t):\n",
    "    d = {t.tag: {} if t.attrib else None}\n",
    "    children = list(t)\n",
    "    if children:\n",
    "        dd = defaultdict(list)\n",
    "        for dc in map(etree_to_dict, children):\n",
    "            for k, v in dc.items():\n",
    "                dd[k].append(v)\n",
    "        d = {t.tag: {k:v[0] if len(v) == 1 else v for k, v in dd.items()}}\n",
    "    if t.attrib:\n",
    "        d[t.tag].update(('@' + k, v) for k, v in t.attrib.items())\n",
    "    if t.text:\n",
    "        text = t.text.strip()\n",
    "        if children or t.attrib:\n",
    "            if text:\n",
    "              d[t.tag]['#text'] = text\n",
    "        else:\n",
    "            d[t.tag] = text\n",
    "    return d"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 115,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'CATALOG': {'CD': [{'ARTIST': 'Bob Dylan',\n",
      "                     'COMPANY': 'Columbia',\n",
      "                     'COUNTRY': 'USA',\n",
      "                     'PRICE': '10.90',\n",
      "                     'TITLE': 'Empire Burlesque',\n",
      "                     'YEAR': '1985'},\n",
      "                    {'ARTIST': 'Bonnie Tyler',\n",
      "                     'COMPANY': 'CBS Records',\n",
      "                     'COUNTRY': 'UK',\n",
      "                     'PRICE': '9.90',\n",
      "                     'TITLE': 'Hide your heart',\n",
      "                     'YEAR': '1988'},\n",
      "                    {'ARTIST': 'Dolly Parton',\n",
      "                     'COMPANY': 'RCA',\n",
      "                     'COUNTRY': 'USA',\n",
      "                     'PRICE': '9.90',\n",
      "                     'TITLE': 'Greatest Hits',\n",
      "                     'YEAR': '1982'},\n",
      "                    {'ARTIST': 'Gary Moore',\n",
      "                     'COMPANY': 'Virgin records',\n",
      "                     'COUNTRY': 'UK',\n",
      "                     'PRICE': '10.20',\n",
      "                     'TITLE': 'Still got the blues',\n",
      "                     'YEAR': '1990'},\n",
      "                    {'ARTIST': 'Eros Ramazzotti',\n",
      "                     'COMPANY': 'BMG',\n",
      "                     'COUNTRY': 'EU',\n",
      "                     'PRICE': '9.90',\n",
      "                     'TITLE': 'Eros',\n",
      "                     'YEAR': '1997'},\n",
      "                    {'ARTIST': 'Bee Gees',\n",
      "                     'COMPANY': 'Polydor',\n",
      "                     'COUNTRY': 'UK',\n",
      "                     'PRICE': '10.90',\n",
      "                     'TITLE': 'One night only',\n",
      "                     'YEAR': '1998'},\n",
      "                    {'ARTIST': 'Dr.Hook',\n",
      "                     'COMPANY': 'CBS',\n",
      "                     'COUNTRY': 'UK',\n",
      "                     'PRICE': '8.10',\n",
      "                     'TITLE': 'Sylvias Mother',\n",
      "                     'YEAR': '1973'},\n",
      "                    {'ARTIST': 'Rod Stewart',\n",
      "                     'COMPANY': 'Pickwick',\n",
      "                     'COUNTRY': 'UK',\n",
      "                     'PRICE': '8.50',\n",
      "                     'TITLE': 'Maggie May',\n",
      "                     'YEAR': '1990'},\n",
      "                    {'ARTIST': 'Andrea Bocelli',\n",
      "                     'COMPANY': 'Polydor',\n",
      "                     'COUNTRY': 'EU',\n",
      "                     'PRICE': '10.80',\n",
      "                     'TITLE': 'Romanza',\n",
      "                     'YEAR': '1996'},\n",
      "                    {'ARTIST': 'Percy Sledge',\n",
      "                     'COMPANY': 'Atlantic',\n",
      "                     'COUNTRY': 'USA',\n",
      "                     'PRICE': '8.70',\n",
      "                     'TITLE': 'When a man loves a woman',\n",
      "                     'YEAR': '1987'},\n",
      "                    {'ARTIST': 'Savage Rose',\n",
      "                     'COMPANY': 'Mega',\n",
      "                     'COUNTRY': 'EU',\n",
      "                     'PRICE': '10.90',\n",
      "                     'TITLE': 'Black angel',\n",
      "                     'YEAR': '1995'},\n",
      "                    {'ARTIST': 'Many',\n",
      "                     'COMPANY': 'Grammy',\n",
      "                     'COUNTRY': 'USA',\n",
      "                     'PRICE': '10.20',\n",
      "                     'TITLE': '1999 Grammy Nominees',\n",
      "                     'YEAR': '1999'},\n",
      "                    {'ARTIST': 'Kenny Rogers',\n",
      "                     'COMPANY': 'Mucik Master',\n",
      "                     'COUNTRY': 'UK',\n",
      "                     'PRICE': '8.70',\n",
      "                     'TITLE': 'For the good times',\n",
      "                     'YEAR': '1995'},\n",
      "                    {'ARTIST': 'Will Smith',\n",
      "                     'COMPANY': 'Columbia',\n",
      "                     'COUNTRY': 'USA',\n",
      "                     'PRICE': '9.90',\n",
      "                     'TITLE': 'Big Willie style',\n",
      "                     'YEAR': '1997'},\n",
      "                    {'ARTIST': 'Van Morrison',\n",
      "                     'COMPANY': 'Polydor',\n",
      "                     'COUNTRY': 'UK',\n",
      "                     'PRICE': '8.20',\n",
      "                     'TITLE': 'Tupelo Honey',\n",
      "                     'YEAR': '1971'},\n",
      "                    {'ARTIST': 'Jorn Hoel',\n",
      "                     'COMPANY': 'WEA',\n",
      "                     'COUNTRY': 'Norway',\n",
      "                     'PRICE': '7.90',\n",
      "                     'TITLE': 'Soulsville',\n",
      "                     'YEAR': '1996'},\n",
      "                    {'ARTIST': 'Cat Stevens',\n",
      "                     'COMPANY': 'Island',\n",
      "                     'COUNTRY': 'UK',\n",
      "                     'PRICE': '8.90',\n",
      "                     'TITLE': 'The very best of',\n",
      "                     'YEAR': '1990'},\n",
      "                    {'ARTIST': 'Sam Brown',\n",
      "                     'COMPANY': 'A and M',\n",
      "                     'COUNTRY': 'UK',\n",
      "                     'PRICE': '8.90',\n",
      "                     'TITLE': 'Stop',\n",
      "                     'YEAR': '1988'},\n",
      "                    {'ARTIST': \"T'Pau\",\n",
      "                     'COMPANY': 'Siren',\n",
      "                     'COUNTRY': 'UK',\n",
      "                     'PRICE': '7.90',\n",
      "                     'TITLE': 'Bridge of Spies',\n",
      "                     'YEAR': '1987'},\n",
      "                    {'ARTIST': 'Tina Turner',\n",
      "                     'COMPANY': 'Capitol',\n",
      "                     'COUNTRY': 'UK',\n",
      "                     'PRICE': '8.90',\n",
      "                     'TITLE': 'Private Dancer',\n",
      "                     'YEAR': '1983'},\n",
      "                    {'ARTIST': 'Kim Larsen',\n",
      "                     'COMPANY': 'Medley',\n",
      "                     'COUNTRY': 'EU',\n",
      "                     'PRICE': '7.80',\n",
      "                     'TITLE': 'Midt om natten',\n",
      "                     'YEAR': '1983'},\n",
      "                    {'ARTIST': 'Luciano Pavarotti',\n",
      "                     'COMPANY': 'DECCA',\n",
      "                     'COUNTRY': 'UK',\n",
      "                     'PRICE': '9.90',\n",
      "                     'TITLE': 'Pavarotti Gala Concert',\n",
      "                     'YEAR': '1991'},\n",
      "                    {'ARTIST': 'Otis Redding',\n",
      "                     'COMPANY': 'Stax Records',\n",
      "                     'COUNTRY': 'USA',\n",
      "                     'PRICE': '7.90',\n",
      "                     'TITLE': 'The dock of the bay',\n",
      "                     'YEAR': '1968'},\n",
      "                    {'ARTIST': 'Simply Red',\n",
      "                     'COMPANY': 'Elektra',\n",
      "                     'COUNTRY': 'EU',\n",
      "                     'PRICE': '7.20',\n",
      "                     'TITLE': 'Picture book',\n",
      "                     'YEAR': '1985'},\n",
      "                    {'ARTIST': 'The Communards',\n",
      "                     'COMPANY': 'London',\n",
      "                     'COUNTRY': 'UK',\n",
      "                     'PRICE': '7.80',\n",
      "                     'TITLE': 'Red',\n",
      "                     'YEAR': '1987'},\n",
      "                    {'ARTIST': 'Joe Cocker',\n",
      "                     'COMPANY': 'EMI',\n",
      "                     'COUNTRY': 'USA',\n",
      "                     'PRICE': '8.20',\n",
      "                     'TITLE': 'Unchain my heart',\n",
      "                     'YEAR': '1987'}]}}\n"
     ]
    }
   ],
   "source": [
    "pprint(etree_to_dict(e))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 116,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Element node: CATALOG\n",
      "text node:\n",
      "  \n",
      "Element node: CD\n",
      "text node:\n",
      "    \n",
      "Element node: TITLE\n",
      "text node:Empire Burlesque\n",
      "text node:\n",
      "    \n",
      "Element node: ARTIST\n",
      "text node:Bob Dylan\n",
      "text node:\n",
      "    \n",
      "Element node: COUNTRY\n",
      "text node:USA\n",
      "text node:\n",
      "    \n",
      "Element node: COMPANY\n",
      "text node:Columbia\n",
      "text node:\n",
      "    \n",
      "Element node: PRICE\n",
      "text node:10.90\n",
      "text node:\n",
      "    \n",
      "Element node: YEAR\n",
      "text node:1985\n",
      "text node:\n",
      "  \n",
      "text node:\n",
      "  \n",
      "Element node: CD\n",
      "text node:\n",
      "    \n",
      "Element node: TITLE\n",
      "text node:Hide your heart\n",
      "text node:\n",
      "    \n",
      "Element node: ARTIST\n",
      "text node:Bonnie Tyler\n",
      "text node:\n",
      "    \n",
      "Element node: COUNTRY\n",
      "text node:UK\n",
      "text node:\n",
      "    \n",
      "Element node: COMPANY\n",
      "text node:CBS Records\n",
      "text node:\n",
      "    \n",
      "Element node: PRICE\n",
      "text node:9.90\n",
      "text node:\n",
      "    \n",
      "Element node: YEAR\n",
      "text node:1988\n",
      "text node:\n",
      "  \n",
      "text node:\n",
      "  \n",
      "Element node: CD\n",
      "text node:\n",
      "    \n",
      "Element node: TITLE\n",
      "text node:Greatest Hits\n",
      "text node:\n",
      "    \n",
      "Element node: ARTIST\n",
      "text node:Dolly Parton\n",
      "text node:\n",
      "    \n",
      "Element node: COUNTRY\n",
      "text node:USA\n",
      "text node:\n",
      "    \n",
      "Element node: COMPANY\n",
      "text node:RCA\n",
      "text node:\n",
      "    \n",
      "Element node: PRICE\n",
      "text node:9.90\n",
      "text node:\n",
      "    \n",
      "Element node: YEAR\n",
      "text node:1982\n",
      "text node:\n",
      "  \n",
      "text node:\n",
      "  \n",
      "Element node: CD\n",
      "text node:\n",
      "    \n",
      "Element node: TITLE\n",
      "text node:Still got the blues\n",
      "text node:\n",
      "    \n",
      "Element node: ARTIST\n",
      "text node:Gary Moore\n",
      "text node:\n",
      "    \n",
      "Element node: COUNTRY\n",
      "text node:UK\n",
      "text node:\n",
      "    \n",
      "Element node: COMPANY\n",
      "text node:Virgin records\n",
      "text node:\n",
      "    \n",
      "Element node: PRICE\n",
      "text node:10.20\n",
      "text node:\n",
      "    \n",
      "Element node: YEAR\n",
      "text node:1990\n",
      "text node:\n",
      "  \n",
      "text node:\n",
      "  \n",
      "Element node: CD\n",
      "text node:\n",
      "    \n",
      "Element node: TITLE\n",
      "text node:Eros\n",
      "text node:\n",
      "    \n",
      "Element node: ARTIST\n",
      "text node:Eros Ramazzotti\n",
      "text node:\n",
      "    \n",
      "Element node: COUNTRY\n",
      "text node:EU\n",
      "text node:\n",
      "    \n",
      "Element node: COMPANY\n",
      "text node:BMG\n",
      "text node:\n",
      "    \n",
      "Element node: PRICE\n",
      "text node:9.90\n",
      "text node:\n",
      "    \n",
      "Element node: YEAR\n",
      "text node:1997\n",
      "text node:\n",
      "  \n",
      "text node:\n",
      "  \n",
      "Element node: CD\n",
      "text node:\n",
      "    \n",
      "Element node: TITLE\n",
      "text node:One night only\n",
      "text node:\n",
      "    \n",
      "Element node: ARTIST\n",
      "text node:Bee Gees\n",
      "text node:\n",
      "    \n",
      "Element node: COUNTRY\n",
      "text node:UK\n",
      "text node:\n",
      "    \n",
      "Element node: COMPANY\n",
      "text node:Polydor\n",
      "text node:\n",
      "    \n",
      "Element node: PRICE\n",
      "text node:10.90\n",
      "text node:\n",
      "    \n",
      "Element node: YEAR\n",
      "text node:1998\n",
      "text node:\n",
      "  \n",
      "text node:\n",
      "  \n",
      "Element node: CD\n",
      "text node:\n",
      "    \n",
      "Element node: TITLE\n",
      "text node:Sylvias Mother\n",
      "text node:\n",
      "    \n",
      "Element node: ARTIST\n",
      "text node:Dr.Hook\n",
      "text node:\n",
      "    \n",
      "Element node: COUNTRY\n",
      "text node:UK\n",
      "text node:\n",
      "    \n",
      "Element node: COMPANY\n",
      "text node:CBS\n",
      "text node:\n",
      "    \n",
      "Element node: PRICE\n",
      "text node:8.10\n",
      "text node:\n",
      "    \n",
      "Element node: YEAR\n",
      "text node:1973\n",
      "text node:\n",
      "  \n",
      "text node:\n",
      "  \n",
      "Element node: CD\n",
      "text node:\n",
      "    \n",
      "Element node: TITLE\n",
      "text node:Maggie May\n",
      "text node:\n",
      "    \n",
      "Element node: ARTIST\n",
      "text node:Rod Stewart\n",
      "text node:\n",
      "    \n",
      "Element node: COUNTRY\n",
      "text node:UK\n",
      "text node:\n",
      "    \n",
      "Element node: COMPANY\n",
      "text node:Pickwick\n",
      "text node:\n",
      "    \n",
      "Element node: PRICE\n",
      "text node:8.50\n",
      "text node:\n",
      "    \n",
      "Element node: YEAR\n",
      "text node:1990\n",
      "text node:\n",
      "  \n",
      "text node:\n",
      "  \n",
      "Element node: CD\n",
      "text node:\n",
      "    \n",
      "Element node: TITLE\n",
      "text node:Romanza\n",
      "text node:\n",
      "    \n",
      "Element node: ARTIST\n",
      "text node:Andrea Bocelli\n",
      "text node:\n",
      "    \n",
      "Element node: COUNTRY\n",
      "text node:EU\n",
      "text node:\n",
      "    \n",
      "Element node: COMPANY\n",
      "text node:Polydor\n",
      "text node:\n",
      "    \n",
      "Element node: PRICE\n",
      "text node:10.80\n",
      "text node:\n",
      "    \n",
      "Element node: YEAR\n",
      "text node:1996\n",
      "text node:\n",
      "  \n",
      "text node:\n",
      "  \n",
      "Element node: CD\n",
      "text node:\n",
      "    \n",
      "Element node: TITLE\n",
      "text node:When a man loves a woman\n",
      "text node:\n",
      "    \n",
      "Element node: ARTIST\n",
      "text node:Percy Sledge\n",
      "text node:\n",
      "    \n",
      "Element node: COUNTRY\n",
      "text node:USA\n",
      "text node:\n",
      "    \n",
      "Element node: COMPANY\n",
      "text node:Atlantic\n",
      "text node:\n",
      "    \n",
      "Element node: PRICE\n",
      "text node:8.70\n",
      "text node:\n",
      "    \n",
      "Element node: YEAR\n",
      "text node:1987\n",
      "text node:\n",
      "  \n",
      "text node:\n",
      "  \n",
      "Element node: CD\n",
      "text node:\n",
      "    \n",
      "Element node: TITLE\n",
      "text node:Black angel\n",
      "text node:\n",
      "    \n",
      "Element node: ARTIST\n",
      "text node:Savage Rose\n",
      "text node:\n",
      "    \n",
      "Element node: COUNTRY\n",
      "text node:EU\n",
      "text node:\n",
      "    \n",
      "Element node: COMPANY\n",
      "text node:Mega\n",
      "text node:\n",
      "    \n",
      "Element node: PRICE\n",
      "text node:10.90\n",
      "text node:\n",
      "    \n",
      "Element node: YEAR\n",
      "text node:1995\n",
      "text node:\n",
      "  \n",
      "text node:\n",
      "  \n",
      "Element node: CD\n",
      "text node:\n",
      "    \n",
      "Element node: TITLE\n",
      "text node:1999 Grammy Nominees\n",
      "text node:\n",
      "    \n",
      "Element node: ARTIST\n",
      "text node:Many\n",
      "text node:\n",
      "    \n",
      "Element node: COUNTRY\n",
      "text node:USA\n",
      "text node:\n",
      "    \n",
      "Element node: COMPANY\n",
      "text node:Grammy\n",
      "text node:\n",
      "    \n",
      "Element node: PRICE\n",
      "text node:10.20\n",
      "text node:\n",
      "    \n",
      "Element node: YEAR\n",
      "text node:1999\n",
      "text node:\n",
      "  \n",
      "text node:\n",
      "  \n",
      "Element node: CD\n",
      "text node:\n",
      "    \n",
      "Element node: TITLE\n",
      "text node:For the good times\n",
      "text node:\n",
      "    \n",
      "Element node: ARTIST\n",
      "text node:Kenny Rogers\n",
      "text node:\n",
      "    \n",
      "Element node: COUNTRY\n",
      "text node:UK\n",
      "text node:\n",
      "    \n",
      "Element node: COMPANY\n",
      "text node:Mucik Master\n",
      "text node:\n",
      "    \n",
      "Element node: PRICE\n",
      "text node:8.70\n",
      "text node:\n",
      "    \n",
      "Element node: YEAR\n",
      "text node:1995\n",
      "text node:\n",
      "  \n",
      "text node:\n",
      "  \n",
      "Element node: CD\n",
      "text node:\n",
      "    \n",
      "Element node: TITLE\n",
      "text node:Big Willie style\n",
      "text node:\n",
      "    \n",
      "Element node: ARTIST\n",
      "text node:Will Smith\n",
      "text node:\n",
      "    \n",
      "Element node: COUNTRY\n",
      "text node:USA\n",
      "text node:\n",
      "    \n",
      "Element node: COMPANY\n",
      "text node:Columbia\n",
      "text node:\n",
      "    \n",
      "Element node: PRICE\n",
      "text node:9.90\n",
      "text node:\n",
      "    \n",
      "Element node: YEAR\n",
      "text node:1997\n",
      "text node:\n",
      "  \n",
      "text node:\n",
      "  \n",
      "Element node: CD\n",
      "text node:\n",
      "    \n",
      "Element node: TITLE\n",
      "text node:Tupelo Honey\n",
      "text node:\n",
      "    \n",
      "Element node: ARTIST\n",
      "text node:Van Morrison\n",
      "text node:\n",
      "    \n",
      "Element node: COUNTRY\n",
      "text node:UK\n",
      "text node:\n",
      "    \n",
      "Element node: COMPANY\n",
      "text node:Polydor\n",
      "text node:\n",
      "    \n",
      "Element node: PRICE\n",
      "text node:8.20\n",
      "text node:\n",
      "    \n",
      "Element node: YEAR\n",
      "text node:1971\n",
      "text node:\n",
      "  \n",
      "text node:\n",
      "  \n",
      "Element node: CD\n",
      "text node:\n",
      "    \n",
      "Element node: TITLE\n",
      "text node:Soulsville\n",
      "text node:\n",
      "    \n",
      "Element node: ARTIST\n",
      "text node:Jorn Hoel\n",
      "text node:\n",
      "    \n",
      "Element node: COUNTRY\n",
      "text node:Norway\n",
      "text node:\n",
      "    \n",
      "Element node: COMPANY\n",
      "text node:WEA\n",
      "text node:\n",
      "    \n",
      "Element node: PRICE\n",
      "text node:7.90\n",
      "text node:\n",
      "    \n",
      "Element node: YEAR\n",
      "text node:1996\n",
      "text node:\n",
      "  \n",
      "text node:\n",
      "  \n",
      "Element node: CD\n",
      "text node:\n",
      "    \n",
      "Element node: TITLE\n",
      "text node:The very best of\n",
      "text node:\n",
      "    \n",
      "Element node: ARTIST\n",
      "text node:Cat Stevens\n",
      "text node:\n",
      "    \n",
      "Element node: COUNTRY\n",
      "text node:UK\n",
      "text node:\n",
      "    \n",
      "Element node: COMPANY\n",
      "text node:Island\n",
      "text node:\n",
      "    \n",
      "Element node: PRICE\n",
      "text node:8.90\n",
      "text node:\n",
      "    \n",
      "Element node: YEAR\n",
      "text node:1990\n",
      "text node:\n",
      "  \n",
      "text node:\n",
      "  \n",
      "Element node: CD\n",
      "text node:\n",
      "    \n",
      "Element node: TITLE\n",
      "text node:Stop\n",
      "text node:\n",
      "    \n",
      "Element node: ARTIST\n",
      "text node:Sam Brown\n",
      "text node:\n",
      "    \n",
      "Element node: COUNTRY\n",
      "text node:UK\n",
      "text node:\n",
      "    \n",
      "Element node: COMPANY\n",
      "text node:A and M\n",
      "text node:\n",
      "    \n",
      "Element node: PRICE\n",
      "text node:8.90\n",
      "text node:\n",
      "    \n",
      "Element node: YEAR\n",
      "text node:1988\n",
      "text node:\n",
      "  \n",
      "text node:\n",
      "  \n",
      "Element node: CD\n",
      "text node:\n",
      "    \n",
      "Element node: TITLE\n",
      "text node:Bridge of Spies\n",
      "text node:\n",
      "    \n",
      "Element node: ARTIST\n",
      "text node:T'Pau\n",
      "text node:\n",
      "    \n",
      "Element node: COUNTRY\n",
      "text node:UK\n",
      "text node:\n",
      "    \n",
      "Element node: COMPANY\n",
      "text node:Siren\n",
      "text node:\n",
      "    \n",
      "Element node: PRICE\n",
      "text node:7.90\n",
      "text node:\n",
      "    \n",
      "Element node: YEAR\n",
      "text node:1987\n",
      "text node:\n",
      "  \n",
      "text node:\n",
      "  \n",
      "Element node: CD\n",
      "text node:\n",
      "    \n",
      "Element node: TITLE\n",
      "text node:Private Dancer\n",
      "text node:\n",
      "    \n",
      "Element node: ARTIST\n",
      "text node:Tina Turner\n",
      "text node:\n",
      "    \n",
      "Element node: COUNTRY\n",
      "text node:UK\n",
      "text node:\n",
      "    \n",
      "Element node: COMPANY\n",
      "text node:Capitol\n",
      "text node:\n",
      "    \n",
      "Element node: PRICE\n",
      "text node:8.90\n",
      "text node:\n",
      "    \n",
      "Element node: YEAR\n",
      "text node:1983\n",
      "text node:\n",
      "  \n",
      "text node:\n",
      "  \n",
      "Element node: CD\n",
      "text node:\n",
      "    \n",
      "Element node: TITLE\n",
      "text node:Midt om natten\n",
      "text node:\n",
      "    \n",
      "Element node: ARTIST\n",
      "text node:Kim Larsen\n",
      "text node:\n",
      "    \n",
      "Element node: COUNTRY\n",
      "text node:EU\n",
      "text node:\n",
      "    \n",
      "Element node: COMPANY\n",
      "text node:Medley\n",
      "text node:\n",
      "    \n",
      "Element node: PRICE\n",
      "text node:7.80\n",
      "text node:\n",
      "    \n",
      "Element node: YEAR\n",
      "text node:1983\n",
      "text node:\n",
      "  \n",
      "text node:\n",
      "  \n",
      "Element node: CD\n",
      "text node:\n",
      "    \n",
      "Element node: TITLE\n",
      "text node:Pavarotti Gala Concert\n",
      "text node:\n",
      "    \n",
      "Element node: ARTIST\n",
      "text node:Luciano Pavarotti\n",
      "text node:\n",
      "    \n",
      "Element node: COUNTRY\n",
      "text node:UK\n",
      "text node:\n",
      "    \n",
      "Element node: COMPANY\n",
      "text node:DECCA\n",
      "text node:\n",
      "    \n",
      "Element node: PRICE\n",
      "text node:9.90\n",
      "text node:\n",
      "    \n",
      "Element node: YEAR\n",
      "text node:1991\n",
      "text node:\n",
      "  \n",
      "text node:\n",
      "  \n",
      "Element node: CD\n",
      "text node:\n",
      "    \n",
      "Element node: TITLE\n",
      "text node:The dock of the bay\n",
      "text node:\n",
      "    \n",
      "Element node: ARTIST\n",
      "text node:Otis Redding\n",
      "text node:\n",
      "    \n",
      "Element node: COUNTRY\n",
      "text node:USA\n",
      "text node:\n",
      "    \n",
      "Element node: COMPANY\n",
      "text node:Stax Records\n",
      "text node:\n",
      "    \n",
      "Element node: PRICE\n",
      "text node:7.90\n",
      "text node:\n",
      "    \n",
      "Element node: YEAR\n",
      "text node:1968\n",
      "text node:\n",
      "  \n",
      "text node:\n",
      "  \n",
      "Element node: CD\n",
      "text node:\n",
      "    \n",
      "Element node: TITLE\n",
      "text node:Picture book\n",
      "text node:\n",
      "    \n",
      "Element node: ARTIST\n",
      "text node:Simply Red\n",
      "text node:\n",
      "    \n",
      "Element node: COUNTRY\n",
      "text node:EU\n",
      "text node:\n",
      "    \n",
      "Element node: COMPANY\n",
      "text node:Elektra\n",
      "text node:\n",
      "    \n",
      "Element node: PRICE\n",
      "text node:7.20\n",
      "text node:\n",
      "    \n",
      "Element node: YEAR\n",
      "text node:1985\n",
      "text node:\n",
      "  \n",
      "text node:\n",
      "  \n",
      "Element node: CD\n",
      "text node:\n",
      "    \n",
      "Element node: TITLE\n",
      "text node:Red\n",
      "text node:\n",
      "    \n",
      "Element node: ARTIST\n",
      "text node:The Communards\n",
      "text node:\n",
      "    \n",
      "Element node: COUNTRY\n",
      "text node:UK\n",
      "text node:\n",
      "    \n",
      "Element node: COMPANY\n",
      "text node:London\n",
      "text node:\n",
      "    \n",
      "Element node: PRICE\n",
      "text node:7.80\n",
      "text node:\n",
      "    \n",
      "Element node: YEAR\n",
      "text node:1987\n",
      "text node:\n",
      "  \n",
      "text node:\n",
      "  \n",
      "Element node: CD\n",
      "text node:\n",
      "    \n",
      "Element node: TITLE\n",
      "text node:Unchain my heart\n",
      "text node:\n",
      "    \n",
      "Element node: ARTIST\n",
      "text node:Joe Cocker\n",
      "text node:\n",
      "    \n",
      "Element node: COUNTRY\n",
      "text node:USA\n",
      "text node:\n",
      "    \n",
      "Element node: COMPANY\n",
      "text node:EMI\n",
      "text node:\n",
      "    \n",
      "Element node: PRICE\n",
      "text node:8.20\n",
      "text node:\n",
      "    \n",
      "Element node: YEAR\n",
      "text node:1987\n",
      "text node:\n",
      "  \n",
      "text node:\n",
      "\n"
     ]
    }
   ],
   "source": [
    "raw_page = urllib.request.urlopen(url)\n",
    "doc = minidom.parse(raw_page)\n",
    "\n",
    "def findTextnodes(nodeList):\n",
    "    for subnode in nodeList:\n",
    "        if subnode.nodeType == subnode.ELEMENT_NODE:\n",
    "            print(\"Element node: \" + subnode.tagName)\n",
    "            findTextnodes(subnode.childNodes)\n",
    "        elif subnode.nodeType == subnode.TEXT_NODE:\n",
    "            print(\"text node:\" + subnode.data)\n",
    "            \n",
    "findTextnodes(doc.childNodes)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 117,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Empire Burlesque\n",
      "Hide your heart\n",
      "Greatest Hits\n",
      "Still got the blues\n",
      "Eros\n",
      "One night only\n",
      "Sylvias Mother\n",
      "Maggie May\n",
      "Romanza\n",
      "When a man loves a woman\n",
      "Black angel\n",
      "1999 Grammy Nominees\n",
      "For the good times\n",
      "Big Willie style\n",
      "Tupelo Honey\n",
      "Soulsville\n",
      "The very best of\n",
      "Stop\n",
      "Bridge of Spies\n",
      "Private Dancer\n",
      "Midt om natten\n",
      "Pavarotti Gala Concert\n",
      "The dock of the bay\n",
      "Picture book\n",
      "Red\n",
      "Unchain my heart\n"
     ]
    }
   ],
   "source": [
    "root = ET.fromstring(xml_page)\n",
    "cdtags = root.xpath('//CD/TITLE')\n",
    "for cd in cdtags:\n",
    "    print(cd.text)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 118,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Bob Dylan\tUSA\tEmpire Burlesque\n",
      "Bonnie Tyler\tUK\tHide your heart\n",
      "Dolly Parton\tUSA\tGreatest Hits\n",
      "Gary Moore\tUK\tStill got the blues\n",
      "Eros Ramazzotti\tEU\tEros\n",
      "Bee Gees\tUK\tOne night only\n",
      "Dr.Hook\tUK\tSylvias Mother\n",
      "Rod Stewart\tUK\tMaggie May\n",
      "Andrea Bocelli\tEU\tRomanza\n",
      "Percy Sledge\tUSA\tWhen a man loves a woman\n",
      "Savage Rose\tEU\tBlack angel\n",
      "Many\tUSA\t1999 Grammy Nominees\n",
      "Kenny Rogers\tUK\tFor the good times\n",
      "Will Smith\tUSA\tBig Willie style\n",
      "Van Morrison\tUK\tTupelo Honey\n",
      "Jorn Hoel\tNorway\tSoulsville\n",
      "Cat Stevens\tUK\tThe very best of\n",
      "Sam Brown\tUK\tStop\n",
      "T'Pau\tUK\tBridge of Spies\n",
      "Tina Turner\tUK\tPrivate Dancer\n",
      "Kim Larsen\tEU\tMidt om natten\n",
      "Luciano Pavarotti\tUK\tPavarotti Gala Concert\n",
      "Otis Redding\tUSA\tThe dock of the bay\n",
      "Simply Red\tEU\tPicture book\n",
      "The Communards\tUK\tRed\n",
      "Joe Cocker\tUSA\tUnchain my heart\n"
     ]
    }
   ],
   "source": [
    "for cd in root.findall('CD'):\n",
    "    title = cd.find('TITLE').text\n",
    "    artist = cd.find('ARTIST').text\n",
    "    country = cd.find('COUNTRY').text\n",
    "    print('{}\\t{}\\t{}'.format(artist, country, title))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 119,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[{'TITLE': 'Empire Burlesque',\n",
       "  'ARTIST': 'Bob Dylan',\n",
       "  'COUNTRY': 'USA',\n",
       "  'PRICE': '10.90',\n",
       "  'YEAR': '1985'},\n",
       " {'TITLE': 'Hide your heart',\n",
       "  'ARTIST': 'Bonnie Tyler',\n",
       "  'COUNTRY': 'UK',\n",
       "  'PRICE': '9.90',\n",
       "  'YEAR': '1988'},\n",
       " {'TITLE': 'Greatest Hits',\n",
       "  'ARTIST': 'Dolly Parton',\n",
       "  'COUNTRY': 'USA',\n",
       "  'PRICE': '9.90',\n",
       "  'YEAR': '1982'}]"
      ]
     },
     "execution_count": 119,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "cds = []\n",
    "for cd in root.findall('./CD'):\n",
    "        data = {}\n",
    "        data[\"TITLE\"] = cd.find('./TITLE').text\n",
    "        data[\"ARTIST\"] = cd.find('./ARTIST').text\n",
    "        data[\"COUNTRY\"] = cd.find('./COUNTRY').text\n",
    "        data[\"PRICE\"] = cd.find('./PRICE').text\n",
    "        data[\"YEAR\"] = cd.find('./YEAR').text\n",
    "        cds.append(data)\n",
    "cds[0:3]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Accessing Feeds RSS"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "http://www.feedparser.org/  \n",
    "http://docs.python.org/library/re.html  \n",
    "http://www.pythonware.com/library/pil/handbook/index.htm  "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 71,
   "metadata": {},
   "outputs": [],
   "source": [
    "import feedparser"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 77,
   "metadata": {},
   "outputs": [],
   "source": [
    "d = feedparser.parse('https://oglobo.globo.com/rss.xml')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 78,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "OGlobo\n",
      "https://oglobo.globo.com/\n",
      "10\n"
     ]
    }
   ],
   "source": [
    "print(d['feed']['title'])\n",
    "print(d['feed']['link'])\n",
    "#print(d.feed.subtitle)\n",
    "print(len(d['entries']))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 79,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Operação da PF desbarata quadrilha nos Correios que desviou R$ 1 milhão em celulares e eletrônicos\n",
      "https://oglobo.globo.com/rio/operacao-da-pf-desbarata-quadrilha-nos-correios-que-desviou-1-milhao-em-celulares-eletronicos-24762248\n"
     ]
    }
   ],
   "source": [
    "print(d['entries'][0]['title']) \n",
    "print(d.entries[0]['link'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 80,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'Date': 'Tue, 24 Nov 2020 11:20:09 GMT', 'Server': 'Apache', 'Referrer-Policy': 'no-referrer-when-downgrade', 'Feature-Policy': \"geolocation 'none'; midi 'none'; microphone 'none'; camera 'none'; speaker 'self'; fullscreen 'self';\", 'X-Frame-Options': 'SAMEORIGIN', 'Expires': '1606216819326', 'Content-Type': 'application/rss+xml;charset=UTF-8', 'Access-Control-Allow-Origin': '*', 'Vary': 'Accept-Encoding', 'Content-Encoding': 'gzip', 'Content-Length': '2862', 'cache-control': 'public, max-age=602', 'Age': '113', 'grace': 'none', 'X-Cache': 'HIT', 'X-Cache-Hits': '3', 'Accept-Ranges': 'bytes', 'Strict-Transport-Security': 'max-age=15768000', 'Content-Security-Policy': \"default-src 'self' 'unsafe-inline' https: data: blob:; script-src 'self' 'unsafe-eval' 'unsafe-inline' https: blob:; img-src 'self' data: https:;\", 'X-XSS-Protection': '1; mode=block', 'Connection': 'close'}\n"
     ]
    }
   ],
   "source": [
    "print(d.headers)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 81,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Operação da PF desbarata quadrilha nos Correios que desviou R$ 1 milhão em celulares e eletrônicos: https://oglobo.globo.com/rio/operacao-da-pf-desbarata-quadrilha-nos-correios-que-desviou-1-milhao-em-celulares-eletronicos-24762248\n",
      "\n",
      "Datafolha SP: Boulos reduz diferença para Covas na reta final da campanha: https://oglobo.globo.com/brasil/datafolha-sp-boulos-reduz-diferenca-para-covas-na-reta-final-da-campanha-24762197\n",
      "\n",
      "Datafolha SP: Boulos alcança 40% e reduz vantagem contra Covas, que tem 48%: https://oglobo.globo.com/brasil/eleicoes-2020/datafolha-sp-boulos-alcanca-40-reduz-vantagem-contra-covas-que-tem-48-1-24762198\n",
      "\n",
      "Polícia faz operação contra milícia que explora construções irregulares na Gardênia Azul: https://oglobo.globo.com/rio/policia-faz-operacao-contra-milicia-que-explora-construcoes-irregulares-na-gardenia-azul-24762219\n",
      "\n",
      "Carrefour é cobrado a adotar mudanças por organizações que combatem racismo: https://epoca.globo.com/brasil/carrefour-cobrado-adotar-mudancas-por-organizacoes-que-combatem-racismo-24761762\n",
      "\n",
      "Brinquedos ficam até 50% mais baratos na Black Friday; veja como aproveitar as ofertas: https://oglobo.globo.com/economia/como-economizar/brinquedos-ficam-ate-50-mais-baratos-na-black-friday-veja-como-aproveitar-as-ofertas-24762109\n",
      "\n",
      "Governo do DF alegou 'transparência' para dificultar divulgação de mortes por Covid: https://epoca.globo.com/guilherme-amado/governo-do-df-alegou-transparencia-para-dificultar-divulgacao-de-mortes-por-covid-24762120\n",
      "\n",
      "Após Bolsonaro e Mourão negarem racismo no Brasil, Secretaria de Igualdade Racial silencia: https://epoca.globo.com/guilherme-amado/apos-bolsonaro-mourao-negarem-racismo-no-brasil-secretaria-de-igualdade-racial-silencia-24762093\n",
      "\n",
      "Como Daniel Ortega construiu um império de mídia que enriquece sua família e aumenta seu poder na Nicarágua: https://oglobo.globo.com/mundo/como-daniel-ortega-construiu-um-imperio-de-midia-que-enriquece-sua-familia-aumenta-seu-poder-na-nicaragua-24761282\n",
      "\n",
      "Superstições e simpatias realmente funcionam nos esportes?: https://oglobo.globo.com/esportes/supersticoes-simpatias-realmente-funcionam-nos-esportes-24751938\n",
      "\n"
     ]
    }
   ],
   "source": [
    "for post in d.entries:\n",
    "    print(post.title + \": \" + post.link + \"\\n\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Dealing with Big Files"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 82,
   "metadata": {},
   "outputs": [],
   "source": [
    "import urllib\n",
    "from zipfile import ZipFile\n",
    "import csv"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Retrieving the file:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 83,
   "metadata": {},
   "outputs": [],
   "source": [
    "link = 'http://eforexcel.com/wp/wp-content/uploads/2017/07/1500000%20Sales%20Records.zip'\n",
    "filename = \"bigcsv.zip\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "('../SampleDBs/bigcsv.zip', <http.client.HTTPMessage at 0x7f2d64ef20b8>)"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "urllib.request.urlretrieve(link, os.path.join(datapath, filename))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Using Pandas with chunks:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 84,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "1500000 Sales Records.csv\n",
      "<ZipInfo filename='1500000 Sales Records.csv' compress_type=deflate external_attr=0x820 file_size=187182221 compress_size=54267220>\n",
      "\n",
      "Retrieving a new chunk\n",
      "                         Region           Country Item Type Sales Channel  \\\n",
      "0            Sub-Saharan Africa      South Africa    Fruits       Offline   \n",
      "1  Middle East and North Africa           Morocco   Clothes        Online   \n",
      "2         Australia and Oceania  Papua New Guinea      Meat       Offline   \n",
      "\n",
      "  Order Priority Order Date   Order ID   Ship Date  Units Sold  Unit Price  \\\n",
      "0              M  7/27/2012  443368995   7/28/2012        1593        9.33   \n",
      "1              M  9/14/2013  667593514  10/19/2013        4611      109.28   \n",
      "2              M  5/15/2015  940995585    6/4/2015         360      421.89   \n",
      "\n",
      "   Unit Cost  Total Revenue  Total Cost  Total Profit  \n",
      "0       6.92       14862.69    11023.56       3839.13  \n",
      "1      35.84      503890.08   165258.24     338631.84  \n",
      "2     364.69      151880.40   131288.40      20592.00  \n",
      "\n",
      "Retrieving a new chunk\n",
      "                       Region      Country      Item Type Sales Channel  \\\n",
      "100000     Sub-Saharan Africa   Madagascar         Cereal       Offline   \n",
      "100001  Australia and Oceania  New Zealand  Personal Care       Offline   \n",
      "100002     Sub-Saharan Africa       Malawi         Cereal       Offline   \n",
      "\n",
      "       Order Priority Order Date   Order ID  Ship Date  Units Sold  \\\n",
      "100000              L   1/9/2016  238872230  1/24/2016        6456   \n",
      "100001              C  6/14/2014  976817452  7/21/2014         846   \n",
      "100002              C  4/27/2017  371752727  6/10/2017        8748   \n",
      "\n",
      "        Unit Price  Unit Cost  Total Revenue  Total Cost  Total Profit  \n",
      "100000      205.70     117.11     1327999.20   756062.16     571937.04  \n",
      "100001       81.73      56.67       69143.58    47942.82      21200.76  \n",
      "100002      205.70     117.11     1799463.60  1024478.28     774985.32  \n",
      "\n",
      "Retrieving a new chunk\n",
      "                                   Region         Country        Item Type  \\\n",
      "200000                             Europe  Czech Republic  Office Supplies   \n",
      "200001  Central America and the Caribbean        Honduras           Snacks   \n",
      "200002       Middle East and North Africa    Saudi Arabia           Cereal   \n",
      "\n",
      "       Sales Channel Order Priority Order Date   Order ID   Ship Date  \\\n",
      "200000       Offline              M  8/30/2011  457898414   8/31/2011   \n",
      "200001        Online              L  12/6/2014  820111215  12/17/2014   \n",
      "200002       Offline              M  3/28/2014  134626567    4/2/2014   \n",
      "\n",
      "        Units Sold  Unit Price  Unit Cost  Total Revenue  Total Cost  \\\n",
      "200000        1171      651.21     524.96      762566.91   614728.16   \n",
      "200001        7988      152.58      97.44     1218809.04   778350.72   \n",
      "200002        7848      205.70     117.11     1614333.60   919079.28   \n",
      "\n",
      "        Total Profit  \n",
      "200000     147838.75  \n",
      "200001     440458.32  \n",
      "200002     695254.32  \n",
      "\n"
     ]
    }
   ],
   "source": [
    "chunksize = 10 ** 5\n",
    "num_chunks_to_read = 3\n",
    "\n",
    "with ZipFile(os.path.join(datapath, filename), 'r') as myzip:\n",
    "    unzipped = myzip.namelist()[0]\n",
    "    print(unzipped)\n",
    "    print(myzip.getinfo(unzipped))\n",
    "    print()\n",
    "    with myzip.open(unzipped) as myfile:\n",
    "        chunks = pd.read_csv((myfile), chunksize=chunksize) #, iterator=True)\n",
    "        for i in range(num_chunks_to_read):\n",
    "            print('Retrieving a new chunk')\n",
    "            df = chunks.get_chunk()\n",
    "            print(df.head(3))\n",
    "            print()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Unzipping the file and reading row by row:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "with ZipFile(os.path.join(datapath, filename), 'r') as myzip:\n",
    "    myzip.extractall(datapath)\n",
    "    myzip.close()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 85,
   "metadata": {},
   "outputs": [],
   "source": [
    "def rowgen(filename):\n",
    "    with open(filename, \"r\") as csvfile:\n",
    "        datareader = csv.reader(csvfile)\n",
    "        for row in datareader:\n",
    "            yield(row)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 86,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "['Region', 'Country', 'Item Type', 'Sales Channel', 'Order Priority', 'Order Date', 'Order ID', 'Ship Date', 'Units Sold', 'Unit Price', 'Unit Cost', 'Total Revenue', 'Total Cost', 'Total Profit']\n",
      "['Sub-Saharan Africa', 'South Africa', 'Fruits', 'Offline', 'M', '7/27/2012', '443368995', '7/28/2012', '1593', '9.33', '6.92', '14862.69', '11023.56', '3839.13']\n",
      "['Middle East and North Africa', 'Morocco', 'Clothes', 'Online', 'M', '9/14/2013', '667593514', '10/19/2013', '4611', '109.28', '35.84', '503890.08', '165258.24', '338631.84']\n",
      "['Australia and Oceania', 'Papua New Guinea', 'Meat', 'Offline', 'M', '5/15/2015', '940995585', '6/4/2015', '360', '421.89', '364.69', '151880.40', '131288.40', '20592.00']\n",
      "['Sub-Saharan Africa', 'Djibouti', 'Clothes', 'Offline', 'H', '5/17/2017', '880811536', '7/2/2017', '562', '109.28', '35.84', '61415.36', '20142.08', '41273.28']\n",
      "['Europe', 'Slovakia', 'Beverages', 'Offline', 'L', '10/26/2016', '174590194', '12/4/2016', '3973', '47.45', '31.79', '188518.85', '126301.67', '62217.18']\n",
      "['Asia', 'Sri Lanka', 'Fruits', 'Online', 'L', '11/7/2011', '830192887', '12/18/2011', '1379', '9.33', '6.92', '12866.07', '9542.68', '3323.39']\n",
      "['Sub-Saharan Africa', 'Seychelles ', 'Beverages', 'Online', 'M', '1/18/2013', '425793445', '2/16/2013', '597', '47.45', '31.79', '28327.65', '18978.63', '9349.02']\n",
      "['Sub-Saharan Africa', 'Tanzania', 'Beverages', 'Online', 'L', '11/30/2016', '659878194', '1/16/2017', '1476', '47.45', '31.79', '70036.20', '46922.04', '23114.16']\n",
      "['Sub-Saharan Africa', 'Ghana', 'Office Supplies', 'Online', 'L', '3/23/2017', '601245963', '4/15/2017', '896', '651.21', '524.96', '583484.16', '470364.16', '113120.00']\n",
      "['Sub-Saharan Africa', 'Tanzania', 'Cosmetics', 'Offline', 'L', '5/23/2016', '739008080', '5/24/2016', '7768', '437.20', '263.33', '3396169.60', '2045547.44', '1350622.16']\n"
     ]
    }
   ],
   "source": [
    "count = 0\n",
    "for row in rowgen(os.path.join(datapath, unzipped)):\n",
    "    print(row)\n",
    "    count +=1\n",
    "    if count > 10:\n",
    "        break"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [],
   "source": [
    "#erasing files\n",
    "#os.remove(os.path.join(datapath, unzipped))\n",
    "#os.remove(os.path.join(datapath, filename))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Using [HDF5](https://portal.hdfgroup.org/display/HDF5/HDF5)  \n",
    "### [H5py Tutorial](https://docs.h5py.org/en/stable/quick.html)\n",
    "\n",
    "HDF is a good complement to databases, it may make sense to run a query to produce a roughly memory-sized dataset and then cache it in HDF if the same data would be used more than once. If you have a dataset which is fixed, and usually processed as a whole, storing it as a collection of appropriately sized HDF files is not a bad option. If you have a dataset which is updated often, staging some of it as HDF files periodically might still be helpful.  \n",
    "\n",
    "To summarize, HDF is a good format for data which is read (or written) typically as a whole; it is the lingua franca or common/preferred interchange format for many applications due to wide support and compatibility, decent as an archival format, and very fast.  "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 87,
   "metadata": {},
   "outputs": [],
   "source": [
    "#!sudo pip install -q -U h5py  "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Creating Dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import h5py"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 88,
   "metadata": {},
   "outputs": [],
   "source": [
    "arr = np.random.randn(1000)\n",
    "\n",
    "filename = 'random.hdf5'\n",
    "#with h5py.File(os.path.join(datapath, filename), 'a') as f:   #if we'd like to append data\n",
    "with h5py.File(os.path.join(datapath, filename), 'w') as f:\n",
    "    dset = f.create_dataset(\"default\", data=arr)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Reading Dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 90,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "datasets in this hdf system: default\n"
     ]
    }
   ],
   "source": [
    "with h5py.File(os.path.join(datapath, filename), 'r') as f:\n",
    "    for key in f.keys():\n",
    "        print(\"datasets in this hdf system:\", key)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 92,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<class 'h5py._hl.dataset.Dataset'>\n",
      "-2.9085971441217113\n",
      "3.3224151423291004\n",
      "\n",
      "[ 0.09740022  0.24629433  1.2484955  -1.57179904  0.59453711 -0.20756992\n",
      " -0.49296165  0.78044995  0.63347146  1.60380351 -0.13916422  0.94675755\n",
      " -0.84641552  0.00631172  0.69590968]\n"
     ]
    }
   ],
   "source": [
    "with h5py.File(os.path.join(datapath, filename), 'r') as f:\n",
    "    data = f['default']\n",
    "    print(type(data))\n",
    "    print(min(data))\n",
    "    print(max(data))\n",
    "    print()\n",
    "    print(data[:15])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Recreating dataset file with new data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 94,
   "metadata": {},
   "outputs": [],
   "source": [
    "arr1 = np.random.randn(10000)\n",
    "arr2 = np.random.randn(10000)\n",
    "\n",
    "with h5py.File(os.path.join(datapath, filename), 'w') as f:\n",
    "    f.create_dataset('array_1', data=arr1)\n",
    "    f.create_dataset('array_2', data=arr2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 95,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<class 'h5py._hl.dataset.Dataset'>\n",
      "<class 'h5py._hl.dataset.Dataset'>\n",
      "[ 0.44429956  2.17950519  0.77676391 ... -1.80978667  0.05503899\n",
      " -0.76341002]\n"
     ]
    }
   ],
   "source": [
    "with h5py.File(os.path.join(datapath, filename), 'r') as f:\n",
    "    d1 = f['array_1']\n",
    "    d2 = f['array_2']\n",
    "    print(type(d1))\n",
    "    print(type(d2))\n",
    "    #data = d2[d1>0]  #error\n",
    "    data = d2[d1[:]>0]\n",
    "    print(data)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 96,
   "metadata": {},
   "outputs": [],
   "source": [
    "#erasing files\n",
    "\n",
    "os.remove(os.path.join(datapath, filename))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#https://www.datarevenue.com/en-blog/pandas-vs-dask-vs-vaex-vs-modin-vs-rapids-vs-ray\n",
    "\n",
    "\n",
    "## Using [Dask](https://docs.dask.org/en/latest/dataframe.html)  \n",
    "\n",
    "Dask DataFrame is used in situations where Pandas is commonly needed, usually when Pandas fails due to data size or computation speed:\n",
    "\n",
    "+ Manipulating large datasets, even when those datasets don’t fit in memory\n",
    "+ Accelerating long computations by using many cores\n",
    "+ Distributed computing on large datasets with standard Pandas operations like groupby, join, and time series computations"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 97,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Note: you may need to restart the kernel to use updated packages.\n"
     ]
    }
   ],
   "source": [
    "pip install -q -U dask "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 98,
   "metadata": {},
   "outputs": [],
   "source": [
    "import dask.dataframe as dd"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 100,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU times: user 10.5 ms, sys: 312 µs, total: 10.9 ms\n",
      "Wall time: 9.98 ms\n"
     ]
    }
   ],
   "source": [
    "filename = \"1500000 Sales Records.csv\"\n",
    "\n",
    "%time df = dd.read_csv(os.path.join(datapath, filename))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Comparing reading time with Pandas:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 101,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU times: user 1.92 s, sys: 114 ms, total: 2.04 s\n",
      "Wall time: 2.04 s\n"
     ]
    }
   ],
   "source": [
    "filename = \"1500000 Sales Records.csv\"\n",
    "\n",
    "%time df2 = pd.read_csv(os.path.join(datapath, filename))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 102,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Region</th>\n",
       "      <th>Country</th>\n",
       "      <th>Item Type</th>\n",
       "      <th>Sales Channel</th>\n",
       "      <th>Order Priority</th>\n",
       "      <th>Order Date</th>\n",
       "      <th>Order ID</th>\n",
       "      <th>Ship Date</th>\n",
       "      <th>Units Sold</th>\n",
       "      <th>Unit Price</th>\n",
       "      <th>Unit Cost</th>\n",
       "      <th>Total Revenue</th>\n",
       "      <th>Total Cost</th>\n",
       "      <th>Total Profit</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>Sub-Saharan Africa</td>\n",
       "      <td>South Africa</td>\n",
       "      <td>Fruits</td>\n",
       "      <td>Offline</td>\n",
       "      <td>M</td>\n",
       "      <td>7/27/2012</td>\n",
       "      <td>443368995</td>\n",
       "      <td>7/28/2012</td>\n",
       "      <td>1593</td>\n",
       "      <td>9.33</td>\n",
       "      <td>6.92</td>\n",
       "      <td>14862.69</td>\n",
       "      <td>11023.56</td>\n",
       "      <td>3839.13</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>Middle East and North Africa</td>\n",
       "      <td>Morocco</td>\n",
       "      <td>Clothes</td>\n",
       "      <td>Online</td>\n",
       "      <td>M</td>\n",
       "      <td>9/14/2013</td>\n",
       "      <td>667593514</td>\n",
       "      <td>10/19/2013</td>\n",
       "      <td>4611</td>\n",
       "      <td>109.28</td>\n",
       "      <td>35.84</td>\n",
       "      <td>503890.08</td>\n",
       "      <td>165258.24</td>\n",
       "      <td>338631.84</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>Australia and Oceania</td>\n",
       "      <td>Papua New Guinea</td>\n",
       "      <td>Meat</td>\n",
       "      <td>Offline</td>\n",
       "      <td>M</td>\n",
       "      <td>5/15/2015</td>\n",
       "      <td>940995585</td>\n",
       "      <td>6/4/2015</td>\n",
       "      <td>360</td>\n",
       "      <td>421.89</td>\n",
       "      <td>364.69</td>\n",
       "      <td>151880.40</td>\n",
       "      <td>131288.40</td>\n",
       "      <td>20592.00</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>Sub-Saharan Africa</td>\n",
       "      <td>Djibouti</td>\n",
       "      <td>Clothes</td>\n",
       "      <td>Offline</td>\n",
       "      <td>H</td>\n",
       "      <td>5/17/2017</td>\n",
       "      <td>880811536</td>\n",
       "      <td>7/2/2017</td>\n",
       "      <td>562</td>\n",
       "      <td>109.28</td>\n",
       "      <td>35.84</td>\n",
       "      <td>61415.36</td>\n",
       "      <td>20142.08</td>\n",
       "      <td>41273.28</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>Europe</td>\n",
       "      <td>Slovakia</td>\n",
       "      <td>Beverages</td>\n",
       "      <td>Offline</td>\n",
       "      <td>L</td>\n",
       "      <td>10/26/2016</td>\n",
       "      <td>174590194</td>\n",
       "      <td>12/4/2016</td>\n",
       "      <td>3973</td>\n",
       "      <td>47.45</td>\n",
       "      <td>31.79</td>\n",
       "      <td>188518.85</td>\n",
       "      <td>126301.67</td>\n",
       "      <td>62217.18</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                         Region           Country  Item Type Sales Channel  \\\n",
       "0            Sub-Saharan Africa      South Africa     Fruits       Offline   \n",
       "1  Middle East and North Africa           Morocco    Clothes        Online   \n",
       "2         Australia and Oceania  Papua New Guinea       Meat       Offline   \n",
       "3            Sub-Saharan Africa          Djibouti    Clothes       Offline   \n",
       "4                        Europe          Slovakia  Beverages       Offline   \n",
       "\n",
       "  Order Priority  Order Date   Order ID   Ship Date  Units Sold  Unit Price  \\\n",
       "0              M   7/27/2012  443368995   7/28/2012        1593        9.33   \n",
       "1              M   9/14/2013  667593514  10/19/2013        4611      109.28   \n",
       "2              M   5/15/2015  940995585    6/4/2015         360      421.89   \n",
       "3              H   5/17/2017  880811536    7/2/2017         562      109.28   \n",
       "4              L  10/26/2016  174590194   12/4/2016        3973       47.45   \n",
       "\n",
       "   Unit Cost  Total Revenue  Total Cost  Total Profit  \n",
       "0       6.92       14862.69    11023.56       3839.13  \n",
       "1      35.84      503890.08   165258.24     338631.84  \n",
       "2     364.69      151880.40   131288.40      20592.00  \n",
       "3      35.84       61415.36    20142.08      41273.28  \n",
       "4      31.79      188518.85   126301.67      62217.18  "
      ]
     },
     "execution_count": 102,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 103,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<class 'dask.dataframe.core.DataFrame'>\n",
      "Columns: 14 entries, Region to Total Profit\n",
      "dtypes: object(7), float64(5), int64(2)"
     ]
    }
   ],
   "source": [
    "df.info()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Delayed execution"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 110,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU times: user 1.96 ms, sys: 301 µs, total: 2.26 ms\n",
      "Wall time: 1.48 ms\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "7498957001"
      ]
     },
     "execution_count": 110,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "%time total = df2['Units Sold'].sum()\n",
    "total"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 111,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU times: user 766 µs, sys: 0 ns, total: 766 µs\n",
      "Wall time: 773 µs\n"
     ]
    }
   ],
   "source": [
    "%time total = df['Units Sold'].sum()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 112,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "7498957001"
      ]
     },
     "execution_count": 112,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "total.compute()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 113,
   "metadata": {},
   "outputs": [],
   "source": [
    "df2 = df[df['Region'] == 'Sub-Saharan Africa'].compute()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 114,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<class 'pandas.core.frame.DataFrame'>\n",
      "Int64Index: 389607 entries, 0 to 474265\n",
      "Data columns (total 14 columns):\n",
      " #   Column          Non-Null Count   Dtype  \n",
      "---  ------          --------------   -----  \n",
      " 0   Region          389607 non-null  object \n",
      " 1   Country         389607 non-null  object \n",
      " 2   Item Type       389607 non-null  object \n",
      " 3   Sales Channel   389607 non-null  object \n",
      " 4   Order Priority  389607 non-null  object \n",
      " 5   Order Date      389607 non-null  object \n",
      " 6   Order ID        389607 non-null  int64  \n",
      " 7   Ship Date       389607 non-null  object \n",
      " 8   Units Sold      389607 non-null  int64  \n",
      " 9   Unit Price      389607 non-null  float64\n",
      " 10  Unit Cost       389607 non-null  float64\n",
      " 11  Total Revenue   389607 non-null  float64\n",
      " 12  Total Cost      389607 non-null  float64\n",
      " 13  Total Profit    389607 non-null  float64\n",
      "dtypes: float64(5), int64(2), object(7)\n",
      "memory usage: 44.6+ MB\n"
     ]
    }
   ],
   "source": [
    "df2.info()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 115,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "pandas.core.frame.DataFrame"
      ]
     },
     "execution_count": 115,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "type(df2)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Using [VAEX](https://vaex.readthedocs.io/en/latest/)  \n",
    "#### [Github](https://github.com/vaexio/vaex)\n",
    "\n",
    "Vaex is a python library for lazy Out-of-Core DataFrames (similar to Pandas), to visualize and explore big tabular datasets. It can calculate statistics such as mean, sum, count, standard deviation etc, on an N-dimensional grid up to a billion (109) objects/rows per second. Visualization is done using histograms, density plots and 3d volume rendering, allowing interactive exploration of big data. Vaex uses memory mapping, a zero memory copy policy, and lazy computations for best performance (no memory wasted).  "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 52,
   "metadata": {},
   "outputs": [],
   "source": [
    "! pip install -q -U vaex"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 116,
   "metadata": {},
   "outputs": [],
   "source": [
    "import vaex"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 117,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<style>.vaex-description pre {\n",
       "          max-width : 450px;\n",
       "          white-space : nowrap;\n",
       "          overflow : hidden;\n",
       "          text-overflow: ellipsis;\n",
       "        }\n",
       "\n",
       "        .vex-description pre:hover {\n",
       "          max-width : initial;\n",
       "          white-space: pre;\n",
       "        }</style>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<div><h2>helmi-dezeeuw-2000-FeH-v2-10percent</h2> <b>rows</b>: 330,000</div><div><b>path</b>: <i>/home/rsouza/.vaex/data/helmi-dezeeuw-2000-FeH-v2-10percent.hdf5</i></div><div><b>Description</b>: file exported by vaex, by user maartenbreddels, on date 2020-02-24 16:31:31.900958, from source /has/no/path/arraysprevious description:\n",
       "file exported by vaex, by user maartenbreddels, on date 2020-02-24 16:31:07.946417, from source /has/no/path/arrays</div><h2>Columns:</h2><table class='table-striped'><thead><tr><th>column</th><th>type</th><th>unit</th><th>description</th><th>expression</th></tr></thead><tr><td>id</td><td>uint8</td><td></td><td ><pre></pre></td><td></td></tr><tr><td>x</td><td>float32</td><td></td><td ><pre></pre></td><td></td></tr><tr><td>y</td><td>float32</td><td></td><td ><pre></pre></td><td></td></tr><tr><td>z</td><td>float32</td><td></td><td ><pre></pre></td><td></td></tr><tr><td>vx</td><td>float32</td><td></td><td ><pre></pre></td><td></td></tr><tr><td>vy</td><td>float32</td><td></td><td ><pre></pre></td><td></td></tr><tr><td>vz</td><td>float32</td><td></td><td ><pre></pre></td><td></td></tr><tr><td>E</td><td>float32</td><td></td><td ><pre></pre></td><td></td></tr><tr><td>L</td><td>float32</td><td></td><td ><pre></pre></td><td></td></tr><tr><td>Lz</td><td>float32</td><td></td><td ><pre></pre></td><td></td></tr><tr><td>FeH</td><td>float32</td><td></td><td ><pre></pre></td><td></td></tr></table><h2>Data:</h2><table>\n",
       "<thead>\n",
       "<tr><th>#                                  </th><th>id  </th><th>x                   </th><th>y                   </th><th>z                   </th><th>vx                 </th><th>vy                 </th><th>vz                 </th><th>E             </th><th>L                 </th><th>Lz                 </th><th>FeH                </th></tr>\n",
       "</thead>\n",
       "<tbody>\n",
       "<tr><td><i style='opacity: 0.6'>0</i>      </td><td>0   </td><td>1.2318683862686157  </td><td>-0.39692866802215576</td><td>-0.598057746887207  </td><td>301.1552734375     </td><td>174.05947875976562 </td><td>27.42754554748535  </td><td>-149431.40625 </td><td>407.38897705078125</td><td>333.9555358886719  </td><td>-1.0053852796554565</td></tr>\n",
       "<tr><td><i style='opacity: 0.6'>1</i>      </td><td>23  </td><td>-0.16370061039924622</td><td>3.654221296310425   </td><td>-0.25490644574165344</td><td>-195.00022888183594</td><td>170.47216796875    </td><td>142.5302276611328  </td><td>-124247.953125</td><td>890.2411499023438 </td><td>684.6676025390625  </td><td>-1.7086670398712158</td></tr>\n",
       "<tr><td><i style='opacity: 0.6'>2</i>      </td><td>32  </td><td>-2.120255947113037  </td><td>3.326052665710449   </td><td>1.7078403234481812  </td><td>-48.63423156738281 </td><td>171.6472930908203  </td><td>-2.079437255859375 </td><td>-138500.546875</td><td>372.2410888671875 </td><td>-202.17617797851562</td><td>-1.8336141109466553</td></tr>\n",
       "<tr><td><i style='opacity: 0.6'>3</i>      </td><td>8   </td><td>4.7155890464782715  </td><td>4.5852508544921875  </td><td>2.2515437602996826  </td><td>-232.42083740234375</td><td>-294.850830078125  </td><td>62.85865020751953  </td><td>-60037.0390625</td><td>1297.63037109375  </td><td>-324.6875          </td><td>-1.4786882400512695</td></tr>\n",
       "<tr><td><i style='opacity: 0.6'>4</i>      </td><td>16  </td><td>7.21718692779541    </td><td>11.99471664428711   </td><td>-1.064562201499939  </td><td>-1.6891745328903198</td><td>181.329345703125   </td><td>-11.333610534667969</td><td>-83206.84375  </td><td>1332.7989501953125</td><td>1328.948974609375  </td><td>-1.8570483922958374</td></tr>\n",
       "<tr><td>...                                </td><td>... </td><td>...                 </td><td>...                 </td><td>...                 </td><td>...                </td><td>...                </td><td>...                </td><td>...           </td><td>...               </td><td>...                </td><td>...                </td></tr>\n",
       "<tr><td><i style='opacity: 0.6'>329,995</i></td><td>21  </td><td>1.9938701391220093  </td><td>0.789276123046875   </td><td>0.22205990552902222 </td><td>-216.92990112304688</td><td>16.124420166015625 </td><td>-211.244384765625  </td><td>-146457.4375  </td><td>457.72247314453125</td><td>203.36758422851562 </td><td>-1.7451677322387695</td></tr>\n",
       "<tr><td><i style='opacity: 0.6'>329,996</i></td><td>25  </td><td>3.7180912494659424  </td><td>0.721337616443634   </td><td>1.6415337324142456  </td><td>-185.92160034179688</td><td>-117.25082397460938</td><td>-105.4986572265625 </td><td>-126627.109375</td><td>335.0025634765625 </td><td>-301.8370056152344 </td><td>-0.9822322130203247</td></tr>\n",
       "<tr><td><i style='opacity: 0.6'>329,997</i></td><td>14  </td><td>0.3688507676124573  </td><td>13.029608726501465  </td><td>-3.633934736251831  </td><td>-53.677146911621094</td><td>-145.15771484375   </td><td>76.70909881591797  </td><td>-84912.2578125</td><td>817.1375732421875 </td><td>645.8507080078125  </td><td>-1.7645612955093384</td></tr>\n",
       "<tr><td><i style='opacity: 0.6'>329,998</i></td><td>18  </td><td>-0.11259264498949051</td><td>1.4529125690460205  </td><td>2.168952703475952   </td><td>179.30865478515625 </td><td>205.79710388183594 </td><td>-68.75872802734375 </td><td>-133498.46875 </td><td>724.000244140625  </td><td>-283.6910400390625 </td><td>-1.8808952569961548</td></tr>\n",
       "<tr><td><i style='opacity: 0.6'>329,999</i></td><td>4   </td><td>20.796220779418945  </td><td>-3.331387758255005  </td><td>12.18841552734375   </td><td>42.69000244140625  </td><td>69.20479583740234  </td><td>29.54275131225586  </td><td>-65519.328125 </td><td>1843.07470703125  </td><td>1581.4151611328125 </td><td>-1.1231083869934082</td></tr>\n",
       "</tbody>\n",
       "</table>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "df = vaex.example()\n",
    "df.info()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 118,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<table>\n",
       "<thead>\n",
       "<tr><th>#                                  </th><th>id  </th><th>x                   </th><th>y                   </th><th>z                   </th><th>vx                 </th><th>vy                 </th><th>vz                 </th><th>E             </th><th>L                 </th><th>Lz                 </th><th>FeH                </th></tr>\n",
       "</thead>\n",
       "<tbody>\n",
       "<tr><td><i style='opacity: 0.6'>0</i>      </td><td>0   </td><td>1.2318683862686157  </td><td>-0.39692866802215576</td><td>-0.598057746887207  </td><td>301.1552734375     </td><td>174.05947875976562 </td><td>27.42754554748535  </td><td>-149431.40625 </td><td>407.38897705078125</td><td>333.9555358886719  </td><td>-1.0053852796554565</td></tr>\n",
       "<tr><td><i style='opacity: 0.6'>1</i>      </td><td>23  </td><td>-0.16370061039924622</td><td>3.654221296310425   </td><td>-0.25490644574165344</td><td>-195.00022888183594</td><td>170.47216796875    </td><td>142.5302276611328  </td><td>-124247.953125</td><td>890.2411499023438 </td><td>684.6676025390625  </td><td>-1.7086670398712158</td></tr>\n",
       "<tr><td><i style='opacity: 0.6'>2</i>      </td><td>32  </td><td>-2.120255947113037  </td><td>3.326052665710449   </td><td>1.7078403234481812  </td><td>-48.63423156738281 </td><td>171.6472930908203  </td><td>-2.079437255859375 </td><td>-138500.546875</td><td>372.2410888671875 </td><td>-202.17617797851562</td><td>-1.8336141109466553</td></tr>\n",
       "<tr><td><i style='opacity: 0.6'>3</i>      </td><td>8   </td><td>4.7155890464782715  </td><td>4.5852508544921875  </td><td>2.2515437602996826  </td><td>-232.42083740234375</td><td>-294.850830078125  </td><td>62.85865020751953  </td><td>-60037.0390625</td><td>1297.63037109375  </td><td>-324.6875          </td><td>-1.4786882400512695</td></tr>\n",
       "<tr><td><i style='opacity: 0.6'>4</i>      </td><td>16  </td><td>7.21718692779541    </td><td>11.99471664428711   </td><td>-1.064562201499939  </td><td>-1.6891745328903198</td><td>181.329345703125   </td><td>-11.333610534667969</td><td>-83206.84375  </td><td>1332.7989501953125</td><td>1328.948974609375  </td><td>-1.8570483922958374</td></tr>\n",
       "<tr><td>...                                </td><td>... </td><td>...                 </td><td>...                 </td><td>...                 </td><td>...                </td><td>...                </td><td>...                </td><td>...           </td><td>...               </td><td>...                </td><td>...                </td></tr>\n",
       "<tr><td><i style='opacity: 0.6'>329,995</i></td><td>21  </td><td>1.9938701391220093  </td><td>0.789276123046875   </td><td>0.22205990552902222 </td><td>-216.92990112304688</td><td>16.124420166015625 </td><td>-211.244384765625  </td><td>-146457.4375  </td><td>457.72247314453125</td><td>203.36758422851562 </td><td>-1.7451677322387695</td></tr>\n",
       "<tr><td><i style='opacity: 0.6'>329,996</i></td><td>25  </td><td>3.7180912494659424  </td><td>0.721337616443634   </td><td>1.6415337324142456  </td><td>-185.92160034179688</td><td>-117.25082397460938</td><td>-105.4986572265625 </td><td>-126627.109375</td><td>335.0025634765625 </td><td>-301.8370056152344 </td><td>-0.9822322130203247</td></tr>\n",
       "<tr><td><i style='opacity: 0.6'>329,997</i></td><td>14  </td><td>0.3688507676124573  </td><td>13.029608726501465  </td><td>-3.633934736251831  </td><td>-53.677146911621094</td><td>-145.15771484375   </td><td>76.70909881591797  </td><td>-84912.2578125</td><td>817.1375732421875 </td><td>645.8507080078125  </td><td>-1.7645612955093384</td></tr>\n",
       "<tr><td><i style='opacity: 0.6'>329,998</i></td><td>18  </td><td>-0.11259264498949051</td><td>1.4529125690460205  </td><td>2.168952703475952   </td><td>179.30865478515625 </td><td>205.79710388183594 </td><td>-68.75872802734375 </td><td>-133498.46875 </td><td>724.000244140625  </td><td>-283.6910400390625 </td><td>-1.8808952569961548</td></tr>\n",
       "<tr><td><i style='opacity: 0.6'>329,999</i></td><td>4   </td><td>20.796220779418945  </td><td>-3.331387758255005  </td><td>12.18841552734375   </td><td>42.69000244140625  </td><td>69.20479583740234  </td><td>29.54275131225586  </td><td>-65519.328125 </td><td>1843.07470703125  </td><td>1581.4151611328125 </td><td>-1.1231083869934082</td></tr>\n",
       "</tbody>\n",
       "</table>"
      ],
      "text/plain": [
       "#        id    x                     y                     z                     vx                   vy                   vz                   E               L                   Lz                   FeH\n",
       "0        0     1.2318683862686157    -0.39692866802215576  -0.598057746887207    301.1552734375       174.05947875976562   27.42754554748535    -149431.40625   407.38897705078125  333.9555358886719    -1.0053852796554565\n",
       "1        23    -0.16370061039924622  3.654221296310425     -0.25490644574165344  -195.00022888183594  170.47216796875      142.5302276611328    -124247.953125  890.2411499023438   684.6676025390625    -1.7086670398712158\n",
       "2        32    -2.120255947113037    3.326052665710449     1.7078403234481812    -48.63423156738281   171.6472930908203    -2.079437255859375   -138500.546875  372.2410888671875   -202.17617797851562  -1.8336141109466553\n",
       "3        8     4.7155890464782715    4.5852508544921875    2.2515437602996826    -232.42083740234375  -294.850830078125    62.85865020751953    -60037.0390625  1297.63037109375    -324.6875            -1.4786882400512695\n",
       "4        16    7.21718692779541      11.99471664428711     -1.064562201499939    -1.6891745328903198  181.329345703125     -11.333610534667969  -83206.84375    1332.7989501953125  1328.948974609375    -1.8570483922958374\n",
       "...      ...   ...                   ...                   ...                   ...                  ...                  ...                  ...             ...                 ...                  ...\n",
       "329,995  21    1.9938701391220093    0.789276123046875     0.22205990552902222   -216.92990112304688  16.124420166015625   -211.244384765625    -146457.4375    457.72247314453125  203.36758422851562   -1.7451677322387695\n",
       "329,996  25    3.7180912494659424    0.721337616443634     1.6415337324142456    -185.92160034179688  -117.25082397460938  -105.4986572265625   -126627.109375  335.0025634765625   -301.8370056152344   -0.9822322130203247\n",
       "329,997  14    0.3688507676124573    13.029608726501465    -3.633934736251831    -53.677146911621094  -145.15771484375     76.70909881591797    -84912.2578125  817.1375732421875   645.8507080078125    -1.7645612955093384\n",
       "329,998  18    -0.11259264498949051  1.4529125690460205    2.168952703475952     179.30865478515625   205.79710388183594   -68.75872802734375   -133498.46875   724.000244140625    -283.6910400390625   -1.8808952569961548\n",
       "329,999  4     20.796220779418945    -3.331387758255005    12.18841552734375     42.69000244140625    69.20479583740234    29.54275131225586    -65519.328125   1843.07470703125    1581.4151611328125   -1.1231083869934082"
      ]
     },
     "execution_count": 118,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "The above preview shows that this dataset contains >300,000 rows, and columns named x ,y, z (positions), vx, vy, vz (velocities), E (energy), L (angular momentum), and an id (subgroup of samples). When we print out a columns, we can see that it is not a Numpy array, but an Expression."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 119,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Expression = x\n",
       "Length: 330,000 dtype: float32 (column)\n",
       "---------------------------------------\n",
       "     0    1.23187\n",
       "     1  -0.163701\n",
       "     2   -2.12026\n",
       "     3    4.71559\n",
       "     4    7.21719\n",
       "       ...       \n",
       "329995    1.99387\n",
       "329996    3.71809\n",
       "329997   0.368851\n",
       "329998  -0.112593\n",
       "329999    20.7962"
      ]
     },
     "execution_count": 119,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.x  # df.col.x or df['x'] are equivalent, but df.x may be preferred because it is more tab completion friendly or programming friendly respectively"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "One can use the .values method to get an in-memory representation of an expression. The same method can be applied to a DataFrame as well."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 120,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([ 1.2318684 , -0.16370061, -2.120256  , ...,  0.36885077,\n",
       "       -0.11259264, 20.79622   ], dtype=float32)"
      ]
     },
     "execution_count": 120,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.x.values"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Most Numpy functions (ufuncs) can be performed on expressions, and will not result in a direct result, but in a new expression."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 122,
   "metadata": {},
   "outputs": [],
   "source": [
    "calc = np.sqrt(df.x**2 + df.y**2 + df.z**2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 123,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Expression = sqrt((((x ** 2) + (y ** 2)) + (z ** 2)))\n",
       "Length: 330,000 dtype: float32 (expression)\n",
       "-------------------------------------------\n",
       "     0  1.42574\n",
       "     1  3.66676\n",
       "     2  4.29824\n",
       "     3  6.95203\n",
       "     4   14.039\n",
       "      ...      \n",
       "329995  2.15587\n",
       "329996  4.12785\n",
       "329997  13.5319\n",
       "329998  2.61304\n",
       "329999  24.3339"
      ]
     },
     "execution_count": 123,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "calc"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 124,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([ 1.4257367,  3.6667573,  4.298236 , ..., 13.531897 ,  2.6130419,\n",
       "       24.333895 ], dtype=float32)"
      ]
     },
     "execution_count": 124,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "calc.values"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "We can try data manipulation just as usual with dataframes"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 125,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<table>\n",
       "<thead>\n",
       "<tr><th>#                            </th><th style=\"text-align: right;\">         x</th><th style=\"text-align: right;\">       y</th></tr>\n",
       "</thead>\n",
       "<tbody>\n",
       "<tr><td><i style='opacity: 0.6'>0</i></td><td style=\"text-align: right;\"> -0.163701</td><td style=\"text-align: right;\"> 3.65422</td></tr>\n",
       "<tr><td><i style='opacity: 0.6'>1</i></td><td style=\"text-align: right;\"> -2.12026 </td><td style=\"text-align: right;\"> 3.32605</td></tr>\n",
       "<tr><td><i style='opacity: 0.6'>2</i></td><td style=\"text-align: right;\"> -7.78437 </td><td style=\"text-align: right;\"> 5.98977</td></tr>\n",
       "<tr><td><i style='opacity: 0.6'>3</i></td><td style=\"text-align: right;\"> -3.55719 </td><td style=\"text-align: right;\"> 5.41363</td></tr>\n",
       "<tr><td><i style='opacity: 0.6'>4</i></td><td style=\"text-align: right;\">-20.8139  </td><td style=\"text-align: right;\">-3.29468</td></tr>\n",
       "</tbody>\n",
       "</table>"
      ],
      "text/plain": [
       "  #           x         y\n",
       "  0   -0.163701   3.65422\n",
       "  1   -2.12026    3.32605\n",
       "  2   -7.78437    5.98977\n",
       "  3   -3.55719    5.41363\n",
       "  4  -20.8139    -3.29468"
      ]
     },
     "execution_count": 125,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_negative = df[df.x < 0]\n",
    "df_negative[:5][['x', 'y']]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 126,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "vaex.dataframe.DataFrameArrays"
      ]
     },
     "execution_count": 126,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "type(df_negative)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "When dealing with huge datasets, say a billion rows (109), computations with the data can waste memory, up to 8 GB for a new column. Instead, vaex uses lazy computation, storing only a representation of the computation, and computations are done on the fly when needed. You can just use many of the numpy functions, as if it was a normal array."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 127,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Expression = (x + z)\n",
       "Length: 330,000 dtype: float32 (expression)\n",
       "-------------------------------------------\n",
       "     0   0.633811\n",
       "     1  -0.418607\n",
       "     2  -0.412416\n",
       "     3    6.96713\n",
       "     4    6.15262\n",
       "       ...       \n",
       "329995    2.21593\n",
       "329996    5.35962\n",
       "329997   -3.26508\n",
       "329998    2.05636\n",
       "329999    32.9846"
      ]
     },
     "execution_count": 127,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# creates an expression (nothing is computed)\n",
    "some_expression = df.x + df.z\n",
    "some_expression  # for convenience, we print out some values"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 128,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(array(-0.0632868), array(-0.03853567))"
      ]
     },
     "execution_count": 128,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df['r'] = some_expression  # add a (virtual) column that will be computed on the fly\n",
    "df.mean(df.x), df.mean(df.r)  # calculate statistics on normal and virtual columns"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 129,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([-9.43396488, -8.82963676, -8.14583574, -7.64420812, -6.93267297,\n",
       "       -6.40486861, -5.75335515, -5.06125446, -4.5048235 , -3.92031189,\n",
       "       -3.37898946, -2.80547337, -2.14810986, -1.63193784, -0.93497955,\n",
       "       -0.30048423,  0.32454422,  0.91274671,  1.45780962,  2.10523919,\n",
       "        2.76645729,  3.28189081,  4.0129405 ,  4.59913821,  5.15632131,\n",
       "        5.64241673,  6.31535617,  6.9517402 ,  7.52107727,  8.20589011,\n",
       "        9.08560375,  9.67737866])"
      ]
     },
     "execution_count": 129,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.mean(df.r, binby=df.x, shape=32, limits=[-10, 10]) # create statistics on a regular grid (1d)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 130,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[22, 31, 44, ..., 53, 48, 45],\n",
       "       [35, 30, 43, ..., 55, 46, 58],\n",
       "       [34, 36, 51, ..., 57, 53, 58],\n",
       "       ...,\n",
       "       [59, 59, 71, ..., 45, 45, 31],\n",
       "       [37, 72, 55, ..., 45, 37, 33],\n",
       "       [39, 52, 49, ..., 33, 33, 22]])"
      ]
     },
     "execution_count": 130,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.mean(df.r, binby=[df.x, df.y], shape=32, limits=[-10, 10]) # or 2d\n",
    "df.count(df.r, binby=[df.x, df.y], shape=32, limits=[-10, 10]) # or 2d counts/histogram"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 131,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAaYAAAEYCAYAAAAXsVIGAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjMuMiwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8vihELAAAACXBIWXMAAAsTAAALEwEAmpwYAAApSElEQVR4nO3df7RdZX3n8fcnhBD5kZIYfoQECtjQtcDVImSQtuqyA0JgtSLMyIBLQWUVXYWOrulMi6VTGB3WWCu6irpwYkmFVkAqRbIsFoFxdLpalIApJPyQBENJTIghYsCYhEu+88d+Nve5O/uce+7N+bHvPZ/XWnvdc5796zkn55xvnuf57mcrIjAzM2uKGYOugJmZWc6ByczMGsWByczMGsWByczMGsWByczMGmXmoCvQa5Kcdmhm/bI1Ig7blwMsXbo0tm7d2vH2Dz/88L0RsXRfztk00z4wmZn10bP7eoCtW7eycuXKjreXNH9fz9k0DkxmZo2zZ9AVGCgHJjOzRgkcmMzMrGEcmMzMrDECGBl0JQbKgcnMrHHcYmokSeuBl4BXgZGIWCJpHvBV4FhgPXBhRPx0UHU0M+s+jzE1/QLb346IkyNiSXp+FfBARCwGHkjPzcymmT0TWKafpgemqvOAm9Pjm4F3Da4qZma9ULaYHJiaKIBvSXpY0uWp7IiI2JQebwaOGEzVzMx6abgDU2PHmIC3RMRGSYcD90l6Ml8ZEdFquqEUyC6vW2dm1nzTM+B0qrEtpojYmP5uAe4CTgOel7QAIP3d0mLfZRGxJBubMjObItyV18jAJOkgSYeUj4GzgNXACuDStNmlwN2DqaGZWS8Nd2BqalfeEcBdkqCo460R8Y+SHgLukHQZxWSJFw6wjmZmPeB08UYGpoh4Bvj1mvIXgDP6XyMzs35yYDIzs0ZxYDIzs8bwXHkOTGZmjeIxJgcmM7PGcWAyM7NGGe7A1MjrmMzMhld3L7CVdLSkb0t6XNIaSR9J5fMk3Sfp6fR3biqXpBskrZX0qKRTsmNdmrZ/WtKlrc65rxyYzMwap6sX2I4AfxgRJwKnA1dIOpHWd2s4B1iclsuBG6EIZMA1wJspZuK5pgxm3ebAZGbWKN1tMUXEpoh4JD1+CXgCWEjruzWcB9wShQeBQ9MUcGcD90XEtnQfvPuApfv6aut4jMnMrHEmNMY0X9LK7PmyiFhWt6GkY4E3Ad+j9d0aFgLPZbttSGWtyrvOgcnMrHEmFJi2djJhtaSDgTuBj0bE9jTlG9D+bg2D4K48M7NG6f7s4pL2pwhKX4mIv0/Fre7WsBE4Ott9USprVd51DkxmZo3T1aw8ATcBT0TEZ7JVre7WsAK4JGXnnQ78LHX53QucJWluSno4K5V1nbvyzMwapetTEv0W8D7gMUmrUtmfAJ+k/m4N9wDnAmuBHcAHACJim6RPAA+l7T4eEdu6WdGSA5OZWeN07wLbiPgnQC1W73W3hogI4IoWx1oOLO9a5VpwYDIzaxTPlefAZGbWOA5MZo01g2H/itrwcYvJgckGolXAqZYP99fThtdwf/IdmKyv8sBTF5yG++toVhrub4IDk3VNGWiqASe/WG5P5e94LSd35dnwcVeeA5N1TR5I8oDSqrz6GOoDWr5v9Vz5Pg5iNn0M9yfZgcn2Sd3UITMpLg9sF1iorGt3zGpgq+5XPh7ur7JNH24xOTBZV9S1kNptm28/s/I4D2qtzpEfY7i/wjY9Dfen2oHJJqwuaLQaUxpvv7osvOp2eeBq1Z1XfT7cX2ub+ob7E+zAZB3pZLbfdmM8ZQsH9g4o4wWbdi2o8bL8xquXWfN0fa68KceBycZVDQrtAswMig9V3soptxup7F/dhkp5ud/Mmm3qMvZaZQQ6KNnU4jEmByZrKR/DqUv5rm47i+IDtZv6rr08wJRLOaZUXVduXw1qVNbl9cwTI+peh9nUMdyfWAcmq9Uq0aB8TmXdbIqANEIRoOpaMmUAKoNRq2NXA82erLza6mrVlVcXtNySsqljuD+dDkzW1njZdrPZO9DkLR4YbUHNyp7ngaYuuLQag6rr1ivLWyVgtApKDk7WTO7Kc2CyWq0udM1/8A9mbCtmZmWf3dnjanBrd21Tfoxyuzz45evzbsC6uldfg9nU4MBkVquuKw3gwGz9CKOtofLxzMr+ZXApg0g1saGuhVSXONHuwttqnfNtWm0/3F99ay63mPwfSXvNjMqSl8PY7jgYbcGUQSf/4c+X8hjl+NIsiuA2O62bWVmXJ1LUJUzkiRPlsevGv+peX6tt/EWwZqn7xLda2pO0XNIWSauzsq9KWpWW9eUt1yUdK+kX2bovZvucKukxSWsl3SCp1V1x95lbTPaaunGYPDDMZGzX3aysvHyct0ZmVR5Xj1cGq7zbr9xmvMy8ajCayAe5XbKE2eB1vcX0ZeDzwC2vnSHiP5WPJV0P/Czbfl1EnFxznBuB3wO+B9wDLAW+2c2KlvwfRdurhZSXVxMZRrLt8+613YwmOVQz7GalZTajLaGD09+y1TQ7LWWQmZUdA/YOPnUp4fm4VX5NVatt6167vxDWDN1rMUXEd4FtdetSq+dC4LZ2x5C0AJgTEQ9GRFAEuXd19lomzt9D2+vj3WqcplXLiGx93mVXDQBUysvjzGJsECuPWQax2YwNcHl3YnncEcZm+LUKttU6V1+7W07WDBMKTPMlrcyWyydworcCz0fE01nZcZJ+IOk7kt6ayhYCG7JtNqSynnBXnrWVjy+VY0lkZXmX3x5GW0B1QWE2sJPRQHNg+ruT0Q9ieYFueT3UCGPTy/N6lUGkeoHuRD7U+esza4YJT0m0NSKWTPJkFzO2tbQJOCYiXpB0KvB1SSdN8tiT5sA05Ma7picfC8rLYO/W0yyKIFLthisTI0YYTTHPL7Sdzd6tlgMZm/FXDWDl8Xez94e4rEer15cHOAcka57+ZOVJmglcAJz62pkjdgG70uOHJa0DTgA2Aouy3Relsp5wYBpy1bRsWjyvU3afVZMcYLQbDkaD1UyKgDMHOBQ4CDgE2C875k8pAtAu4CVgO/Ayeyc/VINjniG4m7Gqt9LIx8kcmKyZ+vLJPBN4MiJe66KTdBiwLSJelXQ8sBh4JiK2Sdou6XSK5IdLgM/1qmIOTAbsHYSqMyzkrY5qQkI10aCacFCOCx1O0TqaB8wFfilte/BsmJ36AHfvhk07igB1ELB/2mY7o1171Trm10fV1TEPRLtx951NBd37dEq6DXg7xVjUBuCaiLgJuIi9kx7eBnxc0iupEh+OiDJx4vcpMvxeR5GN15OMPHBgGnp5EkJdiykf26nbJj8OjHbN5d1zBwLzKT5sC4AjgQXzimD0ugPhgFmwfXux7Zw5cPgO2LwZfryz6PrbzGhX3vbKuatTGlVbR3uysuo8e9WvvltQ1gzd7cqLiItblL+/puxO4M4W268E3ti1irXhwDTkWmWktUvHzqcgKoNA2WopWyWzKYJRue9Mik7pX5kN8+fDvHnFMmcOHHww7Ekn37EDtm4tgtYra2HPSNH1t4ux3Ya7qQ9K1Wueqh/wavDJW10OVNYcw/3Jc2AacnU/vu1aSHU/7LPYO5CV2XVQBKgj0zJ/Phx+OBx1FJxyCuiYo4uoNG9esfGG53j+yRfZvLkITgc9AzO3F+f8BaMBp2yZleNX1RymfNyrVTZfeRyobwkO90+DDY6nJJpygUnSUuAvKcbM/yoiPjngKk0Lddcu1bU2qpOmli2oGRTdbuUYTh4QDqUYUzpgJhwwuwhMp54+E07/DTj0UDj8MDjwoGLj1as54qgtHPjgGmalLr5t24vuwIMZDSQ7s/OXwanu+qq8vvm4U6sMRLNmGO5P45QKTJL2A74AvIPiAq+HJK2IiMcHW7Opa7wLa/Mf8TJY5S2qsmUyk9FrlMoPVbnudRTZd4ceWixHHQUsWgRLlsAMwSFnAuuKjd8I/PApDvm1n3Pwi+uZMwfmHwq7X4SfZHXJW0n5tU/VwFR3J93ytQ73V9+abbg/nVMqMAGnAWsj4hkASbcD5wEOTPugXXde/jhPlKiW53YzOgM5FF1wM2fAL82Bgw6Eo4+dAcf8ctFXd8AS4CTgnGLj1y+H+dtgyxbmzIFZs4pxqA0vFll6ebZgPr/ebGBHVr+6BIe67rq61zncPwk2eO7Km2qBaSHwXPZ8A/Dm6kZpSo6JTMsx1OouQM1bJlXVQFWOMc3MyvIP1i7gF+mAIyPA7NcVT3bsKPr3eD2jYWYHzJwJM2bw4otFV96ePaOJFnkd8ySH/Nqluuuv6oJq9VYaw/1TYI2yZ7g/jVMtMHUkIpYBywAkxYCrM2XU/aCXMyvUzfwARVDaydjZHqrp2funvy+/XOQ4/HTDz5l7zPPw440w8i04bC1Fuwr4yaY0sLSNWbOKRtXTa4vWUD6jQ37tUh5gyr/ldtULcPPXMF5ih9lARDgwDboCE7QRODp73tNpMaaL8bqoqskKra5hKretpljn1zDB6HRC5fMX03LUHtiwoeiaO3DVkxwAcPhP4PgdRc44wI9/DI/+Kz99ZD2PP148hSIwwWgAfJnRefTytPF8bInKuny8qbp9u/fHrK8CeGVCc+VNO1MtMD0ELJZ0HEVAugh4z2Cr1HyT/cGtBp38hz3fJp+ANb+26eVUPg/4ObBxK7w6AvvNhJ074cSZT7LfkVth/Xo4MHXvbd7MT374Is88A//2b8V2z48UF9a+yN43J6zLHMxnd6jOAJHXu7qfA5M1g1tMUyowRcSIpCuBeynSxZdHxJoBV2vKqraQ2v1Y562O/Ae+vAdT2WVWdv3lrZENFN19O4FnX4Qfr4QnHofVq2H27K3MmLH1teO9+GIxLdH27bDx5WJqos3AVkZbR2THLoNQtYsv367VjBZ11zjlxzYbmBjuT+GUCkwAEXEPxd0TbR+1y8Qr1+ddemWwKW9tsbuyX36xap5KvpMisOykuMj2aeDFHfDTH8LcA1NyQ/okbk/NrC0Udzbbko5bXrdUrV+ZKl7trivr1Gr9cHeUWKN5jGnqBSbrnmqroVWLobrN7prtqjN2z2J0XGh2Wv8y8GOKa5peoWgJzdpRPE+X1/Iyo62rzYwdQyrnyyvrUW291dW92sLK08mr12iBu/SsIRyYbBjV/Sjn6pIEWo3r5HPYwdgffxjb3beD0euN5mXnSKkP7ARUc6xyXV5eTRfPEx5avZ5qskP1Oqfh/jmwRggcmAZdARuMuhZC9RohatbB3l12dWNR1QSEPRSBJW/1bGP0A7gt/S0TKPLutrLFlM/2UB6v3K5MJa9LG6+2jnK+jsmax115DkxDrlVrqW68qUxwyOXjS2XQqU5JVB6vvAA33zaf5y4/Zp6WXo5rjTD2thd5y6wanOq6I6lZ1y4Ymw1EAK8O96fQgcleM94Pc976INu2LsstDzZlSyfvemuV5ZenpO/Oyl5mbCsqD0xQf7+lct/8eqtWGYh5gDIbLLeYHJjsNe2SHuq2yz88ZdApy3bWPM7L6i5wrR47D1DVQLSb+v2r1ytVu/byc+Tl7cbbzPrOgclsrPFaDdXuOxibil0+r7ZgqjNF5Bfm5i2fmRQJEnnraydjW0XV65SqszpUs/HybduNNQ33z4E1QjD01zG558Jq1Y3D5M/LwJO3WqrXM5WJDNULYHcymp23g2LcqEwR381oyvjutOTr6jLp8oSLuvTxupZV3esd7p8Ca44oZjvudBmHpOWStkhanZVdK2mjpFVpOTdb9zFJayU9JensrHxpKlsr6aquv+yMW0wGtE8fz1sT1Wua8lZInj1X90NfnWy1boqgfOypTIzIb2fRqk7V66hadd1VW02tgpFTx21gun+B7ZeBzwO3VMo/GxGfzgsknUgx1dtJwFHA/ZJOSKv7di88ByYD2gej8nldd1e1Cw327kbLb4eRny/P0Mu78soWVjkulc8+Ue3KK4+1p/K40665dhcVmw1MFwNTRHxX0rEdbn4ecHtE7AJ+JGktxX3woI/3wnNXntVql5iQB6nqpKp1Yzv57A11iQn5+apBqpxBvOzSKx/n52uVpFE9R/VcnXbzmfVVeYFtpwvMl7QyWzq9F92Vkh5NXX1zU1ndPe8WtinvCbeYbNLquszqUrLzC2bzFlZ1JodSu9ZRfuFsPpP5eFMTjVdu1hwx0euYtkbEkgme5EbgE8XJ+ARwPfDBCR6jZxyYrFarC1Gr2+TrWnWLVcd/qvtUs/s66VarBsB2Y0LVoOfgZI3WhymJIuL58rGkLwHfSE/b3fOub/fCc1ee1Wr3tWh1/VC+foSxXW3l4+r1S61aOnmWXXXMqC5gtsocrNbPQcmaL4p08U6XSZC0IHt6PlBm7K0ALpJ0QLrv3WLg+2T3wpM0iyJBYsWkX+I43GKyCWk1njNSU95qu+q6urn2qtl/dft10hIarzVl1khdbDFJug14O8VY1AbgGuDtkk6maJ+tBz4EEBFrJN1BkdQwAlwREa+m4/TtXngOTDZhdeNBVdVgUDfDQpkW3m58qBqgqttUs/uq+9c9N2u0LqeLR8TFNcU3tdn+OuC6mvK+3QvPgckmrBqI2nXrlc/rWlp5SyifsLV63Go6eLmvpxOyactTEplNTF0XWh4k6rrOOp2rLj9GtYuvbvvh/vratOXAZDZ5dS2cuvGmfNyobiaJdt1vrVpbZtNSBLwy/lRD05kDk01IuySD6vrq3/IapvEuiG3V2hru/0Pa0Oj+lERTjgOTTch4X5e6LrZ8Lr18u/FSwDvhjDubloZ8dnEHJuu5Ti6QbbftRI9tNuW5xWTWf9V07+H+Gppl3JXnwGSDN9xfQbMaDkxmZtYYbjE5MJmZNY4Dk5mZNUYw0dteTDsOTGZmjeKuPAcmM7Om8XVMZmbWGE5+cGAyM2ucEc+VZ2ZmTeEWkwOTmVnjDHlWnu8eYGbWJGWLqdNlHJKWS9oiaXVW9heSnpT0qKS7JB2ayo+V9AtJq9LyxWyfUyU9JmmtpBskqZOXI+kgSftN5C1wYDIza5ouBibgy8DSStl9wBsj4teAHwIfy9ati4iT0/LhrPxG4PeAxWmpHhMASTMkvUfSP0jaAjwJbJL0eAqIvzJehR2YzMyaJvZ0vox3qIjvAtsqZd+KiDLD4kFgUbtjSFoAzImIByMigFuAd7XY/NvAGyiC3ZERcXREHA68JZ3rzyW9t935PMZkZtYkE09+mC9pZfZ8WUQsm8D+HwS+mj0/TtIPgO3An0bE/wMWAhuybTaksjpnRsQr1cKI2AbcCdwpaf92FXJgMjNrmokFpq0RsWQyp5F0NcU9PL+SijYBx0TEC5JOBb4u6aSJHDMiXpH0noi4VdJFEXF73TbtjuHAZGbWJH1KF5f0fuB3gDNS9xwRsQvYlR4/LGkdcAKwkbHdfYtSWSsLJV3IOF2ErXiMycysabqb/LAXSUuBPwLeGRE7svLDygw6ScdTJDk8ExGbgO2STk/ZeJcAd7c49jXAPIpW2DxJfzbR+jkwmZk1SdDtdPHbgH8BflXSBkmXAZ8HDgHuq6SFvw14VNIq4GvAh9PYEMDvA38FrAXWAd+srX7E/wBeAN4HvBARH5/oW9C4rjxJ11KkJP4kFf1JRNyT1n0MuAx4FfjPEXHvQCppZtYrEfBK96YkioiLa4pvarHtnRQJCnXrVgJv7PC0myLidkl15x5X4wJT8tmI+HReIOlE4CLgJOAo4H5JJ0TEq4OooJlZb8SUnl1c0sER8RWAiLitzTYvtzrGVOrKOw+4PSJ2RcSPKJqTpw24TmZm3dfjMaYeu1vS9ZLeJumgslDS8ZIuk3QvLS7OLTU1MF2ZpspYLmluKlsIPJdt0y6P3sxsauryGFO/RcQZwAPAh4A1kn4m6QXgb4EFwKUR8bV2xxhIV56k+4Eja1ZdTTHtxSco/nk+AVxPcQHYRI5/OXD5PlbTzGwApv7s4ikv4J7J7j+QwBQRZ3aynaQvAd9ITzcCR2erW+bRp6uel6VjxORramY2AFM8MAFIeiC1ntqW1WlcV16ak6l0PlDOiLsCuEjSAZKOo8iv/36/62dm1lMRxW0vOl0aRtJsSfMopkqaK2leWo6lw+GXJmblfUrSyRRdeesp+imJiDWS7gAep5hC4wpn5JnZtDS1W0wfAj5KkT39MFDeHmM7xfVT42pcYIqI97VZdx1wXR+rY2bWX2XywxQVEX8J/KWkP4iIz03mGOMGJkl/APxtRPx0MicwM7OJmNrXMZUi4nOSfhM4lizWRMQt4+3bSYvpCOAhSY8Ay4F7ywn/zMysB6Zwi6kk6W8o7su0imK2Hijag/semCLiTyX9d+As4APA59NYz00RsW6ylTYzsxoBsWda/N9/CXDiZBoyHWXlpQNvTssIMBf4mqRPTfSEZmbWWgSMvNL50mCrqb9edVydjDF9hGKK860UM8v+t3QjqBnA0xRTp5uZWZdMjwYT84HHJX2fdI8ngIh453g7djLGNA+4ICKezQsjYo+k35loTc3MrLUpnpSXu3ayO3YyxnRNm3VPTPbEZmZWI2DPNLhCMyK+M9l9G3cdk5nZsJsOLSZJL1E0AAFmAfsDP4+IOePt68BkZtYgEdNjjCkiDikfp9uxnwec3sm+jZsrz8xs2E3hu17UisLXgbM72d6BycysQbp9O6Z0X7stklZnZfMk3Sfp6fR3biqXpBskrU33xDsl2+fStP3Tki7t4LwXZMt/lPRJYGcn74EDk5lZk6Tkh06XDnyZve8YexXwQEQsprip31Wp/ByKOzcsprin3Y1QBDLgGuDNFHcOvya7iWsrv5stZwMvUXTnjctjTGZmDRJ0d4wpIr6bbjmROw94e3p8M/B/gT9O5bekSRUelHRouhXR24H7ImIbgKT7KILdbW3O+4HJ1tktJjOzJokJd+XNl7QyWzq5e/cREbEpPd5MMScqFPdLei7bbkMqa1XekqRFku5K3YhbJN0paVEHdXOLycysScopiSZga0Qsmfz5Inp0p++/Bm4F3p2evzeVvWO8Hd1iMjNrmD3R+TJJz5d3C09/t6TyjcDR2XaLUlmr8nYOi4i/joiRtHwZOKyTyjkwmZk1SLez8lpYAZSZdZcCd2fll6TsvNOBn6Uuv3uBs9Kt0udS3G3i3nHO8YKk90raLy3vBV7opHLuyjMza5Lo7vVJkm6jSF6YL2kDRXbdJ4E7JF0GPAtcmDa/BzgXWAvsoLjVERGxTdIngIfSdh8vEyHa+CDwOeCzFPH2n4H3d1JnByYzs4bp5lx5EXFxi1Vn1GwbwBUtjrOc4maxnfo4cGl59/OUcv5pioDVlgOTmVmDdDtdfIB+rQxK8Fqr602d7OjAZGbWJF3uyhugGZLmVlpMHcUcByYzswYJpsdtL4DrgX+R9Hfp+buB6zrZ0YHJzKxJpkmLKSJukbQS+Pep6IKIeLyTfR2YzMwaZpqMMZECUUfBKOfAZGbWIDFNWkz7woHJzKxhHJjMzKwxImBkZNC1GCwHJjOzhnGLyczMGiNi2qSLT5oDk5lZw7jFZGZmjTGNpiSaNAcmM7Mmcbq4A5OZWdM4MJmZWWM4+cGBycyscTzGZGZmjVHeWn2YzRh0BczMLJO68jpdxiPpVyWtypbtkj4q6VpJG7Pyc7N9PiZpraSnJJ3dy5dbxy0mM7MGiYCR7t5a/SngZABJ+wEbgbuADwCfjYhP59tLOhG4CDgJOAq4X9IJEdG3kS+3mMzMGmbPns6XCToDWBcRz7bZ5jzg9ojYFRE/AtYCp03ulUyOA5OZWYOUY0wTCEzzJa3MlsvbHP4i4Lbs+ZWSHpW0XNLcVLYQeC7bZkMq6xsHJjOzJpn4GNPWiFiSLcvqDitpFvBOoLzV+Y3AGyi6+TZR3Aq9ETzGZGbWMD1KFz8HeCQingco/wJI+hLwjfR0I3B0tt+iVNY3A2kxSXq3pDWS9khaUllXmw0iaWkqWyvpqv7X2sys98o72PZgjOlism48SQuydecDq9PjFcBFkg6QdBywGPj+vr2qiRlUi2k1cAHwv/PCVtkgafUXgHdQ9Hc+JGlFup+8mdm00u2ZHyQdRPH7+aGs+FOSTqYY1lpfrouINZLuAB4HRoAr+pmRBwMKTBHxBICk6qrXskGAH0nKs0HWRsQzab/b07YOTGY2rUQPJnGNiJ8Dr6+Uva/N9tcB13W3Fp1r2hjTQuDB7HmeDVLNEnlzq4OkrJR2mSlmZo3lKYl6RNL9wJE1q66OiLt7dV6AlJWyLNVjyP+JzWwq8ZREPQxMEXHmJHZrlw0y0CwRM7O+8P2YGteVtwK4VdJnKJIfymwQAYtThshGigSJ9wyslmZmPRIBI68MuhaDNZDAJOl84HPAYcA/SFoVEWe3ywaRdCVwL7AfsDwi1gyi7mZmvTbsY0yKmN7vgMeYzKyPHo6IJeNv1tox+yv++NDOt79y676fs2ma1pVnZjbcPMbkwGRm1jQOTGZm1hiBx5gcmMzMmsRdeQ5MZmZNEnR/rrypxoHJzKxJ3GJyYDIzaxqPMZmZWWNEuCvPgcnMrGHclWdmZo0RwIgDk5mZNcmQxyUHJjOzJgkcmGYMugJmZjbWngksnZC0XtJjklZJWpnK5km6T9LT6e/cVC5JN0haK+lRSad0+eWNy4HJzKxByhZTNwNT8tsRcXI2E/lVwAMRsRh4ID0HOIfiXniLgcuBGyf/aibHgcnMrGF6FJiqzgNuTo9vBt6Vld8ShQeBQyUt2LdTTYwDk5lZw0wwMM2XtDJbLq85ZADfkvRwtv6IiNiUHm8GjkiPFwLPZftuSGV94+QHM7MGmUTyw9YObhT4lojYKOlw4D5JT445Z0Q06aaqbjGZmTVMt7vyImJj+rsFuAs4DXi+7KJLf7ekzTcCR2e7L0plfePAZGbWIN1OfpB0kKRDysfAWcBqYAVwadrsUuDu9HgFcEnKzjsd+FnW5dcX7sozM2uYLl/HdARwlyQofvNvjYh/lPQQcIeky4BngQvT9vcA5wJrgR3AB7pbnfE5MJmZNUgAI908XsQzwK/XlL8AnFFTHsAVXazChDkwmZk1zLDP/ODAZGbWIJ6SyIHJzKxxHJjMzKxRHJjMzKwx3JXnwGRm1jgOTGZm1hhuMTkwmZk1jgOTmZk1igOTmZk1hrvyHJjMzBrHgcnMzBqj23PlTUUOTGZmDeKuPAcmM7PGcWAyM7NGcWAyM7PGcFeeA5OZWeMMe2CaMegKmJnZqLLF1OkyHklHS/q2pMclrZH0kVR+raSNklal5dxsn49JWivpKUlnd/cVjs8tJjOzhulyi2kE+MOIeETSIcDDku5L6z4bEZ/ON5Z0InARcBJwFHC/pBMi4tXuVqs1t5jMzBqmmy2miNgUEY+kxy8BTwAL2+xyHnB7ROyKiB8Ba4HTJvlSJmUggUnSu1OTco+kJVn5sZJ+kTUtv5itO1XSY6l5eYMkDaLuZma9NImuvPmSVmbL5a2OLelY4E3A91LRlZIelbRc0txUthB4LtttA+0DWdcNqsW0GrgA+G7NunURcXJaPpyV3wj8HrA4LUt7X00zs/6bYGDaGhFLsmVZ3TElHQzcCXw0IrZT/Ka+ATgZ2ARc37tXNDEDCUwR8UREPNXp9pIWAHMi4sGICOAW4F29qp+Z2aCUUxJ1unRC0v4UQekrEfH3ABHxfES8GhF7gC8x2l23ETg6231RKuubJo4xHSfpB5K+I+mtqWwhRXOy1PempZlZv3Q5K0/ATcATEfGZrHxBttn5FD1ZACuAiyQdIOk4ih6q7+/TC5qgnmXlSbofOLJm1dURcXeL3TYBx0TEC5JOBb4u6aRJnPtyoGU/q5lZU/XgAtvfAt4HPCZpVSr7E+BiSSenU64HPgQQEWsk3QE8TtEou6KfGXnQw8AUEWdOYp9dwK70+GFJ64ATKJqRi7JN2zYtUx/rMgBJMdF6mJkNUjcDU0T8E1CXLHZPm32uA67rYjUmpFFdeZIOk7Rfenw8RRPymYjYBGyXdHpqll4CtGp1mZlNWd2+wHYqGlS6+PmSNgC/AfyDpHvTqrcBj6bm5teAD0fEtrTu94G/osipXwd8s7+1NjPrj2EPTCqS3KYvd+WZWR89HBFLxt+stddJcewEtn+yC+dsGk9JZGbWIJ5d3IHJzKxxHJjMzKwx3GJyYDIzaxwHJjMzaxQHJjMza4xyrrxh5sBkZtYgHmNyYDIzaxwHJjMzaxQHJjMzawx35TkwmZk1jgOTmZk1hltMDkxmZo3jwGRmZo3iwGRmZo3hrryG3cHWzMy6f6NASUslPSVpraSrelDlrnKLycysQbo9JZGk/YAvAO8ANgAPSVoREY938TRd5RaTmVnDdLnFdBqwNiKeiYjdwO3AeV2vdBcNQ4tpK/Bs9nx+Khs012Ms12Ms12OsqVKPX+7COe5N5+nUbEkrs+fLImJZ9nwh8Fz2fAPw5n2oX89N+8AUEYflzyWtjIglg6qP6+F6uB6uRzsRsbSXx58K3JVnZja9bQSOzp4vSmWN5cBkZja9PQQslnScpFnARcCKAdeprWnflVdj2fib9IXrMZbrMZbrMZbrMUkRMSLpSoqxq/2A5RGxZsDVaksRMeg6mJmZvcZdeWZm1igOTGZm1ihDEZgkfVXSqrSsl7QqlR8r6RfZui/2uB7XStqYne/cbN3H0nQhT0k6u8f1+AtJT0p6VNJdkg5N5X19P9I5BzJViqSjJX1b0uOS1kj6SCpv+W/Uw7qsl/RYOt/KVDZP0n2Snk5/5/a4Dr+aveZVkrZL+mi/3g9JyyVtkbQ6K6t9D1S4IX1mHpV0So/r0Zjvy9CIiKFagOuBP0uPjwVW9/Hc1wL/tab8ROBfgQOA44B1wH49rMdZwMz0+M+BPx/Q+7Ffeq3HA7PSe3Bin869ADglPT4E+GH6d6j9N+pxXdYD8ytlnwKuSo+vKv+N+vjvspniYtG+vB/A24BT8s9fq/cAOBf4JiDgdOB7Pa5HI74vw7QMRYupJEnAhcBtg65LxXnA7RGxKyJ+BKylmEakJyLiWxFRTsf1IMV1DYMwsKlSImJTRDySHr8EPEFxhXxTnAfcnB7fDLyrj+c+A1gXEc+Ou2WXRMR3gW2V4lbvwXnALVF4EDhU0oJe1aNB35ehMVSBCXgr8HxEPJ2VHSfpB5K+I+mtfajDlalLYHnWPVM3ZUi/fiQ/SPG/z1I/349Bvu7XSDoWeBPwvVRU92/USwF8S9LDki5PZUdExKb0eDNwRB/qUbqIsf956/f7UWr1Hgzr92VoTJvAJOl+Satrlvx/4Bcz9gu3CTgmIt4E/BfgVklzeliPG4E3ACenc1+/L+fah3qU21xNMZHxV1JR19+PppN0MHAn8NGI2E4f/40yb4mIU4BzgCskvS1fGUW/UV+u61BxAeY7gb9LRYN4P/bSz/egFX9f+mfaXGAbEWe2Wy9pJnABcGq2zy5gV3r8sKR1wAnAytqDdKEeWX2+BHwjPe36lCEdvB/vB34HOCN96XvyfoxjoFOlSNqfIih9JSL+HiAins/W5/9GPRMRG9PfLZLuoujifF7SgojYlLqptvS6Hsk5wCPl+zCI9yPT6j3o++emId+XoTFtWkwdOBN4MiI2lAWSDlNxrxIkHQ8sBp7pVQUq/eDnA2XmzwrgIkkHSDou1eP7PazHUuCPgHdGxI6svK/vBwOcKiWNN94EPBERn8nKW/0b9aoeB0k6pHxMMdC+muJ9uDRtdilwdy/rkRnTq9Dv96Oi1XuwArgkZeedDvws6/LrugZ9X4bHoLMv+rUAXwY+XCn7D8AaYBXwCPC7Pa7D3wCPAY9SfLkWZOuupshQewo4p8f1WEvRR78qLV8cxPuRznkuRUbcOuDqPn4e3kLRNfRo9j6c2+7fqEf1OJ4iG/Ff03t/dSp/PfAA8DRwPzCvD+/JQcALwC918pnt8rlvo+gae4VizOiyVu8BRTbeF9Jn5jFgSY/r0Zjvy7AsnpLIzMwaZZi68szMbApwYDIzs0ZxYDIzs0ZxYDIzs0ZxYDIzs0ZxYDIzs0ZxYDIzs0ZxYDJLJP27NFnp7DQjwxpJbxx0vcyGjS+wNctI+p/AbOB1wIaI+F8DrpLZ0HFgMsukOfseAnYCvxkRrw64SmZDx115ZmO9HjiY4q62swdcF7Oh5BaTWUbSCoo76R5HMWHplQOuktnQmTb3YzLbV5IuAV6JiFvT7Qz+WdK/j4j/M+i6mQ0Tt5jMzKxRPMZkZmaN4sBkZmaN4sBkZmaN4sBkZmaN4sBkZmaN4sBkZmaN4sBkZmaN8v8BX+okoPgW6T4AAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 432x288 with 2 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "df.plot(df.x, df.y, show=True);  # make a plot quickly"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### [We can export vaex data and import from other formats](https://vaex.readthedocs.io/en/docs/getting_data_in_vaex.html)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}