Notebooks/01_SQLite.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Introduction to Databases\n",
    "\n",
    "### Accessing Relational Databases - SQLite"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import os\n",
    "import sys\n",
    "import time\n",
    "import datetime\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "\n",
    "import sqlite3"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Acessing [SQLite](https://docs.python.org/3/library/sqlite3.html)  "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "conn = sqlite3.connect(os.path.join(\"..\",\"SampleDBs\",'sqlite_example.db'))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "cur = conn.cursor()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[]\n"
     ]
    }
   ],
   "source": [
    "cur.execute(\"SELECT name FROM sqlite_master WHERE type='table';\")\n",
    "print(cur.fetchall())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Create table\n",
    "cur.execute('''CREATE TABLE IF NOT EXISTS stocks\n",
    "             (date text, trans text, symbol text, qty real, price real)''')\n",
    "\n",
    "# Insert a row of data\n",
    "cur.execute(\"INSERT INTO stocks VALUES ('2006-01-05','BUY','RHAT',100,35.14)\")\n",
    "\n",
    "# Save (commit) the changes\n",
    "conn.commit()\n",
    "\n",
    "# We can also close the connection if we are done with it.\n",
    "# Just be sure any changes have been committed or they will be lost.\n",
    "conn.close()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "conn = sqlite3.connect(os.path.join(\"..\",\"SampleDBs\",'sqlite_example.db'))\n",
    "cur = conn.cursor()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[('stocks',)]\n"
     ]
    }
   ],
   "source": [
    "cur.execute(\"SELECT name FROM sqlite_master WHERE type='table';\")\n",
    "print(cur.fetchall())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "('2006-01-05', 'BUY', 'RHAT', 100.0, 35.14)\n"
     ]
    }
   ],
   "source": [
    "t = ('RHAT',) #tuple with just one element\n",
    "cur.execute('SELECT * FROM stocks WHERE symbol=?', t)\n",
    "print(cur.fetchone())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<sqlite3.Cursor at 0x7fe55c78aa40>"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Larger example that inserts many records at a time\n",
    "purchases = [('2006-03-28', 'BUY', 'IBM', 1000, 45.00),\n",
    "             ('2006-04-05', 'BUY', 'MSFT', 1000, 72.00),\n",
    "             ('2006-04-06', 'SELL', 'IBM', 500, 53.00),\n",
    "            ]\n",
    "cur.executemany('INSERT INTO stocks VALUES (?,?,?,?,?)', purchases)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "('2006-01-05', 'BUY', 'RHAT', 100.0, 35.14)\n",
      "('2006-03-28', 'BUY', 'IBM', 1000.0, 45.0)\n",
      "('2006-04-06', 'SELL', 'IBM', 500.0, 53.0)\n",
      "('2006-04-05', 'BUY', 'MSFT', 1000.0, 72.0)\n"
     ]
    }
   ],
   "source": [
    "for row in cur.execute('SELECT * FROM stocks ORDER BY price'):\n",
    "        print(row)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Using Pandas"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "         date trans symbol    qty  price\n",
      "0  2006-01-05   BUY   RHAT  100.0  35.14\n"
     ]
    }
   ],
   "source": [
    "conn = sqlite3.connect(os.path.join(\"..\",\"SampleDBs\",'sqlite_example.db'))\n",
    "df = pd.read_sql_query(\"SELECT * from stocks ORDER BY price\", conn)\n",
    "\n",
    "# verify that result of SQL query is stored in the dataframe\n",
    "print(df.head())\n",
    "\n",
    "conn.close()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Deleting the database file:  "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "os.remove(os.path.join(\"..\",\"SampleDBs\",'sqlite_example.db'))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Accessing existing [Database](https://www.sqlitetutorial.net/sqlite-sample-database/):\n",
    "\n",
    "![Chinook Schema](../SampleDBs/sqlite-sample-database-color.jpg)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "! wget https://cdn.sqlitetutorial.net/wp-content/uploads/2018/03/chinook.zip\n",
    "! unzip chinook.zip\n",
    "! mv chinook.zip ../SampleDBs/chinook.sqlite\n",
    "! rm chinook.zip"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "conn = sqlite3.connect(os.path.join(\"..\",\"SampleDBs\",'chinook.sqlite'))\n",
    "cur = conn.cursor()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "('albums',)\n",
      "('artists',)\n",
      "('customers',)\n",
      "('employees',)\n",
      "('genres',)\n",
      "('invoices',)\n",
      "('invoice_items',)\n",
      "('media_types',)\n",
      "('playlists',)\n",
      "('playlist_track',)\n",
      "('tracks',)\n"
     ]
    }
   ],
   "source": [
    "query = '''\n",
    "SELECT name\n",
    "FROM sqlite_master \n",
    "WHERE type ='table' AND name NOT LIKE 'sqlite_%';\n",
    "'''\n",
    "\n",
    "cur.execute(query)\n",
    "for c in cur.fetchall():\n",
    "    print(c)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CREATE TABLE \"albums\"\n",
      "(\n",
      "    [AlbumId] INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL,\n",
      "    [Title] NVARCHAR(160)  NOT NULL,\n",
      "    [ArtistId] INTEGER  NOT NULL,\n",
      "    FOREIGN KEY ([ArtistId]) REFERENCES \"artists\" ([ArtistId]) \n",
      "\t\tON DELETE NO ACTION ON UPDATE NO ACTION\n",
      ")\n"
     ]
    }
   ],
   "source": [
    "query = '''\n",
    "SELECT sql \n",
    "FROM sqlite_master \n",
    "WHERE name = 'albums';\n",
    "'''\n",
    "\n",
    "cur.execute(query)\n",
    "for c in cur.fetchall():\n",
    "    print(c[0])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(156, '...And Justice For All')\n",
      "(257, '20th Century Masters - The Millennium Collection: The Best of Scorpions')\n",
      "(296, 'A Copland Celebration, Vol. I')\n",
      "(94, 'A Matter of Life and Death')\n",
      "(95, 'A Real Dead One')\n",
      "(96, 'A Real Live One')\n",
      "(285, 'A Soprano Inspired')\n",
      "(139, 'A TempestadeTempestade Ou O Livro Dos Dias')\n",
      "(203, 'A-Sides')\n",
      "(160, 'Ace Of Spades')\n"
     ]
    }
   ],
   "source": [
    "query = '''\n",
    "SELECT albumid, title\n",
    "FROM albums\n",
    "ORDER BY title\n",
    "'''\n",
    "\n",
    "cur.execute(query)\n",
    "for c in cur.fetchmany(10):\n",
    "    print(c)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(11, 'C.O.D.', 'Angus Young, Malcolm Young, Brian Johnson', 0.99)\n",
      "(12, 'Breaking The Rules', 'Angus Young, Malcolm Young, Brian Johnson', 0.99)\n",
      "(13, 'Night Of The Long Knives', 'Angus Young, Malcolm Young, Brian Johnson', 0.99)\n",
      "(14, 'Spellbound', 'Angus Young, Malcolm Young, Brian Johnson', 0.99)\n",
      "(15, 'Go Down', 'AC/DC', 0.99)\n",
      "(16, 'Dog Eat Dog', 'AC/DC', 0.99)\n",
      "(17, 'Let There Be Rock', 'AC/DC', 0.99)\n",
      "(18, 'Bad Boy Boogie', 'AC/DC', 0.99)\n",
      "(19, 'Problem Child', 'AC/DC', 0.99)\n",
      "(20, 'Overdose', 'AC/DC', 0.99)\n"
     ]
    }
   ],
   "source": [
    "query = '''\n",
    "SELECT trackid, name, composer, unitprice\n",
    "FROM tracks\n",
    "LIMIT 10 OFFSET 10;\n",
    "'''\n",
    "\n",
    "cur.execute(query)\n",
    "for c in cur.fetchall():\n",
    "    print(c)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "('Buenos Aires', 'Argentina')\n",
      "('Sidney', 'Australia')\n",
      "('Vienne', 'Austria')\n",
      "('Brussels', 'Belgium')\n",
      "('São José dos Campos', 'Brazil')\n",
      "('São Paulo', 'Brazil')\n",
      "('Rio de Janeiro', 'Brazil')\n",
      "('Brasília', 'Brazil')\n",
      "('Montréal', 'Canada')\n",
      "('Edmonton', 'Canada')\n"
     ]
    }
   ],
   "source": [
    "query = '''\n",
    "SELECT  DISTINCT city, country\n",
    "FROM customers\n",
    "ORDER BY country;\n",
    "'''\n",
    "\n",
    "cur.execute(query)\n",
    "for c in cur.fetchmany(10):\n",
    "    print(c)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "('Occupation / Precipice', 5286953, 227)\n",
      "('Through a Looking Glass', 5088838, 229)\n",
      "('Greetings from Earth, Pt. 1', 2960293, 253)\n",
      "('The Man With Nine Lives', 2956998, 253)\n",
      "('Battlestar Galactica, Pt. 2', 2956081, 253)\n",
      "('Battlestar Galactica, Pt. 1', 2952702, 253)\n",
      "('Murder On the Rising Star', 2935894, 253)\n",
      "('Battlestar Galactica, Pt. 3', 2927802, 253)\n",
      "('Take the Celestra', 2927677, 253)\n",
      "('Fire In Space', 2926593, 253)\n"
     ]
    }
   ],
   "source": [
    "query = '''\n",
    "SELECT name, milliseconds, albumid\n",
    "FROM tracks\n",
    "ORDER BY milliseconds DESC, albumid ASC;\n",
    "'''\n",
    "\n",
    "cur.execute(query)\n",
    "for c in cur.fetchmany(10):\n",
    "    print(c)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "('For Those About To Rock (We Salute You)', 343719, 11170334, 1)\n",
      "('Evil Walks', 263497, 8611245, 1)\n",
      "('Breaking The Rules', 263288, 8596840, 1)\n",
      "('Spellbound', 270863, 8817038, 1)\n"
     ]
    }
   ],
   "source": [
    "query = '''\n",
    "SELECT name, milliseconds, bytes, albumid\n",
    "FROM tracks\n",
    "WHERE albumid = 1\n",
    "AND milliseconds > 250000;\n",
    "'''\n",
    "\n",
    "cur.execute(query)\n",
    "for c in cur.fetchmany(10):\n",
    "    print(c)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "('Restless and Wild', 3, 'F. Baltes, R.A. Smith-Diesel, S. Kaufman, U. Dirkscneider & W. Hoffman')\n",
      "('Princess of the Dawn', 3, 'Deaffy & R.A. Smith-Diesel')\n",
      "('Killing Floor', 19, 'Adrian Smith')\n",
      "('Machine Men', 19, 'Adrian Smith')\n",
      "('2 Minutes To Midnight', 95, 'Adrian Smith/Bruce Dickinson')\n",
      "('Can I Play With Madness', 96, 'Adrian Smith/Bruce Dickinson/Steve Harris')\n",
      "('The Evil That Men Do', 96, 'Adrian Smith/Bruce Dickinson/Steve Harris')\n",
      "('The Wicker Man', 97, 'Adrian Smith/Bruce Dickinson/Steve Harris')\n",
      "('The Fallen Angel', 97, 'Adrian Smith/Steve Harris')\n",
      "('Wildest Dreams', 98, 'Adrian Smith/Steve Harris')\n"
     ]
    }
   ],
   "source": [
    "query = '''\n",
    "SELECT name, albumid, composer\n",
    "FROM tracks\n",
    "WHERE composer LIKE '%Smith%'\n",
    "ORDER BY albumid;\n",
    "'''\n",
    "\n",
    "cur.execute(query)\n",
    "for c in cur.fetchmany(10):\n",
    "    print(c)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "('Balls to the Wall', 2, 2)\n",
      "('Fast As a Shark', 3, 2)\n",
      "('Restless and Wild', 3, 2)\n",
      "('Princess of the Dawn', 3, 2)\n",
      "('Welcome to the Jungle', 90, 2)\n",
      "(\"It's So Easy\", 90, 2)\n",
      "('Nightrain', 90, 2)\n",
      "('Out Ta Get Me', 90, 2)\n",
      "('Mr. Brownstone', 90, 2)\n",
      "('Paradise City', 90, 2)\n"
     ]
    }
   ],
   "source": [
    "query = '''\n",
    "SELECT name, albumid, mediatypeid\n",
    "FROM tracks\n",
    "WHERE mediatypeid IN (2, 3);\n",
    "'''\n",
    "\n",
    "cur.execute(query)\n",
    "for c in cur.fetchmany(10):\n",
    "    print(c)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(193, 'Berger Straße 10', 14.91)\n",
      "(103, '162 E Superior Street', 15.86)\n",
      "(208, 'Ullevålsveien 14', 15.86)\n",
      "(306, 'Klanova 9/506', 16.86)\n",
      "(313, '68, Rue Jouvence', 16.86)\n",
      "(88, 'Calle Lira, 198', 17.91)\n",
      "(89, 'Rotenturmstraße 4, 1010 Innere Stadt', 18.86)\n",
      "(201, '319 N. Frances Street', 18.86)\n"
     ]
    }
   ],
   "source": [
    "query = '''\n",
    "SELECT InvoiceId, BillingAddress, Total\n",
    "FROM invoices\n",
    "WHERE Total BETWEEN 14.91 and 18.86    \n",
    "ORDER BY Total; \n",
    "'''\n",
    "\n",
    "cur.execute(query)\n",
    "for c in cur.fetchmany(10):\n",
    "    print(c)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(6, 'Berger Straße 10', 0.99)\n",
      "(13, '1600 Amphitheatre Parkway', 0.99)\n",
      "(20, '110 Raeburn Pl', 0.99)\n",
      "(27, '5112 48 Street', 0.99)\n",
      "(34, 'Praça Pio X, 119', 0.99)\n",
      "(41, 'C/ San Bernardo 85', 0.99)\n",
      "(48, '796 Dundas Street West', 0.99)\n",
      "(55, 'Grétrystraat 63', 0.99)\n",
      "(62, '3 Chatham Street', 0.99)\n",
      "(69, '319 N. Frances Street', 0.99)\n"
     ]
    }
   ],
   "source": [
    "query = '''\n",
    "SELECT InvoiceId, BillingAddress, Total\n",
    "FROM invoices\n",
    "WHERE Total NOT BETWEEN 1 and 20\n",
    "ORDER BY Total;  \n",
    "'''\n",
    "\n",
    "cur.execute(query)\n",
    "for c in cur.fetchmany(10):\n",
    "    print(c)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "('For Those About To Rock We Salute You', 'AC/DC')\n",
      "('Balls to the Wall', 'Accept')\n",
      "('Restless and Wild', 'Accept')\n",
      "('Let There Be Rock', 'AC/DC')\n",
      "('Big Ones', 'Aerosmith')\n",
      "('Jagged Little Pill', 'Alanis Morissette')\n",
      "('Facelift', 'Alice In Chains')\n",
      "('Warner 25 Anos', 'Antônio Carlos Jobim')\n",
      "('Plays Metallica By Four Cellos', 'Apocalyptica')\n",
      "('Audioslave', 'Audioslave')\n"
     ]
    }
   ],
   "source": [
    "query = '''\n",
    "SELECT Title, Name\n",
    "FROM albums\n",
    "INNER JOIN artists \n",
    "ON artists.ArtistId = albums.ArtistId;\n",
    "'''\n",
    "\n",
    "cur.execute(query)\n",
    "for c in cur.fetchmany(10):\n",
    "    print(c)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(123, 'Quadrant', 'The Best Of Billy Cobham', 'Billy Cobham')\n",
      "(124, \"Snoopy's search-Red baron\", 'The Best Of Billy Cobham', 'Billy Cobham')\n",
      "(125, 'Spanish moss-\"A sound portrait\"-Spanish moss', 'The Best Of Billy Cobham', 'Billy Cobham')\n",
      "(126, 'Moon germs', 'The Best Of Billy Cobham', 'Billy Cobham')\n",
      "(127, 'Stratus', 'The Best Of Billy Cobham', 'Billy Cobham')\n",
      "(128, 'The pleasant pheasant', 'The Best Of Billy Cobham', 'Billy Cobham')\n",
      "(129, 'Solo-Panhandler', 'The Best Of Billy Cobham', 'Billy Cobham')\n",
      "(130, 'Do what cha wanna', 'The Best Of Billy Cobham', 'Billy Cobham')\n"
     ]
    }
   ],
   "source": [
    "query = '''\n",
    "SELECT tracks.trackid, tracks.name AS track, albums.title AS album, artists.name AS artist\n",
    "FROM tracks\n",
    "INNER JOIN albums \n",
    "ON albums.albumid = tracks.albumid\n",
    "INNER JOIN artists \n",
    "ON artists.artistid = albums.artistid\n",
    "WHERE artists.artistid = 10;\n",
    "'''\n",
    "\n",
    "cur.execute(query)\n",
    "for c in cur.fetchmany(10):\n",
    "    print(c)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "('A Cor Do Som', None)\n",
      "('AC/DC', 'For Those About To Rock We Salute You')\n",
      "('AC/DC', 'Let There Be Rock')\n",
      "('Aaron Copland & London Symphony Orchestra', 'A Copland Celebration, Vol. I')\n",
      "('Aaron Goldberg', 'Worlds')\n",
      "('Academy of St. Martin in the Fields & Sir Neville Marriner', 'The World of Classical Favourites')\n",
      "('Academy of St. Martin in the Fields Chamber Ensemble & Sir Neville Marriner', 'Sir Neville Marriner: A Celebration')\n",
      "('Academy of St. Martin in the Fields, John Birch, Sir Neville Marriner & Sylvia McNair', 'Fauré: Requiem, Ravel: Pavane & Others')\n",
      "('Academy of St. Martin in the Fields, Sir Neville Marriner & Thurston Dart', 'Bach: Orchestral Suites Nos. 1 - 4')\n",
      "('Academy of St. Martin in the Fields, Sir Neville Marriner & William Bennett', None)\n"
     ]
    }
   ],
   "source": [
    "query = '''\n",
    "SELECT Name, Title\n",
    "FROM artists\n",
    "LEFT JOIN albums \n",
    "ON artists.ArtistId = albums.ArtistId\n",
    "ORDER BY Name;\n",
    "'''\n",
    "\n",
    "cur.execute(query)\n",
    "for c in cur.fetchmany(10):\n",
    "    print(c)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [],
   "source": [
    "#https://www.sqlitetutorial.net/"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}