Notebooks/Example_SQLite_search engine.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Introduction to Databases\n",
    "\n",
    "### Databases for SRI Search and Ranking"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": true,
    "jupyter": {
     "outputs_hidden": true
    }
   },
   "outputs": [],
   "source": [
    "import os\n",
    "import re\n",
    "import nltk\n",
    "import matplotlib.pyplot as plt\n",
    "import pandas as pd\n",
    "import pandas.io.sql as psql\n",
    "from IPython.display import Image, HTML, IFrame, FileLink, FileLinks #needed to render in notebook\n",
    "from IPython.core.display import display\n",
    "from bs4 import BeautifulSoup\n",
    "import urllib\n",
    "from urllib.parse import urljoin\n",
    "from sqlite3 import dbapi2 as sqlite\n",
    "\n",
    "%matplotlib inline\n",
    "# Set default figure size for this notebook\n",
    "plt.rcParams['figure.figsize'] = (16.0, 12.8)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Specifying the path to the files"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": true,
    "jupyter": {
     "outputs_hidden": true
    }
   },
   "outputs": [],
   "source": [
    "outputs = \"../outputs/\"\n",
    "\n",
    "dbfile = \"searchindex.sqlite\"\n",
    "db = os.path.join(outputs,dbfile)\n",
    "\n",
    "stoplist_en = nltk.corpus.stopwords.words('english')\n",
    "stoplist_pt = nltk.corpus.stopwords.words('portuguese')\n",
    "ignorewords = stoplist_en + stoplist_pt"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### First block of classes and functions: crawling"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAecAAADrCAIAAADc2snMAAAABmJLR0QA/wD/AP+gvaeTAAAACXBIWXMAAAsTAAALEwEAmpwYAAAAB3RJTUUH3gsZCy0l2VqfsgAAIABJREFUeNrt3XlcU1faB/AnrAGVYongQn2FKCpqWbMobjjvjNqq7czbWjvTqbZOO13QDh/r9GNt7ebW1r1KW1HRyqi4DCjiCk5VkDVshlYgio5sSgkQCUlILrx/3JrGJERUkCT8vh//CLm5R+55Tp5cbs59Dqe9vZ0AAMBGOKALAACQtQEAAFkbAABZG10AAICsDQAAyNoAAMja6AIAAGRtAABA1gYAQNZGFwAAIGsDAEC3cLLvw7tecX39uq/tPoqzZs+ePmMGRjMAsrbNa2pqLCoofCcqyo6P8fSpU1dlMgxlAGRtO/Gkl9e8P79sxwd48+Z/MY4Beg9c1wYAQNYGAABk7UenUCjGBY7JyswkIrVavWbV6urqarOvbJA3jAsck5uTiyECAMjaPUnV0sIwbURUU1OzMza2qKDQwivb2hgMEQCwKk699sj9/PwysjK9fXwwCAAA59o2QKvVZmVm1dXVKZXKo4lJSqUy+dixA/v2t7a2Gr0y89Il9qIKAACydo9RqVRLoqPLy8oVCsWS6OiFCxYcOXRo9aqVK5Z/ZPiyysrKhQte02q1GCsAgKxtFbiuXCIKCg7ZvXfvu1GLUlJSDNet/3rtl3+YPn3S5MnoKACwBk7oAkcnRyKaEDGBiEYEBKhaWpRKJbupsKDw4oULp9NS0UsAgHNtK/v4cnImIldXVyJqa2tjn9wRu32Y3zAej4f+AQBkbRswe86ckpKSjPQMdAUAWMspJrrAgukzZigUii2bNkVMjOBwOOgQAMC5trV78+9v5UsklzIuoSsAAFnbBgSMDIicNm3Lpk2GE0sAAHpK77pC4uHhIbteYfpY/yBiYoTpk7G7dmKgAADOtQEAAFkbAABZGwAArIT9X9duamo6mXLCjg/wquzq00FPYygDIGvbiQa5/MD+fXZ8gDeu30DWBkDWth/D/Pz2xMfb8QF+/eWXGMcAvQeuawMAIGsDAED3QB0SsFJRb79z547Cvo9x+IgRH3/yCWINyNpgD3Kysz/9/PMnPD3t9QDLSq+csOvZTYCsDb1OuFDg7e1tr0fn4uKCrA0PAde1AQCQte2UWq1es2p1dXW14ZPHk5NFYeHoHABA1rY6NTU1O2NjiwoKDZ9kdEyLSoXO6XEKhWJc4JiszEz9A7OfsgC2zlqua7e2tt6qre3yZm/dutWFrfn5+WVkZXr7+GDcWCdVSwvDtOkfsJ+ywcHBgwcPNnolwzC3b98eNGgQOg2QtR+SrLz8/57/48Cufhe1trYOGDBA/2Nzc3Pa2dTpM2ecS00TjRd7eXkR0aWMjLLSsqmRkcP8hmWkZ7i6uoYLwokoXyLp06fvyFEjiehcatoQX19/vn9WZtb4iAne3t7t7e2nT566c0fBccDfK1bKwqfsG68v9Of7f7RiBXoJkLUfnj/fP+XUqa5ts6iw8JOPf3tnNjY0LImOvnDh/JlTp5OOJ3t5eX0XE7Nzx46xY8dtXL9+/8GE1LNniouKjyQlEtGqL1b6+fut27CBYZjFUVEbt2weNHjQkujoPfHx3t7eX61dG783PmDEiLq6Ogyjx3Mefeb0mYmTJ3l5ed24fkMqvfzsrFmGH8OjxwQa7aLVavWfsrU1tadOnuzn0W/mzGdycrIrKira2tqOJiZN+9/f9evXD90LNqR3nSe6crlEVF5WniPJ4/P5jQ0N277ZunHz5rgf9ojEopht20Qi8ZWff2Z0jEqlqq6qkuRJiKji2jWNRiMQCvXtyOXy3bviVq9dczgp8engIAyjx6CpSbEkOrriWgURZWdlfbTsQ/3H8PJlyz5YurSpsdE40atUS6Kjy8vKb9+6NWvmzHNpafv2xtfX/5KXm3dHoaisvHkuLa1FqUTfArK2Ff9l4ehERPNfW+Dm7k5ExcXFKpUqNDSMiELDw3OyssOFAo1GI5PJiouKwsLDNRpNXV1dibRk1OjRnga3exQXFWm1WpFIxOFwpkydimFkDR/D/nx+Ry9LS0vr06fPnvi9R44mPTV06Pv/XPo/w4ZNjYzcvPUbn4ED0Y2AKyTWbsiQIeyD6qpqz/793fu4E5Gvr69cLvfw8ODz+VLp5dqa2tGBgWq1WpKXJ5VKxePHG7ZQW1PL5XIHeHsTkYuzC4aRNXwMaxUd3v7O5/Nv3bpVkJ8fGhaGTgOca9seDofz65maq6tGrWYXX1epVM7Ozk5OTkKRqKy0rKioMDgk5OmgIOllaVnpFbFYfE+ycHbSarU6nQ4DyKo+hjsSLhC8NG/eKy//+STuRQRkbZs2eMhglUpVX19PRFWVlb6+vg4ODgKRUFZeXlZaFhoaIhAKfiopkcmuCkRCwx0HDRrMMMzNmzeJqKGhAcPosWlvb7PwMdzhKHdw+PSLz6MWL1ocFSXJy0M3ArK2rQoLDx84aGDC/gPsKmVznn+eiEQiUV5uro+Pj5u7e3BwcFFhIY/H8/DwMNxRIBR4enoePnhQLpfH7/1Bp9ViJHU3N3c3Ivql7hciUt/9C+mB/sB6Jypq5MiRGenpROTo6KjV4k8lQNa2NU5OTus2bIjbuVMYGvbU0KGv/20hEfkMHOjF44nEYiJyc3f38/c3uqhNRC4uLivXrN4Tt3va5CkvzJ2r1WoZHYPB1K369ev35JNPZmSkKxSK48nJ2gf5pKyqqqq8ebOysrKqquqpp4YSEY/HKywouHPnDjoWbC9x9aqj7f9kf9n1CsNnxOPHZ0vyFE2K/k/21z957vyP+seHE/+tf+zh4aHffcbMmVMjIzkcjqur61tvv42R1O3nFw4Oi//x3meffHr8WPKLc+fmSySdP90+l5r2+aefEtEzzz47+7k5RDR33kvv/P2tsKDg0quy+15gAUDWti6Ojo6GKbvzuFwueu9xeuXVV5//059cXV2dnZ2Xr/jY6GPY8DPV6EHExIgX5r5I7e3sjE8imhoZKSkqbG9rR8oGZG2AbtS3b9+H29HNze2+zwDYxt+dvfOw1Wo1Yg8AyNo24M6dO/NfeWXsqNFfrV376Kkf5bYB4DHrdVdIdu+Kq7hWcTHz0qPXDDJbCBTltrvQ0cQkOy7tdOPGdYQYkLXvLy83RygWdUlhZZTb7m5Xrvxsx1efUS0SkLXvL+1s6o3rN1xcXI8mJk2eMuXC+fOGtbYNC20TkVKpTD1z9vd/+ENq6lkimvnMM/L6+tSzZ32feoqtGGVYCBTltrvDB8uW2fFqv7k5uY9+mQ6Qte1cVlaWQqH4740b59LSRgcGGtbaPpSQYFhoO3DMGIVCsSQ6WiQWOzs7SySSzIxL0suXvXhe6RfTt34bM2PmTLYQKMptAwCydndZ/vFHuTk5ERMjln7wAZte2SKfGo1m2zdbv93+/cRJk95cuDBm27atMTFcVy4RhYSGvv/Ppdu+2bpx/fof/hU/ISLiL/NePplyYsbMmfpm2XLbX61fN2v27EXvvnv+x/OP+bh27dh56OAhOwtWU1MT3p8AvT1r33PkBkU+c3NzDQtt74rdQUSOTo5ENH7CBCIaPmI4EYWFhxMRn88vLy83bMqo3Pbjz9r/9+KLr/z1FTsL0B/nPIf3JwCytjG2yKdpoe3W1tZfO8jJkYicnV30id7N3b2t7Z7Kcz1ebvuJJzyeGjrUzkLjgG8IAMy+NXr58bM3NJsttP0AH30otw0AyNqPk9lC253fHeW2AQBZ+7EyW2i781Bu28rhLlawJ6geRXS30HbU2+9s2bRp0uTJbKHtzmPLbS/5R/S++H+99e476778itEx7JeZYA2s9i7WO3fuZKRndF/7Y8aOMVylGpC1bVJS8jH2gVGtbdNC24aVP6f9bpr+8bLlH5q+AOW2rZnV3sV6q7Z2+3ffdlPj0svSLdu2RUyMwABA1rZbD11om9U7y203NjRUVlZ2R8sMc8/yQM3NzWlnUzu6lzUjPcPV1TVcEE5E+RJJnz59R44aSUTnUtOG+Pr68/2t8y7W4SNG7ImP76bG/zLvZbypkbUBjKWmpq5dvcbX17fLW25WNht9PFi4lzX17JniouIjSYlEtOqLlX7+fus2bGAYZnFU1MYtmwcNHoS7WAFZG+BXUyOnrtuwocubFYaGGf7oyuVSx/eyzpo1++CBBEbHtGpbq6uq5HI5EVVcu6bRaARCob6RHr+LFeDRYQ7JPcxONiDMN7CG8wuDe1mLi4sN72XNycoOFwo0Go1MJisuKgoLD9doNHV1dSXSklGjRxt+HWd0Fyt6FXCubQMYhrl9+3ZHlVrNTjYgVM22Gh3dy+rh4cHn86XSy7U1taMDA9VqtSQvTyqVisePN9y9x+9itZuTm7rbt9EPZnHd3AYMGICs3ZXeeH2hP9//oxUrzG5FyWwrZ3QvK4fD0d/LKhSJykrLKiquzV/wGsMw0svSstIrr85fcM9wv3sXq7OzMzrzoeXl5r71xpsD7LeI7sN/nqlUY8eNi921E1m7y/z4n/9UVFS0tbUdTUya9r+/Y9dJMZyKYFgym4hQNds66e9l5fF4+ntZBSLh0cSkq1evhoaGODhwYr/fLpNdFYiEhjvq72L19/fHXayPIiQ0dO++f6EfjKSePZuw/0B3/y+9Kxnl5ebdUSgqK2+eS0trUSqJ6LuYmPcWLTr/44/PzZr1U0kJWzK7vOzXkn5frV279P33D+zbv2XjJoxI62H2XlaRSJSXm+vj4+Pm7h4cHFxUWMjj8Tw8PAx3xF2sYAd6V9Z+/59L/2fYsKmRkZu3fuMzcGBjQ8O2b7Zu3Lw57oc9IrEoZts2wxez8w1Wr11zOCnx6eAgjBXrwd7LGrdzpzA07KmhQ9l7WX0GDvTi8URiMRG5ubv7+fsbXdSmu3ex7onbPW3ylBfmztVqtYyOQX+CjY3/3nzwRlMR2LLav23t6arZYOi+97IS0bnzP+ofH078t/4x7mK1FceTk7/49LNsSZ5CoYgQj4/ducP0o/dBqdXqjes3zH9tgdEUA5xr2yQLZbUJ8w2s3kPfy8rlcl1dXdGBVshwspaqpYVh2h6+KYapqamhuxPDigoKcYXEHlguq42q2QC2643XF+6MjaW7E8NmPDMTWduGT9C02l8TseWy2qiaDdAZGekZebl57ON8iaT0Sin7+Fxqmv7xpYyM3bvirldcZ39sbm4+mpikVqtPHE+pr69vb28/deLkoYQErc7898NGuxNRdXX1oYSElOPH2YWlSq+U7onbzX5BTXdni8nKZUcTk+RyeVZmlr6AgVFTSqXyaGKSUqlMPnbswL79hn9tW61ed12bx+MVFhTcuXOnX79++qkIr7z6V9Oy2vr5Bn97800rn29w8cJFpVLZI/916ZVSz/5WUQu0C6+EwgOxXARm5KiR38XEGBaNCRwzxqiqzI7t2y0UhzHdPf3ixXfeemvMmDF9+/Z7dtYsIvp4+XJfX9/LxcVxu3alnDxpOFsscEygvgqNaVMKhWJJdHS4IJzL5ebn5xcWFKz9+iuca1uXufNeKi8rCwsKbm9vNzsVQc+G5htwuVwPjyd65J9AKHzm2VlW0g+PeCUUHo5IJL7y88+MjlGpVNVVVZI8CRkUgTE7U8uwqkz//v0tTNYy3V2n0y375wcLXnt9/8GD+vtZDh45vGHzps1bv5GVl+fn5xvNFuuoKSLiunKJKCg4ZPfeve9GLUpJSWEvmeJc24pMjYyUFBW2t7Wzd9mZTkUwnKhgK/MNBELB399+C+kDeoS+CExjY0NYeHh+fr5hEZgL58+bztQyrCqTnZ1tYbKW6USv3Jycmpqav85/1ejXUCqVKpWKiGpra8z+nmbnjLGrl0yImEBEIwICVC0tSqWyb9++ONe2Lm5ubuy8EZblqQiYb2A959FHE5PYLyFuXL+Rcvw4mVweRS/1CB6PxxaBycvNGx0YGBgYaFgExsJMLbaqjOXJWqa7X5XJ+vbta1TrY/euuCkREz//9DMHB4e2Dv7esvCbODk5ExH7TmcvlCNrAzyqpibFkujoimsVRJSdlfXRsg/pbtHt5cuWfbB0aVNjI3qpp7BFYIqKCoNDQp4OCmKLwIjFYrI4U4v9e9fyZC3T3d3c3VtbWw2vY+Tm5K5dvXpfQsLR48muHS9OYnnOmA1B1gYbZnh51J/PR4f0FIFIKCsvLystCw0NEQgFP5WU6IvAWJ6pRfebrGW6+7Bhw1pbW6/KZPrX5GRnDx8+PGBkQFtbm37WgOFssY6acrDN+kLI2mDDDC+Pojd6kIUiMGaLxtyT8S0WhzHdPSQ0lM/nf/HZZ7dv3aqtqSUiDw+PioqK+vr6o4lJWq2WnT6ony1moSkb7W1kbbB57OVR6EEWisBYnqlF95usZbq7g4PD1m9jbt+umyAS/+XleUT0/J/+6OfnFyESn0hJ8ff3V6tUdO9ssU7+JjZzsoIBBzakvd3MN0Xs5VHoWR0VgSFzM7WMqsqYTtYyLB1juvuIgICTZ043Nzf36dOHiPr163fsREpTY5PhtAL9bDH3Pu4WmjL8jyImRhj+VsjaPUZeX39g3347PsCfSn4SCAV2H0c3dzci+qXuFyJS3/1OCWzFfYvGcDv+FrGj3Q3n5zk4OJi+wM3N7SF+E2TtHvbEE55BIcEZGel2fIx9+/XlDx9u92/7fv36PfnkkxkZ6RGTJh5PTtaiNDb0VnaetYf5Dfvm3qrZYKMcHBwW/+O9zz759Pix5Bfnzs2XSHC6DcjaAFbtlVdfff5Pf3J1dXV2dl6+4mO69/Ko4QVKAGRtAKtg5bca9x7FRUXPz56DfjCiUCj43X/fALI2ADww/nD+5ytXoh+M5GRnZ17KRNYGAKvTp0/fsePGoR+M1NbWPoasjbtsAABsCbI2AACyNgAAIGsDAACyNgCALcEcErBeH334oR0vJNQgb+g9odSvxRwcErJx/Yb5ry0YPHiwfuvx5OQvPv0sW5KHMY+sDTZszZdf6heIsldWsrz948GuxVxTU7MzNjY4ONgwazM6pkWlwphH1gbb9rvf/y864RFt3bLlwP59Xd7sL3W/PPTaXX5+fhlZmd4+PogOsjYA3GPxP96T18u7o+UrV37Ol+QbnkSfOX1m4uRJXl5eN67fkEovPztrVnNzc9rZ1OkzZ5xLTRONFzs7O7Mv1mq1WZlZ4yMmeHt7t7e3nz556s4dBccBX7AhawP0euzKMt2hn0c/w6zNLsR84NAhLy+v7KysNatWPTtrFrsQ84UL58+cOp10PFm/pLpKpVoSHb0nPt7b2/urtWvj98YHjBhRV1eHeHUePuIAoOsZLsRstqCSXC7fvStu9do1h5MSnw4OQo8hawNAj/4Vf7+FmIuLirRarUgk4nA4U6ZORY8hawNAz7OwEHNtTS2Xyx3g7U1ELs4u6CtkbQB4rB50IWYnZyetVqvT6dB1yNoA8Fg93ELMgwYNZhjm5s2bRNTQ0IBuRNYGgMdEvxCzQqHo/ELMAqHA09Pz8MGDcrk8fu8POizfjKwNAI8piTg4LP7Hewn7D0yeEBEUFKTRaDpzuu3i4rJyzeo9cbunTZ7ywty5Wq2W0THozM7AfG0AeFSWF2Kme9di1j+YMXPm1MhIDofj6ur61ttvoxuRtQHg8Xm4hZi5XC667oH/uEEXAAAgawMAALI2ANi148nJorBwIlKr1WtWra6urja7FVkbAMAq6Atts2W4iwoKzW7t5fBtJABYHZThxrk2AHSXjPSMvNxfFw/Ll0hKr5Syj8+lpukfX8rI2L0r7nrFdfbH5ubmo4lJarX6xPGU+vr69vb2UydOHkpI0Op+vdeGLcPNVnA13YpzbQCwQ+dS027fvt0dLctk5YY/pp49U1xUfCQpkYhWfbHSz99v3YYNDMMsjorauGXzyFEjv4uJ2bljx9ix4zauX7//YELgmDFG1bd3bN9uVGgbZbiRtQF6nZ07drS1tfEG8Lq8ZaMVyEQi8cEDCYyOadW2VldVyeVyIqq4dk2j0QiEwsaGhm3fbP12+/cTJ016c+HCmG3btsbEGFbfVqnVu3fFfbV+3azZsxe9++75H88b/l9sGe6OtiJrA4BdiVq8OGJiRJc3m37x4vfffqf/MVwo0Gg0MpmssbEhLDw8Pz+/rq6uRFoyavRoT0/PC+fPq1Sq0NAwIgoND98Vu4Purb6dnZ1tWGjbKC8bleFG1kbWBoBHxePx+Hy+VHq5tqZ2dGCgWq2W5OVJpVLx+PFEVF1V7dm/v3sfdyLy9fWVy+Wtra3sjmz1bcuFtlGG2xS+jQSARyUUicpKy4qKCoNDQp4OCpJelpaVXhGLxUTk6uqquVu+VaVSOTs766+usNW3LRfaRhluZG0A6HoCkVBWXl5WWhYaGiIQCn4qKZHJrgpEQiIaPGSwSqWqr68noqrKSl9fX4d7V2S3XGgbZbiRtQGg64lEorzcXB8fHzd39+Dg4KLCQh6P5+HhQURh4eEDBw1M2H+gqanpZMqJOc8/b5zxLRbaRhluZG0A6Ho+Awd68XgisZiI3Nzd/fz92YvaROTk5LRuw4a4nTuFoWFPDR36+t8WGu1rudA2ynCbwreRANAFzp3/Uf/4cOK/DTeJx4/PluQpmhT9n+zPPmNUfdu00LZhPW6U4UbWBoDHzdHRUZ+yzbJcaBtluA3hCgkAALI2AAAgawOAFVIoFOMCx2RlZnZJa2YrawOyNgB0JVVLC8O0PUoLDMPU1NRQB5W1AVkbAKzLG68v3BkbS3cra894Zib6BFkbAB4To2raRFRdXX0oISHl+PG2trbSK6V74naz992wW3/8z38qKipk5bKjiUlyuVxfWdu0KaVSeTQxSalUJh87dmDffn1JE2RtAICH9F1MzHuLFp3/8cfnZs36qaSEiNIvXpzx+9//+8iRpH8nOjg4fLx8eVFh4Y7t21964UX2lpm83Lw7CkVl5c1zaWn1v/yyJDq6vKzcbFMKhWJJdPTCBQuOHDq0etXKFcs/QtYGAHh4bDXtjZs3x/2wRyQWxWzbptPplv3zgwWvvb7/4MHYXTuJ6OCRwxs2b9q89RtZeXl+fj4Rvf/Ppf8zbNjUyMjNW7/xGTiwo6aIiOvKJaKg4JDde/e+G7UoJSWFrUuFrA0A8DCKi4sNq2nnZGXn5uTU1NT8df6rhi9TKpUqlYqIamtrOt8UETk6ORLRhIgJRDQiIEDV0qJUKntbJ+PeSADoMqbVtK/KZH379h0wYID+Nbt3xW3dsmWIr6+Dg0NbxzNPLBTmdnJyJiJXV1ciamtr622djKwNAF1GX02bw+Gw1bTd3N1bW1vZZ4goNyd37erVx1JSAkYGjAsc80BNGS571pvhCgkAdBnTatrDhg1rbW29KpOxL8jJzh4+fHjAyIC2tjbDsquOjo5arc5yU0aFuZG1AQAelWk17ZDQUD6f/8Vnn92+dau2ptbDw6OioqK+vv5oYpJWq9Xqfk3cPB6vsKDgzp07FppC9yJrA0AXM62m7eDgsPXbmNu36yaIxH95ed7zf/qjn59fhEh8IiXF399frVKxO86d91J5WVlYULB+Tsh9C3P33k5GFwDAozCshU3mqmmPCAg4eeZ0c3Nznz59OBzOsRMpTY1NRoVbp0ZGSooK29va3fu461szbcrw/4qYGGH4/yJrAwA8JLPVtPv27fvrH/gODmZrbbu5uXWyqV4OV0gAAJC1AQAAWRsAAOz8urb08uX5r/zVXo+ub9++5zPS7TuCx44e/WzFJ3b/Ply3cUPktGnIR4CsTQzDDB48OP7Afvs7tKbGxudmzbb7Adra2hoxaeIXq1bZ8TFGL35Pa3C/iU3IyckOCwpGAjWi1WlFIjGy9qNydHJ64okn7O+42ntN+QUXFxe7jOBvb0JnG3sbisePz5FIkKPNcnZyRtYGACv7mLHTMyFbgW8jAQCQta2PfhlpyytAN8gbxgWOyc3J7Y7f4XhysigsHGPOSrDhMFxfHKuDA7K2dWGXkb7vCtCqlpa2NqY7fgFGx7TcrboAPU4fDv364hbGhn4FcYAe1+uua7MrQHv7+CD20Pmx8cbrC/35/h+tWIFeApxrP25arZZdAfq+iz1nXrqUlZmZkZ6Rl5vHPpMvkZReKWUfn0tN0z82Wka6ubn5aGKSWq0+cTylvr6+vb391ImThxIS9EUpiai2pnb3rrgjhw+3KFswCjvvQcNhFAsiMhsOo7FhFCDDFcQNS4kCIGs/luskKhW7ArTlxZ4rKysXLnhNq9Wmnj2z5u5k4VVfrIzd/j379/LiqKj//vcGmVtGurGhYUl09PJlyz5YurSxsfGrtWuXvv/+gX37t2zcxLZz+9atWTNnnktL27c3vr7+F4zCznvQcBjFgohMw2E6NowCZLiCeEvvW6UQkLWtheXFnr9e++Ufpk+fNHmySCS+8vPPjI5RqVTVVVWSPAkRVVy7ptFoBEKh2WWkXblcIiovK8+R5PXv33/3rrjVa9ccTkp8OjiIbTwtLa1Pnz574vceOZr01NChGIWd96DhMIwFn8+Xy+Wm4TBlFCCzK4gDIGs/bhYWey4sKLx44cLyFR8TUbhQoNFoZDJZcVFRWHi4RqOpq6srkZaMGj3a09PT7DLSTo5ORDT/tQVu7u7FRUVarVYkEnE4nClTp7Lt8/n8W7duFeTnY/w9qAcNh2EsiMhsOEwhQICsbb3MLva8I3b7ML9hPB6PiHg8Hp/Pl0ov5+XmjQ4MDAwMlOTlSaVS8fjxZHEZ6SFDhhBRbU0tl8sd4O1NRC7OLr+mHoHgpXnzXnn5zydTTmAIPpCHCwcbi47CYeazAQECZG3bMnvOnJKSkoz0DPZHoUhUVlpWVFQYHBLydFCQ9LK0rPSKWCwmg2WkichoGWl2RWonZyetVqvT3bOMqYODw6dffB61eNHiqChJXh46/IE8RDjYWHQUDjPvCgQIkLVty/QZM2bPmbNl0yb2/S8QCWXl5WWlZaGhIQKh4KeSEpnsqkAkpE4sIz1o0GCGYW7evEmDXIkRAAAJi0lEQVREDQ0N+uc5HM47UVEjR47MSE9Hhz+Q7giHKaMAma4gDoCsbV3e/Ptb+RLJpYxLRCQSifJyc318fNzc3YODg4sKC3k8noeHB3ViGWmBUODp6Xn44EG5XB6/9wedVktEVVVVlTdvVlZWVlVVPfUUvo18MF0eDlOmATJdQRwAWdu6BIwMiJw2jT3d9hk40IvHE4nFROTm7u7n789eRaVOLCPt4uKycs3qPXG7p02e8sLcuVqtltEx51LTIidPiZw0efKUKbOfm4PefiBdHg7T/8I0QKYriAP0mHbrUCKVPjN9epc3W1hQ8Nys2d39y+t0Onm93MILVCqVWq02fKalpaVFqXyU/7RBLg8eO67d3h06eHBJdHR3h8OUaYBaWlqUzcruOMY3Fi48fepUlzf755fmpV9Mbwe7g0qtXeC+y0hzuVyjZ8wuRw09FQ5TpgFCyABXSAAAAFkbAMCu2f8Vkqqqyo8+/ND+jkuj0fSSMVpYUGCXEdQr/fkKMhEga//Gjes2duw4+zsulaol9czZ3jBGPT3722UEf/tYyi9AJgJk7d886eU1788v299xNTY0bNm0uTeM0WF+w+wygnrnzqUhE0Hn4bo2AACyNgAAIGsDAACyNgAAsjYAACBrAwAAsvaDOZ6cLAoLJyK1Wr1m1erq6mqzW8E6IWqArN3rMDqmRaUiopqamp2xsUUFhWa3gnVC1MAOoObfQ/Lz88vIyvT28UFXIGoAONfuehnpGXm5vy4AmC+RlF4pZR+fS03TP76UkbF7V9z1iutE1NzcfDQxSa1WnzieUl9f397efurEyUMJCVrdr6ufaLXarMysuro6IjLdCt3EKC5GUbtvoBE1wLm2zUg9e6a4qPhIUiIRrfpipZ+/37oNGxiGWRwVtXHL5pGjRn4XE7Nzx46xY8dtXL9+/8EEDw+PJdHRFy6cP3PqdNLx5B3bt8fvjQ8YMYJ9wxORSqVaEh29Jz7e29v7q7VrjbY+Hlqt9sC+/fYduHyJxPDHxoYGw7gcSkgwjFrgmDGWAz1o8KAej5r5s4qL6fJ6ede2efvWLSQ4ZG0bJhKJDx5IYHRMq7a1uqpKLpcTUcW1axqNRiAUNjY0bPtm67fbv584adKbCxfGbNv2yWefEVF5WXmOJE+lVu/eFffV+nWzZs9e9O675388b9iyXC63sLVbMQwjlV6278BVVlYOHjJY/6Mrl6uPi0ajMYra1pgYy4G2hqiZdePGDaaN6eIxP148cCCuBSFr26xwoUCj0chkssbGhrDw8Pz8/Lq6uhJpyajRoz09PS+cP69SqUJDw4goNDx8V+wOJ0cnIpr/2gI3d/fs7GytVisSiTgczpSpU43e4cVFRRa2disul7ty9Wr7DtzhQ4eyMjN/G68GccnNzTWK2n0DrVAoejxqZv35lb/8Yfp05CNA1v4Nj8fj8/lS6eXamtrRgYFqtVqSlyeVStmFYqurqj3793fv405Evr6+crm8tbWViIYMGUJEtTW1XC53gLc3Ebk4uxi1bHkrdAc2LmajZjnQiBrYgV40808oEpWVlhUVFQaHhDwdFCS9LC0rvSIWi4nI1dVVo1azy2+rVCpnZ2cnJyci4nA4ROTk7KTVanU6nfnPPYtboTuwcekoahYCjagBsrYtEYiEsvLystKy0NAQgVDwU0mJTHZVIBIS0eAhg1UqFTsnoaqy0tfX18Hht54ZNGgwwzA3b94kooaGBqNmLW+F7tNR1CwEGlEDZG1bIhKJ8nJzfXx83Nzdg4ODiwoLeTyeh4cHEYWFhw8cNDBh/4GmpqaTKSfmPP/8PeleKPD09Dx88KBcLo/f+4NOq+38Vug+HUXNQqARNUDWtiU+Awd68XgisZiI3Nzd/fz99dc6nZyc1m3YELdzpzA07KmhQ1//20LDHV1cXFauWb0nbve0yVNemDtXq9UyOqaTW6H7dBQ1C4FG1MAetFuHEqn0menTu7zZwoKC52bN7uSLdTqdvF7e0VaVSqVWqx9ua3dokMuDx45rt3eHDh5cEh390FGz7PFHzaw3Fi48fepUO0Dn4I723zg6OvZ/sn9HW7lcroV9LW+FnoqaZYga4AoJAAAgawMAQG/L2gqFYlzgmKzMTLMVlvUa5A3jAsfk5uTqn+no9SjKbOsQQUDWtnaqlhaGaTNbYdnoZW0GFSE6ej2KMts6RBBsVK/7NvJBKyyjIjMA4Fy7J+krLCuVyqOJSUqlMvnYsQP79rOFRwxlXrqUlZlpWJGZUJS5pz1inXREEHCubXv0dbH9+f5LoqPDBeFcLjc/P7+woGDt11/pX1ZZWblwwWvf74g1rKNNRFZVlLkXesQ66V5eXogg4FzbVnFduUQUFByye+/ed6MWpaSksHWIWF+v/fIP06dPmjzZcBe2KPPqtWsOJyU+HRyE0fP4iUTiKz//zOgYlUpVXVUlyZOQSZ30jZs3x/2wRyQWxWzbZliPm8/nI4KArG3DHJ0ciWhCxAQiGhEQoGppUSqV7KbCgsKLFy4sX/Gx0S5GRZkxeh4/ffns4qKisPBwjUZjWD67uLjYsOJ2Tla2YT1uRBBwhaSLMUxbU1NT17bZ3Ky8z/E7ORORq6srEbW1tbFP7ojdPsxvGI/HM3oxijL3uEepk44IArJ2l57zOzrW3b49bfKUrm1Wp9P5+fs/6F6z58z5V3x8RnrGxEkT7+msu0WZnZ2dMXR6Cls+u6Li2vwFrzEMw5bPfnX+AjKouM3hcEzrpCOCgKzdlUaNGiUpKuzyZosKCz/5eMWD7jV9xgyFQrFl06aIiRGGz+uLMvv7+6Moc08RiIRHE5OuXr0aGhri4MCJ/X67aZ10Ho9nWicdEQT7gDvazXvz72/lSySXMi7dky9QlNkKPHSddEQQkLXtWcDIgMhp07Zs2mQ4sQRFma3BQ9dJRwTBPnAMs5L9Ya+QJCUf68I21Wo1h8Nhv8DsQY0NDZGTpxRcLrbvAcqu0b5uw4bO78IwjKJJYaF8q5VE8Lc/7P72txdefBFrtEMnob72A0NRZit334rbiCDYNFwhAQBA1gYAAGRtAABA1gYAQNYGAABkbQAAQNYGALAl9j9fu6y0NLKri1JZgzamt9zUd+b0abuMoN4vdXUvvPgikhF0kp3fG9na2nqrttZug+fg4Ovra98DVKlUyuvr7f59yOPx2ArgAL09awMA2Blc1wYAQNYGAABkbQAAQNYGAEDWBgAAZG0AAEDWBgBA1gYAAGRtAABA1gYAQNYGAIDu8f/mOo3f9Tr27wAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<IPython.core.display.Image object>"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "Image(filename='../datasets/Figs/db_schema.png')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "collapsed": true,
    "jupyter": {
     "outputs_hidden": true
    }
   },
   "outputs": [],
   "source": [
    "class crawler:\n",
    "    def __init__(self,dbname):\n",
    "        self.con=sqlite.connect(dbname)\n",
    "        \n",
    "    def __del__(self):\n",
    "        self.con.close()\n",
    "\n",
    "    def dbcommit(self):\n",
    "        self.con.commit()\n",
    "\n",
    "    def createindextables(self):\n",
    "        self.con.execute('create table if not exists urllist(url)')\n",
    "        self.con.execute('create table if not exists wordlist(word)')\n",
    "        self.con.execute('create table if not exists wordlocation(urlid,wordid,location)')\n",
    "        self.con.execute('create table if not exists link(fromid integer,toid integer)')\n",
    "        self.con.execute('create table if not exists linkwords(wordid,linkid)')\n",
    "        self.con.execute('create index if not exists wordidx on wordlist(word)')\n",
    "        self.con.execute('create index if not exists urlidx on urllist(url)')\n",
    "        self.con.execute('create index if not exists wordurlidx on wordlocation(wordid)')\n",
    "        self.con.execute('create index if not exists urltoidx on link(toid)')\n",
    "        self.con.execute('create index if not exists urlfromidx on link(fromid)')\n",
    "        self.dbcommit()\n",
    "\n",
    "    def isindexed(self,url):\n",
    "        '''Verify whether url is already indexed'''\n",
    "        q = \"select rowid from urllist where url='{}'\"\n",
    "        u = self.con.execute(q.format(url)).fetchone()\n",
    "        if u != None:\n",
    "            q = \"select * from wordlocation where urlid={}\"\n",
    "            v = self.con.execute(q.format(u[0])).fetchone()\n",
    "            if v != None:\n",
    "                return True\n",
    "        return False        \n",
    "\n",
    "    def gettextonly(self,soup):\n",
    "        '''Extract raw text from html page'''\n",
    "        v = soup.string\n",
    "        if v == None:   \n",
    "            c = soup.contents\n",
    "            resulttext = ''\n",
    "            for t in c:\n",
    "                subtext = self.gettextonly(t)\n",
    "                resulttext += subtext+'\\n'\n",
    "            return resulttext\n",
    "        else:\n",
    "            return v.strip()\n",
    "\n",
    "    def separatewords(self,text):\n",
    "        '''splits the sentences by the non alpha characters\n",
    "        and converts all words to lowercase'''\n",
    "        splitter = re.compile('\\\\W*', flags=re.U)\n",
    "        return [s.lower() for s in splitter.split(text) if s != '']\n",
    "\n",
    "    def getentryid(self, table, field, value, createnew=True):\n",
    "        '''Add page id to the database, if not present'''\n",
    "        q = \"select rowid from {} where {}='{}'\"\n",
    "        cursor = self.con.execute(q.format(table,field,value))\n",
    "        result = cursor.fetchone()\n",
    "        if result == None:\n",
    "            q = \"insert into {} ({}) values ('{}')\"\n",
    "            cursor = self.con.execute(q.format(table,field,value))\n",
    "            return cursor.lastrowid\n",
    "        else:\n",
    "            return result[0]     \n",
    "\n",
    "    def addtoindex(self,url,sopa):\n",
    "        '''Add url to the index if not there'''\n",
    "        if self.isindexed(url): \n",
    "            print('Page {} already indexed...'.format(url))\n",
    "            return\n",
    "        print('Indexing: {}'.format(url))\n",
    "        text = self.gettextonly(sopa)\n",
    "        words = self.separatewords(text)\n",
    "        urlid = self.getentryid('urllist', 'url', url)\n",
    "        for i in range(len(words)):\n",
    "            word = words[i]\n",
    "            if word in ignorewords: \n",
    "                continue\n",
    "            wordid = self.getentryid('wordlist','word',word)\n",
    "            q = \"insert into wordlocation (urlid,wordid,location) values ({},{},{})\"\n",
    "            self.con.execute(q.format(urlid,wordid,i))\n",
    "        \n",
    "    def addlinkref(self,urlFrom,urlTo,linkText):\n",
    "        '''Add a link between two pages'''\n",
    "        words = self.separatewords(linkText)\n",
    "        fromid = self.getentryid('urllist','url', urlFrom)\n",
    "        toid = self.getentryid('urllist','url', urlTo)\n",
    "        if fromid == toid:\n",
    "            return\n",
    "        q = \"insert into link(fromid,toid) values ({},{})\"\n",
    "        cursor = self.con.execute(q.format(fromid,toid))\n",
    "        linkid = cursor.lastrowid\n",
    "        for word in words:\n",
    "            if word in ignorewords: \n",
    "                continue\n",
    "            wordid = self.getentryid('wordlist','word', word)\n",
    "            q = \"insert into linkwords(linkid,wordid) values ({},{})\"\n",
    "            self.con.execute(q.format(linkid,wordid))\n",
    "    \n",
    "    def crawl(self,pages,depth=1):\n",
    "        '''Starts indexing seed page(s), goes indexing all pages following\n",
    "        breadth first, until the desired depth'''\n",
    "        print('Seed URL(s)')        \n",
    "        for p in pages: print(p)\n",
    "        print('\\nIndexing from seed with depth of {}\\n'.format(depth))\n",
    "        for i in range(depth+1):\n",
    "            newpages = {}\n",
    "            for page in pages:\n",
    "                try:\n",
    "                    c = urllib.request.urlopen(page)\n",
    "                except:\n",
    "                    print(u'Could not access page: {}'.format(page))\n",
    "                    continue\n",
    "                try:\n",
    "                    p = c.read()\n",
    "                    soup = BeautifulSoup(p, \"lxml\")\n",
    "                    self.addtoindex(page,soup)\n",
    "                    links = soup('a')\n",
    "                    for link in links:\n",
    "                        if ('href' in dict(link.attrs)):\n",
    "                            url = urljoin(page,link['href'])\n",
    "                            if url.find(\"'\") != -1:\n",
    "                                continue\n",
    "                            url = url.split('#')[0]  #Keeps base url\n",
    "                            if url[0:4] == 'http' and not self.isindexed(url):\n",
    "                                newpages[url] = 1\n",
    "                            linkText = self.gettextonly(link)\n",
    "                            self.addlinkref(page,url,linkText)\n",
    "                    self.dbcommit()\n",
    "                except:\n",
    "                    print(\"Could not parse page {}\".format(page))\n",
    "                    raise\n",
    "            self.dbcommit()\n",
    "            pages = newpages\n",
    "\n",
    "    def calculatepagerank(self,iterations=20):\n",
    "        '''Initialize each url with pagerank = 1, and iterates until\n",
    "        it reaches the limit. Calculates pagerank with damping factor'''\n",
    "        self.con.execute('drop table if exists pagerank')\n",
    "        self.dbcommit()\n",
    "        self.con.execute('create table pagerank(urlid primary key,score)')\n",
    "        for (urlid,) in self.con.execute('select rowid from urllist'):\n",
    "            q = 'insert into pagerank(urlid,score) values ({},1.0)'\n",
    "            self.con.execute(q.format(urlid))\n",
    "        self.dbcommit()\n",
    "        for i in range(iterations):\n",
    "            print(\"Iteration {}\".format(i))\n",
    "            for (urlid,) in self.con.execute('select rowid from urllist'):\n",
    "                pr = 0.15\n",
    "                q1 = 'select distinct fromid from link where toid = {}'\n",
    "                for (linker,) in self.con.execute(q1.format(urlid)):\n",
    "                    q2 = 'select score from pagerank where urlid = {}'\n",
    "                    linkingpr = self.con.execute(q2.format(linker)).fetchone()[0]\n",
    "                    q3 = 'select count(*) from link where fromid = {}'\n",
    "                    linkingcount = self.con.execute(q3.format(linker)).fetchone()[0]\n",
    "                    pr += 0.85 * (linkingpr/linkingcount)\n",
    "                q4 = 'update pagerank set score = {} where urlid = {}'\n",
    "                self.con.execute(q4.format(pr,urlid))\n",
    "            self.dbcommit()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Second block of classes and functions: searching"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "collapsed": true,
    "jupyter": {
     "outputs_hidden": true
    }
   },
   "outputs": [],
   "source": [
    "class searcher:\n",
    "    def __init__(self,dbname):\n",
    "        self.con=sqlite.connect(dbname)\n",
    "\n",
    "    def __del__(self):\n",
    "        self.con.close()\n",
    "\n",
    "    def getmatchrows(self,q):\n",
    "        fieldlist='w0.urlid'\n",
    "        tablelist=''  \n",
    "        clauselist=''\n",
    "        wordids=[]\n",
    "        words=q.split(' ')  \n",
    "        tablenumber=0\n",
    "        for word in words:\n",
    "            q = \"select rowid from wordlist where word='{}'\"\n",
    "            wordrow = self.con.execute(q.format(word)).fetchone()\n",
    "            if wordrow != None:\n",
    "                wordid = wordrow[0]\n",
    "                wordids.append(wordid)\n",
    "                if tablenumber > 0:\n",
    "                    tablelist += ','\n",
    "                    clauselist += ' and '\n",
    "                    clauselist += 'w{}.urlid=w{}.urlid and '.format(tablenumber-1,tablenumber)\n",
    "                fieldlist += ',w{}.location'.format(tablenumber)\n",
    "                tablelist += 'wordlocation w{}'.format(tablenumber)   \n",
    "                clauselist += 'w{}.wordid={}'.format(tablenumber,wordid)\n",
    "                tablenumber += 1\n",
    "        fullquery = 'select {} from {} where {}'.format(fieldlist,tablelist,clauselist)\n",
    "        cursor = self.con.execute(fullquery)\n",
    "        rows = [row for row in cursor]\n",
    "        return rows, wordids\n",
    "\n",
    "    def getscoredlist(self,rows,wordids):\n",
    "        totalscores = dict([(row[0],0) for row in rows])\n",
    "        weights=[(1.0,self.frequencyscore(rows)),\n",
    "                 (1.0,self.locationscore(rows)), \n",
    "                 (1.0,self.distancescore(rows)),\n",
    "                 (1.0,self.inboundlinkscore(rows)),\n",
    "                 (1.0,self.linktextscore(rows,wordids)),\n",
    "                 (1.0,self.pagerankscore(rows))]\n",
    "        for (weight,scores) in weights:\n",
    "            for url in totalscores:\n",
    "                totalscores[url] += weight*scores[url]\n",
    "        return totalscores\n",
    "\n",
    "    def geturlname(self,id):\n",
    "        q = \"select url from urllist where rowid = {}\"\n",
    "        return self.con.execute(q.format(id)).fetchone()[0]\n",
    "\n",
    "    def query(self,q):\n",
    "        try:\n",
    "            rows,wordids = self.getmatchrows(q)\n",
    "        except:\n",
    "            print('No results in the database...')\n",
    "            return\n",
    "        scores = self.getscoredlist(rows,wordids)\n",
    "        rankedscores = [(score,url) for (url,score) in scores.items()]\n",
    "        rankedscores.sort()\n",
    "        rankedscores.reverse()\n",
    "        print('\\nResultados para busca por {}:'.format(q))\n",
    "        for (score,urlid) in rankedscores[0:10]:\n",
    "            print('{}\\t{}'.format(score, self.geturlname(urlid)))\n",
    "        return wordids,[r[1] for r in rankedscores[0:10]]\n",
    "\n",
    "    def normalizescores(self,scores,smallIsBetter=0):\n",
    "        vsmall=0.00001 # Avoiding division by zero\n",
    "        if smallIsBetter:\n",
    "            minscore = min(scores.values())\n",
    "            return dict([(u,float(minscore)/max(vsmall,l)) for (u,l) in scores.items()])\n",
    "        else:\n",
    "            maxscore = max(scores.values())\n",
    "            if maxscore == 0:\n",
    "                maxscore = vsmall\n",
    "            return dict([(u,float(c)/maxscore) for (u,c) in scores.items()])\n",
    "\n",
    "    def frequencyscore(self,rows):\n",
    "        counts = dict([(row[0],0) for row in rows])\n",
    "        for row in rows:\n",
    "            counts[row[0]] += 1\n",
    "        return self.normalizescores(counts)\n",
    "\n",
    "    def locationscore(self,rows):\n",
    "        locations = dict([(row[0],1000000) for row in rows])\n",
    "        for row in rows:\n",
    "            loc = sum(row[1:])\n",
    "            if loc < locations[row[0]]:\n",
    "                locations[row[0]] = loc\n",
    "        return self.normalizescores(locations,smallIsBetter=1)\n",
    "\n",
    "    def distancescore(self,rows):\n",
    "        if len(rows[0]) <= 2: \n",
    "            return dict([(row[0],1.0) for row in rows])\n",
    "        mindistance = dict([(row[0],1000000) for row in rows])\n",
    "        for row in rows:\n",
    "            dist = sum([abs(row[i]-row[i-1]) for i in range(1,len(row))])\n",
    "            if dist < mindistance[row[0]]:\n",
    "                mindistance[row[0]]=dist\n",
    "        return self.normalizescores(mindistance,smallIsBetter=1)\n",
    "\n",
    "    def inboundlinkscore(self,rows):\n",
    "        uniqueurls = dict([(row[0],1) for row in rows])\n",
    "        q = 'select count(*) from link where toid = {}'\n",
    "        inboundcount = dict([(u,self.con.execute(q.format(u)).fetchone()[0]) for u in uniqueurls])   \n",
    "        return self.normalizescores(inboundcount)\n",
    "\n",
    "    def linktextscore(self,rows,wordids):\n",
    "        linkscores = dict([(row[0],0) for row in rows])\n",
    "        for wordid in wordids:\n",
    "            q = 'select link.fromid,link.toid from linkwords,'\n",
    "            q += 'link where wordid={} and linkwords.linkid=link.rowid'\n",
    "            cursor = self.con.execute(q.format(wordid))\n",
    "            for (fromid,toid) in cursor:\n",
    "                if toid in linkscores:\n",
    "                    q = 'select score from pagerank where urlid={}'\n",
    "                    pr=self.con.execute(q.format(fromid)).fetchone()[0]\n",
    "                    linkscores[toid] += pr\n",
    "        maxscore = max(linkscores.values())\n",
    "        normalizedscores = dict([(u,float(l)/maxscore) for (u,l) in linkscores.items()])\n",
    "        return normalizedscores\n",
    "\n",
    "    def pagerankscore(self,rows):\n",
    "        q = 'select score from pagerank where urlid={}'\n",
    "        pageranks=dict([(row[0],self.con.execute(q.format(row[0])).fetchone()[0]) for row in rows])\n",
    "        maxrank = max(pageranks.values())\n",
    "        normalizedscores = dict([(u,float(l)/maxrank) for (u,l) in pageranks.items()])\n",
    "        return normalizedscores"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Defining seed pages"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "collapsed": true,
    "jupyter": {
     "outputs_hidden": true
    }
   },
   "outputs": [],
   "source": [
    "seed1 = ['http://www.oglobo.com/']\n",
    "seed2 = ['http://emap.fgv.br/']"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Instantiating the crawler"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "collapsed": true,
    "jupyter": {
     "outputs_hidden": true
    }
   },
   "outputs": [],
   "source": [
    "crawl = crawler(db)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Creating tables - needed only in the first time"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "collapsed": true,
    "jupyter": {
     "outputs_hidden": true
    }
   },
   "outputs": [],
   "source": [
    "crawl.createindextables()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Crawling to a specific depth level"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Seed URL(s)\n",
      "http://emap.fgv.br/\n",
      "\n",
      "Indexing from seed with depth of 1\n",
      "\n",
      "Indexing: http://emap.fgv.br/\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/usr/local/lib/python3.6/dist-packages/ipykernel/__main__.py:52: FutureWarning: split() requires a non-empty pattern match.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Indexing: http://emap.fgv.br/quem-somos\n",
      "Indexing: http://emap.fgv.br/institucional\n",
      "Indexing: http://emap.fgv.br/estrutura-organizacional\n",
      "Indexing: http://emap.fgv.br/comissao-propria-de-avaliacao\n",
      "Indexing: http://emap.fgv.br/pessoas\n",
      "Indexing: http://emap.fgv.br/noticias\n",
      "Indexing: http://emap.fgv.br/graduacao/apresentacao\n",
      "Indexing: http://emap.fgv.br/vestibular\n",
      "Indexing: http://emap.fgv.br/graduacao/grade-curricular\n",
      "Indexing: http://emap.fgv.br/horarios-calendarios\n",
      "Indexing: http://emap.fgv.br/graduacao/corpo-docente\n",
      "Indexing: http://emap.fgv.br/transferencia-portadores-de-diploma\n",
      "Indexing: http://emap.fgv.br/colocacao-profissional\n",
      "Indexing: http://emap.fgv.br/nucleo-de-apoio-pedagogico-nap\n",
      "Indexing: http://emap.fgv.br/academia-matematica\n",
      "Indexing: http://emap.fgv.br/mestrado/modelagem-matematica\n",
      "Indexing: http://emap.fgv.br/mestrado/disciplinas\n",
      "Indexing: http://emap.fgv.br/mestrado/corpo-docente\n",
      "Indexing: http://emap.fgv.br/processo-seletivo\n",
      "Indexing: http://emap.fgv.br/mestrado/dissertacoes\n",
      "Indexing: http://emap.fgv.br/curso-de-verao\n",
      "Indexing: http://emap.fgv.br/projetos-de-pesquisa-aplicada\n",
      "Indexing: http://emap.fgv.br/linhas-de-pesquisa\n",
      "Indexing: http://emap.fgv.br/publicacoes\n",
      "Indexing: http://emap.fgv.br/projetos-de-pesquisa-academica\n",
      "Indexing: http://emap.fgv.br/pesquisa-aplicada-andamento\n",
      "Indexing: http://emap.fgv.br/pesquisa-academica-andamento\n",
      "Indexing: http://emap.fgv.br/eventos\n",
      "Indexing: http://emap.fgv.br/seminarios\n",
      "Indexing: http://emap.fgv.br/cursos\n",
      "Indexing: http://emap.fgv.br/defesa-de-dissertacao\n",
      "Indexing: http://emap.fgv.br/contato\n",
      "Indexing: http://vestibular.fgv.br/dia-fgv-rj\n",
      "Indexing: http://emap.fgv.br/postdoctoral-position-applied-mathematics\n",
      "Indexing: http://vestibular.fgv.br/avise-me-proximo-vestibular-fgv\n",
      "Indexing: http://emap.fgv.br/mestrado/apresentacao\n",
      "Indexing: http://emap.fgv.br/noticias/fgv-emap-organiza-competicao-financas-quantitativas-para-alunos-de-mestrado-inscreva-se\n",
      "Indexing: http://emap.fgv.br/noticias/workshop-data-science-revolution-finance-ocorrera-dia-2411\n",
      "Indexing: http://emap.fgv.br/noticias/cnmai-comeca-segunda-feira-dia-2310\n",
      "Indexing: https://www.facebook.com/emap.fgv\n",
      "Indexing: https://twitter.com/fgv\n",
      "Indexing: https://www.youtube.com/channel/UC8DhN1cJi0QklBQot5hv_kg\n",
      "Indexing: https://eclass.fgv.br/\n",
      "Indexing: https://www.google.com/maps?ll=-22.940808,-43.180153&spn=0.011441,0.01929&cid=3352914085499141244&t=m&z=16&vpsrc=0\n",
      "Indexing: http://www.fgv.br/politica-privacidade/sites-fgv/politica-de-privacidade.pdf\n",
      "Indexing: http://portal.fgv.br\n",
      "Indexing: http://portal.fgv.br/en\n",
      "Indexing: http://portal.fgv.br/educacao/graduacao\n",
      "Indexing: http://graduacaotecnologica.fgv.br/\n",
      "Indexing: http://portal.fgv.br/educacao/mestrado-doutorado\n",
      "Indexing: http://portal.fgv.br/educacao/mba-pos\n",
      "Indexing: http://portal.fgv.br/educacao/cursos-extensao\n",
      "Indexing: http://incompany.fgv.br/\n",
      "Indexing: http://www5.fgv.br/fgvonline/\n",
      "Indexing: http://vestibular.fgv.br\n",
      "Indexing: http://ensinomediodigital.fgv.br/\n",
      "Indexing: http://portal.fgv.br/livros\n",
      "Indexing: http://editora.fgv.br\n",
      "Indexing: http://bibliotecadigital.fgv.br/\n",
      "Indexing: http://cpdoc.fgv.br\n",
      "Indexing: http://direitorio.fgv.br\n",
      "Indexing: http://direitosp.fgv.br\n",
      "Indexing: http://eaesp.fgvsp.br\n",
      "Indexing: http://ebape.fgv.br\n",
      "Indexing: http://eesp.fgv.br\n",
      "Indexing: http://emap.fgv.br\n",
      "Indexing: http://epge.fgv.br\n",
      "Indexing: https://portal.fgv.br/pesquisas-teses-publicacoes\n",
      "Indexing: http://fgvprojetos.fgv.br/\n",
      "Indexing: http://portalibre.fgv.br\n",
      "Indexing: http://ceri.fgv.br\n",
      "Indexing: http://dapp.fgv.br\n",
      "Indexing: http://dint.fgv.br\n",
      "Indexing: http://fgvenergia.fgv.br\n",
      "Indexing: http://cps.fgv.br\n",
      "Indexing: http://portal.fgv.br/areas/administracao\n",
      "Indexing: http://portal.fgv.br/areas/ciencias-sociais\n",
      "Indexing: http://portal.fgv.br/areas/direito\n",
      "Indexing: http://portal.fgv.br/areas/economia\n",
      "Indexing: http://portal.fgv.br/areas/matematica\n",
      "Indexing: https://www.facebook.com/Vestibular.FGV/\n",
      "Could not access page: https://www.linkedin.com/school/8175?pathWildcard=8175\n",
      "Indexing: https://www.instagram.com/fgv.oficial/\n",
      "Indexing: https://plus.google.com/+FGV\n",
      "Indexing: https://www.youtube.com/user/fgvtv\n",
      "Indexing: http://portal.fgv.br/noticias\n",
      "Indexing: http://portal.fgv.br/eventos\n"
     ]
    }
   ],
   "source": [
    "crawl.crawl(seed2,1)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Calculates pagerank with n iterations"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Iteration 0\n",
      "Iteration 1\n",
      "Iteration 2\n",
      "Iteration 3\n",
      "Iteration 4\n",
      "Iteration 5\n",
      "Iteration 6\n",
      "Iteration 7\n",
      "Iteration 8\n",
      "Iteration 9\n",
      "Iteration 10\n",
      "Iteration 11\n",
      "Iteration 12\n",
      "Iteration 13\n",
      "Iteration 14\n",
      "Iteration 15\n",
      "Iteration 16\n",
      "Iteration 17\n",
      "Iteration 18\n",
      "Iteration 19\n",
      "Iteration 20\n",
      "Iteration 21\n",
      "Iteration 22\n",
      "Iteration 23\n",
      "Iteration 24\n"
     ]
    }
   ],
   "source": [
    "crawl.calculatepagerank(25)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Instantiating the searcher"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {
    "collapsed": true,
    "jupyter": {
     "outputs_hidden": true
    }
   },
   "outputs": [],
   "source": [
    "search = searcher(db)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Querying the index"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Resultados para busca por matemática:\n",
      "5.349990613415032\thttp://emap.fgv.br/graduacao/apresentacao\n",
      "5.275601205098057\thttp://emap.fgv.br/mestrado/modelagem-matematica\n",
      "4.395477485408656\thttp://emap.fgv.br/projetos-de-pesquisa-aplicada\n",
      "4.361390773371286\thttp://emap.fgv.br/quem-somos\n",
      "4.325050670951551\thttp://emap.fgv.br/\n",
      "4.082276804822115\thttp://emap.fgv.br/linhas-de-pesquisa\n",
      "3.968000123200454\thttp://emap.fgv.br\n",
      "3.828234198370277\thttp://emap.fgv.br/processo-seletivo\n",
      "3.797541474954284\thttp://emap.fgv.br/graduacao/grade-curricular\n",
      "3.7771173077403812\thttp://emap.fgv.br/curso-de-verao\n",
      "([26], [8, 17, 23, 2, 1, 24, 67, 20, 10, 22])\n"
     ]
    }
   ],
   "source": [
    "query = 'matemática'\n",
    "print(search.query(query))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {
    "collapsed": true,
    "jupyter": {
     "outputs_hidden": true
    }
   },
   "outputs": [],
   "source": [
    "conn=sqlite.connect(db)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>url</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>http://emap.fgv.br/</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>http://emap.fgv.br/quem-somos</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>http://emap.fgv.br/institucional</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>http://emap.fgv.br/estrutura-organizacional</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>http://emap.fgv.br/comissao-propria-de-avaliacao</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                                url\n",
       "0                               http://emap.fgv.br/\n",
       "1                     http://emap.fgv.br/quem-somos\n",
       "2                  http://emap.fgv.br/institucional\n",
       "3       http://emap.fgv.br/estrutura-organizacional\n",
       "4  http://emap.fgv.br/comissao-propria-de-avaliacao"
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_mysql1 = psql.read_sql('select * from urllist;', con=conn)\n",
    "df_mysql1.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "url    http://emap.fgv.br/\n",
       "Name: 0, dtype: object"
      ]
     },
     "execution_count": 21,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_mysql1.loc[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>fromid</th>\n",
       "      <th>toid</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>1</td>\n",
       "      <td>3</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>1</td>\n",
       "      <td>4</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>1</td>\n",
       "      <td>5</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   fromid  toid\n",
       "0       1     2\n",
       "1       1     2\n",
       "2       1     3\n",
       "3       1     4\n",
       "4       1     5"
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_mysql2 = psql.read_sql('select * from link;', con=conn)\n",
    "df_mysql2.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Link entre as páginas http://emap.fgv.br/quem-somos --> http://emap.fgv.br/institucional\n",
      "Link entre as páginas http://emap.fgv.br/quem-somos --> http://emap.fgv.br/institucional\n",
      "Link entre as páginas http://emap.fgv.br/quem-somos --> http://emap.fgv.br/estrutura-organizacional\n",
      "Link entre as páginas http://emap.fgv.br/quem-somos --> http://emap.fgv.br/comissao-propria-de-avaliacao\n",
      "Link entre as páginas http://emap.fgv.br/quem-somos --> http://emap.fgv.br/pessoas\n",
      "Link entre as páginas http://emap.fgv.br/quem-somos --> http://emap.fgv.br/noticias\n",
      "Link entre as páginas http://emap.fgv.br/quem-somos --> http://emap.fgv.br/graduacao/apresentacao\n",
      "Link entre as páginas http://emap.fgv.br/quem-somos --> http://emap.fgv.br/vestibular\n",
      "Link entre as páginas http://emap.fgv.br/quem-somos --> http://emap.fgv.br/vestibular\n",
      "Link entre as páginas http://emap.fgv.br/quem-somos --> http://emap.fgv.br/graduacao/grade-curricular\n"
     ]
    }
   ],
   "source": [
    "for i in range(10):\n",
    "    print('Link entre as páginas {} --> {}'.format(df_mysql1['url'].loc[df_mysql2['fromid'].loc[i]],\n",
    "                                                   df_mysql1['url'].loc[df_mysql2['toid'].loc[i]]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {
    "collapsed": true,
    "jupyter": {
     "outputs_hidden": true
    }
   },
   "outputs": [],
   "source": [
    "conn.close()"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}