data_prep.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "%matplotlib inline\n",
    "\n",
    "import pandas as pd\n",
    "import seaborn as sns\n",
    "import numpy as np\n",
    "import matplotlib.pyplot as plt\n",
    "import _pickle as cPickle\n",
    "import nltk"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "df = pd.read_csv('lsapp.tsv', sep='\\t')\n",
    "df_open = df.loc[df['event_type'] == 'Opened']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/maliann/anaconda3/lib/python3.7/site-packages/pandas/core/indexing.py:494: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: http://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "  self.obj[item] = s\n"
     ]
    }
   ],
   "source": [
    "df_open.loc[:,'timestamp'] = pd.to_datetime(df_open['timestamp'])\n",
    "df.loc[:,'timestamp'] = pd.to_datetime(df['timestamp'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>user_id</th>\n",
       "      <th>session_id</th>\n",
       "      <th>timestamp</th>\n",
       "      <th>app_name</th>\n",
       "      <th>event_type</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>2018-01-16 06:01:05</td>\n",
       "      <td>Minesweeper Classic (Mines)</td>\n",
       "      <td>Opened</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>2018-01-16 06:01:05</td>\n",
       "      <td>Minesweeper Classic (Mines)</td>\n",
       "      <td>Closed</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>2018-01-16 06:01:07</td>\n",
       "      <td>Minesweeper Classic (Mines)</td>\n",
       "      <td>Opened</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>2018-01-16 06:01:07</td>\n",
       "      <td>Minesweeper Classic (Mines)</td>\n",
       "      <td>Closed</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>2018-01-16 06:01:08</td>\n",
       "      <td>Minesweeper Classic (Mines)</td>\n",
       "      <td>Opened</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3658584</th>\n",
       "      <td>291</td>\n",
       "      <td>76247</td>\n",
       "      <td>2018-04-06 14:35:15</td>\n",
       "      <td>Facebook</td>\n",
       "      <td>Closed</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3658585</th>\n",
       "      <td>291</td>\n",
       "      <td>76247</td>\n",
       "      <td>2018-04-06 14:35:15</td>\n",
       "      <td>Facebook</td>\n",
       "      <td>Opened</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3658586</th>\n",
       "      <td>291</td>\n",
       "      <td>76247</td>\n",
       "      <td>2018-04-06 14:35:37</td>\n",
       "      <td>Facebook</td>\n",
       "      <td>Closed</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3658587</th>\n",
       "      <td>291</td>\n",
       "      <td>76247</td>\n",
       "      <td>2018-04-06 14:35:37</td>\n",
       "      <td>Facebook</td>\n",
       "      <td>Opened</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3658588</th>\n",
       "      <td>291</td>\n",
       "      <td>76247</td>\n",
       "      <td>2018-04-06 14:35:52</td>\n",
       "      <td>Facebook Messenger</td>\n",
       "      <td>User Interaction</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>3658589 rows × 5 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "         user_id  session_id           timestamp                     app_name  \\\n",
       "0              0           1 2018-01-16 06:01:05  Minesweeper Classic (Mines)   \n",
       "1              0           1 2018-01-16 06:01:05  Minesweeper Classic (Mines)   \n",
       "2              0           1 2018-01-16 06:01:07  Minesweeper Classic (Mines)   \n",
       "3              0           1 2018-01-16 06:01:07  Minesweeper Classic (Mines)   \n",
       "4              0           1 2018-01-16 06:01:08  Minesweeper Classic (Mines)   \n",
       "...          ...         ...                 ...                          ...   \n",
       "3658584      291       76247 2018-04-06 14:35:15                     Facebook   \n",
       "3658585      291       76247 2018-04-06 14:35:15                     Facebook   \n",
       "3658586      291       76247 2018-04-06 14:35:37                     Facebook   \n",
       "3658587      291       76247 2018-04-06 14:35:37                     Facebook   \n",
       "3658588      291       76247 2018-04-06 14:35:52           Facebook Messenger   \n",
       "\n",
       "               event_type  \n",
       "0                  Opened  \n",
       "1                  Closed  \n",
       "2                  Opened  \n",
       "3                  Closed  \n",
       "4                  Opened  \n",
       "...                   ...  \n",
       "3658584            Closed  \n",
       "3658585            Opened  \n",
       "3658586            Closed  \n",
       "3658587            Opened  \n",
       "3658588  User Interaction  \n",
       "\n",
       "[3658589 rows x 5 columns]"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/maliann/anaconda3/lib/python3.7/site-packages/pandas/core/indexing.py:376: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: http://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "  self.obj[key] = _infer_fill_value(value)\n"
     ]
    }
   ],
   "source": [
    "df['time_dff']  = df[['timestamp']].diff()\n",
    "df['interaction_id'] = (((df.timestamp-df.timestamp.shift(1) > pd.Timedelta(1, 'm')) \n",
    "                              & (df.event_type == 'Opened')) \n",
    "                              | ~(df.app_name == df.app_name.shift(1))\n",
    "                              | ~(df.user_id == df.user_id.shift(1))).cumsum()\n",
    "df['session_id'] = (((df.timestamp-df.timestamp.shift(1) > pd.Timedelta(5, 'm')) \n",
    "                              & (df.event_type == 'Opened')) \n",
    "                              | ~(df.user_id == df.user_id.shift(1))).cumsum()\n",
    "df_start = df.drop_duplicates(subset=['interaction_id'], keep='first')\n",
    "df_end = df.drop_duplicates(subset=['interaction_id'], keep='last')\n",
    "df_start.set_index('interaction_id', inplace=True)\n",
    "df_end.set_index('interaction_id', inplace=True)\n",
    "df_start.loc[:, 'open_time'] = df_start['timestamp']\n",
    "df_start.loc[:, 'close_time']  = df_end['timestamp']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>user_id</th>\n",
       "      <th>session_id</th>\n",
       "      <th>timestamp</th>\n",
       "      <th>app_name</th>\n",
       "      <th>event_type</th>\n",
       "      <th>time_dff</th>\n",
       "      <th>open_time</th>\n",
       "      <th>close_time</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>interaction_id</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>2018-01-16 06:01:05</td>\n",
       "      <td>Minesweeper Classic (Mines)</td>\n",
       "      <td>Opened</td>\n",
       "      <td>NaT</td>\n",
       "      <td>2018-01-16 06:01:05</td>\n",
       "      <td>2018-01-16 06:01:09</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>2018-01-16 06:03:44</td>\n",
       "      <td>Minesweeper Classic (Mines)</td>\n",
       "      <td>Opened</td>\n",
       "      <td>00:02:35</td>\n",
       "      <td>2018-01-16 06:03:44</td>\n",
       "      <td>2018-01-16 06:04:17</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>2018-01-16 06:25:54</td>\n",
       "      <td>Gmail</td>\n",
       "      <td>User Interaction</td>\n",
       "      <td>00:21:37</td>\n",
       "      <td>2018-01-16 06:25:54</td>\n",
       "      <td>2018-01-16 06:25:54</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>2018-01-16 06:26:05</td>\n",
       "      <td>Google</td>\n",
       "      <td>Opened</td>\n",
       "      <td>00:00:11</td>\n",
       "      <td>2018-01-16 06:26:05</td>\n",
       "      <td>2018-01-16 06:26:10</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>2018-01-16 06:26:10</td>\n",
       "      <td>Instagram</td>\n",
       "      <td>Opened</td>\n",
       "      <td>00:00:00</td>\n",
       "      <td>2018-01-16 06:26:10</td>\n",
       "      <td>2018-01-16 06:26:21</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>599631</th>\n",
       "      <td>291</td>\n",
       "      <td>33218</td>\n",
       "      <td>2018-04-06 12:47:28</td>\n",
       "      <td>Facebook Messenger</td>\n",
       "      <td>Opened</td>\n",
       "      <td>00:03:29</td>\n",
       "      <td>2018-04-06 12:47:28</td>\n",
       "      <td>2018-04-06 12:53:13</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>599632</th>\n",
       "      <td>291</td>\n",
       "      <td>33219</td>\n",
       "      <td>2018-04-06 13:20:12</td>\n",
       "      <td>Settings</td>\n",
       "      <td>Opened</td>\n",
       "      <td>00:26:59</td>\n",
       "      <td>2018-04-06 13:20:12</td>\n",
       "      <td>2018-04-06 13:20:14</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>599633</th>\n",
       "      <td>291</td>\n",
       "      <td>33220</td>\n",
       "      <td>2018-04-06 14:34:15</td>\n",
       "      <td>Settings</td>\n",
       "      <td>Opened</td>\n",
       "      <td>01:14:01</td>\n",
       "      <td>2018-04-06 14:34:15</td>\n",
       "      <td>2018-04-06 14:34:17</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>599634</th>\n",
       "      <td>291</td>\n",
       "      <td>33220</td>\n",
       "      <td>2018-04-06 14:34:34</td>\n",
       "      <td>Facebook</td>\n",
       "      <td>Opened</td>\n",
       "      <td>00:00:17</td>\n",
       "      <td>2018-04-06 14:34:34</td>\n",
       "      <td>2018-04-06 14:35:37</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>599635</th>\n",
       "      <td>291</td>\n",
       "      <td>33220</td>\n",
       "      <td>2018-04-06 14:35:52</td>\n",
       "      <td>Facebook Messenger</td>\n",
       "      <td>User Interaction</td>\n",
       "      <td>00:00:15</td>\n",
       "      <td>2018-04-06 14:35:52</td>\n",
       "      <td>2018-04-06 14:35:52</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>599635 rows × 8 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                user_id  session_id           timestamp  \\\n",
       "interaction_id                                            \n",
       "1                     0           1 2018-01-16 06:01:05   \n",
       "2                     0           1 2018-01-16 06:03:44   \n",
       "3                     0           1 2018-01-16 06:25:54   \n",
       "4                     0           1 2018-01-16 06:26:05   \n",
       "5                     0           1 2018-01-16 06:26:10   \n",
       "...                 ...         ...                 ...   \n",
       "599631              291       33218 2018-04-06 12:47:28   \n",
       "599632              291       33219 2018-04-06 13:20:12   \n",
       "599633              291       33220 2018-04-06 14:34:15   \n",
       "599634              291       33220 2018-04-06 14:34:34   \n",
       "599635              291       33220 2018-04-06 14:35:52   \n",
       "\n",
       "                                   app_name        event_type time_dff  \\\n",
       "interaction_id                                                           \n",
       "1               Minesweeper Classic (Mines)            Opened      NaT   \n",
       "2               Minesweeper Classic (Mines)            Opened 00:02:35   \n",
       "3                                     Gmail  User Interaction 00:21:37   \n",
       "4                                    Google            Opened 00:00:11   \n",
       "5                                 Instagram            Opened 00:00:00   \n",
       "...                                     ...               ...      ...   \n",
       "599631                   Facebook Messenger            Opened 00:03:29   \n",
       "599632                             Settings            Opened 00:26:59   \n",
       "599633                             Settings            Opened 01:14:01   \n",
       "599634                             Facebook            Opened 00:00:17   \n",
       "599635                   Facebook Messenger  User Interaction 00:00:15   \n",
       "\n",
       "                         open_time          close_time  \n",
       "interaction_id                                          \n",
       "1              2018-01-16 06:01:05 2018-01-16 06:01:09  \n",
       "2              2018-01-16 06:03:44 2018-01-16 06:04:17  \n",
       "3              2018-01-16 06:25:54 2018-01-16 06:25:54  \n",
       "4              2018-01-16 06:26:05 2018-01-16 06:26:10  \n",
       "5              2018-01-16 06:26:10 2018-01-16 06:26:21  \n",
       "...                            ...                 ...  \n",
       "599631         2018-04-06 12:47:28 2018-04-06 12:53:13  \n",
       "599632         2018-04-06 13:20:12 2018-04-06 13:20:14  \n",
       "599633         2018-04-06 14:34:15 2018-04-06 14:34:17  \n",
       "599634         2018-04-06 14:34:34 2018-04-06 14:35:37  \n",
       "599635         2018-04-06 14:35:52 2018-04-06 14:35:52  \n",
       "\n",
       "[599635 rows x 8 columns]"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_start"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}