1_Data_Preprocessing.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "WmyzGg7qyYb1"
   },
   "source": [
    "# Importing Libraries"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-08-04T20:05:24.549445Z",
     "start_time": "2020-08-04T20:05:23.851738Z"
    },
    "colab": {},
    "colab_type": "code",
    "id": "E2rgQMvCyYb3"
   },
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import pandas as pd\n",
    "\n",
    "import os\n",
    "\n",
    "import matplotlib.pyplot as plt\n",
    "import email\n",
    "import re\n",
    "from datetime import datetime as dt\n",
    "# import sns"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "- emails.csv\n",
    "- preprocessed_data.csv\n",
    "- vectorized_folders_df\n",
    "- vectorized_folders_df_w_label\n",
    "- preprocessed_data_v2\n",
    "- NEW_vectorized_folders_df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-08-04T20:05:33.222196Z",
     "start_time": "2020-08-04T20:05:33.219195Z"
    }
   },
   "outputs": [],
   "source": [
    "dataset_directory = \"../Datasets/\""
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Loading Data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-08-04T18:31:24.109445Z",
     "start_time": "2020-08-04T18:31:10.819739Z"
    },
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 202
    },
    "colab_type": "code",
    "executionInfo": {
     "elapsed": 56296,
     "status": "ok",
     "timestamp": 1557214469199,
     "user": {
      "displayName": "Arjun Patel",
      "photoUrl": "https://lh5.googleusercontent.com/-BlLPIwYkZoQ/AAAAAAAAAAI/AAAAAAAABy0/fGnMWZAEyF0/s64/photo.jpg",
      "userId": "15471468418524645217"
     },
     "user_tz": 420
    },
    "id": "aMCePl4bUSq2",
    "outputId": "3580a455-a605-4591-ba8d-e479a60c8140"
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>file</th>\n",
       "      <th>message</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>allen-p/_sent_mail/1.</td>\n",
       "      <td>Message-ID: &lt;18782981.1075855378110.JavaMail.e...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>allen-p/_sent_mail/10.</td>\n",
       "      <td>Message-ID: &lt;15464986.1075855378456.JavaMail.e...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>allen-p/_sent_mail/100.</td>\n",
       "      <td>Message-ID: &lt;24216240.1075855687451.JavaMail.e...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>allen-p/_sent_mail/1000.</td>\n",
       "      <td>Message-ID: &lt;13505866.1075863688222.JavaMail.e...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>allen-p/_sent_mail/1001.</td>\n",
       "      <td>Message-ID: &lt;30922949.1075863688243.JavaMail.e...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                       file                                            message\n",
       "0     allen-p/_sent_mail/1.  Message-ID: <18782981.1075855378110.JavaMail.e...\n",
       "1    allen-p/_sent_mail/10.  Message-ID: <15464986.1075855378456.JavaMail.e...\n",
       "2   allen-p/_sent_mail/100.  Message-ID: <24216240.1075855687451.JavaMail.e...\n",
       "3  allen-p/_sent_mail/1000.  Message-ID: <13505866.1075863688222.JavaMail.e...\n",
       "4  allen-p/_sent_mail/1001.  Message-ID: <30922949.1075863688243.JavaMail.e..."
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "dataset_OG = pd.read_csv(os.path.join(dataset_directory, 'emails.csv'))\n",
    "\n",
    "data = dataset_OG.copy() ## keeping a safe copy\n",
    "data.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-08-04T18:31:24.115437Z",
     "start_time": "2020-08-04T18:31:24.110436Z"
    },
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 35
    },
    "colab_type": "code",
    "executionInfo": {
     "elapsed": 56277,
     "status": "ok",
     "timestamp": 1557214469202,
     "user": {
      "displayName": "Arjun Patel",
      "photoUrl": "https://lh5.googleusercontent.com/-BlLPIwYkZoQ/AAAAAAAAAAI/AAAAAAAABy0/fGnMWZAEyF0/s64/photo.jpg",
      "userId": "15471468418524645217"
     },
     "user_tz": 420
    },
    "id": "q9JZVe_yeap3",
    "outputId": "ceacd587-0d87-45e6-eeab-71e7701ec250"
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(517401, 2)"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data.shape"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "xVmJF-pyyYcC"
   },
   "source": [
    "# Extracting Features \n",
    "\n",
    "This functions we're taken from Dondald's E-mail categorization notebook in Kaggle. You can find it <a href=\"https://www.kaggle.com/donaldtfung/e-mail-categorization/notebook?fbclid=IwAR3gqVcP7ZUpKlyqVXkaXYK0SOoZqrEfjxpBLT3VXnCzARk_RYiwsBFQIws\">here.</a>"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "LdScXVlbyYcD"
   },
   "source": [
    "### Extracting Headers"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-08-04T18:31:24.123439Z",
     "start_time": "2020-08-04T18:31:24.117447Z"
    }
   },
   "outputs": [],
   "source": [
    "def insert_value(dictionary, key, value):\n",
    "    if key in dictionary:\n",
    "        values = dictionary.get(key)\n",
    "        values.append(value)\n",
    "        dictionary[key] = values\n",
    "    else:\n",
    "        dictionary[key] = [value]\n",
    "    return dictionary\n",
    "\n",
    "def get_headers(df, header_names):\n",
    "    headers = {}\n",
    "    messages = df[\"message\"]\n",
    "    for message in messages:\n",
    "        e = email.message_from_string(message)\n",
    "        for item in header_names:\n",
    "            header = e.get(item)\n",
    "            insert_value(dictionary = headers, key = item, value = header) \n",
    "    print(\"Successfully retrieved header information!\")\n",
    "    return headers"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-08-04T18:32:43.449941Z",
     "start_time": "2020-08-04T18:31:24.124439Z"
    },
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 690
    },
    "colab_type": "code",
    "executionInfo": {
     "elapsed": 156906,
     "status": "ok",
     "timestamp": 1557214569854,
     "user": {
      "displayName": "Arjun Patel",
      "photoUrl": "https://lh5.googleusercontent.com/-BlLPIwYkZoQ/AAAAAAAAAAI/AAAAAAAABy0/fGnMWZAEyF0/s64/photo.jpg",
      "userId": "15471468418524645217"
     },
     "user_tz": 420
    },
    "id": "5Z7N2aauyYcE",
    "outputId": "319b21d3-8c0b-4543-888b-45e6acc83616"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Successfully retrieved header information!\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>file</th>\n",
       "      <th>message</th>\n",
       "      <th>Date</th>\n",
       "      <th>Subject</th>\n",
       "      <th>X-Folder</th>\n",
       "      <th>X-From</th>\n",
       "      <th>X-To</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>allen-p/_sent_mail/1.</td>\n",
       "      <td>Message-ID: &lt;18782981.1075855378110.JavaMail.e...</td>\n",
       "      <td>Mon, 14 May 2001 16:39:00 -0700 (PDT)</td>\n",
       "      <td></td>\n",
       "      <td>\\Phillip_Allen_Jan2002_1\\Allen, Phillip K.\\'Se...</td>\n",
       "      <td>Phillip K Allen</td>\n",
       "      <td>Tim Belden &lt;Tim Belden/Enron@EnronXGate&gt;</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>allen-p/_sent_mail/10.</td>\n",
       "      <td>Message-ID: &lt;15464986.1075855378456.JavaMail.e...</td>\n",
       "      <td>Fri, 4 May 2001 13:51:00 -0700 (PDT)</td>\n",
       "      <td>Re:</td>\n",
       "      <td>\\Phillip_Allen_Jan2002_1\\Allen, Phillip K.\\'Se...</td>\n",
       "      <td>Phillip K Allen</td>\n",
       "      <td>John J Lavorato &lt;John J Lavorato/ENRON@enronXg...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>allen-p/_sent_mail/100.</td>\n",
       "      <td>Message-ID: &lt;24216240.1075855687451.JavaMail.e...</td>\n",
       "      <td>Wed, 18 Oct 2000 03:00:00 -0700 (PDT)</td>\n",
       "      <td>Re: test</td>\n",
       "      <td>\\Phillip_Allen_Dec2000\\Notes Folders\\'sent mail</td>\n",
       "      <td>Phillip K Allen</td>\n",
       "      <td>Leah Van Arsdall</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>allen-p/_sent_mail/1000.</td>\n",
       "      <td>Message-ID: &lt;13505866.1075863688222.JavaMail.e...</td>\n",
       "      <td>Mon, 23 Oct 2000 06:13:00 -0700 (PDT)</td>\n",
       "      <td></td>\n",
       "      <td>\\Phillip_Allen_Dec2000\\Notes Folders\\'sent mail</td>\n",
       "      <td>Phillip K Allen</td>\n",
       "      <td>Randall L Gay</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>allen-p/_sent_mail/1001.</td>\n",
       "      <td>Message-ID: &lt;30922949.1075863688243.JavaMail.e...</td>\n",
       "      <td>Thu, 31 Aug 2000 05:07:00 -0700 (PDT)</td>\n",
       "      <td>Re: Hello</td>\n",
       "      <td>\\Phillip_Allen_Dec2000\\Notes Folders\\'sent mail</td>\n",
       "      <td>Phillip K Allen</td>\n",
       "      <td>Greg Piper</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                       file  \\\n",
       "0     allen-p/_sent_mail/1.   \n",
       "1    allen-p/_sent_mail/10.   \n",
       "2   allen-p/_sent_mail/100.   \n",
       "3  allen-p/_sent_mail/1000.   \n",
       "4  allen-p/_sent_mail/1001.   \n",
       "\n",
       "                                             message  \\\n",
       "0  Message-ID: <18782981.1075855378110.JavaMail.e...   \n",
       "1  Message-ID: <15464986.1075855378456.JavaMail.e...   \n",
       "2  Message-ID: <24216240.1075855687451.JavaMail.e...   \n",
       "3  Message-ID: <13505866.1075863688222.JavaMail.e...   \n",
       "4  Message-ID: <30922949.1075863688243.JavaMail.e...   \n",
       "\n",
       "                                    Date    Subject  \\\n",
       "0  Mon, 14 May 2001 16:39:00 -0700 (PDT)              \n",
       "1   Fri, 4 May 2001 13:51:00 -0700 (PDT)        Re:   \n",
       "2  Wed, 18 Oct 2000 03:00:00 -0700 (PDT)   Re: test   \n",
       "3  Mon, 23 Oct 2000 06:13:00 -0700 (PDT)              \n",
       "4  Thu, 31 Aug 2000 05:07:00 -0700 (PDT)  Re: Hello   \n",
       "\n",
       "                                            X-Folder           X-From  \\\n",
       "0  \\Phillip_Allen_Jan2002_1\\Allen, Phillip K.\\'Se...  Phillip K Allen   \n",
       "1  \\Phillip_Allen_Jan2002_1\\Allen, Phillip K.\\'Se...  Phillip K Allen   \n",
       "2    \\Phillip_Allen_Dec2000\\Notes Folders\\'sent mail  Phillip K Allen   \n",
       "3    \\Phillip_Allen_Dec2000\\Notes Folders\\'sent mail  Phillip K Allen   \n",
       "4    \\Phillip_Allen_Dec2000\\Notes Folders\\'sent mail  Phillip K Allen   \n",
       "\n",
       "                                                X-To  \n",
       "0           Tim Belden <Tim Belden/Enron@EnronXGate>  \n",
       "1  John J Lavorato <John J Lavorato/ENRON@enronXg...  \n",
       "2                                   Leah Van Arsdall  \n",
       "3                                      Randall L Gay  \n",
       "4                                         Greg Piper  "
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "header_names = [\"Date\", \"Subject\", \"X-Folder\", \"X-From\", \"X-To\"] \n",
    "headers = get_headers(data, header_names)\n",
    "\n",
    "for key in list(headers.keys()):\n",
    "    data[key] = headers[key]\n",
    "    \n",
    "data.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "op-Qp1k7yYcG"
   },
   "source": [
    "### Extracting Messages"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-08-04T18:32:43.455737Z",
     "start_time": "2020-08-04T18:32:43.450951Z"
    }
   },
   "outputs": [],
   "source": [
    "def get_messages(df):\n",
    "    messages = []\n",
    "    for item in df[\"message\"]:\n",
    "        # Return a message object structure from a string\n",
    "        e = email.message_from_string(item)    \n",
    "        # get message body  \n",
    "        message_body = e.get_payload()\n",
    "        message_body = message_body.lower()\n",
    "        messages.append(message_body)\n",
    "    print(\"Successfully retrieved message body from e-mails!\")\n",
    "    return messages"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-08-04T18:33:59.094542Z",
     "start_time": "2020-08-04T18:32:43.456737Z"
    },
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 690
    },
    "colab_type": "code",
    "executionInfo": {
     "elapsed": 249551,
     "status": "ok",
     "timestamp": 1557214662518,
     "user": {
      "displayName": "Arjun Patel",
      "photoUrl": "https://lh5.googleusercontent.com/-BlLPIwYkZoQ/AAAAAAAAAAI/AAAAAAAABy0/fGnMWZAEyF0/s64/photo.jpg",
      "userId": "15471468418524645217"
     },
     "user_tz": 420
    },
    "id": "phizJvVPyYcH",
    "outputId": "52c04214-9ae6-4140-9964-28fe5e9cb2c1",
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Successfully retrieved message body from e-mails!\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>file</th>\n",
       "      <th>message</th>\n",
       "      <th>Date</th>\n",
       "      <th>Subject</th>\n",
       "      <th>X-Folder</th>\n",
       "      <th>X-From</th>\n",
       "      <th>X-To</th>\n",
       "      <th>Message-Body</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>allen-p/_sent_mail/1.</td>\n",
       "      <td>Message-ID: &lt;18782981.1075855378110.JavaMail.e...</td>\n",
       "      <td>Mon, 14 May 2001 16:39:00 -0700 (PDT)</td>\n",
       "      <td></td>\n",
       "      <td>\\Phillip_Allen_Jan2002_1\\Allen, Phillip K.\\'Se...</td>\n",
       "      <td>Phillip K Allen</td>\n",
       "      <td>Tim Belden &lt;Tim Belden/Enron@EnronXGate&gt;</td>\n",
       "      <td>here is our forecast\\n\\n</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>allen-p/_sent_mail/10.</td>\n",
       "      <td>Message-ID: &lt;15464986.1075855378456.JavaMail.e...</td>\n",
       "      <td>Fri, 4 May 2001 13:51:00 -0700 (PDT)</td>\n",
       "      <td>Re:</td>\n",
       "      <td>\\Phillip_Allen_Jan2002_1\\Allen, Phillip K.\\'Se...</td>\n",
       "      <td>Phillip K Allen</td>\n",
       "      <td>John J Lavorato &lt;John J Lavorato/ENRON@enronXg...</td>\n",
       "      <td>traveling to have a business meeting takes the...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>allen-p/_sent_mail/100.</td>\n",
       "      <td>Message-ID: &lt;24216240.1075855687451.JavaMail.e...</td>\n",
       "      <td>Wed, 18 Oct 2000 03:00:00 -0700 (PDT)</td>\n",
       "      <td>Re: test</td>\n",
       "      <td>\\Phillip_Allen_Dec2000\\Notes Folders\\'sent mail</td>\n",
       "      <td>Phillip K Allen</td>\n",
       "      <td>Leah Van Arsdall</td>\n",
       "      <td>test successful.  way to go!!!</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>allen-p/_sent_mail/1000.</td>\n",
       "      <td>Message-ID: &lt;13505866.1075863688222.JavaMail.e...</td>\n",
       "      <td>Mon, 23 Oct 2000 06:13:00 -0700 (PDT)</td>\n",
       "      <td></td>\n",
       "      <td>\\Phillip_Allen_Dec2000\\Notes Folders\\'sent mail</td>\n",
       "      <td>Phillip K Allen</td>\n",
       "      <td>Randall L Gay</td>\n",
       "      <td>randy,\\n\\n can you send me a schedule of the s...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>allen-p/_sent_mail/1001.</td>\n",
       "      <td>Message-ID: &lt;30922949.1075863688243.JavaMail.e...</td>\n",
       "      <td>Thu, 31 Aug 2000 05:07:00 -0700 (PDT)</td>\n",
       "      <td>Re: Hello</td>\n",
       "      <td>\\Phillip_Allen_Dec2000\\Notes Folders\\'sent mail</td>\n",
       "      <td>Phillip K Allen</td>\n",
       "      <td>Greg Piper</td>\n",
       "      <td>let's shoot for tuesday at 11:45.</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                       file  \\\n",
       "0     allen-p/_sent_mail/1.   \n",
       "1    allen-p/_sent_mail/10.   \n",
       "2   allen-p/_sent_mail/100.   \n",
       "3  allen-p/_sent_mail/1000.   \n",
       "4  allen-p/_sent_mail/1001.   \n",
       "\n",
       "                                             message  \\\n",
       "0  Message-ID: <18782981.1075855378110.JavaMail.e...   \n",
       "1  Message-ID: <15464986.1075855378456.JavaMail.e...   \n",
       "2  Message-ID: <24216240.1075855687451.JavaMail.e...   \n",
       "3  Message-ID: <13505866.1075863688222.JavaMail.e...   \n",
       "4  Message-ID: <30922949.1075863688243.JavaMail.e...   \n",
       "\n",
       "                                    Date    Subject  \\\n",
       "0  Mon, 14 May 2001 16:39:00 -0700 (PDT)              \n",
       "1   Fri, 4 May 2001 13:51:00 -0700 (PDT)        Re:   \n",
       "2  Wed, 18 Oct 2000 03:00:00 -0700 (PDT)   Re: test   \n",
       "3  Mon, 23 Oct 2000 06:13:00 -0700 (PDT)              \n",
       "4  Thu, 31 Aug 2000 05:07:00 -0700 (PDT)  Re: Hello   \n",
       "\n",
       "                                            X-Folder           X-From  \\\n",
       "0  \\Phillip_Allen_Jan2002_1\\Allen, Phillip K.\\'Se...  Phillip K Allen   \n",
       "1  \\Phillip_Allen_Jan2002_1\\Allen, Phillip K.\\'Se...  Phillip K Allen   \n",
       "2    \\Phillip_Allen_Dec2000\\Notes Folders\\'sent mail  Phillip K Allen   \n",
       "3    \\Phillip_Allen_Dec2000\\Notes Folders\\'sent mail  Phillip K Allen   \n",
       "4    \\Phillip_Allen_Dec2000\\Notes Folders\\'sent mail  Phillip K Allen   \n",
       "\n",
       "                                                X-To  \\\n",
       "0           Tim Belden <Tim Belden/Enron@EnronXGate>   \n",
       "1  John J Lavorato <John J Lavorato/ENRON@enronXg...   \n",
       "2                                   Leah Van Arsdall   \n",
       "3                                      Randall L Gay   \n",
       "4                                         Greg Piper   \n",
       "\n",
       "                                        Message-Body  \n",
       "0                          here is our forecast\\n\\n   \n",
       "1  traveling to have a business meeting takes the...  \n",
       "2                     test successful.  way to go!!!  \n",
       "3  randy,\\n\\n can you send me a schedule of the s...  \n",
       "4                let's shoot for tuesday at 11:45.    "
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data[\"Message-Body\"] = get_messages(data)\n",
    "data.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "VXGbdAfxyYcK"
   },
   "source": [
    "### Getting Employee Names"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-08-04T18:33:59.101542Z",
     "start_time": "2020-08-04T18:33:59.096542Z"
    }
   },
   "outputs": [],
   "source": [
    "def add_name(df, column, labels):\n",
    "    new_data = {}\n",
    "    for item in df[column]: \n",
    "        tokens = item.split('/')\n",
    "        for i in range(0, len(labels)):\n",
    "            value = tokens[i]\n",
    "            key = labels[i]\n",
    "            new_data = insert_value(new_data, key, value)        \n",
    "    for key, value in new_data.items():\n",
    "        df[key] = pd.Series(value)\n",
    "    print(\"Successfully added new column!\")\n",
    "    return df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-08-04T18:33:59.951092Z",
     "start_time": "2020-08-04T18:33:59.103647Z"
    },
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 690
    },
    "colab_type": "code",
    "executionInfo": {
     "elapsed": 250606,
     "status": "ok",
     "timestamp": 1557214663584,
     "user": {
      "displayName": "Arjun Patel",
      "photoUrl": "https://lh5.googleusercontent.com/-BlLPIwYkZoQ/AAAAAAAAAAI/AAAAAAAABy0/fGnMWZAEyF0/s64/photo.jpg",
      "userId": "15471468418524645217"
     },
     "user_tz": 420
    },
    "id": "ssOsgZV_yYcK",
    "outputId": "c6a87474-1f78-46e5-db02-c1d291071a73"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Successfully added new column!\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>file</th>\n",
       "      <th>message</th>\n",
       "      <th>Date</th>\n",
       "      <th>Subject</th>\n",
       "      <th>X-Folder</th>\n",
       "      <th>X-From</th>\n",
       "      <th>X-To</th>\n",
       "      <th>Message-Body</th>\n",
       "      <th>employee</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>allen-p/_sent_mail/1.</td>\n",
       "      <td>Message-ID: &lt;18782981.1075855378110.JavaMail.e...</td>\n",
       "      <td>Mon, 14 May 2001 16:39:00 -0700 (PDT)</td>\n",
       "      <td></td>\n",
       "      <td>\\Phillip_Allen_Jan2002_1\\Allen, Phillip K.\\'Se...</td>\n",
       "      <td>Phillip K Allen</td>\n",
       "      <td>Tim Belden &lt;Tim Belden/Enron@EnronXGate&gt;</td>\n",
       "      <td>here is our forecast\\n\\n</td>\n",
       "      <td>allen-p</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>allen-p/_sent_mail/10.</td>\n",
       "      <td>Message-ID: &lt;15464986.1075855378456.JavaMail.e...</td>\n",
       "      <td>Fri, 4 May 2001 13:51:00 -0700 (PDT)</td>\n",
       "      <td>Re:</td>\n",
       "      <td>\\Phillip_Allen_Jan2002_1\\Allen, Phillip K.\\'Se...</td>\n",
       "      <td>Phillip K Allen</td>\n",
       "      <td>John J Lavorato &lt;John J Lavorato/ENRON@enronXg...</td>\n",
       "      <td>traveling to have a business meeting takes the...</td>\n",
       "      <td>allen-p</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>allen-p/_sent_mail/100.</td>\n",
       "      <td>Message-ID: &lt;24216240.1075855687451.JavaMail.e...</td>\n",
       "      <td>Wed, 18 Oct 2000 03:00:00 -0700 (PDT)</td>\n",
       "      <td>Re: test</td>\n",
       "      <td>\\Phillip_Allen_Dec2000\\Notes Folders\\'sent mail</td>\n",
       "      <td>Phillip K Allen</td>\n",
       "      <td>Leah Van Arsdall</td>\n",
       "      <td>test successful.  way to go!!!</td>\n",
       "      <td>allen-p</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>allen-p/_sent_mail/1000.</td>\n",
       "      <td>Message-ID: &lt;13505866.1075863688222.JavaMail.e...</td>\n",
       "      <td>Mon, 23 Oct 2000 06:13:00 -0700 (PDT)</td>\n",
       "      <td></td>\n",
       "      <td>\\Phillip_Allen_Dec2000\\Notes Folders\\'sent mail</td>\n",
       "      <td>Phillip K Allen</td>\n",
       "      <td>Randall L Gay</td>\n",
       "      <td>randy,\\n\\n can you send me a schedule of the s...</td>\n",
       "      <td>allen-p</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>allen-p/_sent_mail/1001.</td>\n",
       "      <td>Message-ID: &lt;30922949.1075863688243.JavaMail.e...</td>\n",
       "      <td>Thu, 31 Aug 2000 05:07:00 -0700 (PDT)</td>\n",
       "      <td>Re: Hello</td>\n",
       "      <td>\\Phillip_Allen_Dec2000\\Notes Folders\\'sent mail</td>\n",
       "      <td>Phillip K Allen</td>\n",
       "      <td>Greg Piper</td>\n",
       "      <td>let's shoot for tuesday at 11:45.</td>\n",
       "      <td>allen-p</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                       file  \\\n",
       "0     allen-p/_sent_mail/1.   \n",
       "1    allen-p/_sent_mail/10.   \n",
       "2   allen-p/_sent_mail/100.   \n",
       "3  allen-p/_sent_mail/1000.   \n",
       "4  allen-p/_sent_mail/1001.   \n",
       "\n",
       "                                             message  \\\n",
       "0  Message-ID: <18782981.1075855378110.JavaMail.e...   \n",
       "1  Message-ID: <15464986.1075855378456.JavaMail.e...   \n",
       "2  Message-ID: <24216240.1075855687451.JavaMail.e...   \n",
       "3  Message-ID: <13505866.1075863688222.JavaMail.e...   \n",
       "4  Message-ID: <30922949.1075863688243.JavaMail.e...   \n",
       "\n",
       "                                    Date    Subject  \\\n",
       "0  Mon, 14 May 2001 16:39:00 -0700 (PDT)              \n",
       "1   Fri, 4 May 2001 13:51:00 -0700 (PDT)        Re:   \n",
       "2  Wed, 18 Oct 2000 03:00:00 -0700 (PDT)   Re: test   \n",
       "3  Mon, 23 Oct 2000 06:13:00 -0700 (PDT)              \n",
       "4  Thu, 31 Aug 2000 05:07:00 -0700 (PDT)  Re: Hello   \n",
       "\n",
       "                                            X-Folder           X-From  \\\n",
       "0  \\Phillip_Allen_Jan2002_1\\Allen, Phillip K.\\'Se...  Phillip K Allen   \n",
       "1  \\Phillip_Allen_Jan2002_1\\Allen, Phillip K.\\'Se...  Phillip K Allen   \n",
       "2    \\Phillip_Allen_Dec2000\\Notes Folders\\'sent mail  Phillip K Allen   \n",
       "3    \\Phillip_Allen_Dec2000\\Notes Folders\\'sent mail  Phillip K Allen   \n",
       "4    \\Phillip_Allen_Dec2000\\Notes Folders\\'sent mail  Phillip K Allen   \n",
       "\n",
       "                                                X-To  \\\n",
       "0           Tim Belden <Tim Belden/Enron@EnronXGate>   \n",
       "1  John J Lavorato <John J Lavorato/ENRON@enronXg...   \n",
       "2                                   Leah Van Arsdall   \n",
       "3                                      Randall L Gay   \n",
       "4                                         Greg Piper   \n",
       "\n",
       "                                        Message-Body employee  \n",
       "0                          here is our forecast\\n\\n   allen-p  \n",
       "1  traveling to have a business meeting takes the...  allen-p  \n",
       "2                     test successful.  way to go!!!  allen-p  \n",
       "3  randy,\\n\\n can you send me a schedule of the s...  allen-p  \n",
       "4                let's shoot for tuesday at 11:45.    allen-p  "
      ]
     },
     "execution_count": 17,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "x_from = pd.DataFrame(data[\"X-From\"], columns = [\"X-From\"])\n",
    "if \"X-From\" not in data.columns:\n",
    "    data = pd.concat([data, x_from], axis = 1, join = 'inner')\n",
    "\n",
    "data = add_name(df = data, column = \"file\", labels = [\"employee\"])\n",
    "data.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Further Data Processing\n",
    "\n",
    "Once we had the data in a tabular for we had to perform further data extraction. This include extracting the data, converting it to datetime, and extraing the message from the `message` feature. "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "QNrZ6ylyyYcM"
   },
   "source": [
    "### Converting Date to Datetime"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-08-04T18:33:59.958085Z",
     "start_time": "2020-08-04T18:33:59.952082Z"
    }
   },
   "outputs": [],
   "source": [
    "# return the numerical value of the month\n",
    "# (e.g. Jan = 1, Feb = 2, ...)\n",
    "def month_mapping(month_str):\n",
    "    months = np.array(['Jan', 'Feb', 'Mar', 'Apr',\n",
    "              'May', 'Jun', 'Jul', 'Aug',\n",
    "              'Sep', 'Oct', 'Nov', 'Dec'])\n",
    "    \n",
    "    return np.where(months == month_str)[0][0] + 1\n",
    "\n",
    "def extract_date(date_string):\n",
    "    # should be of the format: DoW, Date, Month (MMM), Year (YYYY), \n",
    "    # Time (HH:MM:SS), timezone num, Timezone name\n",
    "    date_array = re.split(\" \", date_string)\n",
    "    time_array = re.split(\":\", date_array[4])\n",
    "    return dt(year = int(date_array[3]),\n",
    "              month = month_mapping(date_array[2]),\n",
    "              day = int(date_array[1]),\n",
    "              hour = int(time_array[0]),\n",
    "              minute = int(time_array[1]),\n",
    "              second = int(time_array[2]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-08-04T18:34:06.691739Z",
     "start_time": "2020-08-04T18:33:59.959085Z"
    },
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 673
    },
    "colab_type": "code",
    "executionInfo": {
     "elapsed": 257940,
     "status": "ok",
     "timestamp": 1557214670930,
     "user": {
      "displayName": "Arjun Patel",
      "photoUrl": "https://lh5.googleusercontent.com/-BlLPIwYkZoQ/AAAAAAAAAAI/AAAAAAAABy0/fGnMWZAEyF0/s64/photo.jpg",
      "userId": "15471468418524645217"
     },
     "user_tz": 420
    },
    "id": "QIpVvyfByYcN",
    "outputId": "49a50e9e-3f6e-4c9b-83ce-e92fe61398a8",
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>file</th>\n",
       "      <th>message</th>\n",
       "      <th>Date</th>\n",
       "      <th>Subject</th>\n",
       "      <th>X-Folder</th>\n",
       "      <th>X-From</th>\n",
       "      <th>X-To</th>\n",
       "      <th>Message-Body</th>\n",
       "      <th>employee</th>\n",
       "      <th>Date_datetime</th>\n",
       "      <th>Day_of_week</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>allen-p/_sent_mail/1.</td>\n",
       "      <td>Message-ID: &lt;18782981.1075855378110.JavaMail.e...</td>\n",
       "      <td>Mon, 14 May 2001 16:39:00 -0700 (PDT)</td>\n",
       "      <td></td>\n",
       "      <td>\\Phillip_Allen_Jan2002_1\\Allen, Phillip K.\\'Se...</td>\n",
       "      <td>Phillip K Allen</td>\n",
       "      <td>Tim Belden &lt;Tim Belden/Enron@EnronXGate&gt;</td>\n",
       "      <td>here is our forecast\\n\\n</td>\n",
       "      <td>allen-p</td>\n",
       "      <td>2001-05-14 16:39:00</td>\n",
       "      <td>Mon</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>allen-p/_sent_mail/10.</td>\n",
       "      <td>Message-ID: &lt;15464986.1075855378456.JavaMail.e...</td>\n",
       "      <td>Fri, 4 May 2001 13:51:00 -0700 (PDT)</td>\n",
       "      <td>Re:</td>\n",
       "      <td>\\Phillip_Allen_Jan2002_1\\Allen, Phillip K.\\'Se...</td>\n",
       "      <td>Phillip K Allen</td>\n",
       "      <td>John J Lavorato &lt;John J Lavorato/ENRON@enronXg...</td>\n",
       "      <td>traveling to have a business meeting takes the...</td>\n",
       "      <td>allen-p</td>\n",
       "      <td>2001-05-04 13:51:00</td>\n",
       "      <td>Fri</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>allen-p/_sent_mail/100.</td>\n",
       "      <td>Message-ID: &lt;24216240.1075855687451.JavaMail.e...</td>\n",
       "      <td>Wed, 18 Oct 2000 03:00:00 -0700 (PDT)</td>\n",
       "      <td>Re: test</td>\n",
       "      <td>\\Phillip_Allen_Dec2000\\Notes Folders\\'sent mail</td>\n",
       "      <td>Phillip K Allen</td>\n",
       "      <td>Leah Van Arsdall</td>\n",
       "      <td>test successful.  way to go!!!</td>\n",
       "      <td>allen-p</td>\n",
       "      <td>2000-10-18 03:00:00</td>\n",
       "      <td>Wed</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>allen-p/_sent_mail/1000.</td>\n",
       "      <td>Message-ID: &lt;13505866.1075863688222.JavaMail.e...</td>\n",
       "      <td>Mon, 23 Oct 2000 06:13:00 -0700 (PDT)</td>\n",
       "      <td></td>\n",
       "      <td>\\Phillip_Allen_Dec2000\\Notes Folders\\'sent mail</td>\n",
       "      <td>Phillip K Allen</td>\n",
       "      <td>Randall L Gay</td>\n",
       "      <td>randy,\\n\\n can you send me a schedule of the s...</td>\n",
       "      <td>allen-p</td>\n",
       "      <td>2000-10-23 06:13:00</td>\n",
       "      <td>Mon</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>allen-p/_sent_mail/1001.</td>\n",
       "      <td>Message-ID: &lt;30922949.1075863688243.JavaMail.e...</td>\n",
       "      <td>Thu, 31 Aug 2000 05:07:00 -0700 (PDT)</td>\n",
       "      <td>Re: Hello</td>\n",
       "      <td>\\Phillip_Allen_Dec2000\\Notes Folders\\'sent mail</td>\n",
       "      <td>Phillip K Allen</td>\n",
       "      <td>Greg Piper</td>\n",
       "      <td>let's shoot for tuesday at 11:45.</td>\n",
       "      <td>allen-p</td>\n",
       "      <td>2000-08-31 05:07:00</td>\n",
       "      <td>Thu</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                       file  \\\n",
       "0     allen-p/_sent_mail/1.   \n",
       "1    allen-p/_sent_mail/10.   \n",
       "2   allen-p/_sent_mail/100.   \n",
       "3  allen-p/_sent_mail/1000.   \n",
       "4  allen-p/_sent_mail/1001.   \n",
       "\n",
       "                                             message  \\\n",
       "0  Message-ID: <18782981.1075855378110.JavaMail.e...   \n",
       "1  Message-ID: <15464986.1075855378456.JavaMail.e...   \n",
       "2  Message-ID: <24216240.1075855687451.JavaMail.e...   \n",
       "3  Message-ID: <13505866.1075863688222.JavaMail.e...   \n",
       "4  Message-ID: <30922949.1075863688243.JavaMail.e...   \n",
       "\n",
       "                                    Date    Subject  \\\n",
       "0  Mon, 14 May 2001 16:39:00 -0700 (PDT)              \n",
       "1   Fri, 4 May 2001 13:51:00 -0700 (PDT)        Re:   \n",
       "2  Wed, 18 Oct 2000 03:00:00 -0700 (PDT)   Re: test   \n",
       "3  Mon, 23 Oct 2000 06:13:00 -0700 (PDT)              \n",
       "4  Thu, 31 Aug 2000 05:07:00 -0700 (PDT)  Re: Hello   \n",
       "\n",
       "                                            X-Folder           X-From  \\\n",
       "0  \\Phillip_Allen_Jan2002_1\\Allen, Phillip K.\\'Se...  Phillip K Allen   \n",
       "1  \\Phillip_Allen_Jan2002_1\\Allen, Phillip K.\\'Se...  Phillip K Allen   \n",
       "2    \\Phillip_Allen_Dec2000\\Notes Folders\\'sent mail  Phillip K Allen   \n",
       "3    \\Phillip_Allen_Dec2000\\Notes Folders\\'sent mail  Phillip K Allen   \n",
       "4    \\Phillip_Allen_Dec2000\\Notes Folders\\'sent mail  Phillip K Allen   \n",
       "\n",
       "                                                X-To  \\\n",
       "0           Tim Belden <Tim Belden/Enron@EnronXGate>   \n",
       "1  John J Lavorato <John J Lavorato/ENRON@enronXg...   \n",
       "2                                   Leah Van Arsdall   \n",
       "3                                      Randall L Gay   \n",
       "4                                         Greg Piper   \n",
       "\n",
       "                                        Message-Body employee  \\\n",
       "0                          here is our forecast\\n\\n   allen-p   \n",
       "1  traveling to have a business meeting takes the...  allen-p   \n",
       "2                     test successful.  way to go!!!  allen-p   \n",
       "3  randy,\\n\\n can you send me a schedule of the s...  allen-p   \n",
       "4                let's shoot for tuesday at 11:45.    allen-p   \n",
       "\n",
       "         Date_datetime Day_of_week  \n",
       "0  2001-05-14 16:39:00         Mon  \n",
       "1  2001-05-04 13:51:00         Fri  \n",
       "2  2000-10-18 03:00:00         Wed  \n",
       "3  2000-10-23 06:13:00         Mon  \n",
       "4  2000-08-31 05:07:00         Thu  "
      ]
     },
     "execution_count": 19,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data['Date_datetime'] = data['Date'].apply(extract_date)\n",
    "data['Day_of_week'] = data['Date'].apply(lambda x: re.split(\" \", x)[0][:-1])\n",
    "data.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "kVf2HtuxyYcO"
   },
   "source": [
    "### X-Folder Contents"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-08-04T18:34:06.696728Z",
     "start_time": "2020-08-04T18:34:06.693728Z"
    }
   },
   "outputs": [],
   "source": [
    "def extract_folder_contents(x): \n",
    "    return re.split(\"\\\\\\\\\", str(x))[-1].lower()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-08-04T18:34:07.594961Z",
     "start_time": "2020-08-04T18:34:06.697728Z"
    },
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 690
    },
    "colab_type": "code",
    "executionInfo": {
     "elapsed": 258920,
     "status": "ok",
     "timestamp": 1557214671924,
     "user": {
      "displayName": "Arjun Patel",
      "photoUrl": "https://lh5.googleusercontent.com/-BlLPIwYkZoQ/AAAAAAAAAAI/AAAAAAAABy0/fGnMWZAEyF0/s64/photo.jpg",
      "userId": "15471468418524645217"
     },
     "user_tz": 420
    },
    "id": "uEpUGdIL_VYy",
    "outputId": "b3f92b2e-dd4f-4840-93eb-66197bf14897"
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>file</th>\n",
       "      <th>message</th>\n",
       "      <th>Date</th>\n",
       "      <th>Subject</th>\n",
       "      <th>X-Folder</th>\n",
       "      <th>X-From</th>\n",
       "      <th>X-To</th>\n",
       "      <th>Message-Body</th>\n",
       "      <th>employee</th>\n",
       "      <th>Date_datetime</th>\n",
       "      <th>Day_of_week</th>\n",
       "      <th>X-Folder-split</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>allen-p/_sent_mail/1.</td>\n",
       "      <td>Message-ID: &lt;18782981.1075855378110.JavaMail.e...</td>\n",
       "      <td>Mon, 14 May 2001 16:39:00 -0700 (PDT)</td>\n",
       "      <td></td>\n",
       "      <td>\\Phillip_Allen_Jan2002_1\\Allen, Phillip K.\\'Se...</td>\n",
       "      <td>Phillip K Allen</td>\n",
       "      <td>Tim Belden &lt;Tim Belden/Enron@EnronXGate&gt;</td>\n",
       "      <td>here is our forecast\\n\\n</td>\n",
       "      <td>allen-p</td>\n",
       "      <td>2001-05-14 16:39:00</td>\n",
       "      <td>Mon</td>\n",
       "      <td>'sent mail</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>allen-p/_sent_mail/10.</td>\n",
       "      <td>Message-ID: &lt;15464986.1075855378456.JavaMail.e...</td>\n",
       "      <td>Fri, 4 May 2001 13:51:00 -0700 (PDT)</td>\n",
       "      <td>Re:</td>\n",
       "      <td>\\Phillip_Allen_Jan2002_1\\Allen, Phillip K.\\'Se...</td>\n",
       "      <td>Phillip K Allen</td>\n",
       "      <td>John J Lavorato &lt;John J Lavorato/ENRON@enronXg...</td>\n",
       "      <td>traveling to have a business meeting takes the...</td>\n",
       "      <td>allen-p</td>\n",
       "      <td>2001-05-04 13:51:00</td>\n",
       "      <td>Fri</td>\n",
       "      <td>'sent mail</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>allen-p/_sent_mail/100.</td>\n",
       "      <td>Message-ID: &lt;24216240.1075855687451.JavaMail.e...</td>\n",
       "      <td>Wed, 18 Oct 2000 03:00:00 -0700 (PDT)</td>\n",
       "      <td>Re: test</td>\n",
       "      <td>\\Phillip_Allen_Dec2000\\Notes Folders\\'sent mail</td>\n",
       "      <td>Phillip K Allen</td>\n",
       "      <td>Leah Van Arsdall</td>\n",
       "      <td>test successful.  way to go!!!</td>\n",
       "      <td>allen-p</td>\n",
       "      <td>2000-10-18 03:00:00</td>\n",
       "      <td>Wed</td>\n",
       "      <td>'sent mail</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>allen-p/_sent_mail/1000.</td>\n",
       "      <td>Message-ID: &lt;13505866.1075863688222.JavaMail.e...</td>\n",
       "      <td>Mon, 23 Oct 2000 06:13:00 -0700 (PDT)</td>\n",
       "      <td></td>\n",
       "      <td>\\Phillip_Allen_Dec2000\\Notes Folders\\'sent mail</td>\n",
       "      <td>Phillip K Allen</td>\n",
       "      <td>Randall L Gay</td>\n",
       "      <td>randy,\\n\\n can you send me a schedule of the s...</td>\n",
       "      <td>allen-p</td>\n",
       "      <td>2000-10-23 06:13:00</td>\n",
       "      <td>Mon</td>\n",
       "      <td>'sent mail</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>allen-p/_sent_mail/1001.</td>\n",
       "      <td>Message-ID: &lt;30922949.1075863688243.JavaMail.e...</td>\n",
       "      <td>Thu, 31 Aug 2000 05:07:00 -0700 (PDT)</td>\n",
       "      <td>Re: Hello</td>\n",
       "      <td>\\Phillip_Allen_Dec2000\\Notes Folders\\'sent mail</td>\n",
       "      <td>Phillip K Allen</td>\n",
       "      <td>Greg Piper</td>\n",
       "      <td>let's shoot for tuesday at 11:45.</td>\n",
       "      <td>allen-p</td>\n",
       "      <td>2000-08-31 05:07:00</td>\n",
       "      <td>Thu</td>\n",
       "      <td>'sent mail</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                       file  \\\n",
       "0     allen-p/_sent_mail/1.   \n",
       "1    allen-p/_sent_mail/10.   \n",
       "2   allen-p/_sent_mail/100.   \n",
       "3  allen-p/_sent_mail/1000.   \n",
       "4  allen-p/_sent_mail/1001.   \n",
       "\n",
       "                                             message  \\\n",
       "0  Message-ID: <18782981.1075855378110.JavaMail.e...   \n",
       "1  Message-ID: <15464986.1075855378456.JavaMail.e...   \n",
       "2  Message-ID: <24216240.1075855687451.JavaMail.e...   \n",
       "3  Message-ID: <13505866.1075863688222.JavaMail.e...   \n",
       "4  Message-ID: <30922949.1075863688243.JavaMail.e...   \n",
       "\n",
       "                                    Date    Subject  \\\n",
       "0  Mon, 14 May 2001 16:39:00 -0700 (PDT)              \n",
       "1   Fri, 4 May 2001 13:51:00 -0700 (PDT)        Re:   \n",
       "2  Wed, 18 Oct 2000 03:00:00 -0700 (PDT)   Re: test   \n",
       "3  Mon, 23 Oct 2000 06:13:00 -0700 (PDT)              \n",
       "4  Thu, 31 Aug 2000 05:07:00 -0700 (PDT)  Re: Hello   \n",
       "\n",
       "                                            X-Folder           X-From  \\\n",
       "0  \\Phillip_Allen_Jan2002_1\\Allen, Phillip K.\\'Se...  Phillip K Allen   \n",
       "1  \\Phillip_Allen_Jan2002_1\\Allen, Phillip K.\\'Se...  Phillip K Allen   \n",
       "2    \\Phillip_Allen_Dec2000\\Notes Folders\\'sent mail  Phillip K Allen   \n",
       "3    \\Phillip_Allen_Dec2000\\Notes Folders\\'sent mail  Phillip K Allen   \n",
       "4    \\Phillip_Allen_Dec2000\\Notes Folders\\'sent mail  Phillip K Allen   \n",
       "\n",
       "                                                X-To  \\\n",
       "0           Tim Belden <Tim Belden/Enron@EnronXGate>   \n",
       "1  John J Lavorato <John J Lavorato/ENRON@enronXg...   \n",
       "2                                   Leah Van Arsdall   \n",
       "3                                      Randall L Gay   \n",
       "4                                         Greg Piper   \n",
       "\n",
       "                                        Message-Body employee  \\\n",
       "0                          here is our forecast\\n\\n   allen-p   \n",
       "1  traveling to have a business meeting takes the...  allen-p   \n",
       "2                     test successful.  way to go!!!  allen-p   \n",
       "3  randy,\\n\\n can you send me a schedule of the s...  allen-p   \n",
       "4                let's shoot for tuesday at 11:45.    allen-p   \n",
       "\n",
       "         Date_datetime Day_of_week X-Folder-split  \n",
       "0  2001-05-14 16:39:00         Mon     'sent mail  \n",
       "1  2001-05-04 13:51:00         Fri     'sent mail  \n",
       "2  2000-10-18 03:00:00         Wed     'sent mail  \n",
       "3  2000-10-23 06:13:00         Mon     'sent mail  \n",
       "4  2000-08-31 05:07:00         Thu     'sent mail  "
      ]
     },
     "execution_count": 21,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data['X-Folder-split'] = data['X-Folder'].apply(extract_folder_contents)\n",
    "data.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-08-04T18:34:08.578012Z",
     "start_time": "2020-08-04T18:34:07.595961Z"
    },
    "colab": {},
    "colab_type": "code",
    "id": "LADI7N99Vx32"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Message-ID: <15464986.1075855378456.JavaMail.evans@thyme>\n",
      "Date: Fri, 4 May 2001 13:51:00 -0700 (PDT)\n",
      "From: phillip.allen@enron.com\n",
      "To: john.lavorato@enron.com\n",
      "Subject: Re:\n",
      "Mime-Version: 1.0\n",
      "Content-Type: text/plain; charset=us-ascii\n",
      "Content-Transfer-Encoding: 7bit\n",
      "X-From: Phillip K Allen\n",
      "X-To: John J Lavorato <John J Lavorato/ENRON@enronXgate@ENRON>\n",
      "X-cc: \n",
      "X-bcc: \n",
      "X-Folder: \\Phillip_Allen_Jan2002_1\\Allen, Phillip K.\\'Sent Mail\n",
      "X-Origin: Allen-P\n",
      "X-FileName: pallen (Non-Privileged).pst\n",
      "\n",
      "Traveling to have a business meeting takes the fun out of the trip.  Especially if you have to prepare a presentation.  I would suggest holding the business plan meetings here then take a trip without any formal business meetings.  I would even try and get some honest opinions on whether a trip is even desired or necessary.\n",
      "\n",
      "As far as the business meetings, I think it would be more productive to try and stimulate discussions across the different groups about what is working and what is not.  Too often the presenter speaks and the others are quiet just waiting for their turn.   The meetings might be better if held in a round table discussion format.  \n",
      "\n",
      "My suggestion for where to go is Austin.  Play golf and rent a ski boat and jet ski's.  Flying somewhere takes too much time.\n",
      "\n"
     ]
    }
   ],
   "source": [
    "#Add column with message length\n",
    "data['message_length']  = data['Message-Body'].str.len()\n",
    "\n",
    "#drop columns that are not needed anymore\n",
    "columns_to_drop = ['file', 'X-From', 'X-Folder', 'X-To', 'employee']\n",
    "data = data.drop(columns_to_drop, axis=1)\n",
    "\n",
    "# Remaining columns to focus on\n",
    "data.columns\n",
    "\n",
    "# shows one entire message with all fields\n",
    "print(data['message'][1])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-08-04T18:34:11.207751Z",
     "start_time": "2020-08-04T18:34:08.579011Z"
    },
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 690
    },
    "colab_type": "code",
    "executionInfo": {
     "elapsed": 261623,
     "status": "ok",
     "timestamp": 1557214674669,
     "user": {
      "displayName": "Arjun Patel",
      "photoUrl": "https://lh5.googleusercontent.com/-BlLPIwYkZoQ/AAAAAAAAAAI/AAAAAAAABy0/fGnMWZAEyF0/s64/photo.jpg",
      "userId": "15471468418524645217"
     },
     "user_tz": 420
    },
    "id": "HvXp5A7wCPxS",
    "outputId": "bc2cdde1-f9b2-439b-9705-a14148df4abf"
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>message</th>\n",
       "      <th>Date</th>\n",
       "      <th>Subject</th>\n",
       "      <th>Message-Body</th>\n",
       "      <th>Date_datetime</th>\n",
       "      <th>Day_of_week</th>\n",
       "      <th>X-Folder-split</th>\n",
       "      <th>message_length</th>\n",
       "      <th>contains-cc</th>\n",
       "      <th>contains-bcc</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>Message-ID: &lt;18782981.1075855378110.JavaMail.e...</td>\n",
       "      <td>Mon, 14 May 2001 16:39:00 -0700 (PDT)</td>\n",
       "      <td></td>\n",
       "      <td>here is our forecast\\n\\n</td>\n",
       "      <td>2001-05-14 16:39:00</td>\n",
       "      <td>Mon</td>\n",
       "      <td>'sent mail</td>\n",
       "      <td>23</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>Message-ID: &lt;15464986.1075855378456.JavaMail.e...</td>\n",
       "      <td>Fri, 4 May 2001 13:51:00 -0700 (PDT)</td>\n",
       "      <td>Re:</td>\n",
       "      <td>traveling to have a business meeting takes the...</td>\n",
       "      <td>2001-05-04 13:51:00</td>\n",
       "      <td>Fri</td>\n",
       "      <td>'sent mail</td>\n",
       "      <td>786</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>Message-ID: &lt;24216240.1075855687451.JavaMail.e...</td>\n",
       "      <td>Wed, 18 Oct 2000 03:00:00 -0700 (PDT)</td>\n",
       "      <td>Re: test</td>\n",
       "      <td>test successful.  way to go!!!</td>\n",
       "      <td>2000-10-18 03:00:00</td>\n",
       "      <td>Wed</td>\n",
       "      <td>'sent mail</td>\n",
       "      <td>30</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>Message-ID: &lt;13505866.1075863688222.JavaMail.e...</td>\n",
       "      <td>Mon, 23 Oct 2000 06:13:00 -0700 (PDT)</td>\n",
       "      <td></td>\n",
       "      <td>randy,\\n\\n can you send me a schedule of the s...</td>\n",
       "      <td>2000-10-23 06:13:00</td>\n",
       "      <td>Mon</td>\n",
       "      <td>'sent mail</td>\n",
       "      <td>187</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>Message-ID: &lt;30922949.1075863688243.JavaMail.e...</td>\n",
       "      <td>Thu, 31 Aug 2000 05:07:00 -0700 (PDT)</td>\n",
       "      <td>Re: Hello</td>\n",
       "      <td>let's shoot for tuesday at 11:45.</td>\n",
       "      <td>2000-08-31 05:07:00</td>\n",
       "      <td>Thu</td>\n",
       "      <td>'sent mail</td>\n",
       "      <td>35</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                             message  \\\n",
       "0  Message-ID: <18782981.1075855378110.JavaMail.e...   \n",
       "1  Message-ID: <15464986.1075855378456.JavaMail.e...   \n",
       "2  Message-ID: <24216240.1075855687451.JavaMail.e...   \n",
       "3  Message-ID: <13505866.1075863688222.JavaMail.e...   \n",
       "4  Message-ID: <30922949.1075863688243.JavaMail.e...   \n",
       "\n",
       "                                    Date    Subject  \\\n",
       "0  Mon, 14 May 2001 16:39:00 -0700 (PDT)              \n",
       "1   Fri, 4 May 2001 13:51:00 -0700 (PDT)        Re:   \n",
       "2  Wed, 18 Oct 2000 03:00:00 -0700 (PDT)   Re: test   \n",
       "3  Mon, 23 Oct 2000 06:13:00 -0700 (PDT)              \n",
       "4  Thu, 31 Aug 2000 05:07:00 -0700 (PDT)  Re: Hello   \n",
       "\n",
       "                                        Message-Body        Date_datetime  \\\n",
       "0                          here is our forecast\\n\\n   2001-05-14 16:39:00   \n",
       "1  traveling to have a business meeting takes the...  2001-05-04 13:51:00   \n",
       "2                     test successful.  way to go!!!  2000-10-18 03:00:00   \n",
       "3  randy,\\n\\n can you send me a schedule of the s...  2000-10-23 06:13:00   \n",
       "4                let's shoot for tuesday at 11:45.    2000-08-31 05:07:00   \n",
       "\n",
       "  Day_of_week X-Folder-split  message_length  contains-cc  contains-bcc  \n",
       "0         Mon     'sent mail              23            0             0  \n",
       "1         Fri     'sent mail             786            0             0  \n",
       "2         Wed     'sent mail              30            0             0  \n",
       "3         Mon     'sent mail             187            0             0  \n",
       "4         Thu     'sent mail              35            0             0  "
      ]
     },
     "execution_count": 23,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data['contains-cc'] = data.message.str.contains(r'X-cc: .+').astype(int)\n",
    "\n",
    "data['contains-bcc'] = data.message.str.contains(r'X-bcc: .+').astype(int)\n",
    "\n",
    "data.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-08-04T20:02:17.379445Z",
     "start_time": "2020-08-04T20:01:40.147481Z"
    }
   },
   "outputs": [],
   "source": [
    "# data.to_csv(os.path.join(dataset_directory, \"preprocessed_data.csv\"), index=False)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "VesmiudwdnhA"
   },
   "source": [
    "## Creating Embedding"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-08-04T21:08:02.746944Z",
     "start_time": "2020-08-04T21:07:44.907819Z"
    }
   },
   "outputs": [],
   "source": [
    "data = pd.read_csv(os.path.join(dataset_directory, \"preprocessed_data.csv\"))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-08-04T20:06:02.375643Z",
     "start_time": "2020-08-04T20:05:58.852231Z"
    },
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 401
    },
    "colab_type": "code",
    "executionInfo": {
     "elapsed": 269398,
     "status": "ok",
     "timestamp": 1557214682474,
     "user": {
      "displayName": "Arjun Patel",
      "photoUrl": "https://lh5.googleusercontent.com/-BlLPIwYkZoQ/AAAAAAAAAAI/AAAAAAAABy0/fGnMWZAEyF0/s64/photo.jpg",
      "userId": "15471468418524645217"
     },
     "user_tz": 420
    },
    "id": "wkOBt8oLdqpd",
    "outputId": "97c98252-7755-46ca-8c2e-6b4784fd7576"
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[nltk_data] Downloading package stopwords to\n",
      "[nltk_data]     C:\\Users\\Pedro\\AppData\\Roaming\\nltk_data...\n",
      "[nltk_data]   Unzipping corpora\\stopwords.zip.\n",
      "[nltk_data] Downloading package punkt to\n",
      "[nltk_data]     C:\\Users\\Pedro\\AppData\\Roaming\\nltk_data...\n",
      "[nltk_data]   Unzipping tokenizers\\punkt.zip.\n"
     ]
    }
   ],
   "source": [
    "import gensim\n",
    "from gensim.models import KeyedVectors, Word2Vec\n",
    "import nltk\n",
    "\n",
    "nltk.download('stopwords')\n",
    "nltk.download('punkt')\n",
    "\n",
    "from nltk.corpus import stopwords\n",
    "from nltk.tokenize import word_tokenize\n",
    "import string"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-08-04T21:08:02.791954Z",
     "start_time": "2020-08-04T21:08:02.747944Z"
    },
    "colab": {},
    "colab_type": "code",
    "id": "064OTXMjeqy6"
   },
   "outputs": [],
   "source": [
    "data2 = data.copy()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-08-04T20:51:50.327736Z",
     "start_time": "2020-08-04T20:49:38.017547Z"
    },
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 208
    },
    "colab_type": "code",
    "executionInfo": {
     "elapsed": 425438,
     "status": "ok",
     "timestamp": 1557214838535,
     "user": {
      "displayName": "Arjun Patel",
      "photoUrl": "https://lh5.googleusercontent.com/-BlLPIwYkZoQ/AAAAAAAAAAI/AAAAAAAABy0/fGnMWZAEyF0/s64/photo.jpg",
      "userId": "15471468418524645217"
     },
     "user_tz": 420
    },
    "id": "Jz7_Lx1cehn3",
    "outputId": "84846a95-7c9e-4739-abbc-4f9d0c3c5af9"
   },
   "outputs": [],
   "source": [
    "# !wget -nd https://s3.amazonaws.com/dl4j-distribution/GoogleNews-vectors-negative300.bin.gz \n",
    "model = KeyedVectors.load_word2vec_format('../Models/GoogleNews-vectors-negative300.bin.gz', binary=True)\n",
    "# Words are encoded into length 300 vector"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-08-04T21:08:02.805958Z",
     "start_time": "2020-08-04T21:08:02.793955Z"
    },
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 72
    },
    "colab_type": "code",
    "executionInfo": {
     "elapsed": 159770,
     "status": "ok",
     "timestamp": 1557215298044,
     "user": {
      "displayName": "Arjun Patel",
      "photoUrl": "https://lh5.googleusercontent.com/-BlLPIwYkZoQ/AAAAAAAAAAI/AAAAAAAABy0/fGnMWZAEyF0/s64/photo.jpg",
      "userId": "15471468418524645217"
     },
     "user_tz": 420
    },
    "id": "c2YilfLmehlu",
    "outputId": "bdc2d3ec-f306-4764-9aa5-7078f7cd20c9"
   },
   "outputs": [],
   "source": [
    "# RUN THIS CELL IF DO NOT HAVE EMBEDDINGS YET\n",
    "# RUN THE CELL BELOW IF ALREADY DO\n",
    "\n",
    "def remove_unnecessary_words(s):\n",
    "    \"\"\"\n",
    "    Remove certain words that are irrelevant\n",
    "    e.g. the NLTK stopwords, 'items', 'inbox'\n",
    "    \"\"\"\n",
    "    tokenized_s = word_tokenize(s)\n",
    "    add_to_stopwords = ['items', 'item', 'mails', 'mail', \n",
    "                      'inboxes', 'inbox', 'threads', 'thread']\n",
    "    to_remove = stopwords.words('english') + add_to_stopwords\n",
    "    new_sent = [c for c in tokenized_s if not c in to_remove]\n",
    "    return ' '.join(new_sent)\n",
    "\n",
    "def remove_punctuation(s):\n",
    "    return ''.join([s[i] for i in np.arange(len(s)) if not s[i] in string.punctuation])\n",
    "\n",
    "\n",
    "#    Find vector representation of every folder\n",
    "#    If folder has multiple words after cleaning,\n",
    "#    then just average the vectors\n",
    "\n",
    "def vector_rep(word):\n",
    "    if word in model.wv.vocab:\n",
    "        return model.wv[word]\n",
    "    return np.zeros(shape=(1, 300))\n",
    "\n",
    "def general_vector_rep(phrase):\n",
    "    tokenized = word_tokenize(phrase)\n",
    "    if (len(tokenized) == 0):\n",
    "        return np.zeros(shape=(1, 300))\n",
    "    if (len(tokenized) == 1):\n",
    "        return vector_rep(tokenized[0])\n",
    "    vectors = np.zeros(shape = (len(tokenized), 300))\n",
    "    for i, word in enumerate(tokenized):\n",
    "        vectors[i] = vector_rep(word).reshape((1, 300))\n",
    "    return np.average(vectors, axis=0)\n",
    "\n",
    "def make_vector_rep_df(labels):\n",
    "    d = dict.fromkeys(labels)\n",
    "    for key in list(d.keys()):\n",
    "        d[key] = general_vector_rep(key)\n",
    "    df = pd.DataFrame.from_dict(d, orient='index')\n",
    "    del d\n",
    "    for i in np.arange(len(df)):\n",
    "        if(pd.isnull(df.iloc[i, 1])):\n",
    "            df.iloc[i, :] = pd.Series(np.zeros(300))\n",
    "    return df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-08-04T21:11:13.457429Z",
     "start_time": "2020-08-04T21:08:02.807958Z"
    }
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\Pedro\\Anaconda3\\envs\\data_mining_gpu\\lib\\site-packages\\ipykernel_launcher.py:25: DeprecationWarning: Call to deprecated `wv` (Attribute will be removed in 4.0.0, use self instead).\n",
      "C:\\Users\\Pedro\\Anaconda3\\envs\\data_mining_gpu\\lib\\site-packages\\ipykernel_launcher.py:26: DeprecationWarning: Call to deprecated `wv` (Attribute will be removed in 4.0.0, use self instead).\n"
     ]
    }
   ],
   "source": [
    "data2['X-Folder-split_cleaned'] = data2['X-Folder-split'].apply(remove_unnecessary_words)\n",
    "data2['X-Folder-split_cleaned'] = data2['X-Folder-split_cleaned'].apply(remove_punctuation)\n",
    "data2.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-08-04T21:35:11.864326Z",
     "start_time": "2020-08-04T21:34:33.024202Z"
    }
   },
   "outputs": [],
   "source": [
    "# data2.to_csv(os.path.join(dataset_directory, \"preprocessed_data_v2.csv\"), index=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-08-04T21:11:13.479435Z",
     "start_time": "2020-08-04T21:11:13.458430Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>0</th>\n",
       "      <th>1</th>\n",
       "      <th>2</th>\n",
       "      <th>3</th>\n",
       "      <th>4</th>\n",
       "      <th>5</th>\n",
       "      <th>6</th>\n",
       "      <th>7</th>\n",
       "      <th>8</th>\n",
       "      <th>9</th>\n",
       "      <th>...</th>\n",
       "      <th>290</th>\n",
       "      <th>291</th>\n",
       "      <th>292</th>\n",
       "      <th>293</th>\n",
       "      <th>294</th>\n",
       "      <th>295</th>\n",
       "      <th>296</th>\n",
       "      <th>297</th>\n",
       "      <th>298</th>\n",
       "      <th>299</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>sent</th>\n",
       "      <td>-0.0170898</td>\n",
       "      <td>0.021729</td>\n",
       "      <td>0.225586</td>\n",
       "      <td>-0.129883</td>\n",
       "      <td>-0.199219</td>\n",
       "      <td>-0.107910</td>\n",
       "      <td>-0.184570</td>\n",
       "      <td>-0.083984</td>\n",
       "      <td>0.004974</td>\n",
       "      <td>0.111328</td>\n",
       "      <td>...</td>\n",
       "      <td>-0.148438</td>\n",
       "      <td>-0.283203</td>\n",
       "      <td>-0.051758</td>\n",
       "      <td>0.112305</td>\n",
       "      <td>0.267578</td>\n",
       "      <td>0.059082</td>\n",
       "      <td>0.144531</td>\n",
       "      <td>-0.091309</td>\n",
       "      <td>-0.069824</td>\n",
       "      <td>-0.021606</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>documents</th>\n",
       "      <td>0.0976562</td>\n",
       "      <td>0.063965</td>\n",
       "      <td>0.024658</td>\n",
       "      <td>-0.151367</td>\n",
       "      <td>-0.032471</td>\n",
       "      <td>0.110352</td>\n",
       "      <td>0.044678</td>\n",
       "      <td>0.195312</td>\n",
       "      <td>0.089844</td>\n",
       "      <td>-0.285156</td>\n",
       "      <td>...</td>\n",
       "      <td>0.099121</td>\n",
       "      <td>-0.245117</td>\n",
       "      <td>-0.113770</td>\n",
       "      <td>0.026367</td>\n",
       "      <td>-0.156250</td>\n",
       "      <td>0.119141</td>\n",
       "      <td>0.073730</td>\n",
       "      <td>-0.292969</td>\n",
       "      <td>-0.035889</td>\n",
       "      <td>0.032959</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>contacts</th>\n",
       "      <td>0.0742188</td>\n",
       "      <td>0.001274</td>\n",
       "      <td>0.065918</td>\n",
       "      <td>-0.086914</td>\n",
       "      <td>-0.070801</td>\n",
       "      <td>0.249023</td>\n",
       "      <td>-0.320312</td>\n",
       "      <td>-0.174805</td>\n",
       "      <td>0.091797</td>\n",
       "      <td>-0.052734</td>\n",
       "      <td>...</td>\n",
       "      <td>0.022583</td>\n",
       "      <td>0.097656</td>\n",
       "      <td>0.023071</td>\n",
       "      <td>-0.063965</td>\n",
       "      <td>0.114258</td>\n",
       "      <td>0.015869</td>\n",
       "      <td>0.416016</td>\n",
       "      <td>-0.140625</td>\n",
       "      <td>-0.050537</td>\n",
       "      <td>-0.033447</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>deleted</th>\n",
       "      <td>-0.172852</td>\n",
       "      <td>-0.047852</td>\n",
       "      <td>0.316406</td>\n",
       "      <td>0.145508</td>\n",
       "      <td>-0.220703</td>\n",
       "      <td>0.009521</td>\n",
       "      <td>-0.028076</td>\n",
       "      <td>0.216797</td>\n",
       "      <td>0.507812</td>\n",
       "      <td>0.092285</td>\n",
       "      <td>...</td>\n",
       "      <td>0.181641</td>\n",
       "      <td>0.076660</td>\n",
       "      <td>0.363281</td>\n",
       "      <td>-0.200195</td>\n",
       "      <td>0.287109</td>\n",
       "      <td>-0.263672</td>\n",
       "      <td>0.196289</td>\n",
       "      <td>-0.361328</td>\n",
       "      <td>-0.050537</td>\n",
       "      <td>-0.070801</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>discussion</th>\n",
       "      <td>-0.0390625</td>\n",
       "      <td>0.042236</td>\n",
       "      <td>0.014465</td>\n",
       "      <td>0.115234</td>\n",
       "      <td>0.132812</td>\n",
       "      <td>-0.101074</td>\n",
       "      <td>0.164062</td>\n",
       "      <td>-0.100098</td>\n",
       "      <td>0.123535</td>\n",
       "      <td>0.005585</td>\n",
       "      <td>...</td>\n",
       "      <td>-0.011292</td>\n",
       "      <td>-0.175781</td>\n",
       "      <td>0.326172</td>\n",
       "      <td>0.152344</td>\n",
       "      <td>0.042236</td>\n",
       "      <td>-0.011536</td>\n",
       "      <td>-0.125000</td>\n",
       "      <td>-0.229492</td>\n",
       "      <td>-0.112305</td>\n",
       "      <td>-0.158203</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 300 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                  0         1         2         3         4         5    \\\n",
       "sent       -0.0170898  0.021729  0.225586 -0.129883 -0.199219 -0.107910   \n",
       "documents   0.0976562  0.063965  0.024658 -0.151367 -0.032471  0.110352   \n",
       "contacts    0.0742188  0.001274  0.065918 -0.086914 -0.070801  0.249023   \n",
       "deleted     -0.172852 -0.047852  0.316406  0.145508 -0.220703  0.009521   \n",
       "discussion -0.0390625  0.042236  0.014465  0.115234  0.132812 -0.101074   \n",
       "\n",
       "                 6         7         8         9    ...       290       291  \\\n",
       "sent       -0.184570 -0.083984  0.004974  0.111328  ... -0.148438 -0.283203   \n",
       "documents   0.044678  0.195312  0.089844 -0.285156  ...  0.099121 -0.245117   \n",
       "contacts   -0.320312 -0.174805  0.091797 -0.052734  ...  0.022583  0.097656   \n",
       "deleted    -0.028076  0.216797  0.507812  0.092285  ...  0.181641  0.076660   \n",
       "discussion  0.164062 -0.100098  0.123535  0.005585  ... -0.011292 -0.175781   \n",
       "\n",
       "                 292       293       294       295       296       297  \\\n",
       "sent       -0.051758  0.112305  0.267578  0.059082  0.144531 -0.091309   \n",
       "documents  -0.113770  0.026367 -0.156250  0.119141  0.073730 -0.292969   \n",
       "contacts    0.023071 -0.063965  0.114258  0.015869  0.416016 -0.140625   \n",
       "deleted     0.363281 -0.200195  0.287109 -0.263672  0.196289 -0.361328   \n",
       "discussion  0.326172  0.152344  0.042236 -0.011536 -0.125000 -0.229492   \n",
       "\n",
       "                 298       299  \n",
       "sent       -0.069824 -0.021606  \n",
       "documents  -0.035889  0.032959  \n",
       "contacts   -0.050537 -0.033447  \n",
       "deleted    -0.050537 -0.070801  \n",
       "discussion -0.112305 -0.158203  \n",
       "\n",
       "[5 rows x 300 columns]"
      ]
     },
     "execution_count": 23,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "vectorized_folders_df = make_vector_rep_df(list(data2['X-Folder-split_cleaned'].unique()))\n",
    "vectorized_folders_df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-08-04T21:23:37.222413Z",
     "start_time": "2020-08-04T21:23:36.523134Z"
    }
   },
   "outputs": [],
   "source": [
    "vectorized_folders_df.to_csv(os.path.join(dataset_directory, \"vectorized_folders_df.csv\"))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Performing Clustering"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-08-04T21:23:38.576907Z",
     "start_time": "2020-08-04T21:23:38.574108Z"
    }
   },
   "outputs": [],
   "source": [
    "from sklearn.cluster import KMeans\n",
    "from sklearn.decomposition import PCA"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-08-04T21:23:50.765395Z",
     "start_time": "2020-08-04T21:23:49.606104Z"
    },
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 356
    },
    "colab_type": "code",
    "executionInfo": {
     "elapsed": 159773,
     "status": "ok",
     "timestamp": 1557215299684,
     "user": {
      "displayName": "Arjun Patel",
      "photoUrl": "https://lh5.googleusercontent.com/-BlLPIwYkZoQ/AAAAAAAAAAI/AAAAAAAABy0/fGnMWZAEyF0/s64/photo.jpg",
      "userId": "15471468418524645217"
     },
     "user_tz": 420
    },
    "id": "naKk2KMAehj6",
    "outputId": "2dafaaa3-1f8e-4cf3-8972-0eb00df81d30"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[-0.47388022 -0.48940984 -0.47170528 -0.58598457  3.12847338  0.2773695\n",
      " -0.08098323 -0.52677513 -0.48181311 -0.2952915 ]\n",
      "[-0.47341678 -0.44348545 -0.48547811 -0.37163591 -0.17621657  0.21203515\n",
      " -0.05867861 -0.23623446 -0.40466185  2.43777258]\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXwAAAD4CAYAAADvsV2wAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+WH4yJAAAQ2UlEQVR4nO3df4hl5X3H8fcn64RuqrApO4m6/tgUFiEmJCuDVYRgaVJ1SVCCBfNHLFJYIgYSCIKmoG3/SVohf1hDZGkkEdKEQMxmSdduTUgx+UPjuP5YzWabrSR1dyVOTHdVHBrXfvvH3E3H8c7Onb1n7p2d5/2Cy5x7zjPn+fI4fvbc554fqSokSWvf28ZdgCRpNAx8SWqEgS9JjTDwJakRBr4kNeKMcRdwMhs3bqzNmzePuwxJOm08/vjjv6mqyX7bVnXgb968menp6XGXIUmnjSS/WmybUzqS1IihAz/J+Ul+lGR/kmeTfKZPmyuTHEvyZO91x7D9SpKWp4spnePA56pqb5KzgMeTPFRVP1vQ7sdV9dEO+pMknYKhj/Cr6oWq2ttbfgXYD2wadr+SpG51OoefZDOwFXi0z+bLkzyV5MEkF59kH9uTTCeZnpmZ6bI8SWpaZ2fpJDkT+A7w2ap6ecHmvcCFVfVqkm3ATmBLv/1U1Q5gB8DU1NSqu7PbzicOc9eeAxw5Osu5G9Zz61UXcd1WP9BIWv06OcJPMsFc2H+jqh5YuL2qXq6qV3vLu4GJJBu76HuUdj5xmNsf2Mfho7MUcPjoLLc/sI+dTxwed2mStKQuztIJ8FVgf1V9aZE2Z/fakeTSXr8vDdv3qN215wCzr7/xpnWzr7/BXXsOjKkiSRpcF1M6VwCfBPYlebK37vPABQBVdS9wPXBzkuPALHBDnYY34j9ydHZZ6yVpNRk68KvqJ0CWaHMPcM+wfY3buRvWc7hPuJ+7Yf0YqpGk5fFK22W49aqLWD+x7k3r1k+s49arLhpTRZI0uFV9L53V5sTZOJ6lI+l0ZOAv03VbNxnwkk5LTulIUiMMfElqhIEvSY0w8CWpEQa+JDXCwJekRhj4ktQIA1+SGmHgS1IjDHxJaoSBL0mNMPAlqREGviQ1wsCXpEYY+JLUCANfkhph4EtSIwx8SWqEgS9JjTDwJakRBr4kNcLAl6RGDB34Sc5P8qMk+5M8m+Qzfdokyd1JDiZ5Osklw/YrSVqeMzrYx3Hgc1W1N8lZwONJHqqqn81rcw2wpff6E+ArvZ+SpBEZ+gi/ql6oqr295VeA/cCmBc2uBe6vOY8AG5KcM2zfkqTBdTqHn2QzsBV4dMGmTcDz894f4q3/KJzYx/Yk00mmZ2ZmuixPkprWWeAnORP4DvDZqnp54eY+v1L99lNVO6pqqqqmJicnuypPkprXSeAnmWAu7L9RVQ/0aXIIOH/e+/OAI130LUkaTBdn6QT4KrC/qr60SLNdwI29s3UuA45V1QvD9i1JGlwXZ+lcAXwS2Jfkyd66zwMXAFTVvcBuYBtwEHgNuKmDfiVJyzB04FfVT+g/Rz+/TQG3DNuXJOnUeaWtJDXCwJekRhj4ktQIA1+SGmHgS1IjDHxJaoSBL0mNMPAlqREGviQ1wsCXpEYY+JLUCANfkhph4EtSIwx8SWqEgS9JjTDwJakRBr4kNcLAl6RGGPiS1AgDX5IaYeBLUiMMfElqhIEvSY0w8CWpEZ0EfpL7kryY5JlFtl+Z5FiSJ3uvO7roV5I0uDM62s/XgHuA+0/S5sdV9dGO+pMkLVMnR/hV9TDw2y72JUlaGaOcw788yVNJHkxy8Qj7lSTR3ZTOUvYCF1bVq0m2ATuBLf0aJtkObAe44IILRlSeJK19IznCr6qXq+rV3vJuYCLJxkXa7qiqqaqampycHEV5ktSEkQR+krOTpLd8aa/fl0bRtyRpTidTOkm+CVwJbExyCLgTmACoqnuB64GbkxwHZoEbqqq66FuSNJhOAr+qPrHE9nuYO21TkjQmXmkrSY0w8CWpEQa+JDXCwJekRhj4ktQIA1+SGmHgS1IjDHxJaoSBL0mNMPAlqREGviQ1wsCXpEYY+JLUCANfkhph4EtSIwx8SWqEgS9JjTDwJakRBr4kNcLAl6RGGPiS1AgDX5IaYeBLUiMMfElqhIEvSY3oJPCT3JfkxSTPLLI9Se5OcjDJ00ku6aJfSdLgujrC/xpw9Um2XwNs6b22A1/pqF9J0oA6Cfyqehj47UmaXAvcX3MeATYkOaeLviVJgxnVHP4m4Pl57w/11r1Fku1JppNMz8zMjKQ4SWrBqAI/fdZVv4ZVtaOqpqpqanJycoXLkqR2jCrwDwHnz3t/HnBkRH1Lkhhd4O8CbuydrXMZcKyqXhhR35Ik4IwudpLkm8CVwMYkh4A7gQmAqroX2A1sAw4CrwE3ddGvJGlwnQR+VX1iie0F3NJFX5KkU+OVtpLUCANfkhph4EtSIwx8SWqEgS9JjTDwJakRBr4kNcLAl6RGGPiS1AgDX5IaYeBLUiMMfElqhIEvSY0w8CWpEQa+JDXCwJekRhj4ktQIA1+SGmHgS1IjDHxJaoSBL0mNMPAlqREGviQ1wsCXpEYY+JLUiE4CP8nVSQ4kOZjktj7br0xyLMmTvdcdXfQrSRrcGcPuIMk64MvAR4BDwGNJdlXVzxY0/XFVfXTY/iRJp6aLI/xLgYNV9VxV/Q74FnBtB/uVJHWoi8DfBDw/7/2h3rqFLk/yVJIHk1y82M6SbE8ynWR6Zmamg/IkSdBN4KfPulrwfi9wYVV9APhHYOdiO6uqHVU1VVVTk5OTHZQnSYJuAv8QcP689+cBR+Y3qKqXq+rV3vJuYCLJxg76liQNqIvAfwzYkuQ9Sd4O3ADsmt8gydlJ0lu+tNfvSx30LUka0NBn6VTV8SSfBvYA64D7qurZJJ/qbb8XuB64OclxYBa4oaoWTvtIklZQVnPuTk1N1fT09LjLkKTTRpLHq2qq3zavtJWkRhj4ktQIA1+SGmHgS1IjDHxJaoSBL0mNMPAlqREGviQ1wsCXpEYY+JLUCANfkhox9M3TdPrY+cRh7tpzgCNHZzl3w3puveoirtva71k1ktYiA78RO584zO0P7GP29TcAOHx0ltsf2Adg6EuNcEqnEXftOfD7sD9h9vU3uGvPgTFVJGnUDPxGHDk6u6z1ktYeA78R525Yv6z1ktYeA78Rt151Eesn1r1p3fqJddx61UVjqkjSqPmlbSNOfDHrWTpSuwz8hly3dZMBLzXMKR1JaoSBL0mNMPAlqREGviQ1wsCXpEYY+JLUiE4CP8nVSQ4kOZjktj7bk+Tu3vank1zSRb+SpMENHfhJ1gFfBq4B3gt8Isl7FzS7BtjSe20HvjJsv5Kk5eniCP9S4GBVPVdVvwO+BVy7oM21wP015xFgQ5JzOuhbkjSgLgJ/E/D8vPeHeuuW2waAJNuTTCeZnpmZ6aA8SRJ0c2uF9FlXp9BmbmXVDmAHwNTUVN82pzufPCVpHLoI/EPA+fPenwccOYU2TfDJU5LGpYspnceALUnek+TtwA3ArgVtdgE39s7WuQw4VlUvdND3accnT0kal6GP8KvqeJJPA3uAdcB9VfVskk/1tt8L7Aa2AQeB14Cbhu33dOWTpySNSye3R66q3cyF+vx1985bLuCWLvo63Z27YT2H+4S7T56StNK80nbEfPKUpHHxASgj5pOnJI2LgT8GPnlK0jgY+JK0Sqz0NToGviStAqO4RscvbSVpFRjFNToGviStAqO4RqeJKR3vXSNptRvFNTpr/gj/xLzY4aOzFP8/L7bzicPjLk2Sfm8U1+is+cD33jWSTgfXbd3EFz7+fjZtWE+ATRvW84WPv9+zdJbDe9dIOl2s9DU6a/4If7H5L+9dI6k1az7wvXeNJM1Z81M63rtGkuas+cAH710jSdDAlI4kac6aO8L3IitJ6m9NBb4PCJekxa2pwD/ZRVYnAt9PAJJataYCf6mLrPwEIKlla+pL26UusvI2C5JatqYCf6mLrLzNgqSWrakpnaUustrwjgn++7XX3/J7G94xMdI6JWkc1lTgw8kvsqrq/zv/s2CaR5LWojU1pbOUY7NvPboHeO31//X++JLWvKECP8kfJXkoyS96P9+5SLtfJtmX5Mkk08P0OYyT3SHzb3Y9O8JKJGn0hj3Cvw34YVVtAX7Ye7+YP62qD1bV1JB9nrKT3SHz6CJH/5K0Vgwb+NcCX+8tfx24bsj9rSjPtZfUsmED/91V9QJA7+e7FmlXwL8leTzJ9pPtMMn2JNNJpmdmZoYs763eucgZOYutl6S1YsnAT/KDJM/0eV27jH6uqKpLgGuAW5J8aLGGVbWjqqaqampycnIZXQzmzo9dzMS6vGndxLpw58cu7rwvSVpNljwts6o+vNi2JL9Ock5VvZDkHODFRfZxpPfzxSTfBS4FHj7FmofiA1EktWrY8/B3AX8JfLH383sLGyT5Q+BtVfVKb/nPgb8bst+h+EAUSS0adg7/i8BHkvwC+EjvPUnOTbK71+bdwE+SPAX8FPiXqvrXIfuVJC3TUEf4VfUS8Gd91h8BtvWWnwM+MEw/kqThNXWlrSS1zMCXpEYY+JLUiNRit5BcBZLMAL9agV1vBH6zAvvtkjV2wxq7YY3dGEWNF1ZV34uYVnXgr5Qk0+O8p88grLEb1tgNa+zGuGt0SkeSGmHgS1IjWg38HeMuYADW2A1r7IY1dmOsNTY5hy9JLWr1CF+SmmPgS1Ijmgj81frs3SRXJzmQ5GCStzweMnPu7m1/OsklK13TKdR4ZZJjvTF7MskdY6jxviQvJnlmke2rYRyXqnGs45jk/CQ/SrI/ybNJPtOnzVjHccAaxz2Of5Dkp0me6tX4t33ajG8cq2rNv4B/AG7rLd8G/P0i7X4JbBxRTeuA/wT+GHg78BTw3gVttgEPAgEuAx4d8bgNUuOVwPfH/N/3Q8AlwDOLbB/rOA5Y41jHETgHuKS3fBbwH6vw73GQGsc9jgHO7C1PAI8Cl62WcWziCJ/V+ezdS4GDVfVcVf0O+BZzdc53LXB/zXkE2NB70MxqqnHsquph4LcnaTLucRykxrGqqheqam9v+RVgP7DwoRFjHccBaxyr3ti82ns70XstPDNmbOPYSuB3/uzdDmwCnp/3/hBv/eMdpM1KGrT/y3sfYR9MshqfFTnucRzUqhjHJJuBrcwdnc63asbxJDXCmMcxybokTzL3BMCHqmrVjOOwT7xaNZL8ADi7z6a/XsZurqiqI0neBTyU5Oe9I7OVkD7rFh4JDNJmJQ3S/17m7t3xapJtwE5gy4pXtjzjHsdBrIpxTHIm8B3gs1X18sLNfX5l5OO4RI1jH8eqegP4YJINwHeTvK+q5n93M7ZxXDNH+FX14ap6X5/X94Bfn/jIlAGfvQucePbuSjkEnD/v/XnAkVNos5KW7L+qXj7xEbaqdgMTSTaOrsSBjHscl7QaxjHJBHNB+o2qeqBPk7GP41I1roZxnFfLUeDfgasXbBrbOK6ZwF/CiWfvwkmevZvkrBPLzD17t+8ZFR15DNiS5D1J3g7c0Ktzvl3Ajb1v9S8Djp2YmhqRJWtMcnaS9JYvZe5v6qUR1jiIcY/jksY9jr2+vwrsr6ovLdJsrOM4SI2rYBwne0f2JFkPfBj4+YJmYxvHNTOls4QvAt9O8lfAfwF/AXPP3gX+qaq2Mffs3e/2/lbOAP65VvDZu1V1PMmngT3MnQ1zX1U9m+RTve33AruZ+0b/IPAacNNK1TNEjdcDNyc5DswCN1TvVIRRSfJN5s7O2JjkEHAnc1+WrYpxHLDGcY/jFcAngX29+WeAzwMXzKtx3OM4SI3jHsdzgK8nWcfcPzbfrqrvr5b/r721giQ1opUpHUlqnoEvSY0w8CWpEQa+JDXCwJekRhj4ktQIA1+SGvF/jIbe5QqtpyMAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "kmeans1 = KMeans(n_clusters = 10)\n",
    "kmeans1.fit(vectorized_folders_df)\n",
    "labels1 = kmeans1.predict(vectorized_folders_df)\n",
    "\n",
    "# HERE WE APPEND THE CLUSTER LABEL\n",
    "vectorized_folders_df['label'] = labels1\n",
    "\n",
    "centers1 = kmeans1.cluster_centers_\n",
    "pca_model1 = PCA(n_components=2)\n",
    "centers_pca1 = pca_model1.fit_transform(centers1)\n",
    "centers_pca1_X = centers_pca1.T[0]\n",
    "centers_pca1_y = centers_pca1.T[1]\n",
    "\n",
    "print(centers_pca1_X)\n",
    "print(centers_pca1_y)\n",
    "\n",
    "plt.scatter(centers_pca1_X, centers_pca1_y)\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-08-04T21:23:57.192912Z",
     "start_time": "2020-08-04T21:23:57.181910Z"
    },
    "colab": {},
    "colab_type": "code",
    "id": "6F04DLzBehh0"
   },
   "outputs": [],
   "source": [
    "# Remove the word embedding columns from vectorized_folders_df\n",
    "vectorized_folders_df = vectorized_folders_df.reset_index().rename(columns = {'index': 'subject'})\n",
    "# vectorized_folders_df = vectorized_folders_df[['subject', 'label']]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-08-04T21:23:58.730195Z",
     "start_time": "2020-08-04T21:23:58.709199Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>subject</th>\n",
       "      <th>0</th>\n",
       "      <th>1</th>\n",
       "      <th>2</th>\n",
       "      <th>3</th>\n",
       "      <th>4</th>\n",
       "      <th>5</th>\n",
       "      <th>6</th>\n",
       "      <th>7</th>\n",
       "      <th>8</th>\n",
       "      <th>...</th>\n",
       "      <th>291</th>\n",
       "      <th>292</th>\n",
       "      <th>293</th>\n",
       "      <th>294</th>\n",
       "      <th>295</th>\n",
       "      <th>296</th>\n",
       "      <th>297</th>\n",
       "      <th>298</th>\n",
       "      <th>299</th>\n",
       "      <th>label</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>sent</td>\n",
       "      <td>-0.0170898</td>\n",
       "      <td>0.021729</td>\n",
       "      <td>0.225586</td>\n",
       "      <td>-0.129883</td>\n",
       "      <td>-0.199219</td>\n",
       "      <td>-0.107910</td>\n",
       "      <td>-0.184570</td>\n",
       "      <td>-0.083984</td>\n",
       "      <td>0.004974</td>\n",
       "      <td>...</td>\n",
       "      <td>-0.283203</td>\n",
       "      <td>-0.051758</td>\n",
       "      <td>0.112305</td>\n",
       "      <td>0.267578</td>\n",
       "      <td>0.059082</td>\n",
       "      <td>0.144531</td>\n",
       "      <td>-0.091309</td>\n",
       "      <td>-0.069824</td>\n",
       "      <td>-0.021606</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>documents</td>\n",
       "      <td>0.0976562</td>\n",
       "      <td>0.063965</td>\n",
       "      <td>0.024658</td>\n",
       "      <td>-0.151367</td>\n",
       "      <td>-0.032471</td>\n",
       "      <td>0.110352</td>\n",
       "      <td>0.044678</td>\n",
       "      <td>0.195312</td>\n",
       "      <td>0.089844</td>\n",
       "      <td>...</td>\n",
       "      <td>-0.245117</td>\n",
       "      <td>-0.113770</td>\n",
       "      <td>0.026367</td>\n",
       "      <td>-0.156250</td>\n",
       "      <td>0.119141</td>\n",
       "      <td>0.073730</td>\n",
       "      <td>-0.292969</td>\n",
       "      <td>-0.035889</td>\n",
       "      <td>0.032959</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>contacts</td>\n",
       "      <td>0.0742188</td>\n",
       "      <td>0.001274</td>\n",
       "      <td>0.065918</td>\n",
       "      <td>-0.086914</td>\n",
       "      <td>-0.070801</td>\n",
       "      <td>0.249023</td>\n",
       "      <td>-0.320312</td>\n",
       "      <td>-0.174805</td>\n",
       "      <td>0.091797</td>\n",
       "      <td>...</td>\n",
       "      <td>0.097656</td>\n",
       "      <td>0.023071</td>\n",
       "      <td>-0.063965</td>\n",
       "      <td>0.114258</td>\n",
       "      <td>0.015869</td>\n",
       "      <td>0.416016</td>\n",
       "      <td>-0.140625</td>\n",
       "      <td>-0.050537</td>\n",
       "      <td>-0.033447</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>deleted</td>\n",
       "      <td>-0.172852</td>\n",
       "      <td>-0.047852</td>\n",
       "      <td>0.316406</td>\n",
       "      <td>0.145508</td>\n",
       "      <td>-0.220703</td>\n",
       "      <td>0.009521</td>\n",
       "      <td>-0.028076</td>\n",
       "      <td>0.216797</td>\n",
       "      <td>0.507812</td>\n",
       "      <td>...</td>\n",
       "      <td>0.076660</td>\n",
       "      <td>0.363281</td>\n",
       "      <td>-0.200195</td>\n",
       "      <td>0.287109</td>\n",
       "      <td>-0.263672</td>\n",
       "      <td>0.196289</td>\n",
       "      <td>-0.361328</td>\n",
       "      <td>-0.050537</td>\n",
       "      <td>-0.070801</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>discussion</td>\n",
       "      <td>-0.0390625</td>\n",
       "      <td>0.042236</td>\n",
       "      <td>0.014465</td>\n",
       "      <td>0.115234</td>\n",
       "      <td>0.132812</td>\n",
       "      <td>-0.101074</td>\n",
       "      <td>0.164062</td>\n",
       "      <td>-0.100098</td>\n",
       "      <td>0.123535</td>\n",
       "      <td>...</td>\n",
       "      <td>-0.175781</td>\n",
       "      <td>0.326172</td>\n",
       "      <td>0.152344</td>\n",
       "      <td>0.042236</td>\n",
       "      <td>-0.011536</td>\n",
       "      <td>-0.125000</td>\n",
       "      <td>-0.229492</td>\n",
       "      <td>-0.112305</td>\n",
       "      <td>-0.158203</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 302 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "      subject          0         1         2         3         4         5  \\\n",
       "0        sent -0.0170898  0.021729  0.225586 -0.129883 -0.199219 -0.107910   \n",
       "1   documents  0.0976562  0.063965  0.024658 -0.151367 -0.032471  0.110352   \n",
       "2    contacts  0.0742188  0.001274  0.065918 -0.086914 -0.070801  0.249023   \n",
       "3     deleted  -0.172852 -0.047852  0.316406  0.145508 -0.220703  0.009521   \n",
       "4  discussion -0.0390625  0.042236  0.014465  0.115234  0.132812 -0.101074   \n",
       "\n",
       "          6         7         8  ...       291       292       293       294  \\\n",
       "0 -0.184570 -0.083984  0.004974  ... -0.283203 -0.051758  0.112305  0.267578   \n",
       "1  0.044678  0.195312  0.089844  ... -0.245117 -0.113770  0.026367 -0.156250   \n",
       "2 -0.320312 -0.174805  0.091797  ...  0.097656  0.023071 -0.063965  0.114258   \n",
       "3 -0.028076  0.216797  0.507812  ...  0.076660  0.363281 -0.200195  0.287109   \n",
       "4  0.164062 -0.100098  0.123535  ... -0.175781  0.326172  0.152344  0.042236   \n",
       "\n",
       "        295       296       297       298       299  label  \n",
       "0  0.059082  0.144531 -0.091309 -0.069824 -0.021606      1  \n",
       "1  0.119141  0.073730 -0.292969 -0.035889  0.032959      0  \n",
       "2  0.015869  0.416016 -0.140625 -0.050537 -0.033447      0  \n",
       "3 -0.263672  0.196289 -0.361328 -0.050537 -0.070801      0  \n",
       "4 -0.011536 -0.125000 -0.229492 -0.112305 -0.158203      1  \n",
       "\n",
       "[5 rows x 302 columns]"
      ]
     },
     "execution_count": 28,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "vectorized_folders_df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-08-04T21:24:12.832294Z",
     "start_time": "2020-08-04T21:24:12.142800Z"
    }
   },
   "outputs": [],
   "source": [
    "vectorized_folders_df.to_csv(os.path.join(dataset_directory, \"vectorized_folders_df_w_label.csv\"), index=False)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "sf2g61C5f06Z"
   },
   "source": [
    "Unfortunately, many of these items were not placed in the correct label. For this we wen't one by one and fix some of these errors. After much manual feature engineering to modify the labels, a new csv with the name of `NEW_vectorized_folders_df.csv` was created."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-08-04T21:57:50.056413Z",
     "start_time": "2020-08-04T21:57:50.043419Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>message</th>\n",
       "      <th>Date</th>\n",
       "      <th>Subject</th>\n",
       "      <th>Message-Body</th>\n",
       "      <th>Date_datetime</th>\n",
       "      <th>Day_of_week</th>\n",
       "      <th>X-Folder-split</th>\n",
       "      <th>message_length</th>\n",
       "      <th>contains-cc</th>\n",
       "      <th>contains-bcc</th>\n",
       "      <th>X-Folder-split_cleaned</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>Message-ID: &lt;18782981.1075855378110.JavaMail.e...</td>\n",
       "      <td>Mon, 14 May 2001 16:39:00 -0700 (PDT)</td>\n",
       "      <td>NaN</td>\n",
       "      <td>here is our forecast\\n\\n</td>\n",
       "      <td>2001-05-14 16:39:00</td>\n",
       "      <td>Mon</td>\n",
       "      <td>'sent mail</td>\n",
       "      <td>23</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>sent</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>Message-ID: &lt;15464986.1075855378456.JavaMail.e...</td>\n",
       "      <td>Fri, 4 May 2001 13:51:00 -0700 (PDT)</td>\n",
       "      <td>Re:</td>\n",
       "      <td>traveling to have a business meeting takes the...</td>\n",
       "      <td>2001-05-04 13:51:00</td>\n",
       "      <td>Fri</td>\n",
       "      <td>'sent mail</td>\n",
       "      <td>786</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>sent</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>Message-ID: &lt;24216240.1075855687451.JavaMail.e...</td>\n",
       "      <td>Wed, 18 Oct 2000 03:00:00 -0700 (PDT)</td>\n",
       "      <td>Re: test</td>\n",
       "      <td>test successful.  way to go!!!</td>\n",
       "      <td>2000-10-18 03:00:00</td>\n",
       "      <td>Wed</td>\n",
       "      <td>'sent mail</td>\n",
       "      <td>30</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>sent</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>Message-ID: &lt;13505866.1075863688222.JavaMail.e...</td>\n",
       "      <td>Mon, 23 Oct 2000 06:13:00 -0700 (PDT)</td>\n",
       "      <td>NaN</td>\n",
       "      <td>randy,\\n\\n can you send me a schedule of the s...</td>\n",
       "      <td>2000-10-23 06:13:00</td>\n",
       "      <td>Mon</td>\n",
       "      <td>'sent mail</td>\n",
       "      <td>187</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>sent</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>Message-ID: &lt;30922949.1075863688243.JavaMail.e...</td>\n",
       "      <td>Thu, 31 Aug 2000 05:07:00 -0700 (PDT)</td>\n",
       "      <td>Re: Hello</td>\n",
       "      <td>let's shoot for tuesday at 11:45.</td>\n",
       "      <td>2000-08-31 05:07:00</td>\n",
       "      <td>Thu</td>\n",
       "      <td>'sent mail</td>\n",
       "      <td>35</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>sent</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                             message  \\\n",
       "0  Message-ID: <18782981.1075855378110.JavaMail.e...   \n",
       "1  Message-ID: <15464986.1075855378456.JavaMail.e...   \n",
       "2  Message-ID: <24216240.1075855687451.JavaMail.e...   \n",
       "3  Message-ID: <13505866.1075863688222.JavaMail.e...   \n",
       "4  Message-ID: <30922949.1075863688243.JavaMail.e...   \n",
       "\n",
       "                                    Date    Subject  \\\n",
       "0  Mon, 14 May 2001 16:39:00 -0700 (PDT)        NaN   \n",
       "1   Fri, 4 May 2001 13:51:00 -0700 (PDT)        Re:   \n",
       "2  Wed, 18 Oct 2000 03:00:00 -0700 (PDT)   Re: test   \n",
       "3  Mon, 23 Oct 2000 06:13:00 -0700 (PDT)        NaN   \n",
       "4  Thu, 31 Aug 2000 05:07:00 -0700 (PDT)  Re: Hello   \n",
       "\n",
       "                                        Message-Body        Date_datetime  \\\n",
       "0                          here is our forecast\\n\\n   2001-05-14 16:39:00   \n",
       "1  traveling to have a business meeting takes the...  2001-05-04 13:51:00   \n",
       "2                     test successful.  way to go!!!  2000-10-18 03:00:00   \n",
       "3  randy,\\n\\n can you send me a schedule of the s...  2000-10-23 06:13:00   \n",
       "4                let's shoot for tuesday at 11:45.    2000-08-31 05:07:00   \n",
       "\n",
       "  Day_of_week X-Folder-split  message_length  contains-cc  contains-bcc  \\\n",
       "0         Mon     'sent mail              23            0             0   \n",
       "1         Fri     'sent mail             786            0             0   \n",
       "2         Wed     'sent mail              30            0             0   \n",
       "3         Mon     'sent mail             187            0             0   \n",
       "4         Thu     'sent mail              35            0             0   \n",
       "\n",
       "  X-Folder-split_cleaned  \n",
       "0                   sent  \n",
       "1                   sent  \n",
       "2                   sent  \n",
       "3                   sent  \n",
       "4                   sent  "
      ]
     },
     "execution_count": 34,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data2.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-08-04T21:58:44.283687Z",
     "start_time": "2020-08-04T21:58:26.313482Z"
    }
   },
   "outputs": [],
   "source": [
    "data_testing = pd.read_csv(os.path.join(dataset_directory, \"preprocessed_data_v2.csv\"))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-08-04T21:58:44.296681Z",
     "start_time": "2020-08-04T21:58:44.284678Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>message</th>\n",
       "      <th>Date</th>\n",
       "      <th>Subject</th>\n",
       "      <th>Message-Body</th>\n",
       "      <th>Date_datetime</th>\n",
       "      <th>Day_of_week</th>\n",
       "      <th>X-Folder-split</th>\n",
       "      <th>message_length</th>\n",
       "      <th>contains-cc</th>\n",
       "      <th>contains-bcc</th>\n",
       "      <th>X-Folder-split_cleaned</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>Message-ID: &lt;18782981.1075855378110.JavaMail.e...</td>\n",
       "      <td>Mon, 14 May 2001 16:39:00 -0700 (PDT)</td>\n",
       "      <td>NaN</td>\n",
       "      <td>here is our forecast\\n\\n</td>\n",
       "      <td>2001-05-14 16:39:00</td>\n",
       "      <td>Mon</td>\n",
       "      <td>'sent mail</td>\n",
       "      <td>23</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>sent</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>Message-ID: &lt;15464986.1075855378456.JavaMail.e...</td>\n",
       "      <td>Fri, 4 May 2001 13:51:00 -0700 (PDT)</td>\n",
       "      <td>Re:</td>\n",
       "      <td>traveling to have a business meeting takes the...</td>\n",
       "      <td>2001-05-04 13:51:00</td>\n",
       "      <td>Fri</td>\n",
       "      <td>'sent mail</td>\n",
       "      <td>786</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>sent</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>Message-ID: &lt;24216240.1075855687451.JavaMail.e...</td>\n",
       "      <td>Wed, 18 Oct 2000 03:00:00 -0700 (PDT)</td>\n",
       "      <td>Re: test</td>\n",
       "      <td>test successful.  way to go!!!</td>\n",
       "      <td>2000-10-18 03:00:00</td>\n",
       "      <td>Wed</td>\n",
       "      <td>'sent mail</td>\n",
       "      <td>30</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>sent</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>Message-ID: &lt;13505866.1075863688222.JavaMail.e...</td>\n",
       "      <td>Mon, 23 Oct 2000 06:13:00 -0700 (PDT)</td>\n",
       "      <td>NaN</td>\n",
       "      <td>randy,\\n\\n can you send me a schedule of the s...</td>\n",
       "      <td>2000-10-23 06:13:00</td>\n",
       "      <td>Mon</td>\n",
       "      <td>'sent mail</td>\n",
       "      <td>187</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>sent</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>Message-ID: &lt;30922949.1075863688243.JavaMail.e...</td>\n",
       "      <td>Thu, 31 Aug 2000 05:07:00 -0700 (PDT)</td>\n",
       "      <td>Re: Hello</td>\n",
       "      <td>let's shoot for tuesday at 11:45.</td>\n",
       "      <td>2000-08-31 05:07:00</td>\n",
       "      <td>Thu</td>\n",
       "      <td>'sent mail</td>\n",
       "      <td>35</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>sent</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                             message  \\\n",
       "0  Message-ID: <18782981.1075855378110.JavaMail.e...   \n",
       "1  Message-ID: <15464986.1075855378456.JavaMail.e...   \n",
       "2  Message-ID: <24216240.1075855687451.JavaMail.e...   \n",
       "3  Message-ID: <13505866.1075863688222.JavaMail.e...   \n",
       "4  Message-ID: <30922949.1075863688243.JavaMail.e...   \n",
       "\n",
       "                                    Date    Subject  \\\n",
       "0  Mon, 14 May 2001 16:39:00 -0700 (PDT)        NaN   \n",
       "1   Fri, 4 May 2001 13:51:00 -0700 (PDT)        Re:   \n",
       "2  Wed, 18 Oct 2000 03:00:00 -0700 (PDT)   Re: test   \n",
       "3  Mon, 23 Oct 2000 06:13:00 -0700 (PDT)        NaN   \n",
       "4  Thu, 31 Aug 2000 05:07:00 -0700 (PDT)  Re: Hello   \n",
       "\n",
       "                                        Message-Body        Date_datetime  \\\n",
       "0                          here is our forecast\\n\\n   2001-05-14 16:39:00   \n",
       "1  traveling to have a business meeting takes the...  2001-05-04 13:51:00   \n",
       "2                     test successful.  way to go!!!  2000-10-18 03:00:00   \n",
       "3  randy,\\n\\n can you send me a schedule of the s...  2000-10-23 06:13:00   \n",
       "4                let's shoot for tuesday at 11:45.    2000-08-31 05:07:00   \n",
       "\n",
       "  Day_of_week X-Folder-split  message_length  contains-cc  contains-bcc  \\\n",
       "0         Mon     'sent mail              23            0             0   \n",
       "1         Fri     'sent mail             786            0             0   \n",
       "2         Wed     'sent mail              30            0             0   \n",
       "3         Mon     'sent mail             187            0             0   \n",
       "4         Thu     'sent mail              35            0             0   \n",
       "\n",
       "  X-Folder-split_cleaned  \n",
       "0                   sent  \n",
       "1                   sent  \n",
       "2                   sent  \n",
       "3                   sent  \n",
       "4                   sent  "
      ]
     },
     "execution_count": 36,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data_testing.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "H3mXfCf5dq69"
   },
   "source": [
    "## Joining Clustering Results with Main DF"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "4tkoagYtfw3T"
   },
   "outputs": [],
   "source": [
    "NEW_vectorized_folders_df = pd.read_csv(os.path.join()'NEW_vectorized_folders_df.csv')\n",
    "NEW_vectorized_folders_df = NEW_vectorized_folders_df[['subject', 'label']]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "PhvIzXejd130"
   },
   "outputs": [],
   "source": [
    "data2 = data2.merge(right = NEW_vectorized_folders_df,\n",
    "                    how = 'left',\n",
    "                    left_on = 'X-Folder-split_cleaned',\n",
    "                    right_on = 'subject')\n",
    "\n",
    "data = data2.copy()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "zeK-H25CjA96"
   },
   "source": [
    "## Drop Labels 1 and 2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "zDTwpvRejBRM"
   },
   "outputs": [],
   "source": [
    "data_filtered = data[(data['label'] != 1) & \n",
    "                     (data['label'] != 2)]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "_XYzBHDMO4LV"
   },
   "source": [
    "The data_filtered is ready to go as our main dataset. Unfortunately, we cannot export to CSV because there aren't any delimiters that work -- the message bodies contain the special characters we'd use as delimiters and so they cause errors in reading back in the exported CSV.\n",
    "\n",
    "As such, the preprocessing must be run each time the notebook is opened."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "rDb8QODER7T5"
   },
   "source": [
    "## One Hot Encoding and Adding in Embedded Dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "GtNlDWu-RtoD"
   },
   "outputs": [],
   "source": [
    "dfDummies = pd.get_dummies(data_filtered['Day_of_week'], prefix = 'Day= ')\n",
    "data_filtered = pd.concat([data_filtered, dfDummies], axis=1)\n",
    "data_filtered = data_filtered.drop(labels='Day_of_week', axis=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 269
    },
    "colab_type": "code",
    "executionInfo": {
     "elapsed": 1676,
     "status": "ok",
     "timestamp": 1557216522905,
     "user": {
      "displayName": "Arjun Patel",
      "photoUrl": "https://lh5.googleusercontent.com/-BlLPIwYkZoQ/AAAAAAAAAAI/AAAAAAAABy0/fGnMWZAEyF0/s64/photo.jpg",
      "userId": "15471468418524645217"
     },
     "user_tz": 420
    },
    "id": "Tr4iU_sXaMzu",
    "outputId": "be8644f2-a9f1-4068-f47f-417188348cf5"
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Unnamed: 0</th>\n",
       "      <th>message_length</th>\n",
       "      <th>label</th>\n",
       "      <th>0</th>\n",
       "      <th>1</th>\n",
       "      <th>2</th>\n",
       "      <th>3</th>\n",
       "      <th>4</th>\n",
       "      <th>5</th>\n",
       "      <th>6</th>\n",
       "      <th>...</th>\n",
       "      <th>247</th>\n",
       "      <th>248</th>\n",
       "      <th>249</th>\n",
       "      <th>Day= _Fri</th>\n",
       "      <th>Day= _Mon</th>\n",
       "      <th>Day= _Sat</th>\n",
       "      <th>Day= _Sun</th>\n",
       "      <th>Day= _Thu</th>\n",
       "      <th>Day= _Tue</th>\n",
       "      <th>Day= _Wed</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0</td>\n",
       "      <td>262</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>5</td>\n",
       "      <td>13</td>\n",
       "      <td>5</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1</td>\n",
       "      <td>1255</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>56189</td>\n",
       "      <td>17</td>\n",
       "      <td>76024</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>2</td>\n",
       "      <td>1120</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>374</td>\n",
       "      <td>1365</td>\n",
       "      <td>6669</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>3</td>\n",
       "      <td>541</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>32619</td>\n",
       "      <td>17</td>\n",
       "      <td>26431</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>4</td>\n",
       "      <td>1415</td>\n",
       "      <td>0</td>\n",
       "      <td>366</td>\n",
       "      <td>137</td>\n",
       "      <td>1915</td>\n",
       "      <td>47</td>\n",
       "      <td>10180</td>\n",
       "      <td>1061</td>\n",
       "      <td>8563</td>\n",
       "      <td>...</td>\n",
       "      <td>86</td>\n",
       "      <td>1957</td>\n",
       "      <td>76038</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 260 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "   Unnamed: 0  message_length  label    0    1     2   3      4     5     6  \\\n",
       "0           0             262      0    0    0     0   0      0     0     0   \n",
       "1           1            1255      0    0    0     0   0      0     0     0   \n",
       "2           2            1120      0    0    0     0   0      0     0     0   \n",
       "3           3             541      0    0    0     0   0      0     0     0   \n",
       "4           4            1415      0  366  137  1915  47  10180  1061  8563   \n",
       "\n",
       "   ...    247   248    249  Day= _Fri  Day= _Mon  Day= _Sat  Day= _Sun  \\\n",
       "0  ...      5    13      5          0          0          0          0   \n",
       "1  ...  56189    17  76024          0          0          0          0   \n",
       "2  ...    374  1365   6669          0          0          0          0   \n",
       "3  ...  32619    17  26431          0          0          0          0   \n",
       "4  ...     86  1957  76038          1          0          0          0   \n",
       "\n",
       "   Day= _Thu  Day= _Tue  Day= _Wed  \n",
       "0          0          1          0  \n",
       "1          1          0          0  \n",
       "2          1          0          0  \n",
       "3          1          0          0  \n",
       "4          0          0          0  \n",
       "\n",
       "[5 rows x 260 columns]"
      ]
     },
     "execution_count": 34,
     "metadata": {
      "tags": []
     },
     "output_type": "execute_result"
    }
   ],
   "source": [
    "embedded_df = pd.read_csv('Embedded_Dataset.csv')\n",
    "embedded_df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 52
    },
    "colab_type": "code",
    "executionInfo": {
     "elapsed": 975,
     "status": "ok",
     "timestamp": 1557216522907,
     "user": {
      "displayName": "Arjun Patel",
      "photoUrl": "https://lh5.googleusercontent.com/-BlLPIwYkZoQ/AAAAAAAAAAI/AAAAAAAABy0/fGnMWZAEyF0/s64/photo.jpg",
      "userId": "15471468418524645217"
     },
     "user_tz": 420
    },
    "id": "db1AyA5Td75J",
    "outputId": "6bfde6e2-b0b8-4c78-9b2a-cae061f118bb"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(71374, 19)\n",
      "(29141, 260)\n"
     ]
    }
   ],
   "source": [
    "print(data_filtered.shape)\n",
    "print(embedded_df.shape)"
   ]
  }
 ],
 "metadata": {
  "colab": {
   "collapsed_sections": [],
   "name": "MAIN_NOTEBOOK.ipynb",
   "provenance": [
    {
     "file_id": "0ByYbtoChpm1SVlNhVmU0STFNWUhKbzdsVHJPT2pidDdVdUVF",
     "timestamp": 1555110976262
    }
   ],
   "toc_visible": true,
   "version": "0.3.2"
  },
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.7"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}