Pytorch_RNN_Fake_News.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 71,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import numpy as np\n",
    "import matplotlib.pyplot as plt\n",
    "import os\n",
    "import tqdm\n",
    "import torch\n",
    "from sklearn.model_selection import train_test_split\n",
    "import torch.nn as nn\n",
    "from unidecode import unidecode\n",
    "import random\n",
    "import torch.nn.functional as F\n",
    "import contractions\n",
    "from torch.nn.utils.rnn import pack_padded_sequence,pad_packed_sequence\n",
    "from torchtext.data import Field,TabularDataset,LabelField,BucketIterator\n",
    "from torchtext import data\n",
    "import re\n",
    "import spacy\n",
    "from spacy.tokenizer import Tokenizer"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Unnamed: 0</th>\n",
       "      <th>title</th>\n",
       "      <th>text</th>\n",
       "      <th>label</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>8476</td>\n",
       "      <td>You Can Smell Hillary’s Fear</td>\n",
       "      <td>Daniel Greenfield, a Shillman Journalism Fello...</td>\n",
       "      <td>FAKE</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>10294</td>\n",
       "      <td>Watch The Exact Moment Paul Ryan Committed Pol...</td>\n",
       "      <td>Google Pinterest Digg Linkedin Reddit Stumbleu...</td>\n",
       "      <td>FAKE</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>3608</td>\n",
       "      <td>Kerry to go to Paris in gesture of sympathy</td>\n",
       "      <td>U.S. Secretary of State John F. Kerry said Mon...</td>\n",
       "      <td>REAL</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>10142</td>\n",
       "      <td>Bernie supporters on Twitter erupt in anger ag...</td>\n",
       "      <td>— Kaydee King (@KaydeeKing) November 9, 2016 T...</td>\n",
       "      <td>FAKE</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>875</td>\n",
       "      <td>The Battle of New York: Why This Primary Matters</td>\n",
       "      <td>It's primary day in New York and front-runners...</td>\n",
       "      <td>REAL</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   Unnamed: 0                                              title  \\\n",
       "0        8476                       You Can Smell Hillary’s Fear   \n",
       "1       10294  Watch The Exact Moment Paul Ryan Committed Pol...   \n",
       "2        3608        Kerry to go to Paris in gesture of sympathy   \n",
       "3       10142  Bernie supporters on Twitter erupt in anger ag...   \n",
       "4         875   The Battle of New York: Why This Primary Matters   \n",
       "\n",
       "                                                text label  \n",
       "0  Daniel Greenfield, a Shillman Journalism Fello...  FAKE  \n",
       "1  Google Pinterest Digg Linkedin Reddit Stumbleu...  FAKE  \n",
       "2  U.S. Secretary of State John F. Kerry said Mon...  REAL  \n",
       "3  — Kaydee King (@KaydeeKing) November 9, 2016 T...  FAKE  \n",
       "4  It's primary day in New York and front-runners...  REAL  "
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df=pd.read_csv(\"real_nd_fake_news/news.csv\")\n",
    "df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "df.drop([\"Unnamed: 0\"],axis=1,inplace=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "df[\"label\"]=(df[\"label\"] == \"FAKE\").astype(\"int\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>title</th>\n",
       "      <th>text</th>\n",
       "      <th>label</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>You Can Smell Hillary’s Fear</td>\n",
       "      <td>Daniel Greenfield, a Shillman Journalism Fello...</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>Watch The Exact Moment Paul Ryan Committed Pol...</td>\n",
       "      <td>Google Pinterest Digg Linkedin Reddit Stumbleu...</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>Kerry to go to Paris in gesture of sympathy</td>\n",
       "      <td>U.S. Secretary of State John F. Kerry said Mon...</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>Bernie supporters on Twitter erupt in anger ag...</td>\n",
       "      <td>— Kaydee King (@KaydeeKing) November 9, 2016 T...</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>The Battle of New York: Why This Primary Matters</td>\n",
       "      <td>It's primary day in New York and front-runners...</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                               title  \\\n",
       "0                       You Can Smell Hillary’s Fear   \n",
       "1  Watch The Exact Moment Paul Ryan Committed Pol...   \n",
       "2        Kerry to go to Paris in gesture of sympathy   \n",
       "3  Bernie supporters on Twitter erupt in anger ag...   \n",
       "4   The Battle of New York: Why This Primary Matters   \n",
       "\n",
       "                                                text  label  \n",
       "0  Daniel Greenfield, a Shillman Journalism Fello...      1  \n",
       "1  Google Pinterest Digg Linkedin Reddit Stumbleu...      1  \n",
       "2  U.S. Secretary of State John F. Kerry said Mon...      0  \n",
       "3  — Kaydee King (@KaydeeKing) November 9, 2016 T...      1  \n",
       "4  It's primary day in New York and front-runners...      0  "
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "df[\"titletext\"]=df[\"title\"]+\". \"+df[\"text\"]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>title</th>\n",
       "      <th>text</th>\n",
       "      <th>label</th>\n",
       "      <th>titletext</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>You Can Smell Hillary’s Fear</td>\n",
       "      <td>Daniel Greenfield, a Shillman Journalism Fello...</td>\n",
       "      <td>1</td>\n",
       "      <td>you can smell hillary’s fear. daniel greenfiel...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>Watch The Exact Moment Paul Ryan Committed Pol...</td>\n",
       "      <td>Google Pinterest Digg Linkedin Reddit Stumbleu...</td>\n",
       "      <td>1</td>\n",
       "      <td>watch the exact moment paul ryan committed pol...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>Kerry to go to Paris in gesture of sympathy</td>\n",
       "      <td>U.S. Secretary of State John F. Kerry said Mon...</td>\n",
       "      <td>0</td>\n",
       "      <td>kerry to go to paris in gesture of sympathy. u...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>Bernie supporters on Twitter erupt in anger ag...</td>\n",
       "      <td>— Kaydee King (@KaydeeKing) November 9, 2016 T...</td>\n",
       "      <td>1</td>\n",
       "      <td>bernie supporters on twitter erupt in anger ag...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>The Battle of New York: Why This Primary Matters</td>\n",
       "      <td>It's primary day in New York and front-runners...</td>\n",
       "      <td>0</td>\n",
       "      <td>the battle of new york: why this primary matte...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                               title  \\\n",
       "0                       You Can Smell Hillary’s Fear   \n",
       "1  Watch The Exact Moment Paul Ryan Committed Pol...   \n",
       "2        Kerry to go to Paris in gesture of sympathy   \n",
       "3  Bernie supporters on Twitter erupt in anger ag...   \n",
       "4   The Battle of New York: Why This Primary Matters   \n",
       "\n",
       "                                                text  label  \\\n",
       "0  Daniel Greenfield, a Shillman Journalism Fello...      1   \n",
       "1  Google Pinterest Digg Linkedin Reddit Stumbleu...      1   \n",
       "2  U.S. Secretary of State John F. Kerry said Mon...      0   \n",
       "3  — Kaydee King (@KaydeeKing) November 9, 2016 T...      1   \n",
       "4  It's primary day in New York and front-runners...      0   \n",
       "\n",
       "                                           titletext  \n",
       "0  you can smell hillary’s fear. daniel greenfiel...  \n",
       "1  watch the exact moment paul ryan committed pol...  \n",
       "2  kerry to go to paris in gesture of sympathy. u...  \n",
       "3  bernie supporters on twitter erupt in anger ag...  \n",
       "4  the battle of new york: why this primary matte...  "
      ]
     },
     "execution_count": 26,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df[\"titletext\"]=df[\"titletext\"].apply(lambda x:x.lower())\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [],
   "source": [
    "def expand_contractions(text):\n",
    "    expanded_words=[]\n",
    "    for word in text.split():\n",
    "        expanded_words.append(contractions.fix(word))\n",
    "\n",
    "    expanded_text=\" \".join(expanded_words)\n",
    "    return expanded_text"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [],
   "source": [
    "df[\"titletext\"]=df[\"titletext\"].apply(lambda x:expand_contractions(x))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {},
   "outputs": [],
   "source": [
    "df[\"titletext\"]=df[\"titletext\"].apply(lambda x: re.sub(\"[^A-Z a-z 0-9]+\",'',x))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {},
   "outputs": [],
   "source": [
    "df[\"titletext\"]=df[\"titletext\"].apply(lambda x:([t for t in x.split() if not t.isdigit()]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {},
   "outputs": [],
   "source": [
    "df[\"titletext\"]=df[\"titletext\"].apply(lambda x:\" \".join(x))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {},
   "outputs": [],
   "source": [
    "df[\"titletext\"]=df[\"titletext\"].apply(lambda x: re.sub(r'([a-zA-Z0-9+._-]+@[a-zA-Z0-9+._-]+\\.[a-zA-Z0-9+._-]+)',\"\",x))   "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "metadata": {},
   "outputs": [],
   "source": [
    "df[\"titletext\"]=df[\"titletext\"].apply(lambda x: re.sub(r'([a-zA-Z0-9+._-]+@[a-zA-Z0-9+._-]+\\.[a-zA-Z0-9+._-]+)',\"\",x))   "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "metadata": {},
   "outputs": [],
   "source": [
    "df[\"titletext\"]=df[\"titletext\"].apply(lambda x:re.sub(r\"(http|https|ftp)://([\\w_-]+(?:(?:\\.[\\w_-]+)+))([\\w.,@?^=%&:/~+#-]*[\\w@?^=%&/~+#-])\",\"\",str(x)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "metadata": {},
   "outputs": [],
   "source": [
    "df[\"titletext\"]=df[\"titletext\"].apply(lambda x:unidecode(x))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "metadata": {},
   "outputs": [],
   "source": [
    "df.drop([\"title\",\"text\"],axis=1,inplace=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 61,
   "metadata": {},
   "outputs": [],
   "source": [
    "df.to_csv(\"real_nd_fake_news/real_train.csv\",index=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 62,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>label</th>\n",
       "      <th>titletext</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1</td>\n",
       "      <td>you can smell hillarys fear daniel greenfield ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1</td>\n",
       "      <td>watch the exact moment paul ryan committed pol...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>0</td>\n",
       "      <td>kerry to go to paris in gesture of sympathy yo...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>1</td>\n",
       "      <td>bernie supporters on twitter erupt in anger ag...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>0</td>\n",
       "      <td>the battle of new york why this primary matter...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   label                                          titletext\n",
       "0      1  you can smell hillarys fear daniel greenfield ...\n",
       "1      1  watch the exact moment paul ryan committed pol...\n",
       "2      0  kerry to go to paris in gesture of sympathy yo...\n",
       "3      1  bernie supporters on twitter erupt in anger ag...\n",
       "4      0  the battle of new york why this primary matter..."
      ]
     },
     "execution_count": 62,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 63,
   "metadata": {},
   "outputs": [],
   "source": [
    "nlp=spacy.load(\"en_core_web_sm\")\n",
    "tokenizer=Tokenizer(nlp.vocab)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 64,
   "metadata": {},
   "outputs": [],
   "source": [
    "def spacy_tokenizer(x):\n",
    "    return [tok.text for tok in tokenizer(x)]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 65,
   "metadata": {},
   "outputs": [],
   "source": [
    "TEXT=Field(tokenize=spacy_tokenizer,include_lengths=True)\n",
    "LABEL=LabelField(dtype=torch.float)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 66,
   "metadata": {},
   "outputs": [],
   "source": [
    "fields=[(\"label\",LABEL),(\"text\",TEXT)]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 69,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'label': '0', 'text': ['the', 'battle', 'of', 'new', 'york', 'why', 'this', 'primary', 'matters', 'it', 'is', 'primary', 'day', 'in', 'new', 'york', 'and', 'frontrunners', 'hillary', 'clinton', 'and', 'donald', 'trump', 'are', 'leading', 'in', 'the', 'polls', 'trump', 'is', 'now', 'vowing', 'to', 'win', 'enough', 'delegates', 'to', 'clinch', 'the', 'republican', 'nomination', 'and', 'prevent', 'a', 'contested', 'convention', 'but', 'sensted', 'cruz', 'rtexas', 'bernie', 'sanders', 'dvt', 'and', 'ohio', 'gov', 'john', 'kasich', 'and', 'are', 'not', 'giving', 'up', 'just', 'yet', 'a', 'big', 'win', 'in', 'new', 'york', 'could', 'tip', 'the', 'scales', 'for', 'both', 'the', 'republican', 'and', 'democratic', 'frontrunners', 'in', 'this', 'years', 'race', 'for', 'the', 'white', 'house', 'clinton', 'and', 'trump', 'have', 'each', 'suffered', 'losses', 'in', 'recent', 'contests', 'shifting', 'the', 'momentum', 'to', 'their', 'rivals', 'we', 'have', 'won', 'eight', 'out', 'of', 'the', 'last', 'nine', 'caucuses', 'and', 'primaries', 'cheer', 'sanders', 'recently', 'told', 'supporters', 'while', 'wins', 'in', 'new', 'york', 'for', 'trump', 'and', 'clinton', 'are', 'expected', 'the', 'margins', 'of', 'those', 'victories', 'are', 'also', 'important', 'trump', 'needs', 'to', 'capture', 'more', 'than', 'percent', 'of', 'the', 'vote', 'statewide', 'if', 'he', 'wants', 'to', 'be', 'positioned', 'to', 'win', 'all', 'of', 'the', 'states', 'gop', 'delegates', 'that', 'would', 'put', 'him', 'one', 'step', 'closer', 'to', 'avoiding', 'a', 'contested', 'convention', 'we', 'have', 'got', 'to', 'vote', 'and', 'you', 'know', 'cruz', 'is', 'way', 'way', 'down', 'in', 'the', 'polls', 'trump', 'urged', 'supporters', 'meanwhile', 'sanders', 'is', 'hoping', 'for', 'a', 'close', 'race', 'in', 'the', 'empire', 'state', 'a', 'loss', 'by', 'points', 'means', 'he', 'will', 'need', 'to', 'win', 'percent', 'of', 'the', 'remaining', 'delegates', 'to', 'clinch', 'the', 'nomination', 'despite', 'a', 'predicted', 'loss', 'in', 'new', 'york', 'cruz', 'has', 'not', 'lost', 'momentum', 'he', 'is', 'hoping', 'to', 'sweep', 'up', 'more', 'delegates', 'this', 'weekend', 'while', 'he', 'is', 'talking', 'about', 'how', 'he', 'can', 'win', 'in', 'november', 'because', 'if', 'I', 'am', 'the', 'nominee', 'we', 'win', 'the', 'general', 'election', 'cruz', 'promised', 'his', 'supporters', 'we', 'are', 'beating', 'hillary', 'in', 'the', 'key', 'swing', 'states', 'we', 'are', 'beating', 'hillary', 'with', 'independents', 'we', 'are', 'beating', 'hillary', 'with', 'young', 'people', 'for', 'now', 'cruz', 'kasich', 'and', 'sanders', 'have', 'all', 'moved', 'on', 'from', 'new', 'york', 'to', 'other', 'states', 'trump', 'and', 'clinton', 'are', 'the', 'only', 'two', 'staying', 'in', 'their', 'home', 'state', 'to', 'watch', 'the', 'results', 'come', 'in']}\n"
     ]
    }
   ],
   "source": [
    "training_data=TabularDataset(path=\"real_nd_fake_news/real_train.csv\",format=\"csv\",fields=fields,skip_header=False)\n",
    "\n",
    "print(vars(training_data.examples[5]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 72,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_data,valid_data=training_data.split(split_ratio=0.7,random_state=random.seed(2020))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 73,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(4435, 1901)"
      ]
     },
     "execution_count": 73,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(train_data),len(valid_data)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 74,
   "metadata": {},
   "outputs": [],
   "source": [
    "TEXT.build_vocab(train_data,min_freq=3)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 75,
   "metadata": {},
   "outputs": [],
   "source": [
    "LABEL.build_vocab(train_data)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 77,
   "metadata": {},
   "outputs": [],
   "source": [
    "device=torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 78,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Size of text vocabulary: 32059\n",
      "Size of LABEL vocabulary: 3\n"
     ]
    }
   ],
   "source": [
    "print(\"Size of text vocabulary:\",len(TEXT.vocab))\n",
    "\n",
    "#No. of unique tokens in label\n",
    "print(\"Size of LABEL vocabulary:\",len(LABEL.vocab))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 146,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_iteration,valid_iteration=BucketIterator.splits((train_data,valid_data),\n",
    "                                    batch_size=32,\n",
    "                                    sort_key=lambda x:len(x.text),\n",
    "                                    sort=True,\n",
    "                                    sort_within_batch=True,\n",
    "                                    device=device)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 147,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(139, 60)"
      ]
     },
     "execution_count": 147,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(train_iteration),len(valid_iteration)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 125,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "128 128\n",
      "tensor([23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 22, 22, 22,\n",
      "        22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 21, 21, 21, 21, 21, 21,\n",
      "        21, 21, 21, 21, 21, 21, 20, 20, 20, 20, 20, 20, 20, 20, 20, 19, 19, 19,\n",
      "        19, 19, 19, 19, 19, 19, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 17,\n",
      "        17, 17, 17, 17, 17, 17, 16, 16, 16, 16, 16, 16, 15, 15, 15, 15, 15, 15,\n",
      "        15, 15, 14, 14, 14, 14, 14, 13, 13, 13, 13, 13, 13, 13, 12, 12, 12, 12,\n",
      "        11, 10, 10, 10, 10, 10,  9,  9,  8,  8,  8,  8,  7,  6,  6,  4,  4,  4,\n",
      "         2,  1])\n",
      "tensor([0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 1.,\n",
      "        0., 0., 0., 0., 1., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0.,\n",
      "        0., 1., 0., 0., 0., 0., 0., 0., 0., 0., 0., 1., 0., 0., 0., 1., 0., 1.,\n",
      "        0., 0., 0., 0., 1., 0., 1., 1., 1., 0., 1., 1., 0., 0., 0., 0., 0., 1.,\n",
      "        1., 1., 0., 1., 1., 0., 0., 0., 0., 1., 0., 0., 0., 0., 0., 0., 0., 0.,\n",
      "        0., 1., 1., 1., 1., 0., 0., 1., 0., 1., 1., 1., 0., 1., 1., 1., 1., 1.,\n",
      "        1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1.,\n",
      "        1., 2.])\n"
     ]
    }
   ],
   "source": [
    "for batch in train_iteration:\n",
    "    text,text_len=batch.text\n",
    "#     label,label_len=batch.label\n",
    "    print(len(text_len),len(batch.label))\n",
    "    print(text_len)\n",
    "    print(batch.label)\n",
    "    break"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 119,
   "metadata": {},
   "outputs": [],
   "source": [
    "# df[\"titletext\"][0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 126,
   "metadata": {},
   "outputs": [],
   "source": [
    "train,test=train_test_split(df,test_size=0.2,random_state=0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 127,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(5068, 1267)"
      ]
     },
     "execution_count": 127,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(train),len(test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 130,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_iter,test_iter=BucketIterator.splits((train,test),\n",
    "                                          device=device,\n",
    "                                          sort_within_batch=True,\n",
    "                                          sort_key=True,\n",
    "                                          batch_size=32)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 131,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(159, 40)"
      ]
     },
     "execution_count": 131,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(train_iter),len(test_iter)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 156,
   "metadata": {},
   "outputs": [],
   "source": [
    "class Classifier(nn.Module):\n",
    "    def __init__(self,vocab_size,embedding_dim,hidden_dim,output_dim,n_layers,bidirectional,dropout):\n",
    "        super(Classifier,self).__init__()\n",
    "        self.embedding=nn.Embedding(vocab_size,embedding_dim)\n",
    "        self.lstm=nn.LSTM(embedding_dim,hidden_dim,num_layers=n_layers,bidirectional=bidirectional,batch_first=True)\n",
    "        \n",
    "        self.fc=nn.Linear(2*hidden_dim,output_dim)\n",
    "        self.act=nn.Sigmoid()    \n",
    "            \n",
    "    def forward(self,text,text_len):\n",
    "        embedding=self.embedding(text)\n",
    "        \n",
    "        packed_embedd=pack_padded_sequence(embedding,text_len)\n",
    "        packed_output,(hidden,cell)=self.lstm(packed_embedd)\n",
    "        \n",
    "        \n",
    "        #hidden = [batch size, num layers * num directions,hid dim]\n",
    "        #cell = [batch size, num layers * num directions,hid dim]\n",
    "        \n",
    "        #concat the final forward and backward hidden state\n",
    "        hidden=torch.cat((hidden[-2,:,:],hidden[-1,:,:]),dim=1)\n",
    "        \n",
    "        dense_output=self.fc(hidden)\n",
    "        \n",
    "        #final activation functions\n",
    "        outputs=self.act(dense_output)\n",
    "        \n",
    "        return outputs\n",
    "        "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 157,
   "metadata": {},
   "outputs": [],
   "source": [
    "size_of_vocab=len(TEXT.vocab)\n",
    "embedding_dim=100\n",
    "num_hidden_nodes=32\n",
    "num_output_nodes=1\n",
    "num_layers=2\n",
    "bidirection=True\n",
    "dropout=0.2\n",
    "\n",
    "\n",
    "model=Classifier(size_of_vocab,embedding_dim,num_hidden_nodes,num_output_nodes,num_layers,bidirection,dropout)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 158,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Classifier(\n",
       "  (embedding): Embedding(32059, 100)\n",
       "  (lstm): LSTM(100, 32, num_layers=2, batch_first=True, bidirectional=True)\n",
       "  (fc): Linear(in_features=64, out_features=1, bias=True)\n",
       "  (act): Sigmoid()\n",
       ")"
      ]
     },
     "execution_count": 158,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 135,
   "metadata": {},
   "outputs": [],
   "source": [
    "optimizer=torch.optim.Adam(model.parameters(),lr=0.01)\n",
    "criterion=nn.BCELoss()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 136,
   "metadata": {},
   "outputs": [],
   "source": [
    "def binary_accuracy(pred,y):\n",
    "    rounded_pred=torch.round(pred)\n",
    "    correct=(rounded_pred==y).float()\n",
    "    acc=correct.sum() / len(correct)\n",
    "    return acc"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 159,
   "metadata": {},
   "outputs": [],
   "source": [
    "def train(model, iterator, optimizer, criterion):\n",
    "    \n",
    "    #initialize every epoch \n",
    "    epoch_loss = 0\n",
    "    epoch_acc = 0\n",
    "    \n",
    "    #set the model in training phase\n",
    "    model.train()  \n",
    "    \n",
    "    for batch in iterator:\n",
    "        \n",
    "        #resets the gradients after every batch\n",
    "        optimizer.zero_grad()   \n",
    "        \n",
    "        #retrieve text and no. of words\n",
    "        text, text_lengths = batch.text   \n",
    "        \n",
    "        #convert to 1D tensor\n",
    "        predictions = model(text, text_lengths).squeeze()\n",
    "        \n",
    "        #compute the loss\n",
    "        loss = criterion(predictions, batch.label)        \n",
    "        \n",
    "        #compute the binary accuracy\n",
    "        acc = binary_accuracy(predictions, batch.label)   \n",
    "        \n",
    "        #backpropage the loss and compute the gradients\n",
    "        loss.backward()       \n",
    "        \n",
    "        #update the weights\n",
    "        optimizer.step()      \n",
    "        \n",
    "        #loss and accuracy\n",
    "        epoch_loss += loss.item()  \n",
    "        epoch_acc += acc.item()    \n",
    "        \n",
    "    return epoch_loss / len(iterator), epoch_acc / len(iterator)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 160,
   "metadata": {},
   "outputs": [],
   "source": [
    "def evaluate(model,iterator,criterion):\n",
    "    \n",
    "    epoch_loss=0\n",
    "    \n",
    "    epoch_acc=0\n",
    "    \n",
    "    model.eval()\n",
    "    \n",
    "    with torch.no_grad():\n",
    "        for batch in iterator:\n",
    "            #text or no. of words\n",
    "            text,text_len=batch.text\n",
    "            \n",
    "            #convert to 1d array\n",
    "            predictions=model(text,text_len).squeeze()\n",
    "            \n",
    "            #compute loss and accuracy\n",
    "            \n",
    "            loss=criterion(predictions,batch.label)\n",
    "            acc=binary_accuracy(predictions,batch.label)\n",
    "            \n",
    "            #keep track of loss and accuracy\n",
    "            epoch_loss += loss.item()\n",
    "            epoch_acc += acc.item()\n",
    "        \n",
    "    return epoch_loss / len(iterator), epoch_acc / len(iterator)\n",
    "            "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 163,
   "metadata": {},
   "outputs": [
    {
     "ename": "KeyboardInterrupt",
     "evalue": "",
     "output_type": "error",
     "traceback": [
      "\u001b[1;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[1;31mKeyboardInterrupt\u001b[0m                         Traceback (most recent call last)",
      "\u001b[1;32m<ipython-input-163-02ab93b0e8e7>\u001b[0m in \u001b[0;36m<module>\u001b[1;34m\u001b[0m\n\u001b[0;32m      5\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m      6\u001b[0m     \u001b[1;31m#train the model\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m----> 7\u001b[1;33m     \u001b[0mtrain_loss\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mtrain_acc\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mtrain\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mmodel\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mtrain_iteration\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0moptimizer\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mcriterion\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m      8\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m      9\u001b[0m     \u001b[1;31m#evaluate the model\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32m<ipython-input-159-b51846f8b20e>\u001b[0m in \u001b[0;36mtrain\u001b[1;34m(model, iterator, optimizer, criterion)\u001b[0m\n\u001b[0;32m     26\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m     27\u001b[0m         \u001b[1;31m#backpropage the loss and compute the gradients\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m---> 28\u001b[1;33m         \u001b[0mloss\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mbackward\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m     29\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m     30\u001b[0m         \u001b[1;31m#update the weights\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32mc:\\users\\parikalp\\appdata\\local\\programs\\python\\lib\\site-packages\\torch\\tensor.py\u001b[0m in \u001b[0;36mbackward\u001b[1;34m(self, gradient, retain_graph, create_graph)\u001b[0m\n\u001b[0;32m    116\u001b[0m                 \u001b[0mproducts\u001b[0m\u001b[1;33m.\u001b[0m \u001b[0mDefaults\u001b[0m \u001b[0mto\u001b[0m\u001b[0;31m \u001b[0m\u001b[0;31m`\u001b[0m\u001b[0;31m`\u001b[0m\u001b[1;32mFalse\u001b[0m\u001b[0;31m`\u001b[0m\u001b[0;31m`\u001b[0m\u001b[1;33m.\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    117\u001b[0m         \"\"\"\n\u001b[1;32m--> 118\u001b[1;33m         \u001b[0mtorch\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mautograd\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mbackward\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mself\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mgradient\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mretain_graph\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mcreate_graph\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m    119\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    120\u001b[0m     \u001b[1;32mdef\u001b[0m \u001b[0mregister_hook\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mself\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mhook\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32mc:\\users\\parikalp\\appdata\\local\\programs\\python\\lib\\site-packages\\torch\\autograd\\__init__.py\u001b[0m in \u001b[0;36mbackward\u001b[1;34m(tensors, grad_tensors, retain_graph, create_graph, grad_variables)\u001b[0m\n\u001b[0;32m     91\u001b[0m     Variable._execution_engine.run_backward(\n\u001b[0;32m     92\u001b[0m         \u001b[0mtensors\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mgrad_tensors\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mretain_graph\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mcreate_graph\u001b[0m\u001b[1;33m,\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m---> 93\u001b[1;33m         allow_unreachable=True)  # allow_unreachable flag\n\u001b[0m\u001b[0;32m     94\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m     95\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;31mKeyboardInterrupt\u001b[0m: "
     ]
    }
   ],
   "source": [
    "\n",
    "N_EPOCHS = 5\n",
    "best_valid_loss = float('inf')\n",
    "\n",
    "for epoch in range(N_EPOCHS):\n",
    "     \n",
    "    #train the model\n",
    "    train_loss, train_acc = train(model, train_iteration, optimizer, criterion)\n",
    "    \n",
    "    #evaluate the model\n",
    "    valid_loss, valid_acc = evaluate(model, valid_iteration, criterion)\n",
    "    \n",
    "    #save the best model\n",
    "    if valid_loss < best_valid_loss:\n",
    "        best_valid_loss = valid_loss\n",
    "        torch.save(model.state_dict(), 'real_nd_fake_news/saved_weights.pt')\n",
    "    \n",
    "    print(f'\\tTrain Loss: {train_loss:.3f} | Train Acc: {train_acc*100:.2f}%')\n",
    "    print(f'\\t Val. Loss: {valid_loss:.3f} |  Val. Acc: {valid_acc*100:.2f}%')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}