02-Normalization.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## TEXT NORMALIZATION FUNCTION\n",
    "\n",
    "### Text Mining on Earnings Calls during a Pandemic as a Means to Predict End-Of-The-Month Stock Performances\n",
    "####  Olin School of Business <br> Jose Luis Rodriguez  <br> jlr@wustl.edu <br> Fall 2021"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "**Code Source** \n",
    "\n",
    "DAT562: Text Mining\n",
    "\n",
    "Dr. Yulia Nevskaya"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "test_text = \"<p>The circus dog in a plissé skirt jumped over Python who wasn't that large, just 3 feet long.</p>\""
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "First, let's get Python modules ready and available (remember, you can always look up the modules in Python documentation to learn about what they do in detail):"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "%%capture\n",
    "#module for supressing warnings about future changes in Python:\n",
    "import warnings\n",
    "warnings.simplefilter(action='ignore', category=FutureWarning)\n",
    "\n",
    "#module sys helps install modules from inside Jupyter:\n",
    "import sys\n",
    "\n",
    "#html parser for digesting text taken from web-pages\n",
    "!{sys.executable} -m pip install html.parser\n",
    "import html.parser\n",
    "from html.parser import HTMLParser\n",
    "\n",
    "#module for handling regular expressions and special characters\n",
    "import re\n",
    "\n",
    "#Natural Language ToolKit\n",
    "import nltk\n",
    "from nltk.stem import WordNetLemmatizer\n",
    "from nltk.corpus import wordnet as wn\n",
    "\n",
    "#modules for handling unicode characters and strings\n",
    "import unicodedata\n",
    "import string\n",
    "!{sys.executable} -m pip install pattern3\n",
    "import pattern3\n",
    "\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "\n",
    "from sklearn.model_selection import GridSearchCV\n",
    "from sklearn.feature_extraction.text import  CountVectorizer\n",
    "from sklearn.decomposition import LatentDirichletAllocation\n",
    "\n",
    "#pretty printing\n",
    "from pprint import pprint\n",
    "\n",
    "# Plotting tools\n",
    "!{sys.executable} -m pip install pyLDAvis\n",
    "import pyLDAvis\n",
    "import pyLDAvis.sklearn\n",
    "import matplotlib.pyplot as plt\n",
    "%matplotlib inline"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Define functions for lemmatization and HTML parsing:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "wnl = WordNetLemmatizer()\n",
    "html_parser = HTMLParser()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Defining the list of word contractions:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "contraction_mapping = {\n",
    "\"ain't\": \"is not\",\n",
    "\"aren't\": \"are not\",\n",
    "\"can't\": \"cannot\",\n",
    "\"can't've\": \"cannot have\",\n",
    "\"'cause\": \"because\",\n",
    "\"could've\": \"could have\",\n",
    "\"couldn't\": \"could not\",\n",
    "\"couldn't've\": \"could not have\",\n",
    "\"didn't\": \"did not\",\n",
    "\"doesn't\": \"does not\",\n",
    "\"don't\": \"do not\",\n",
    "\"hadn't\": \"had not\",\n",
    "\"hadn't've\": \"had not have\",\n",
    "\"hasn't\": \"has not\",\n",
    "\"haven't\": \"have not\",\n",
    "\"he'd\": \"he would\",\n",
    "\"he'd've\": \"he would have\",\n",
    "\"he'll\": \"he will\",\n",
    "\"he'll've\": \"he he will have\",\n",
    "\"he's\": \"he is\",\n",
    "\"how'd\": \"how did\",\n",
    "\"how'd'y\": \"how do you\",\n",
    "\"how'll\": \"how will\",\n",
    "\"how's\": \"how is\",\n",
    "\"I'd\": \"I would\",\n",
    "\"I'd've\": \"I would have\",\n",
    "\"I'll\": \"I will\",\n",
    "\"I'll've\": \"I will have\",\n",
    "\"I'm\": \"I am\",\n",
    "\"I've\": \"I have\",\n",
    "\"i'd\": \"i would\",\n",
    "\"i'd've\": \"i would have\",\n",
    "\"i'll\": \"i will\",\n",
    "\"i'll've\": \"i will have\",\n",
    "\"i'm\": \"i am\",\n",
    "\"i've\": \"i have\",\n",
    "\"isn't\": \"is not\",\n",
    "\"it'd\": \"it would\",\n",
    "\"it'd've\": \"it would have\",\n",
    "\"it'll\": \"it will\",\n",
    "\"it'll've\": \"it will have\",\n",
    "\"it's\": \"it is\",\n",
    "\"let's\": \"let us\",\n",
    "\"ma'am\": \"madam\",\n",
    "\"mayn't\": \"may not\",\n",
    "\"might've\": \"might have\",\n",
    "\"mightn't\": \"might not\",\n",
    "\"mightn't've\": \"might not have\",\n",
    "\"must've\": \"must have\",\n",
    "\"mustn't\": \"must not\",\n",
    "\"mustn't've\": \"must not have\",\n",
    "\"needn't\": \"need not\",\n",
    "\"needn't've\": \"need not have\",\n",
    "\"o'clock\": \"of the clock\",\n",
    "\"oughtn't\": \"ought not\",\n",
    "\"oughtn't've\": \"ought not have\",\n",
    "\"shan't\": \"shall not\",\n",
    "\"sha'n't\": \"shall not\",\n",
    "\"shan't've\": \"shall not have\",\n",
    "\"she'd\": \"she would\",\n",
    "\"she'd've\": \"she would have\",\n",
    "\"she'll\": \"she will\",\n",
    "\"she'll've\": \"she will have\",\n",
    "\"she's\": \"she is\",\n",
    "\"should've\": \"should have\",\n",
    "\"shouldn't\": \"should not\",\n",
    "\"shouldn't've\": \"should not have\",\n",
    "\"so've\": \"so have\",\n",
    "\"so's\": \"so as\",\n",
    "\"that'd\": \"that would\",\n",
    "\"that'd've\": \"that would have\",\n",
    "\"that's\": \"that is\",\n",
    "\"there'd\": \"there would\",\n",
    "\"there'd've\": \"there would have\",\n",
    "\"there's\": \"there is\",\n",
    "\"they'd\": \"they would\",\n",
    "\"they'd've\": \"they would have\",\n",
    "\"they'll\": \"they will\",\n",
    "\"they'll've\": \"they will have\",\n",
    "\"they're\": \"they are\",\n",
    "\"they've\": \"they have\",\n",
    "\"to've\": \"to have\",\n",
    "\"wasn't\": \"was not\",\n",
    "\"we'd\": \"we would\",\n",
    "\"we'd've\": \"we would have\",\n",
    "\"we'll\": \"we will\",\n",
    "\"we'll've\": \"we will have\",\n",
    "\"we're\": \"we are\",\n",
    "\"we've\": \"we have\",\n",
    "\"weren't\": \"were not\",\n",
    "\"what'll\": \"what will\",\n",
    "\"what'll've\": \"what will have\",\n",
    "\"what're\": \"what are\",\n",
    "\"what's\": \"what is\",\n",
    "\"what've\": \"what have\",\n",
    "\"when's\": \"when is\",\n",
    "\"when've\": \"when have\",\n",
    "\"where'd\": \"where did\",\n",
    "\"where's\": \"where is\",\n",
    "\"where've\": \"where have\",\n",
    "\"who'll\": \"who will\",\n",
    "\"who'll've\": \"who will have\",\n",
    "\"who's\": \"who is\",\n",
    "\"who've\": \"who have\",\n",
    "\"why's\": \"why is\",\n",
    "\"why've\": \"why have\",\n",
    "\"will've\": \"will have\",\n",
    "\"won't\": \"will not\",\n",
    "\"won't've\": \"will not have\",\n",
    "\"would've\": \"would have\",\n",
    "\"wouldn't\": \"would not\",\n",
    "\"wouldn't've\": \"would not have\",\n",
    "\"y'all\": \"you all\",\n",
    "\"y'all'd\": \"you all would\",\n",
    "\"y'all'd've\": \"you all would have\",\n",
    "\"y'all're\": \"you all are\",\n",
    "\"y'all've\": \"you all have\",\n",
    "\"you'd\": \"you would\",\n",
    "\"you'd've\": \"you would have\",\n",
    "\"you'll\": \"you will\",\n",
    "\"you'll've\": \"you will have\",\n",
    "\"you're\": \"you are\",\n",
    "\"you've\": \"you have\"\n",
    "}"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Select the list of stopwords from NLTK and amend it by adding more stopwords to it:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from nltk.corpus import stopwords\n",
    "nltk.download('stopwords')\n",
    "\n",
    "stopword_list = nltk.corpus.stopwords.words('english')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "stopword_list = stopword_list + ['mr', 'mrs', 'come', 'go', 'get',\n",
    "                                 'tell', 'listen', 'one', 'two', 'three',\n",
    "                                 'four', 'five', 'six', 'seven', 'eight',\n",
    "                                 'nine', 'zero', 'join', 'find', 'make',\n",
    "                                 'say', 'ask', 'tell', 'see', 'try', 'back',\n",
    "                                 'also','would']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "stopword_list = stopword_list + ['marketintelligence','marketintelligences','ceo',\n",
    "                                 'www','global','quarter','cfo','vp','question','spglobal',\n",
    "                                'earnings','copyright','intelligence','research','call','division',\n",
    "                                'market','content','use','financial','party','reserve','inc','right',\n",
    "                                'think','next','director','us','fy','na','nm','consensus',\n",
    "                                'index','com','surprice','statement','business','year','foward','looking',\n",
    "                                'thank', 'million', 'ebitda', 'net', 'third', 'first', 'result', 'last', 'prior',\n",
    "                                'month', 'per', 'include', 'period', 'flow', 'level', 'compare', 'basis',\n",
    "                                'gaap', 'morning', 'update', 'thanks', 'may', 'investor', 'please', 'non', 'like',\n",
    "                                'website','company', 'officer', 'chief', 'provide', 'actual', 'performance',\n",
    "                                'today', 'everyone', 'release', 'information', 'forward','good','operator',\n",
    "                                'bit', 'discuss', 'well', 'much', 'number', 'really', 'thing', 'want', 'talk',\n",
    "                                'give', 'term', 'know', 'mean', 'take', 'point', 'member', 'remark']"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Split text into word tokens:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "nltk.download('punkt')\n",
    "\n",
    "def tokenize_text(text):\n",
    "    tokens = nltk.word_tokenize(text) \n",
    "    tokens = [token.strip() for token in tokens]\n",
    "    return tokens\n",
    "\n",
    "print(\"Original:  \", test_text)\n",
    "print(\"Processed: \", tokenize_text(test_text))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Expand contractions:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def expand_contractions(text, contraction_mapping):\n",
    "    \n",
    "    contractions_pattern = re.compile('({})'.format('|'.join(contraction_mapping.keys())), \n",
    "                                      flags=re.IGNORECASE|re.DOTALL)\n",
    "    def expand_match(contraction):\n",
    "        match = contraction.group(0)\n",
    "        first_char = match[0]\n",
    "        expanded_contraction = contraction_mapping.get(match)\\\n",
    "                                if contraction_mapping.get(match)\\\n",
    "                                else contraction_mapping.get(match.lower())                       \n",
    "        expanded_contraction = first_char+expanded_contraction[1:]\n",
    "        return expanded_contraction\n",
    "        \n",
    "    expanded_text = contractions_pattern.sub(expand_match, text)\n",
    "    expanded_text = re.sub(\"'\", \"\", expanded_text)\n",
    "    return expanded_text\n",
    " \n",
    "print(\"Original:  \", test_text)\n",
    "print(\"Processed: \", expand_contractions(test_text,contraction_mapping))\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Annotate text tokens with Part-Of-Speach tags:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "nltk.download('averaged_perceptron_tagger')\n",
    "nltk.download('wordnet')\n",
    "\n",
    "def pos_tag_text(text_tokens):\n",
    "    def penn_to_wn_tags(pos_tag):\n",
    "        if pos_tag.startswith('J'):\n",
    "            return wn.ADJ\n",
    "        elif pos_tag.startswith('V'):\n",
    "            return wn.VERB\n",
    "        elif pos_tag.startswith('N'):\n",
    "            return wn.NOUN\n",
    "        elif pos_tag.startswith('R'):\n",
    "            return wn.ADV\n",
    "        else:\n",
    "            return None  \n",
    "    tagged_text = nltk.pos_tag(text_tokens)\n",
    "    tagged_lower_text = [(word.lower(), penn_to_wn_tags(pos_tag))\n",
    "                         for word, pos_tag in\n",
    "                         tagged_text]\n",
    "    return tagged_lower_text\n",
    "\n",
    "\n",
    "print(\"Original:  \", test_text)\n",
    "print(\"Processed: \", pos_tag_text(tokenize_text(test_text)))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Lemmatize text based on Part-Of-Speach (POS) tags: "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def lemmatize_text(text):\n",
    "    pos_tagged_text = pos_tag_text(text)\n",
    "    lemmatized_tokens = [wnl.lemmatize(word, pos_tag) if pos_tag\n",
    "                         else word                     \n",
    "                         for word, pos_tag in pos_tagged_text]\n",
    "    lemmatized_text = ' '.join(lemmatized_tokens)\n",
    "    return lemmatized_text\n",
    " \n",
    "print(\"Original:  \", test_text)\n",
    "print(\"Processed: \", lemmatize_text(tokenize_text(test_text)))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Remove special characters, such as punctuation marks:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def remove_special_characters(text):\n",
    "    tokens = tokenize_text(text)\n",
    "    pattern = re.compile('[{}]'.format(re.escape(string.punctuation)))\n",
    "    filtered_tokens = filter(None, [pattern.sub(' ', token) for token in tokens])\n",
    "    filtered_text = ' '.join(filtered_tokens)\n",
    "    return filtered_text \n",
    "\n",
    "print(\"Original:  \", test_text)\n",
    "print(\"Processed: \", remove_special_characters(test_text))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Get rid of stopwords:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def remove_stopwords(text):\n",
    "    tokens = tokenize_text(text)\n",
    "    filtered_tokens = [token for token in tokens if token not in stopword_list]\n",
    "    filtered_text = ' '.join(filtered_tokens)    \n",
    "    return filtered_text\n",
    "\n",
    "print(\"Original:  \", test_text)\n",
    "print(\"Processed: \", remove_stopwords(test_text))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Remove all non-text characters (numbers, etc.):"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def keep_text_characters(text):\n",
    "    filtered_tokens = []\n",
    "    tokens = tokenize_text(text)\n",
    "    for token in tokens:\n",
    "        if re.search('[a-zA-Z]', token):\n",
    "            filtered_tokens.append(token)\n",
    "    filtered_text = ' '.join(filtered_tokens)\n",
    "    return filtered_text\n",
    "\n",
    "print(\"Original:  \", test_text)\n",
    "print(\"Processed: \", keep_text_characters(test_text))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Clean up HTML markups: "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "class MLStripper(HTMLParser):\n",
    "    def __init__(self):\n",
    "        super().__init__()\n",
    "        self.reset()\n",
    "        self.fed = []\n",
    "    def handle_data(self, d):\n",
    "        self.fed.append(d)\n",
    "    def get_data(self):\n",
    "        return ' '.join(self.fed)\n",
    "    \n",
    "def strip_html(text):\n",
    "    html_stripper = MLStripper()\n",
    "    html_stripper.feed(text)\n",
    "    return html_stripper.get_data()\n",
    "\n",
    "print(\"Original:  \", test_text)\n",
    "print(\"Processed: \", strip_html(test_text))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Removing accents from characters:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def normalize_accented_characters(text):\n",
    "    text = unicodedata.normalize('NFKD', text).encode('ascii', 'ignore').decode('utf8')\n",
    "    return text\n",
    "\n",
    "print(\"Original:  \", test_text)\n",
    "print(\"Processed: \", normalize_accented_characters(test_text))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Putting all functions together:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def normalize_corpus(corpus, only_text_chars=True):\n",
    "    normalized_corpus = []  \n",
    "    for index, text in enumerate(corpus):\n",
    "        text = normalize_accented_characters(text)\n",
    "        text = html.unescape(text)\n",
    "        text = strip_html(text)\n",
    "        text = expand_contractions(text, contraction_mapping)\n",
    "        text = tokenize_text(text)\n",
    "        text = lemmatize_text(text)\n",
    "        text = remove_special_characters(text)\n",
    "        text = remove_stopwords(text)\n",
    "        if only_text_chars:\n",
    "            text = keep_text_characters(text)\n",
    "        #text = tokenize_text(text)\n",
    "        normalized_corpus.append(text)    \n",
    "    return normalized_corpus"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Create a small corpus consisting of 2 test sentences and testing the normalize_corpus function on it (uncomment the lines below first):"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "test_corpus = [test_text]\n",
    "test_corpus.append(test_text)\n",
    "\n",
    "#print(\"Original:  \", test_corpus,\"\\n\")\n",
    "#print(\"Processed: \", normalize_corpus(test_corpus))"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.11"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}