Splitting Twitter.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Split Twitter text into individual tweets\n",
    "\n",
    "Based on https://github.com/paulfurley/python-tweet-splitter, https://github.com/glyph/twitter-text-py & https://github.com/twitter/twitter-text/\n",
    "\n",
    "First run all the cells below to load classes."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 85,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Place text below\n",
    "text = \"\"\"\n",
    "\"\"\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 86,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "\n",
      "\n"
     ]
    }
   ],
   "source": [
    "ts = TweetSplitter(text)\n",
    "for t in ts.split():\n",
    "    print(t)\n",
    "    print(\"\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 79,
   "metadata": {},
   "outputs": [],
   "source": [
    "\"\"\"\n",
    "Enhancements includes:\n",
    "\n",
    "* tweet-text validation (based on https://github.com/twitter/twitter-text/) \n",
    "* natural sentence breaks, plus maintains paragraph structure\n",
    "\"\"\"\n",
    "\n",
    "from nltk import tokenize\n",
    "\n",
    "config = {\n",
    "    \"defaults\": {\n",
    "        \"version\": 3,\n",
    "        \"maxWeightedTweetLength\": 280,\n",
    "        \"scale\": 100,\n",
    "        \"defaultWeight\": 200,\n",
    "        \"emojiParsingEnabled\": True,\n",
    "        \"transformedURLLength\": 23,\n",
    "        \"ranges\": [\n",
    "          { \"start\": 0, \"end\": 4351, \"weight\": 100 },\n",
    "          { \"start\": 8192, \"end\": 8205, \"weight\": 100 },\n",
    "          { \"start\": 8208, \"end\": 8223, \"weight\": 100 },\n",
    "          { \"start\": 8242, \"end\": 8247, \"weight\": 100 }\n",
    "        ]\n",
    "    }\n",
    "}\n",
    "\n",
    "class TweetSplitter:\n",
    "    \n",
    "    def __init__(self, text, **kwargs):\n",
    "        self.max_length = config.get(\"defaults\", {}).get(\"maxWeightedTweetLength\")\n",
    "        # Respect existing structure\n",
    "        self.text = text\n",
    "        self.split_text = []\n",
    "        self.dangle = kwargs.get(\"dangle\", 3) # minimum hanging start of sentence\n",
    "\n",
    "    def split(self):\n",
    "        if len(self.text) <= self.max_length:\n",
    "            return [self.text]\n",
    "        text = [t for t in self.text.splitlines() if t.strip()]\n",
    "        for t in text:\n",
    "            if self._tweet_length(t) <= self.max_length:\n",
    "                self.split_text.append(t)\n",
    "            else:\n",
    "                mid_split = self._split_sentences(tokenize.sent_tokenize(t))\n",
    "                self.split_text.extend(mid_split)\n",
    "        return self.split_text\n",
    "\n",
    "    def _split_sentences(self, sentences):\n",
    "        \"\"\"\n",
    "        Return a list of tweets that are each less than max-length, but split\n",
    "        appropriately.\n",
    "        \n",
    "        * Split and test each sentence. \n",
    "        * If only two, and each < max_length, return as is.\n",
    "        * If > 2, test combination of sentences that < max_length.\n",
    "        * As last resort, split a sentence ...\n",
    "        * 1st test if can add to previous sentence/tweet, then split this one.\n",
    "        * Optimise tweet length.\n",
    "        \"\"\"\n",
    "        new_tweets = []\n",
    "        hanging_twit = None\n",
    "        if self._tweet_length(sentences[0]) > self.max_length:\n",
    "            first_sentence = sentences.pop(0)\n",
    "            first_twits = list(self._generate_split_tweets(first_sentence.split(\" \")))\n",
    "            new_tweets.extend(first_twits[:-1])\n",
    "            hanging_twit = first_twits[-1].strip()\n",
    "        while len(sentences):\n",
    "            text = sentences.pop(0)\n",
    "            if hanging_twit:\n",
    "                sub_twits = self._create_tweets(hanging_twit, text)\n",
    "            else:\n",
    "                sub_twits = self._create_tweets(text, sentences.pop(0))\n",
    "            hanging_twit = sub_twits[-1].strip()\n",
    "            new_tweets.extend(sub_twits[:-1])\n",
    "        new_tweets.append(hanging_twit)\n",
    "        return new_tweets\n",
    "            \n",
    "    def _create_tweets(self, text1, text2, ignore_dangle = False):\n",
    "        \"\"\"\n",
    "        Receive two strings. Optimise their correspondence to fit max_length.\n",
    "        \n",
    "        * Join them if their combined length < max_length (i.e. +1 for the ' ')\n",
    "        * Split and test the number of words that can be added to text1 subject to dangle\n",
    "        * Otherwise return text2 split ... if text2 split still > max_length, recurse.\n",
    "        \"\"\"\n",
    "        if self._tweet_length(text1) + self._tweet_length(text2) < self.max_length:\n",
    "            return [\" \".join([text1, text2])]\n",
    "        text2_words = text2.split(\" \")\n",
    "        if not ignore_dangle and len(text2_words) > self.dangle:\n",
    "            dangle_length = sum([self._tweet_length(text2_words[d]) for d in range(self.dangle)])\n",
    "            if self._tweet_length(text1) + dangle_length <= self.max_length:\n",
    "                ignore_dangle = True\n",
    "        while ignore_dangle:\n",
    "            if self._tweet_length(text1) + self._tweet_length(text2_words[0]) < self.max_length:\n",
    "                text1 += \" \" + text2_words.pop(0)\n",
    "            else:\n",
    "                break\n",
    "        if self._tweet_length(\" \".join(text2_words)) <= self.max_length:\n",
    "            return [text1, \" \".join(text2_words)]\n",
    "        # Edge case of text2 still being > max_length ...\n",
    "        return [text1] + list(self._generate_split_tweets(text2_words))\n",
    "\n",
    "    def _generate_split_tweets(self, words):\n",
    "        this_tweet = None\n",
    "        while True:\n",
    "            if this_tweet:\n",
    "                this_tweet = \" \".join([this_tweet, words.pop(0)])\n",
    "            else:\n",
    "                this_tweet = words.pop(0)\n",
    "            if not words:\n",
    "                break\n",
    "            if self._tweet_length(this_tweet) + self._tweet_length(words[0]) > self.max_length:\n",
    "                yield this_tweet\n",
    "                this_tweet = None\n",
    "        yield this_tweet\n",
    "\n",
    "    def _tweet_length(self, text):\n",
    "        \"\"\"\n",
    "        Returns the length of the string as it would be displayed. This is equivilent to the length of the Unicode NFC\n",
    "        (See: http://www.unicode.org/reports/tr15). This is needed in order to consistently calculate the length of a\n",
    "        string no matter which actual form was transmitted. For example:\n",
    "             U+0065  Latin Small Letter E\n",
    "         +   U+0301  Combining Acute Accent\n",
    "         ----------\n",
    "         =   2 bytes, 2 characters, displayed as é (1 visual glyph)\n",
    "             … The NFC of {U+0065, U+0301} is {U+00E9}, which is a single chracter and a +display_length+ of 1\n",
    "         The string could also contain U+00E9 already, in which case the canonicalization will not change the value.\n",
    "\n",
    "        Source: https://github.com/glyph/twitter-text-py/blob/master/twitter_text/validation.py\n",
    "        \"\"\"\n",
    "        length = len(text)\n",
    "        for url in Extractor(text).extract_urls_with_indices():\n",
    "            # remove the link of the original URL\n",
    "            length += url['indices'][0] - url['indices'][1]\n",
    "            # add the length of the t.co URL that will replace it\n",
    "            length += config.get(\"defaults\", {}).get(\"transformedURLLength\")\n",
    "        return length"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [],
   "source": [
    "class Extractor:\n",
    "    \"\"\"\n",
    "    A module for including Tweet parsing in a class. This module provides function for the extraction and processing\n",
    "    of usernames, lists, URLs and hashtags.\n",
    "    \n",
    "    Source: https://github.com/glyph/twitter-text-py/blob/master/twitter_text/extractor.py\n",
    "    \"\"\"\n",
    "    def __init__(self, text):\n",
    "        self.text = text\n",
    "        \n",
    "    def extract_urls_with_indices(self, options = {'extract_url_without_protocol': True}):\n",
    "        \"\"\"\n",
    "        Extracts a list of all URLs included in the Tweet text along\n",
    "        with the indices. If the text is None or contains no\n",
    "        URLs an empty list will be returned.\n",
    "        If a block is given then it will be called for each URL.\n",
    "        \"\"\"\n",
    "        urls = []\n",
    "        for match in REGEXEN['valid_url'].finditer(self.text):\n",
    "            complete, before, url, protocol, domain, port, path, query = match.groups()\n",
    "            start_position = match.start() + len(before or '')\n",
    "            end_position = match.end()\n",
    "            # If protocol is missing and domain contains non-ASCII characters,\n",
    "            # extract ASCII-only domains.\n",
    "            if not protocol:\n",
    "                if not options.get('extract_url_without_protocol') or REGEXEN['invalid_url_without_protocol_preceding_chars'].search(before):\n",
    "                    continue\n",
    "                last_url = None\n",
    "                last_url_invalid_match = None\n",
    "                for ascii_domain in REGEXEN['valid_ascii_domain'].finditer(domain):\n",
    "                    ascii_domain = ascii_domain.group()\n",
    "                    last_url = {\n",
    "                        'url':      ascii_domain,\n",
    "                        'indices':  [start_position - len(before or '') + complete.find(ascii_domain), start_position - len(before or '') + complete.find(ascii_domain) + len(ascii_domain)]\n",
    "                    }\n",
    "                    last_url_invalid_match = REGEXEN['invalid_short_domain'].search(ascii_domain) is not None\n",
    "                    if not last_url_invalid_match:\n",
    "                        urls.append(last_url)\n",
    "                # no ASCII-only domain found. Skip the entire URL\n",
    "                if not last_url:\n",
    "                    continue\n",
    "                if path:\n",
    "                    last_url['url'] = url.replace(domain, last_url['url'])\n",
    "                    last_url['indices'][1] = end_position\n",
    "                    if last_url_invalid_match:\n",
    "                        urls.append(last_url)\n",
    "            else:\n",
    "                if REGEXEN['valid_tco_url'].match(url):\n",
    "                    url = REGEXEN['valid_tco_url'].match(url).group()\n",
    "                    end_position = start_position + len(url)\n",
    "                urls.append({\n",
    "                    'url':      url,\n",
    "                    'indices':  [start_position, end_position]\n",
    "                })\n",
    "        return urls\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Source: https://github.com/glyph/twitter-text-py/blob/master/twitter_text/regex.py\n",
    "\n",
    "import re\n",
    "try:\n",
    "    unichr\n",
    "except NameError:\n",
    "    unichr = chr\n",
    "from functools import reduce\n",
    "\n",
    "def regex_range(start, end = None):\n",
    "    if end:\n",
    "        return u'%s-%s' % (unichr(start), unichr(end))\n",
    "    else:\n",
    "        return u'%s' % unichr(start)\n",
    "\n",
    "REGEXEN = {}\n",
    "PUNCTUATION_CHARS = u'!\"#$%&\\'()*+,-./:;<=>?@\\\\[\\\\]^_\\\\`{|}~'\n",
    "SPACE_CHARS = u\" \\\\t\\\\n\\\\x0B\\\\f\\\\r\"\n",
    "CTRL_CHARS = u\"\\\\x00-\\\\x1F\\\\x7F\"\n",
    "# Space is more than %20, U+3000 for example is the full-width space used with Kanji. Provide a short-hand\n",
    "# to access both the list of characters and a pattern suitible for use with String#split\n",
    "#  Taken from: ActiveSupport::Multibyte::Handlers::UTF8Handler::UNICODE_WHITESPACE\n",
    "UNICODE_SPACES = []\n",
    "for space in reduce(lambda x,y: x + y if type(y) == list else x + [y], [\n",
    "        list(range(0x0009, 0x000D)),  # White_Space # Cc   [5] <control-0009>..<control-000D>\n",
    "        0x0020,                 # White_Space # Zs       SPACE\n",
    "        0x0085,                 # White_Space # Cc       <control-0085>\n",
    "        0x00A0,                 # White_Space # Zs       NO-BREAK SPACE\n",
    "        0x1680,                 # White_Space # Zs       OGHAM SPACE MARK\n",
    "        0x180E,                 # White_Space # Zs       MONGOLIAN VOWEL SEPARATOR\n",
    "        list(range(0x2000, 0x200A)),  # White_Space # Zs  [11] EN QUAD..HAIR SPACE\n",
    "        0x2028,                 # White_Space # Zl       LINE SEPARATOR\n",
    "        0x2029,                 # White_Space # Zp       PARAGRAPH SEPARATOR\n",
    "        0x202F,                 # White_Space # Zs       NARROW NO-BREAK SPACE\n",
    "        0x205F,                 # White_Space # Zs       MEDIUM MATHEMATICAL SPACE\n",
    "        0x3000,                 # White_Space # Zs       IDEOGRAPHIC SPACE\n",
    "    ]):\n",
    "    UNICODE_SPACES.append(unichr(space))\n",
    "REGEXEN['spaces'] = re.compile(u''.join(UNICODE_SPACES))\n",
    "# Latin accented characters\n",
    "# Excludes 0xd7 from the range (the multiplication sign, confusable with \"x\").\n",
    "# Also excludes 0xf7, the division sign\n",
    "LATIN_ACCENTS = [\n",
    "    regex_range(0x00c0, 0x00d6),\n",
    "    regex_range(0x00d8, 0x00f6),\n",
    "    regex_range(0x00f8, 0x00ff),\n",
    "    regex_range(0x0100, 0x024f),\n",
    "    regex_range(0x0253, 0x0254),\n",
    "    regex_range(0x0256, 0x0257),\n",
    "    regex_range(0x0259),\n",
    "    regex_range(0x025b),\n",
    "    regex_range(0x0263),\n",
    "    regex_range(0x0268),\n",
    "    regex_range(0x026f),\n",
    "    regex_range(0x0272),\n",
    "    regex_range(0x0289),\n",
    "    regex_range(0x028b),\n",
    "    regex_range(0x02bb),\n",
    "    regex_range(0x0300, 0x036f),\n",
    "    regex_range(0x1e00, 0x1eff),\n",
    "]\n",
    "REGEXEN['latin_accents'] = re.compile(u''.join(LATIN_ACCENTS), re.IGNORECASE | re.UNICODE)\n",
    "LATIN_ACCENTS = u''.join(LATIN_ACCENTS)\n",
    "# URL related hash regex collection\n",
    "REGEXEN['valid_url_preceding_chars'] = re.compile(u'(?:[^A-Z0-9@＠$#＃%s]|^)', re.IGNORECASE | re.UNICODE)\n",
    "REGEXEN['invalid_url_without_protocol_preceding_chars'] = re.compile(u'[-_.\\\\/]$')\n",
    "DOMAIN_VALID_CHARS = u'[^%s%s%s%s]' % (PUNCTUATION_CHARS, SPACE_CHARS, CTRL_CHARS, u''.join(UNICODE_SPACES))\n",
    "REGEXEN['valid_subdomain'] = re.compile(u'(?:(?:%s(?:[_-]|%s)*)?%s\\\\.)' % (DOMAIN_VALID_CHARS, DOMAIN_VALID_CHARS, DOMAIN_VALID_CHARS), re.IGNORECASE | re.UNICODE)\n",
    "REGEXEN['valid_domain_name'] = re.compile(u'(?:(?:%s(?:[-]|%s)*)?%s\\\\.)' % (DOMAIN_VALID_CHARS, DOMAIN_VALID_CHARS, DOMAIN_VALID_CHARS), re.IGNORECASE | re.UNICODE)\n",
    "REGEXEN['valid_gTLD'] = re.compile(u'(?:(?:academy|actor|aero|agency|arpa|asia|bar|bargains|berlin|best|bid|bike|biz|blue|boutique|build|builders|buzz|cab|camera|camp|cards|careers|cat|catering|center|ceo|cheap|christmas|cleaning|clothing|club|codes|coffee|com|community|company|computer|construction|contractors|cool|coop|cruises|dance|dating|democrat|diamonds|directory|domains|edu|education|email|enterprises|equipment|estate|events|expert|exposed|farm|fish|flights|florist|foundation|futbol|gallery|gift|glass|gov|graphics|guitars|guru|holdings|holiday|house|immobilien|industries|info|institute|int|international|jobs|kaufen|kim|kitchen|kiwi|koeln|kred|land|lighting|limo|link|luxury|management|mango|marketing|menu|mil|mobi|moda|monash|museum|nagoya|name|net|neustar|ninja|okinawa|onl|org|partners|parts|photo|photography|photos|pics|pink|plumbing|post|pro|productions|properties|pub|qpon|recipes|red|rentals|repair|report|reviews|rich|ruhr|sexy|shiksha|shoes|singles|social|solar|solutions|supplies|supply|support|systems|tattoo|technology|tel|tienda|tips|today|tokyo|tools|training|travel|uno|vacations|ventures|viajes|villas|vision|vote|voting|voto|voyage|wang|watch|wed|wien|wiki|works|xxx|xyz|zone|дети|онлайн|орг|сайт|بازار|شبكة|みんな|中信|中文网|公司|公>益|在线|我爱你|政务|游戏|移动|网络|集团|삼성)(?=[^0-9a-z]|$))', re.IGNORECASE | re.UNICODE)\n",
    "REGEXEN['valid_ccTLD'] = re.compile(u'(?:(?:ac|ad|ae|af|ag|ai|al|am|an|ao|aq|ar|as|at|au|aw|ax|az|ba|bb|bd|be|bf|bg|bh|bi|bj|bl|bm|bn|bo|bq|br|bs|bt|bv|bw|by|bz|ca|cc|cd|cf|cg|ch|ci|ck|cl|cm|cn|co|cr|cu|cv|cw|cx|cy|cz|de|dj|dk|dm|do|dz|ec|ee|eg|eh|er|es|et|eu|fi|fj|fk|fm|fo|fr|ga|gb|gd|ge|gf|gg|gh|gi|gl|gm|gn|gp|gq|gr|gs|gt|gu|gw|gy|hk|hm|hn|hr|ht|hu|id|ie|il|im|in|io|iq|ir|is|it|je|jm|jo|jp|ke|kg|kh|ki|km|kn|kp|kr|kw|ky|kz|la|lb|lc|li|lk|lr|ls|lt|lu|lv|ly|ma|mc|md|me|mf|mg|mh|mk|ml|mm|mn|mo|mp|mq|mr|ms|mt|mu|mv|mw|mx|my|mz|na|nc|ne|nf|ng|ni|nl|no|np|nr|nu|nz|om|pa|pe|pf|pg|ph|pk|pl|pm|pn|pr|ps|pt|pw|py|qa|re|ro|rs|ru|rw|sa|sb|sc|sd|se|sg|sh|si|sj|sk|sl|sm|sn|so|sr|ss|st|su|sv|sx|sy|sz|tc|td|tf|tg|th|tj|tk|tl|tm|tn|to|tp|tr|tt|tv|tw|tz|ua|ug|uk|um|us|uy|uz|va|vc|ve|vg|vi|vn|vu|wf|ws|ye|yt|za|zm|zw|мон|рф|срб|укр|қаз|الاردن|الجزائر|السعودية|المغرب|امارات|ایران|بھارت|تونس|سودان|سورية|عمان|فلسطين|قطر|مصر|مليسيا|پاکستان|भारत|বাংলা|ভারত|ਭਾਰਤ|ભારત|இந்தியா|இலங்கை|சிங்கப்பூர்|భారత్|ලංකා|ไทย|გე|中国|中加坡|湾|台灣|新香港|한국)(?=[^0-9a-z]|$))', re.IGNORECASE | re.UNICODE)\n",
    "REGEXEN['valid_punycode'] = re.compile(u'(?:xn--[0-9a-z]+)', re.IGNORECASE | re.UNICODE)\n",
    "\n",
    "REGEXEN['valid_domain'] = re.compile(u'(?:%s*%s(?:%s|%s|%s))' % (REGEXEN['valid_subdomain'].pattern, REGEXEN['valid_domain_name'].pattern, REGEXEN['valid_gTLD'].pattern, REGEXEN['valid_ccTLD'].pattern, REGEXEN['valid_punycode'].pattern), re.IGNORECASE | re.UNICODE)\n",
    "\n",
    "# This is used in Extractor\n",
    "REGEXEN['valid_ascii_domain'] = re.compile(u'(?:(?:[A-Za-z0-9\\\\-_]|[%s])+\\\\.)+(?:%s|%s|%s)' % (REGEXEN['latin_accents'].pattern, REGEXEN['valid_gTLD'].pattern, REGEXEN['valid_ccTLD'].pattern, REGEXEN['valid_punycode'].pattern), re.IGNORECASE | re.UNICODE)\n",
    "\n",
    "# This is used in Extractor for stricter t.co URL extraction\n",
    "REGEXEN['valid_tco_url'] = re.compile(u'^https?:\\\\/\\\\/t\\\\.co\\\\/[a-z0-9]+', re.IGNORECASE | re.UNICODE)\n",
    "\n",
    "# This is used in Extractor to filter out unwanted URLs.\n",
    "REGEXEN['invalid_short_domain'] = re.compile(u'\\\\A%s%s\\\\Z' % (REGEXEN['valid_domain_name'].pattern, REGEXEN['valid_ccTLD'].pattern), re.IGNORECASE | re.UNICODE)\n",
    "\n",
    "REGEXEN['valid_port_number'] = re.compile(u'[0-9]+')\n",
    "\n",
    "REGEXEN['valid_general_url_path_chars'] = re.compile(u\"[a-z0-9!\\\\*';:=\\\\+\\\\,\\\\.\\\\$\\\\/%%#\\\\[\\\\]\\\\-_~&|@%s]\" % LATIN_ACCENTS, re.IGNORECASE | re.UNICODE)\n",
    "# Allow URL paths to contain balanced parens\n",
    "#  1. Used in Wikipedia URLs like /Primer_(film)\n",
    "#  2. Used in IIS sessions like /S(dfd346)/\n",
    "REGEXEN['valid_url_balanced_parens'] = re.compile(u'\\\\(%s+\\\\)' % REGEXEN['valid_general_url_path_chars'].pattern, re.IGNORECASE | re.UNICODE)\n",
    "# Valid end-of-path chracters (so /foo. does not gobble the period).\n",
    "#   1. Allow =&# for empty URL parameters and other URL-join artifacts\n",
    "REGEXEN['valid_url_path_ending_chars'] = re.compile(u'[a-z0-9=_#\\\\/\\\\+\\\\-%s]|(?:%s)' % (LATIN_ACCENTS, REGEXEN['valid_url_balanced_parens'].pattern), re.IGNORECASE | re.UNICODE)\n",
    "REGEXEN['valid_url_path'] = re.compile(u'(?:(?:%s*(?:%s %s*)*%s)|(?:%s+\\\\/))' % (REGEXEN['valid_general_url_path_chars'].pattern, REGEXEN['valid_url_balanced_parens'].pattern, REGEXEN['valid_general_url_path_chars'].pattern, REGEXEN['valid_url_path_ending_chars'].pattern, REGEXEN['valid_general_url_path_chars'].pattern), re.IGNORECASE | re.UNICODE)\n",
    "\n",
    "REGEXEN['valid_url_query_chars'] = re.compile(u\"[a-z0-9!?\\\\*'\\\\(\\\\);:&=\\\\+\\\\$\\\\/%#\\\\[\\\\]\\\\-_\\\\.,~|@]\", re.IGNORECASE | re.UNICODE)\n",
    "REGEXEN['valid_url_query_ending_chars'] = re.compile(u'[a-z0-9_&=#\\\\/]', re.IGNORECASE | re.UNICODE)\n",
    "REGEXEN['valid_url'] = re.compile(u'((%s)((https?:\\\\/\\\\/)?(%s)(?::(%s))?(/%s*)?(\\\\?%s*%s)?))' % (\n",
    "    REGEXEN['valid_url_preceding_chars'].pattern,\n",
    "    REGEXEN['valid_domain'].pattern,\n",
    "    REGEXEN['valid_port_number'].pattern,\n",
    "    REGEXEN['valid_url_path'].pattern,\n",
    "    REGEXEN['valid_url_query_chars'].pattern,\n",
    "    REGEXEN['valid_url_query_ending_chars'].pattern\n",
    "), re.IGNORECASE | re.UNICODE)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.8"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}