Working with Text

%matplotlib inline

Libraries for I/O

import os
import glob

Libraries for numerics

import numpy as np
import pandas as pd
import scipy.stats as stats

Libraries for plotting

import matplotlib.pyplot as plt
import seaborn as sns

Libraries for string manipulation

import string
import re

Libraries for functional programming

from functools import reduce, partial
import itertools as it
import operator as op
import toolz as tz
import toolz.curried as c

String methods

s = "  Avoid taking unnecessary gambles. Lucky numbers: 12, 15, 23, 28, 37\n"

Removing leading and trailing whitespace

s.strip()

'Avoid taking unnecessary gambles. Lucky numbers: 12, 15, 23, 28, 37'

s.lstrip()

'Avoid taking unnecessary gambles. Lucky numbers: 12, 15, 23, 28, 37\n'

s.rstrip()

'  Avoid taking unnecessary gambles. Lucky numbers: 12, 15, 23, 28, 37'

s = s.strip()

Changing case

s.lower()

'avoid taking unnecessary gambles. lucky numbers: 12, 15, 23, 28, 37'

s.upper()

'AVOID TAKING UNNECESSARY GAMBLES. LUCKY NUMBERS: 12, 15, 23, 28, 37'

s.title()

'Avoid Taking Unnecessary Gambles. Lucky Numbers: 12, 15, 23, 28, 37'

Checking conditions

s.startswith('Avoid')

True

s.endswith('37')

True

s.isalpha()

False

s.isnumeric()

False

s.isspace()

False

s.isprintable()

True

Counting and indexing

s.count('a')

3

s.count('gambles')

1

s.find('gambles')

25

s[27:]

'mbles. Lucky numbers: 12, 15, 23, 28, 37'

s.find('foobar')

-1

s.index('gambles')

25

try:
    s.index('foobar')
except ValueError as e:
    print(e)

substring not found

Splitting and joining

s.split()

['Avoid',
 'taking',
 'unnecessary',
 'gambles.',
 'Lucky',
 'numbers:',
 '12,',
 '15,',
 '23,',
 '28,',
 '37']

s.split(':')

['Avoid taking unnecessary gambles. Lucky numbers', ' 12, 15, 23, 28, 37']

'-'.join(s.split())

'Avoid-taking-unnecessary-gambles.-Lucky-numbers:-12,-15,-23,-28,-37'

Replacing

s.replace('gambles', 'risk')

'Avoid taking unnecessary risk. Lucky numbers: 12, 15, 23, 28, 37'

Translating

table = str.maketrans(string.ascii_lowercase, string.ascii_uppercase, string.punctuation)
s.translate(table)

'AVOID TAKING UNNECESSARY GAMBLES LUCKY NUMBERS 12 15 23 28 37'

table = str.maketrans('', '', string.punctuation)
s.translate(table)

'Avoid taking unnecessary gambles Lucky numbers 12 15 23 28 37'

Exercise: Caesar Cipher

A Caesar cipher with offset \(k\) converts a character into the character \(k\) letters down, looping around if this goes past z. Non-characters (numbers, spaces, punctuation) are left intact. For instance, with offset=3, we get abcXYZ being coded as defABC. Write an function encode(k, s) where k is the offset and s the string to be coded. Write a decode(k, s) function that decodes encrypted ciphers. Test it out on the fortune.

def encode(k, s):
    table = str.maketrans(
        string.ascii_lowercase + string.ascii_uppercase,
        string.ascii_lowercase[k:] + string.ascii_lowercase[:k] +
        string.ascii_uppercase[k:] + string.ascii_uppercase[:k])
    return s.translate(table)

encode(3, 'abcXYZ')

'defABC'

def decode(k, s):
    return encode(-k, s)

code = encode(3, s)

code

'Dyrlg wdnlqj xqqhfhvvdub jdpeohv. Oxfnb qxpehuv: 12, 15, 23, 28, 37'

decode(3, code)

'Avoid taking unnecessary gambles. Lucky numbers: 12, 15, 23, 28, 37'

Counting words

To count words, we typically do the following preprocessing:

• Convert to lower (or upper) case
• Remove punctuation
• Split on blank space
• Count each word in list

s

'Avoid taking unnecessary gambles. Lucky numbers: 12, 15, 23, 28, 37'

Preprocessing

words = s.lower().translate(str.maketrans('','',string.punctuation)).split()

Using a Counter (bag)

from collections import Counter

Counter(words)

Counter({'12': 1,
         '15': 1,
         '23': 1,
         '28': 1,
         '37': 1,
         'avoid': 1,
         'gambles': 1,
         'lucky': 1,
         'numbers': 1,
         'taking': 1,
         'unnecessary': 1})

Using a dictionary

counter = {}
for word in words:
    counter[word] = counter.get(word, 0) + 1

counter

{'12': 1,
 '15': 1,
 '23': 1,
 '28': 1,
 '37': 1,
 'avoid': 1,
 'gambles': 1,
 'lucky': 1,
 'numbers': 1,
 'taking': 1,
 'unnecessary': 1}

Using a defaultdict

from collections import defaultdict

d = defaultdict(int)

for word in words:
    d[word] += 1

d

defaultdict(int,
            {'12': 1,
             '15': 1,
             '23': 1,
             '28': 1,
             '37': 1,
             'avoid': 1,
             'gambles': 1,
             'lucky': 1,
             'numbers': 1,
             'taking': 1,
             'unnecessary': 1})

Using a functional pipe

tz.pipe(
    s,
    lambda s: s.lower(),
    lambda s: s.translate(str.maketrans('', '', string.punctuation)),
    lambda s: s.split(),
    tz.frequencies
)

{'12': 1,
 '15': 1,
 '23': 1,
 '28': 1,
 '37': 1,
 'avoid': 1,
 'gambles': 1,
 'lucky': 1,
 'numbers': 1,
 'taking': 1,
 'unnecessary': 1}

Modification for collection of strings

ss = [s, s, s]

ss

['Avoid taking unnecessary gambles. Lucky numbers: 12, 15, 23, 28, 37',
 'Avoid taking unnecessary gambles. Lucky numbers: 12, 15, 23, 28, 37',
 'Avoid taking unnecessary gambles. Lucky numbers: 12, 15, 23, 28, 37']

tz.pipe(
    ss,
    c.map(lambda s: s.lower()),
    c.map(lambda s: s.translate(str.maketrans('', '', string.punctuation))),
    c.mapcat(lambda s: s.split()),
    tz.frequencies
)

{'12': 3,
 '15': 3,
 '23': 3,
 '28': 3,
 '37': 3,
 'avoid': 3,
 'gambles': 3,
 'lucky': 3,
 'numbers': 3,
 'taking': 3,
 'unnecessary': 3}

String to vector

To analyze text, we typically need to convert it to a vector format. There are several ways to do so. Here we show the most obvious method known as one-hot encoding.

One hot character encoding

We first encode the string ‘abcabc’ as the vector [0,1,2,0,1,2]. For one-hot encoding, we next convert this to the one-hot encoded matrix

array([[1, 0, 0],
       [0, 1, 0],
       [0, 0, 1],
       [1, 0, 0],
       [0, 1, 0],
       [0, 0, 1]])

idx = 0
index = {}
for ch in s:
    if not ch in index:
        index[ch] = idx
        idx += 1

index

{' ': 5,
 ',': 25,
 '.': 20,
 '1': 23,
 '2': 24,
 '3': 27,
 '5': 26,
 '7': 29,
 '8': 28,
 ':': 22,
 'A': 0,
 'L': 21,
 'a': 7,
 'b': 18,
 'c': 13,
 'd': 4,
 'e': 12,
 'g': 10,
 'i': 3,
 'k': 8,
 'l': 19,
 'm': 17,
 'n': 9,
 'o': 2,
 'r': 15,
 's': 14,
 't': 6,
 'u': 11,
 'v': 1,
 'y': 16}

Categorical encoding

nchars = len(index)

vs = np.array([index[ch] for ch in s])

vs

array([ 0,  1,  2,  3,  4,  5,  6,  7,  8,  3,  9, 10,  5, 11,  9,  9, 12,
       13, 12, 14, 14,  7, 15, 16,  5, 10,  7, 17, 18, 19, 12, 14, 20,  5,
       21, 11, 13,  8, 16,  5,  9, 11, 17, 18, 12, 15, 14, 22,  5, 23, 24,
       25,  5, 23, 26, 25,  5, 24, 27, 25,  5, 24, 28, 25,  5, 27, 29])

One-hot encoding

n = len(vs)
p = len(index)
m = np.zeros((n,p), dtype='int')
i = np.arange(len(vs))
m[i, vs] = 1
m

array([[1, 0, 0, ..., 0, 0, 0],
       [0, 1, 0, ..., 0, 0, 0],
       [0, 0, 1, ..., 0, 0, 0],
       ...,
       [0, 0, 0, ..., 0, 0, 0],
       [0, 0, 0, ..., 1, 0, 0],
       [0, 0, 0, ..., 0, 0, 1]])

Reverse index lookup

reverse_index = dict(zip(index.values(), index.keys()))

''.join(reverse_index[v] for v in vs)

'Avoid taking unnecessary gambles. Lucky numbers: 12, 15, 23, 28, 37'

One hot encoding for words.

words = ' '.join([s,s]).lower().translate(str.maketrans('', '', string.punctuation)).split()

pos = 0
index = {}
for word in words:
    if word not in index:
        index[word] = pos
        pos += 1

Categorical encoding

ws = np.array([index[word] for word in words])

ws

array([ 0,  1,  2,  3,  4,  5,  6,  7,  8,  9, 10,  0,  1,  2,  3,  4,  5,
        6,  7,  8,  9, 10])

One-hot encoding

n = len(ws)
p = len(index)
m = np.zeros((n,p), dtype='int')
i = np.arange(len(ws))
m[i, ws] = 1
m

array([[1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0],
       [0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0],
       [0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0],
       [0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0],
       [0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0],
       [0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0],
       [0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0],
       [0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0],
       [0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0],
       [0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0],
       [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1],
       [1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0],
       [0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0],
       [0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0],
       [0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0],
       [0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0],
       [0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0],
       [0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0],
       [0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0],
       [0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0],
       [0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0],
       [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1]])

Reverse lookup

reverse_index = dict(zip(index.values(), index.keys()))

' '.join(reverse_index[w] for w in ws)

'avoid taking unnecessary gambles lucky numbers 12 15 23 28 37 avoid taking unnecessary gambles lucky numbers 12 15 23 28 37'

Regular expressions

s

'Avoid taking unnecessary gambles. Lucky numbers: 12, 15, 23, 28, 37'

Literal match

re.findall(r'gambles', s)

['gambles']

Quantifiers ., {m,n}, +, *

re.findall(r'gam.les', s)

['gambles']

re.findall(r'g.*s', s)

['g unnecessary gambles. Lucky numbers']

Non-greedy quantifier.

re.findall(r'g.*?s', s)

['g unneces', 'gambles']

Special characters

re.findall(r'\bg.*?s\b', s)

['gambles']

re.findall(r'\b\w+?\b', s)

['Avoid',
 'taking',
 'unnecessary',
 'gambles',
 'Lucky',
 'numbers',
 '12',
 '15',
 '23',
 '28',
 '37']

re.findall(r'\b\d+?\b', s)

['12', '15', '23', '28', '37']

re.findall(r'\b[a-zA-Z]+?\b', s)

['Avoid', 'taking', 'unnecessary', 'gambles', 'Lucky', 'numbers']

Begin and end anchors

re.findall(r'\w+', s)

['Avoid',
 'taking',
 'unnecessary',
 'gambles',
 'Lucky',
 'numbers',
 '12',
 '15',
 '23',
 '28',
 '37']

re.findall(r'^\w+', s)

['Avoid']

re.findall(r'\w+$', s)

['37']

Capture groups

pat = r'\b(\d)(\d)?\b'

re.findall(pat, s)

[('1', '2'), ('1', '5'), ('2', '3'), ('2', '8'), ('3', '7')]

Using search and match objects

re.search(pat, s)

<_sre.SRE_Match object; span=(49, 51), match='12'>

m = re.search(pat, s)

m.string

'Avoid taking unnecessary gambles. Lucky numbers: 12, 15, 23, 28, 37'

m.group()

'12'

m.groups()

('1', '2')

Replacement using capture groups

rep = r'\2\1'
re.sub(pat, rep, s)

'Avoid taking unnecessary gambles. Lucky numbers: 21, 51, 32, 82, 73'

Using compiled patterns

pat = re.compile(r'\b[a-zA-Z]+?\b')
pat.findall(s)

['Avoid', 'taking', 'unnecessary', 'gambles', 'Lucky', 'numbers']