Election Demo - Text Analysis¶

#We're going to use a natural language toolkit
import nltk
# Downloading nltk packages used in this example
nltk.download('punkt')

# Downloading nltk packages used in this example
# This library can be used to identify different grammatical elements in a sentence (noun, verb etc)
nltk.download('maxent_treebank_pos_tagger')

#This routine allows us to generate a summary of a text

N = 100  # Number of words to consider
CLUSTER_THRESHOLD = 5  # Distance between words to consider
TOP_SENTENCES = 5  # Number of sentences to return for a "top n" summary

# Download nltk packages used in this example
nltk.download('stopwords')
stop_words = nltk.corpus.stopwords.words('english') + ['.',',','--]','\'s','?',')','(',':','\\','"','-','}','{']

# Approach taken from "The Automatic Creation of Literature Abstracts" by H.P. Luhn

def _score_sentences(sentences, important_words):
    scores = []
    sentence_idx = -1

    for s in [nltk.tokenize.word_tokenize(s) for s in sentences]:

        sentence_idx += 1
        word_idx = []

        # For each word in the word list...
        for w in important_words:
            try:
                # Compute an index for where any important words occur in the sentence.

                word_idx.append(s.index(w))
            except ValueError, e: # w not in this particular sentence
                pass

        word_idx.sort()

        # It is possible that some sentences may not contain any important words at all.
        if len(word_idx)== 0: continue

        # Using the word index, compute clusters by using a max distance threshold
        # for any two consecutive words.

        clusters = []
        cluster = [word_idx[0]]
        i = 1
        while i < len(word_idx):
            if word_idx[i] - word_idx[i - 1] < CLUSTER_THRESHOLD:
                cluster.append(word_idx[i])
            else:
                clusters.append(cluster[:])
                cluster = [word_idx[i]]
            i += 1
        clusters.append(cluster)

        # Score each cluster. The max score for any given cluster is the score 
        # for the sentence.

        max_cluster_score = 0
        for c in clusters:
            significant_words_in_cluster = len(c)
            total_words_in_cluster = c[-1] - c[0] + 1
            score = 1.0 * significant_words_in_cluster \
                * significant_words_in_cluster / total_words_in_cluster

            if score > max_cluster_score:
                max_cluster_score = score

        scores.append((sentence_idx, score))

    return scores

def summarize(txt):
    sentences = [s for s in nltk.tokenize.sent_tokenize(txt)]
    normalized_sentences = [s.lower() for s in sentences]

    words = [w.lower() for sentence in normalized_sentences for w in
             nltk.tokenize.word_tokenize(sentence)]

    fdist = nltk.FreqDist(words)

    top_n_words = [w[0] for w in fdist.items() 
            if w[0] not in nltk.corpus.stopwords.words('english')][:N]

    scored_sentences = _score_sentences(normalized_sentences, top_n_words)

    # Summarization Approach 1:
    # Filter out nonsignificant sentences by using the average score plus a
    # fraction of the std dev as a filter

    avg = numpy.mean([s[1] for s in scored_sentences])
    std = numpy.std([s[1] for s in scored_sentences])
    mean_scored = [(sent_idx, score) for (sent_idx, score) in scored_sentences
                   if score > avg + 0.5 * std]

    # Summarization Approach 2:
    # Another approach would be to return only the top N ranked sentences

    top_n_scored = sorted(scored_sentences, key=lambda s: s[1])[-TOP_SENTENCES:]
    top_n_scored = sorted(top_n_scored, key=lambda s: s[0])

    # Decorate the post object with summaries

    return dict(top_n_summary=[sentences[idx] for (idx, score) in top_n_scored],
                mean_scored_summary=[sentences[idx] for (idx, score) in mean_scored])

print txt_summarized['top_n_summary']

#If there are too many tags we get an error. Limit the number of words displayed explicitly
maxwords=50
maxfontsize=30
#After you generate the word cloud, if there are words you want to exlcude, add them to your own stoplist
mystopwords=['000','20']

tags = make_tags(get_tag_counts(txt,mystopwords)[:maxwords], maxsize=maxfontsize)
create_tag_image(tags, 'wordcloud.png', size=(600, 400))

#The word cloud has been saved as an image with the specificed filename.
#We can pull it back in to the cell and render it for display purposes
IPython.display.display(IPython.display.Image(filename='wordcloud.png'))

#Look at the most commonly occurring words that aren't in the stop list
wordcounts=get_tag_counts(txt)
#Or if you want to add further stopwords and have already defined a list: mystopwords
#get_tag_counts(txt,mystopwords)
wordcounts[:20]

#Generate a plot of where particular words appear in the document

#Unfortunately, this doesn't work for me.
#Not sure if it's an issue with the original VM or if I broke something with additional installs eg for pytagcloud
text.dispersion_plot(['development','infrastructure','services','education', 'health', 'security'])

#If we call the *concordance* function on the nltk-ified text, and provide a key word,
# we can get a list of words around occurrences of the that key word.
text.concordance('development')

#Ngrams are groups of N words that commonly occur together (bigrams are N=2, trigrams are N=3)
#If we look for common Ngrams, we can see commonly repeated phrases.
#To find interesting Ngrams, we look for ones where

#sort of via http://stackoverflow.com/a/2453229/454773
finder = nltk.BigramCollocationFinder.from_words(w for w in tokens if w.isalpha())
# only bigrams that appear 3+ times
finder.apply_freq_filter(3) 

# return the 10 n-grams with the highest point wise mutual information (PMI)
#This identifies pairs where the words appear together more frequently than you might expect
#given the frequency of each term taken on its own in the document.
bigram_measures = nltk.collocations.BigramAssocMeasures()
finder.nbest(bigram_measures.pmi, 20)  

#We can also look for interesting trigrams
finder = nltk.TrigramCollocationFinder.from_words(w for w in tokens if w.isalpha())
# only bigrams that appear 3+ times
finder.apply_freq_filter(3) 

# return the 10 n-grams with the highest point wise mutual information (PMI)
trigram_measures = nltk.collocations.TrigramAssocMeasures()
finder.nbest(trigram_measures.pmi, 20)