Code

The following code was used to split the dataset into an array where each element of the array is a string of text representing a speech given by Mark Twain


                import sys
                


                def divide_corpus(f):
                

                count = 1
                

                contents = f.read()
                

                speeches = contents.split("\n\n\n\n\n\n")
                

                return speeches
                


                if __name__ == "__main__":
                

                    filename = sys.argv[1]
                    

                    f = open(filename, "r")
                    

                    divide_corpus(f)
                    

                    f.close()

The following code was used to clean the data further by tokenizing the input, removing a larger set of stopwords than those provided by Voyant, lemmatizing the text, removing numbers, and other various things to clean the data. After cleaning the data, it was then used to train an LDA model that would be able to generate abstract topics for the speeches of Mark Twain.


                def tokenize_corpus(list_of_documents):
    tokenizer = RegexpTokenizer(r'\w+')
    for idx in range(len(list_of_documents)):
    list_of_documents[idx] = list_of_documents[idx].lower()
    list_of_documents[idx] = tokenizer.tokenize(list_of_documents[idx])
return list_of_documents

def remove_single_letters(tokenized_corpus):
    return [[token for token in doc if len(token) > 3] for doc in tokenized_corpus]

def lemmatize_corpus(tokenized_documents):
    docs = tokenized_documents
    lemmatizer = WordNetLemmatizer()
    docs = [[lemmatizer.lemmatize(token) for token in doc] for doc in docs]
    return docs

def remove_stopwords(tokenized_documents):
    stopWords = set(stopwords.words('english'))
    return [[token for token in doc if token not in stopWords and token not in more_stopwords] for doc in tokenized_documents]

def remove_infrequent_words(lemmatized_corpus):
    dictionary = Dictionary(lemmatized_corpus)
    dictionary.filter_extremes(no_below=15, no_above=0.35)
    return dictionary

def remove_numbers(tokenized_documents):
    return [[token for token in doc if not token.isnumeric()] for doc in tokenized_documents]

def corpus_as_bag_of_words(dictionary, docs):
    return [dictionary.doc2bow(doc) for doc in docs]

def add_bigrams(lemmatized_documents):
    bigram = Phrases(lemmatized_documents, min_count=20)
    for idx in range(len(lemmatized_documents)):
        for token in bigram[lemmatized_documents[idx]]:
            if '_' in token:
                lemmatized_documents[idx].append(token)

def train_model(dictionary, corpus):
    num_topics = 4
    chunksize = 3
    passes = 50
    iterations = 400
    eval_every = 1
    
    temp = dictionary[0]
    id2word = dictionary.id2token
    
    model = LdaModel(
        corpus=corpus,
        id2word=id2word,
        chunksize=chunksize,
        alpha='auto',
        eta='auto',
        iterations=iterations,
        num_topics=num_topics,
        passes=passes,
        eval_every=eval_every
    )
    return model

if __name__ == "__main__":
    filename = sys.argv[1]
    f = open(filename, "r")
    list_of_documents = divide_corpus(f)
    f.close()
    tokenized_corpus = tokenize_corpus(list_of_documents)
    tokenized_corpus = remove_single_letters(tokenized_corpus)
    tokenized_corpus = remove_stopwords(tokenized_corpus)
    tokenized_corpus = remove_numbers(tokenized_corpus)
    lemmatized_corpus = lemmatize_corpus(tokenized_corpus)
    add_bigrams(lemmatized_corpus)
    remove_some_words = remove_infrequent_words(lemmatized_corpus)
    corpus_as_bow = corpus_as_bag_of_words(remove_some_words, lemmatized_corpus)
    model = train_model(remove_some_words, corpus_as_bow)
    top_topics = model.top_topics(corpus_as_bow, topn=6)
    from pprint import pprint
    pprint(top_topics)