CLTK Word Tokenization (Latin NLP with Python 11)

The Code:
from cltk.stem.latin.j_v import JVReplacer
from cltk.stem.latin.declension import CollatinusDecliner
from cltk.lemmatize.latin.backoff import BackoffLatinLemmatizer
from cltk.prosody.latin.macronizer import Macronizer
from cltk.tokenize.line import LineTokenizer
from cltk.tokenize.latin.sentence import SentenceTokenizer
from cltk.corpus.utils.formatter import remove_non_latin
from cltk.tokenize.word import WordTokenizer
import re

with open ('data/pl.txt', "r") as f:
text = f.read()

def jvtext(text):
j = JVReplacer()
text = j.replace(text)
return (text)

def clean_pl(text, lower=False):
cleaned = re.sub(r"[\(\[].*?[\)\]]", "", text)
cleaned = cleaned.replace(" ", " ").replace(" ", " ")
if lower==True:
lower_cleaned = cleaned.lower()
return (cleaned, lower_cleaned)
return(cleaned)

def dec(words):
decliner = CollatinusDecliner()
dec_words = {}
try:
for word in words:
dec_word = decliner.decline(word)
dec_words[word] = dec_word
except:
Exception
return (dec_words)

def lemma(tokens):
lemmatizer = BackoffLatinLemmatizer()
tokens = lemmatizer.lemmatize(tokens)
return (tokens)

def macron(text):
macronizer = Macronizer("tag_ngram_123_backoff")
text = macronizer.macronize_text(text)
return (text)

def line_tok(text):
tokenizer = LineTokenizer("latin")
text = tokenizer.tokenize(text)
return (text)

def sent_tok(text, punct=True):
sent_tokenizer = SentenceTokenizer()
sentences = sent_tokenizer.tokenize(text)
new_sents = []
if punct==True:
for sent in sentences:
sent = remove_non_latin(sent)
sent = sent.lower()
new_sents.append(sent)
return(new_sents)
return (sentences)

def word_tok(text):
word_tokenizer = WordTokenizer("latin")
words = word_tokenizer.tokenize(text)
return (words)

text = clean_pl(text)
text = jvtext(text)
sentences = sent_tok(text, punct=True)
sentence = sentences[0]
words = word_tok(sentence)
words = lemma(words)

new = []
for word in words:
new.append(word[1])
decs = dec(new)

If you enjoy this video, please subscribe. I provide all my content at no cost. If you want to support my channel, please donate via
PayPal: https://www.paypal.com/cgi-bin/webscr?cmd=_donations&business=AZ73QW52SUX8N&currency_code=USD&source=url
Patreon: https://www.patreon.com/WJBMattingly (its my www.themedievalworld.com account as well).

If there's a specific video you would like to see or a tutorial series, let me know in the comments and I will try and make it.

If you liked this video, check out www.PythonHumanities.com, where I have Coding Exercises, Lessons, on-site Python shells where you can experiment with code, and a text version of the material discussed here.

You can follow me at:
https://twitter.com/wjb_mattingly