Create tokenization.py

tokenization.py

@@ -0,0 +1,20 @@
+
+
+
+from urllib.request import urlopen
+from bs4 import BeautifulSoup
+
+url = "https://raw.githubusercontent.com/cltk/hindi_text_ltrc/master/tulasidaas/Raamacharita_maanasa/1/main.txt"
+html = urlopen(url).read()
+soup = BeautifulSoup(html, features="html.parser")
+
+# kill all script and style elements
+for script in soup(["script", "style"]):
+    script.extract()    # rip it out
+
+# get text
+text = soup.get_text()
+ramayana_text = text
+print(type(text))
+
+