Mapudungun hangman test

map_avenue/map-test.txt

@@ -0,0 +1,268 @@
+ fillantü witrakeymi fillantü
+ epu, epu antü küla antü ka ta yagel kay ka falikalu tüfanay ngewelay kuyfi ngelafuy yagel anay 
+ welu eymi rumel puliwen witrakeymi 
+ puliwen witraken <*SPA>la <*SPA>hora <*SPA>que <*SPA>quiere <*SPA>levantar witrakey fey iñche tüfa petu mongelen nay ünikerkey ta wenu mapu ta ünikerkey ta mongelealu che nay iñche epu rupa meli rupachi ünin 
+ ¡Aa!
+ ünin mongen, tüfa petu mongelen nay meli rupachi ünin taiñ peuma 
+ ¡Aa!
+ taiñ peuma tati ünikey che nay ngünechen femkunuel kay nay üniy üni ta che mongelealu ta ünikerkellinay 
+ ¡Aa! 
+ mongelealu ünikey <!1üniki> anay refalte mongelekelay che tüfa iñche may petu mongelellen may nay müte fütalan anay newe pelowelan ke llemay
+ chumngelu chi pelowelaymi   
+ feyta ñi kintun nem ka na nay llampaykoketuy chi nay femkey chi nay 
+ amukeymi pitalmu kay 
+ amukelan müte konkelan rume pitalmuanay kiñeke rupamu apuranen kutran fey ngemeken pitalmu 
+ chempikeymu wingka 
+ <*SPA>serio <*SPA>que  <*SPA>tiene <*SPA>tanto <*SPA>años <*SPA>éste Camargo <*SPA>hombre pingeken ka 
+ <Laugh> chumalanta küpaymi pingekeymi
+ feypingeken ka fey <*SPA>tanto, <*SPA>tanto <*SPA>este <*SPA>no <*SPA>quiere <*SPA>entrar niel <*SPA>el <*SPA>kuchillo <*SPA>éste wewon pikeneu ta pu wingka ayekalkeneu ka  
+ eymi müley tami feypiafel iñche ta petu weche wentrulen piafimi 
+ feypikellefiñ may <%> <*SPA>quiero <*SPA>casar <*SPA>hombre pikefiñ engün ayekaukingün   
+ ¡Aa! kurengean pikefimi 
+ may iñche ta petu mongelen llemay 
+ elumulayafun küdau may pikelafimi 
+ feypikelafiñ puwelan küdaumu anay pelotuwelan 
+ welu re feypikantuafiel 
+ may pelowelan tüfa <Noise> 
+ eymi tami chau chunten tripantu mongey 
+ uta <*SPA>más doy <*SPA>cien
+ doy pataka 
+ may 
+ tami ñuke 
+ tañi ñuke ta inangechi laynga  ka epe, ka epe pataka niefuynga 
+ epe pataka niefuy 
+ may 
+ <Cough> ¡Aa! cheu tripapay tami ñuke faupüle müten tuwi 
+ <*SPA>no <*SPA>afuera, <*SPA>afuera
+ chuchipüle 
+ <*SPA>selva <*SPA>oscura <!1selvacura> 
+ üyüu tuwüy 
+ may <*SPA>perquenco pingey tati mapu ta feypüle wingka mapefiñ pilay ta mapu nay 
+ ¡Aa!
+ ka eymingelu tukulelfi üy kakelu femngechi kampiayka <!1cambiayca> 
+ ¡Aa!
+ niekekelafuy ta wingka ta tüfa nay eymiam chuchi mapuche ta ka tüfa nay femyawiymi ka nay weniy 
+ iñche ta Alfarinu pingey tañi mapu 
+ Alfarinu 
+ may 
+ Alfarinu 
+ may 
+ iñche kayta kimlafiñ kayta Alfarino, Alfarinu pikay ta che ka lle 
+ Temuko kay 
+ may, may Temuko ta feyta miauken 
+ kuyfi deuma amuwelaymi ka?
+ tüfa ngemewekelan anay plataam ka fütra <*SPA>escaso nay
+ pensionauküleymi?
+ pensionaukülefun <*SPA>pero pichiken nüken 
+ pichiken llemay 
+ pichiken
+ ¡Aa! welu <uh> fülküleymi wariamu tati 
+ fülkülefun wariamu pekanka wül-lay wingka rume kam paysanumu anay pekanka wül-lay kuyfitu doy pekankakefeyu fütrake aweluyem 
+ ¡Aa! 
+ <*SPA>y pu wingka dpy pekanka tripayal tati pofüre <*SPA>hombre pifuy tüfa pu wingka femwelay anay tenga pülata kontau pipiey 
+ kuyfi doy elungekefuymün
+ doy pichi elungekeyiñ chemu rume tati nielan pülata pingipiy regalangekay iyagel fey tüfa femwelay anay re pülatamu müten amuley 
+ nienole pülata fey amulay?
+ ya amulay, amulay 
+ ¡Aa! 
+ femkey che tüfa anay 
+ tami <uh> chemchi ta kümey ta kutran furinmu 
+ kuyfi 
+ may kutran furinmu chem kümey, kutraniem furi 
+ <%> tati chem fütake awelungati l'awen' ta kimüy ka anay iñchiñ ta kimlayiñ ta anay <%> tüfa kay ta tañi domo nay palusantu piy  
+ palusantu yengu ta nalka 
+ rume kümey pingey <!1pingi> fey feyngey 
+ feyngey, feyngey 
+ eymi putulafimi fey 
+ <*SPA>no putukelafiñ tati
+ ¡Aa!<%> 
+ tati <*SPA>limpiaplata ka müley 
+ <uh> pewen anta l'awen' ngekey kay chachay 
+ fey tati trantupatuy allfeñngelu ta +/wor/+ workükakeychi nay  fey notukey ta che tati allfeñngelulle fey 
+ pewenmu 
+ may pewenmu fayümngey 
+ fey kümey 
+ fey kümey 
+ ¿alüley pewen tapüle?
+ pütrakamaley chi nay repül nentukatuy mawidamu re pewen nentulley chi nay 
+ feymuanta pewenche pingekey?
+ feymulle ta pewenche pingey tati nay 
+ ¡Aa! fentrekeley pewen 
+ pütrakamagel  tüfa kay ta wingka ta apümüy anay apümüy 
+ chumngelu chi 
+ <hm>  seriakefiyngün kay nay 
+ traflafingün 
+ traflafingün fey ta tau newentulelu müten ta paysanu kay feyta küpaymangelay tañi feranamullenay 
+ newentulelu 
+ may feypingey wentrüu mekey kay nay 
+ ¡Aa!
+ femüymamukelchi pipingey feymu ta kapalekellinanay ngülliumu pütrümna küñeke tripantumu müna fünkey ngülliu anay, kiñeke tripantu müten kiñeke tripantu  weda pukemiyem tati wül-lay ta ngünechen müna nongikelley tati ngilliu anay 
+ ngilliu anta ingekey ka 
+ chemuam müna kümey mudayünmu ngülliu anay 
+ ¡Aa!
+ pütrümna kümey ta ngülliu ta müdayünmu lle ka tati füntu pingey tati y deumangey korünmu ellangey tati angküchi medküngeyanay rumeñma kümey 
+ ka kümey 
+ kümey 
+ ¡Aa!
+ fey felekelleychi 
+  ilo korümu ta püstuy 
+ alü niekeymün anta kapüra faupüle 
+ nielayiñ kapüra 
+ kuyfi ta mülekefuy ka tüfa ta ngewelay kiñekenekakefuyiñ kuyfi tüfa ta niewelayiñ anay 
+ niewelay 
+ eskasiay anta chem rume ta wülngerkelaynay keya mongeltual tañi <*SPA>familia ta che 
+ may, may 
+ eskasiafun kompüle ta faltakey ka che  nay 
+ fey femngechi kuyfi niekefuymi 
+ kuyfi niekellefuyiñ pero tüfa ta femwelayiñ anay tüfa rumeñma eskaseakelu itrokom nay 
+ ¡Aa!
+ rumeñma eskaseakeyiñ <Noise> pewelayin ta tüfa anay pekan pengewelay tati 
+ pekan pengewelay 
+ <*SPA>no 
+ ¡Aa! eymi pilukelaymi?
+ <*SPA>no pilukelan chemkülekan müten 
+ kümelkaley tami pilun 
+ may 
+ allküleymi küme 
+ allkülen 
+ ¡Aa! 
+ <*SPA>eso <*SPA>si <*SPA>que pelowelanke anay, pelowenke 
+ welu chafokeymi kay 
+ ¡Aa! 
+ ¿chafokeymi?
+ <*SPA>no 
+ chafokelaymi 
+ <*SPA>no 
+ ya, ya, ya, ya feyti kiñe kutran müley kimlafimi wauyuyem che? 
+ <*SPA>no kimlafiñ anay 
+ ¡Aa!
+ kimlafiñ 
+ wauyukelaymi?
+ <*SPA>no iñche tüfa petu mongelen nay 
+ welu eymi kutranchengelaymi <*SPA>pues <!1pu'>
+ kutranchengekelay kiñekemu rumeñma kutranken rüfanay chuman kay 
+ chem doy kutrankey?
+ tañi piukeyem, tañi pu ngeyem kutrankey kiñekemu tulliuken anay, femkey che anay    
+ piuke kutraniyem chem l'awen' putukeymi 
+ feyti foye workükefiñ 
+ foye
+ may 
+ ¡Aa! 
+ feytamu ngellelay foye nay 
+ kañpüle müley <*SPA>pues <!1pu'>
+ kañpüle <*SPA>afuera müten müley ka 
+ feyta müley ta re ngülliu faupüle 
+ femay re ngilliu ta mülelli nay 
+ welu ngelay foye 
+ ngelay, ngelay 
+ ¡Aa! 
+ ngellelay foye tamu 
+ <uh> pafmu chem kümey pafmuyem?
+ +/pafmuye/+ +/pa/+ pafmuyem 
+ may 
+ fey tati chem ka tukungekey ama 
+ feyta kümey
+ pafmulu 
+  pafmunmu 
+ palki
+ may 
+ müley palki faupüle kay 
+ ngelaychi nay
+ ngelay palki 
+ may 
+ welu mawidamu fentrelepelafuy l'awen' peñi 
+ tüfamu 
+ may 
+ mülellefuy may 
+ <uhm> tüfamu keta müley l'awen' <%> 
+ welu kuyfi kimfuymi l'awen'
+ kimkallefun mayta tati kutran *pütrawe pingey feyta tungefuy kay nay feyta tüfa ta pelotuwelan ka nay 
+ welu üy kimneymafimi l'awen' 
+ llipañmawe pingey kangelu tati chempingekey chi pe fey engün ta mülefuyanay 
+ ya <Noise> welu <uh> alün l'awen' putupelaymi kuyfi mapu l'awen' eymi 
+ worküngey fayümngey tati l'awen' tati, tati *matragelu ta fey tutekey tati anay tutekey 
+ tutekey 
+ may 
+ ¡Aa! feyta <uh> nielu mongen tutey ka 
+ tutellimay, tutey iñche ta petu mongelellen may pütra kuyfiche
+ welu kümey tami alütun mongen tati peñi 
+ kürasia may nay kürasia 
+ kümey tami alünmongen tun 
+ fey kürasia pipiepan tati 
+ müley tami 
+ taiñ choyeteu ka ta ngünechen kayta witrañpüramnieneu feyta mongelen ka nay <*SPA>mucha gürasia tami pipiepafiel 
+ ngillatuleymukeymi welu 
+ ¡Aa!
+ ngillatulekeymi 
+ ngüllatuken nay, workam chem l'awen' nay ngillatu pichiken püntefnentukaken kay ka
+ ¡Aa! 
+ femkey che anay 
+ welu ngillatuley che mongelekey pingey 
+ femkey 
+ eymi rumel ngillatukeymi 
+ ngillatumeken worküayem fey ngillatuken tüfa kay ta felen müte chumlekalafun anay <*SPA>eso <*SPA>si <*SPA>que pelowellelan
+ peumakeymi kay 
+ peumaken ünin melike rupachi ünin.
+ ¡Aa! feyta akukelay pütrümna *wetanki iñamnga akuay pipeyu 
+ ¡Aa!
+ tüfa aku akungepey tañi witan chemuam femekey tüfa fey tüfi llemay pipingepatun ta, ta
+ witaymi 
+ may 
+ ¿küme witaymiam? 
+ müna küme witan tayi pipiepefiñ ta tüfa 
+ fey mayta iñche tañi akual 
+ femüy fey tami akukellal may tati femekepen 
+ pemean ñi peñi doy alütunielu mongen pipenga?
+ tüfey may
+ doy alünielu mongen fey pemeafiñ pipen fey 
+ fey fütra kuyfi che iñche tüfa kay 
+ feymu mayta witapeymi?
+ feymu 
+ welu küme witapeymi?
+ küme witapen <*SPA>porque müna kimyey tañi witan ta che anay 
+ ¡Aa!
+ tüfa man lipang tayiye chemmu amuy fente mekepen pipen 
+ inichi akuay pipeymi 
+ may ta tüfa llemay ngütramkamekepatuymi tüfa tati nay 
+ iñche duamtukeyu pu fey
+ may, may  
+ doy alü nielu mongen feyta ngütramkameafiñ piken   
+ may kümeyka fey kürasia may nay kürasia 
+ kimkelafun müten cheu tami mülen
+ may nay 
+ wingkulmu mülerkeymi <!1mülerkimi>
+ wingkulüukülen, wingkülüukülen 
+ winkülmu müleymi may 
+ may fey 
+ küme dungu tati tami mülen wingkulmeu
+ fey chumngechianta kimüymi ta tüfa femngechi püfkükonpaymi tapüle peñi 
+ ngütramkapefiñ tami fotüm 
+ may, may
+ Domingo 
+ Domingo Cayul 
+ Domingu Camargo 
+ feylle fey 
+ Cayul 
+ Cayul feyti ñi chau pütrüney mongen pipey
+ may, may 
+ <uh> petu allküley piy ngütramkameafimi kichu alü kimney dungu pipey 
+ feyta iney <!1iñi> Domingo feyta ñi fotüm nay 
+ fey pelelmeam ñi chau pipeneu 
+ may, may 
+ pemeafimi piy petu nieyta mongeley tañi fücha chau piy 
+ petu mongelellen may tüfa 
+ fentepuay ngütram peñi feyta pepapeyu feymu fey peumangelmi doy alüniaymi mongen <*SPA>pues 
+ kürasia may nay kürasia 
+ iñche ta rupaletuan tati amuleanta kañpüle 
+ may, may kañpüle ka wariamu amukayaymi ka ka paysanumu ka, ka
+ may 
+ kam pewenko chi tüfachi pewenko ka pewenko, <*SPA>selva <*SPA>oscura <!1selvecura> pingey tati rupauyimi <*SPA>selva <*SPA>oscura <!1selvecura> eymi ta tüfaka
+ feypüle amuan feula 
+ may, may <*SPA>selva <*SPA>oscura <!1selvecura> 
+ may 
+ wiñoltuayminge mapu ka 
+ may 
+ may, may  
+ feyta kümeleaymi pu peñi pepapeyu tati 
+ kürasia may nay kürasia, fey tañi ngütramkapafiel ka nay 
+ feley.

util.py

@@ -0,0 +1,69 @@
+import os
+import re
+import unicodedata
+
+def strip_accents(text: str) -> str:
+    """Removes accents from text."""
+    return ''.join(c for c in unicodedata.normalize('NFD', text)
+                  if unicodedata.category(c) != 'Mn')
+
+
+def load_raw_text(corpus_directory: str, file_names=None) -> str:
+    """Loads all the text files in a directory into one large string"""
+    corpus = ""
+    
+    for file_name in os.listdir(corpus_directory):
+        # Read the file as a string
+        file_path = os.path.join(corpus_directory, file_name)
+        if os.path.isdir(file_path):
+            continue
+            
+        #  Make sure we only read text files
+        if ".txt" not in file_name:
+            continue
+            
+        with open(file_path, 'r') as file:
+            file_contents = file.read()
+            corpus += (file_contents + "\n")
+    return corpus
+
+
+word_regex = r"[\w|\']+"
+def tokenize(text):
+    return re.findall(word_regex, text)
+
+
+def preprocess(text):
+    """Tokenizes and processes text which is already separated by spaces into words. Designed for English punctuation."""
+    text = strip_accents(text)
+    text = text.lower()
+
+    tokens = text.split(" ")
+
+    tokens_filtered = []
+    for token in tokens:
+        # Skip any tokens with special characters
+        if re.match(r"[\w|\']+|[\.|\,|\?|\!]", token):
+            tokens_filtered.append(token)
+    return tokens_filtered
+
+
+def pad(text: list, num_padding: int):
+    """Pads the given text, as a list of strings, with <s> characters between sentences."""
+    padded_text = []
+    
+    # Add initial padding to the first sentence
+    for _ in range(num_padding):
+        padded_text.append("<s>")
+    
+    for word in text:
+        padded_text.append(word)
+
+        # Every time we see an end punctuation mark, add <s> tokens before it
+        # REPLACE IF YOUR LANGUAGE USES DIFFERENT END PUNCTUATION
+        if word in [".", "?", "!"]:
+            for _ in range(num_padding):
+                padded_text.append("<s>")
+        
+        
+    return padded_text