The encoder is working. About to rip it apart to put it in an object.

.gitignore

@@ -0,0 +1,2 @@
+venv/*
+catboost_info/*

.vscode/launch.json

@@ -0,0 +1,47 @@
+{
+    // Use IntelliSense to learn about possible attributes.
+    // Hover to view descriptions of existing attributes.
+    // For more information, visit: https://go.microsoft.com/fwlink/?linkid=830387
+    "version": "0.2.0",
+    "configurations": [
+
+        {
+            "name": "Python: Current File",
+            "type": "python",
+            "request": "launch",
+            "program": "${file}",
+            "console": "integratedTerminal",
+            "justMyCode": true
+        },{
+            "name": "Train phonetic 4000 gpu",
+            "type": "python",
+            "request": "launch",
+            "program": "transmorgrify.py",
+            "console": "integratedTerminal",
+            "justMyCode": true,
+            "args": [
+                "--train",
+                "--in_csv", "/home/lansford/Sync/projects/tf_over/sentance_transmogrifier/examples/phonetic/phonetic.csv",
+                "--a_header", "English",
+                "--b_header", "Phonetic",
+                "--device", "0:1",
+                "--model", "phonetics_forwar.tm"
+            ]
+        },{
+            "name": "Train short phonetic 4000 gpu",
+            "type": "python",
+            "request": "launch",
+            "program": "transmorgrify.py",
+            "console": "integratedTerminal",
+            "justMyCode": true,
+            "args": [
+                "--train",
+                "--in_csv", "/home/lansford/Sync/projects/tf_over/sentance_transmogrifier/examples/phonetic/phonetic_short.csv",
+                "--a_header", "English",
+                "--b_header", "Phonetic",
+                "--device", "0:1",
+                "--model", "phonetics_small.tm"
+            ]
+        }
+    ]
+}

examples/phonetic/phonetic_short.csv

@@ -0,0 +1,1000 @@
+English,Phonetic
+Go.,ɡow.
+Go.,ɡow.
+Go.,ɡow.
+Go.,ɡow.
+Hi.,haj.
+Run!,ɹʌn!
+Run.,ɹʌn.
+Who?,hu?
+Fire!,fajɹ̩!
+Fire!,fajɹ̩!
+Fire!,fajɹ̩!
+Help!,hɛlp!
+Help!,hɛlp!
+Help!,hɛlp!
+Jump!,d͡ʒʌmp!
+Jump.,d͡ʒʌmp.
+Stop!,stɑp!
+Stop!,stɑp!
+Stop!,stɑp!
+Wait!,wejt!
+Wait.,wejt.
+Go on.,ɡow ɑn.
+Go on.,ɡow ɑn.
+Hello!,həlow!
+I ran.,aj ɹæn.
+I ran.,aj ɹæn.
+I try.,aj tɹaj.
+I won!,aj wʌn!
+Oh no!,ow now!
+Relax.,ɹɪlæks.
+Smile.,smajl.
+Attack!,ətæk!
+Attack!,ətæk!
+Get up.,ɡɛt ʌp.
+Go now.,ɡow naw.
+Got it!,ɡɑt ɪt!
+Got it?,ɡɑt ɪt?
+Got it?,ɡɑt ɪt?
+He ran.,hi ɹæn.
+Hop in.,hɑp ɪn.
+Hug me.,hʌɡ mi.
+I fell.,aj fɛl.
+I know.,aj now.
+I left.,aj lɛft.
+I lied.,aj lajd.
+I lost.,aj lɔst.
+I quit.,aj kwɪt.
+I quit.,aj kwɪt.
+I work.,aj wɹ̩k.
+I'm 19.,ɪm 19.
+I'm up.,ɪm ʌp.
+Listen.,lɪsən.
+Listen.,lɪsən.
+Listen.,lɪsən.
+No way!,now wej!
+No way!,now wej!
+No way!,now wej!
+No way!,now wej!
+No way!,now wej!
+No way!,now wej!
+No way!,now wej!
+No way!,now wej!
+No way!,now wej!
+No way!,now wej!
+Really?,ɹɪli?
+Really?,ɹɪli?
+Thanks.,θæŋks.
+Thanks.,θæŋks.
+Try it.,tɹaj ɪt.
+We try.,wi tɹaj.
+We won.,wi wʌn.
+Why me?,waj mi?
+Ask Tom.,æsk tɑm.
+Awesome!,ɑsəm!
+Be calm.,bi kɑm.
+Be cool.,bi kul.
+Be fair.,bi fɛɹ.
+Be kind.,bi kajnd.
+Be nice.,bi najs.
+Beat it.,bit ɪt.
+Call me.,kɔl mi.
+Call me.,kɔl mi.
+Call me.,kɔl mi.
+Call us.,kɔl ʌs.
+Come in.,kʌm ɪn.
+Come in.,kʌm ɪn.
+Come in.,kʌm ɪn.
+Come on!,kʌm ɑn!
+Come on.,kʌm ɑn.
+Come on.,kʌm ɑn.
+Drop it!,dɹɑp ɪt!
+Get Tom.,ɡɛt tɑm.
+Get out!,ɡɛt awt!
+Get out.,ɡɛt awt.
+Get out.,ɡɛt awt.
+Get out.,ɡɛt awt.
+Get out.,ɡɛt awt.
+Get out.,ɡɛt awt.
+Go away!,ɡow əwej!
+Go away!,ɡow əwej!
+Go away!,ɡow əwej!
+Go away!,ɡow əwej!
+Go away!,ɡow əwej!
+Go away!,ɡow əwej!
+Go away!,ɡow əwej!
+Go away.,ɡow əwej.
+Go away.,ɡow əwej.
+Go away.,ɡow əwej.
+Go away.,ɡow əwej.
+Go away.,ɡow əwej.
+Go away.,ɡow əwej.
+Go away.,ɡow əwej.
+Go home.,ɡow howm.
+Go slow.,ɡow slow.
+Goodbye!,ɡʊdbaj!
+Goodbye!,ɡʊdbaj!
+Goodbye!,ɡʊdbaj!
+Hang on!,hæŋ ɑn!
+Hang on!,hæŋ ɑn!
+Hang on!,hæŋ ɑn!
+Hang on.,hæŋ ɑn.
+He came.,hi kejm.
+He quit.,hi kwɪt.
+Help me!,hɛlp mi!
+Help me.,hɛlp mi.
+Help me.,hɛlp mi.
+Help me.,hɛlp mi.
+Help us.,hɛlp ʌs.
+Hit Tom.,hɪt tɑm.
+Hold it!,howld ɪt!
+Hold on.,howld ɑn.
+Hold on.,howld ɑn.
+Hold on.,howld ɑn.
+Hug Tom.,hʌɡ tɑm.
+I agree.,aj əɡɹi.
+I agree.,aj əɡɹi.
+I bowed.,aj bawd.
+I moved.,aj muvd.
+I moved.,aj muvd.
+I moved.,aj muvd.
+I moved.,aj muvd.
+I slept.,aj slɛpt.
+I tried.,aj tɹajd.
+I'll go.,ɪl ɡow.
+I'm Tom.,ɪm tɑm.
+I'm fat.,ɪm fæt.
+I'm fat.,ɪm fæt.
+I'm fit.,ɪm fɪt.
+I'm hit!,ɪm hɪt!
+I'm old.,ɪm owld.
+I'm shy.,ɪm ʃaj.
+I'm wet.,ɪm wɛt.
+It's OK.,ɪts owkej.
+It's me!,ɪts mi!
+It's me.,ɪts mi.
+Join us.,d͡ʒojn ʌs.
+Join us.,d͡ʒojn ʌs.
+Keep it.,kip ɪt.
+Me  too.,mi  tu.
+Open up.,owpən ʌp.
+Perfect!,pɹ̩fəkt!
+See you.,si ju.
+Show me.,ʃow mi.
+Show me.,ʃow mi.
+Show me.,ʃow mi.
+Shut up!,ʃʌt ʌp!
+Shut up!,ʃʌt ʌp!
+Skip it.,skɪp ɪt.
+So long.,sow lɔŋ.
+So long.,sow lɔŋ.
+Stop it.,stɑp ɪt.
+Stop it.,stɑp ɪt.
+Take it.,tejk ɪt.
+Tell me.,tɛl mi.
+Tom ate.,tɑm ejt.
+Tom ran.,tɑm ɹæn.
+Tom won.,tɑm wʌn.
+Wait up.,wejt ʌp.
+Wake up!,wejk ʌp!
+Wake up!,wejk ʌp!
+Wake up!,wejk ʌp!
+Wake up.,wejk ʌp.
+Wash up.,wɑʃ ʌp.
+We care.,wi kɛɹ.
+We know.,wi now.
+We lost.,wi lɔst.
+Welcome.,wɛlkəm.
+Welcome.,wɛlkəm.
+Who ate?,hu ejt?
+Who ran?,hu ɹæn?
+Who ran?,hu ɹæn?
+Who won?,hu wʌn?
+Who won?,hu wʌn?
+Why not?,waj nɑt?
+You run.,ju ɹʌn.
+You won.,ju wʌn.
+Am I fat?,æm aj fæt?
+Ask them.,æsk ðɛm.
+Ask them.,æsk ðɛm.
+Back off!,bæk ɔf!
+Back off.,bæk ɔf.
+Be a man.,bi ə mæn.
+Be brave.,bi bɹejv.
+Be brief.,bi bɹif.
+Be brief.,bi bɹif.
+Be brief.,bi bɹif.
+Be quiet.,bi kwajət.
+Be still.,bi stɪl.
+Call Tom.,kɔl tɑm.
+Call Tom.,kɔl tɑm.
+Call Tom.,kɔl tɑm.
+Cheer up!,t͡ʃɪɹ ʌp!
+Cheer up.,t͡ʃɪɹ ʌp.
+Cool off!,kul ɔf!
+Cuff him.,kʌf hɪm.
+Don't go.,dɑnt ɡow.
+Drive on.,dɹajv ɑn.
+Find Tom.,fajnd tɑm.
+Find Tom.,fajnd tɑm.
+Find Tom.,fajnd tɑm.
+Find Tom.,fajnd tɑm.
+Find Tom.,fajnd tɑm.
+Find Tom.,fajnd tɑm.
+Fix this.,fɪks ðɪs.
+Get away!,ɡɛt əwej!
+Get away!,ɡɛt əwej!
+Get away!,ɡɛt əwej!
+Get away!,ɡɛt əwej!
+Get down!,ɡɛt dawn!
+Get down.,ɡɛt dawn.
+Get down.,ɡɛt dawn.
+Get lost!,ɡɛt lɔst!
+Get lost!,ɡɛt lɔst!
+Get lost!,ɡɛt lɔst!
+Get lost!,ɡɛt lɔst!
+Get lost!,ɡɛt lɔst!
+Get lost.,ɡɛt lɔst.
+Get real!,ɡɛt ɹil!
+Get real!,ɡɛt ɹil!
+Get real.,ɡɛt ɹil.
+Go ahead!,ɡow əhɛd!
+Go ahead.,ɡow əhɛd.
+Go on in.,ɡow ɑn ɪn.
+Go on in.,ɡow ɑn ɪn.
+Go on in.,ɡow ɑn ɪn.
+Go on in.,ɡow ɑn ɪn.
+Go on in.,ɡow ɑn ɪn.
+Good job!,ɡʊd d͡ʒɑb!
+Grab Tom.,ɡɹæb tɑm.
+Grab him.,ɡɹæb hɪm.
+Have fun.,hæv fʌn.
+Have fun.,hæv fʌn.
+Have fun.,hæv fʌn.
+He spoke.,hi spowk.
+He tries.,hi tɹajz.
+He tries.,hi tɹajz.
+Help Tom.,hɛlp tɑm.
+Help him.,hɛlp hɪm.
+Hi  guys.,haj  ɡajz.
+Hi  guys.,haj  ɡajz.
+Hi  guys.,haj  ɡajz.
+Hi  guys.,haj  ɡajz.
+Hi  guys.,haj  ɡajz.
+Hi  guys.,haj  ɡajz.
+How cute!,haw kjut!
+How deep?,haw dip?
+How deep?,haw dip?
+Humor me.,hjumɹ̩ mi.
+Humor me.,hjumɹ̩ mi.
+Hurry up.,hɹ̩i ʌp.
+Hurry up.,hɹ̩i ʌp.
+I agreed.,aj əɡɹid.
+I agreed.,aj əɡɹid.
+I agreed.,aj əɡɹid.
+I am fat.,aj æm fæt.
+I am old.,aj æm owld.
+I ate it.,aj ejt ɪt.
+I ate it.,aj ejt ɪt.
+I can go.,aj kæn ɡow.
+I did OK.,aj dɪd owkej.
+I did it.,aj dɪd ɪt.
+I failed.,aj fejld.
+I forgot.,aj fɹ̩ɡɑt.
+I get by.,aj ɡɛt baj.
+I get it.,aj ɡɛt ɪt.
+I got it.,aj ɡɑt ɪt.
+I phoned.,aj fownd.
+I refuse.,aj ɹɛfjus.
+I resign.,aj ɹɪzajn.
+I resign.,aj ɹɪzajn.
+I saw it.,aj sɔ ɪt.
+I smiled.,aj smajld.
+I stayed.,aj stejd.
+I talked.,aj tɔkt.
+I talked.,aj tɔkt.
+I talked.,aj tɔkt.
+I talked.,aj tɔkt.
+I use it.,aj jus ɪt.
+I waited.,aj wejtəd.
+I'll pay.,ɪl pej.
+I'm back.,ɪm bæk.
+I'm back.,ɪm bæk.
+I'm bald.,ɪm bɔld.
+I'm bald.,ɪm bɔld.
+I'm calm.,ɪm kɑm.
+I'm cool.,ɪm kul.
+I'm done.,ɪm dʌn.
+I'm easy.,ɪm izi.
+I'm fair.,ɪm fɛɹ.
+I'm fine.,ɪm fajn.
+I'm free!,ɪm fɹi!
+I'm free.,ɪm fɹi.
+I'm full.,ɪm fʊl.
+I'm full.,ɪm fʊl.
+I'm full.,ɪm fʊl.
+I'm here.,ɪm hɪɹ.
+I'm home.,ɪm howm.
+I'm hurt.,ɪm hɹ̩t.
+I'm late.,ɪm lejt.
+I'm lazy.,ɪm lejzi.
+I'm lost.,ɪm lɔst.
+I'm mean.,ɪm min.
+I'm next.,ɪm nɛkst.
+I'm okay.,ɪm owkej.
+I'm poor.,ɪm pʊɹ.
+I'm rich.,ɪm ɹɪt͡ʃ.
+I'm rich.,ɪm ɹɪt͡ʃ.
+I'm safe.,ɪm sejf.
+I'm sick.,ɪm sɪk.
+I'm thin.,ɪm θɪn.
+I'm tidy.,ɪm tajdi.
+I'm warm.,ɪm wɔɹm.
+I'm weak.,ɪm wik.
+I'm wise.,ɪm wajz.
+I've won.,ɪv wʌn.
+It helps.,ɪt hɛlps.
+It hurts.,ɪt hɹ̩ts.
+It works.,ɪt wɹ̩ks.
+It's Tom.,ɪts tɑm.
+It's fun.,ɪts fʌn.
+It's his.,ɪts hɪz.
+It's new.,ɪts nu.
+It's odd.,ɪts ɑd.
+It's old.,ɪts owld.
+It's red.,ɪts ɹɛd.
+It's sad.,ɪts sæd.
+Keep out!,kip awt!
+Keep out.,kip awt.
+Kiss Tom.,kɪs tɑm.
+Kiss Tom.,kɪs tɑm.
+Kiss Tom.,kɪs tɑm.
+Leave it.,liv ɪt.
+Leave me.,liv mi.
+Leave us.,liv ʌs.
+Let's go!,lɛtz ɡow!
+Let's go!,lɛtz ɡow!
+Look out!,lʊk awt!
+Marry me.,mɛɹi mi.
+May I go?,mej aj ɡow?
+Save Tom.,sejv tɑm.
+She came.,ʃi kejm.
+She died.,ʃi dajd.
+She runs.,ʃi ɹʌnz.
+Sit down!,sɪt dawn!
+Sit down.,sɪt dawn.
+Sit here.,sɪt hɪɹ.
+Speak up!,spik ʌp!
+Speak up!,spik ʌp!
+Speak up!,spik ʌp!
+Stand by.,stænd baj.
+Stand by.,stænd baj.
+Stand up!,stænd ʌp!
+Stand up!,stænd ʌp!
+Stay put.,stej pʊt.
+Stop Tom.,stɑp tɑm.
+Take Tom.,tejk tɑm.
+Tell Tom.,tɛl tɑm.
+Terrific!,tɹ̩ɪfɪk!
+Terrific!,tɹ̩ɪfɪk!
+They won.,ðej wʌn.
+Tom came.,tɑm kejm.
+Tom died.,tɑm dajd.
+Tom fell.,tɑm fɛl.
+Tom knew.,tɑm nu.
+Tom knew.,tɑm nu.
+Tom left.,tɑm lɛft.
+Tom lied.,tɑm lajd.
+Tom lies.,tɑm lajz.
+Tom lost.,tɑm lɔst.
+Tom paid.,tɑm pejd.
+Tom quit.,tɑm kwɪt.
+Tom swam.,tɑm swæm.
+Tom wept.,tɑm wɛpt.
+Tom's up.,tɑmz ʌp.
+Too late.,tu lejt.
+Trust me.,tɹʌst mi.
+Try hard.,tɹaj hɑɹd.
+Try some.,tɹaj sʌm.
+Try some.,tɹaj sʌm.
+Try some.,tɹaj sʌm.
+Try this.,tɹaj ðɪs.
+Try this.,tɹaj ðɪs.
+Try this.,tɹaj ðɪs.
+Use this.,jus ðɪs.
+Warn Tom.,wɔɹn tɑm.
+Warn Tom.,wɔɹn tɑm.
+Warn Tom.,wɔɹn tɑm.
+Warn Tom.,wɔɹn tɑm.
+Watch me.,wɑt͡ʃ mi.
+Watch me.,wɑt͡ʃ mi.
+Watch me.,wɑt͡ʃ mi.
+Watch us.,wɑt͡ʃ ʌs.
+Watch us.,wɑt͡ʃ ʌs.
+We agree.,wi əɡɹi.
+We tried.,wi tɹajd.
+We'll go.,wɛl ɡow.
+We're OK.,wɛɹ owkej.
+What for?,wʌt fɔɹ?
+What fun!,wʌt fʌn!
+Who am I?,hu æm aj?
+Who came?,hu kejm?
+Who died?,hu dajd?
+Who fell?,hu fɛl?
+Who fell?,hu fɛl?
+Who quit?,hu kwɪt?
+Who quit?,hu kwɪt?
+Who swam?,hu swæm?
+Who's he?,hɑz hi?
+Write me.,ɹajt mi.
+After you.,æftɹ̩ ju.
+After you.,æftɹ̩ ju.
+After you.,æftɹ̩ ju.
+After you.,æftɹ̩ ju.
+After you.,æftɹ̩ ju.
+Aim. Fire!,ejm. fajɹ̩!
+Answer me.,ænsɹ̩ mi.
+Answer me.,ænsɹ̩ mi.
+Answer me.,ænsɹ̩ mi.
+Birds fly.,bɹ̩dz flaj.
+Bless you.,blɛs ju.
+Call home!,kɔl howm!
+Calm down.,kɑm dawn.
+Calm down.,kɑm dawn.
+Calm down.,kɑm dawn.
+Calm down.,kɑm dawn.
+Can we go?,kæn wi ɡow?
+Can we go?,kæn wi ɡow?
+Can we go?,kæn wi ɡow?
+Catch Tom.,kæt͡ʃ tɑm.
+Catch him.,kæt͡ʃ hɪm.
+Catch him.,kæt͡ʃ hɪm.
+Catch him.,kæt͡ʃ hɪm.
+Catch him.,kæt͡ʃ hɪm.
+Come back.,kʌm bæk.
+Come here.,kʌm hɪɹ.
+Come here.,kʌm hɪɹ.
+Come here.,kʌm hɪɹ.
+Come home.,kʌm howm.
+Come over.,kʌm owvɹ̩.
+Come over.,kʌm owvɹ̩.
+Come over.,kʌm owvɹ̩.
+Come over.,kʌm owvɹ̩.
+Come soon.,kʌm sun.
+Do it now.,du ɪt naw.
+Dogs bark.,dɑɡz bɑɹk.
+Don't ask.,dɑnt æsk.
+Don't cry.,dɑnt kɹaj.
+Don't cry.,dɑnt kɹaj.
+Don't cry.,dɑnt kɹaj.
+Don't cry.,dɑnt kɹaj.
+Don't lie.,dɑnt laj.
+Don't run.,dɑnt ɹʌn.
+Fantastic!,fæntæstɪk!
+Fantastic!,fæntæstɪk!
+Fantastic!,fæntæstɪk!
+Feel this.,fil ðɪs.
+Feel this.,fil ðɪs.
+Feel this.,fil ðɪs.
+Feel this.,fil ðɪs.
+Feel this.,fil ðɪs.
+Follow me.,fɑlow mi.
+Follow me.,fɑlow mi.
+Follow us.,fɑlow ʌs.
+Forget it.,fɹ̩ɡɛt ɪt.
+Forget me.,fɹ̩ɡɛt mi.
+Forget me.,fɹ̩ɡɛt mi.
+Forget me.,fɹ̩ɡɛt mi.
+Forget me.,fɹ̩ɡɛt mi.
+Get ready.,ɡɛt ɹɛdi.
+Go for it.,ɡow fɔɹ ɪt.
+Go for it.,ɡow fɔɹ ɪt.
+Go get it.,ɡow ɡɛt ɪt.
+Go get it.,ɡow ɡɛt ɪt.
+Go get it.,ɡow ɡɛt ɪt.
+Go inside.,ɡow ɪnsajd.
+Go to bed.,ɡow tə bɛd.
+Grab that.,ɡɹæb ðæt.
+Grab this.,ɡɹæb ðɪs.
+Have some.,hæv sʌm.
+Have some.,hæv sʌm.
+Have some.,hæv sʌm.
+He is old.,hi ɪz owld.
+He is old.,hi ɪz owld.
+He shaved.,hi ʃejvd.
+He smiled.,hi smajld.
+He's a DJ.,hɛz ə .
+He's fast.,hɛz fæst.
+He's good.,hɛz ɡʊd.
+He's rich.,hɛz ɹɪt͡ʃ.
+He's rich.,hɛz ɹɪt͡ʃ.
+Here I am.,hɪɹ aj æm.
+Hold this.,howld ðɪs.
+How awful!,haw ɑfəl!
+How weird!,haw wɪɹd!
+How's Tom?,hawz tɑm?
+Humor Tom.,hjumɹ̩ tɑm.
+I am busy.,aj æm bɪzi.
+I am full.,aj æm fʊl.
+I am full.,aj æm fʊl.
+I am good.,aj æm ɡʊd.
+I am here.,aj æm hɪɹ.
+I am sick.,aj æm sɪk.
+I am weak.,aj æm wik.
+I beg you.,aj bɛɡ ju.
+I beg you.,aj bɛɡ ju.
+I can fly.,aj kæn flaj.
+I can run.,aj kæn ɹʌn.
+I can run.,aj kæn ɹʌn.
+I can ski.,aj kæn ski.
+I cringed.,aj kɹɪnd͡ʒd.
+I fainted.,aj fejntɪd.
+I fainted.,aj fejntɪd.
+I gave up.,aj ɡejv ʌp.
+I gave up.,aj ɡejv ʌp.
+I gave up.,aj ɡejv ʌp.
+I gave up.,aj ɡejv ʌp.
+I get you.,aj ɡɛt ju.
+I get you.,aj ɡɛt ju.
+I get you.,aj ɡɛt ju.
+I got hit.,aj ɡɑt hɪt.
+I got hit.,aj ɡɑt hɪt.
+I hate it.,aj hejt ɪt.
+I hate it.,aj hejt ɪt.
+I hate it.,aj hejt ɪt.
+I hit Tom.,aj hɪt tɑm.
+I hope so.,aj howp sow.
+I knew it.,aj nu ɪt.
+I laughed.,aj læft.
+I like it.,aj lajk ɪt.
+I love it!,aj lʌv ɪt!
+I love it.,aj lʌv ɪt.
+I mean it!,aj min ɪt!
+I mean it.,aj min ɪt.
+I mean it.,aj min ɪt.
+I miss it.,aj mɪs ɪt.
+I miss it.,aj mɪs ɪt.
+I miss it.,aj mɪs ɪt.
+I need it.,aj nid ɪt.
+I saw Tom.,aj sɔ tɑm.
+I saw him.,aj sɔ hɪm.
+I saw him.,aj sɔ hɪm.
+I saw one.,aj sɔ wʌn.
+I saw one.,aj sɔ wʌn.
+I saw one.,aj sɔ wʌn.
+I saw you.,aj sɔ ju.
+I saw you.,aj sɔ ju.
+I see Tom.,aj si tɑm.
+I tripped.,aj tɹɪpt.
+I'll cook.,ɪl kʊk.
+I'll cook.,ɪl kʊk.
+I'll live.,ɪl lajv.
+I'll sing.,ɪl sɪŋ.
+I'll stop.,ɪl stɑp.
+I'll stop.,ɪl stɑp.
+I'll wait.,ɪl wejt.
+I'll wait.,ɪl wejt.
+I'll walk.,ɪl wɔk.
+I'll walk.,ɪl wɔk.
+I'll work.,ɪl wɹ̩k.
+I'm a man.,ɪm ə mæn.
+I'm a pro.,ɪm ə pɹow.
+I'm alone.,ɪm əlown.
+I'm alone.,ɪm əlown.
+I'm alone.,ɪm əlown.
+I'm alone.,ɪm əlown.
+I'm angry.,ɪm æŋɡɹi.
+I'm angry.,ɪm æŋɡɹi.
+I'm awake.,ɪm əwejk.
+I'm blind.,ɪm blajnd.
+I'm broke.,ɪm bɹowk.
+I'm broke.,ɪm bɹowk.
+I'm broke.,ɪm bɹowk.
+I'm broke.,ɪm bɹowk.
+I'm crazy.,ɪm kɹejzi.
+I'm drunk.,ɪm dɹʌŋk.
+I'm drunk.,ɪm dɹʌŋk.
+I'm drunk.,ɪm dɹʌŋk.
+I'm dying.,ɪm dajɪŋ.
+I'm first.,ɪm fɹ̩st.
+I'm first.,ɪm fɹ̩st.
+I'm first.,ɪm fɹ̩st.
+I'm first.,ɪm fɹ̩st.
+I'm happy.,ɪm hæpi.
+I'm happy.,ɪm hæpi.
+I'm loved.,ɪm lʌvd.
+I'm loved.,ɪm lʌvd.
+I'm obese.,ɪm owbis.
+I'm ready.,ɪm ɹɛdi.
+I'm sorry.,ɪm sɑɹi.
+I'm tired.,ɪm tajɹ̩d.
+I'm tired.,ɪm tajɹ̩d.
+I'm yours.,ɪm jʊɹz.
+I'm yours.,ɪm jʊɹz.
+I'm yours.,ɪm jʊɹz.
+I've lost.,ɪv lɔst.
+Ignore it.,ɪɡnɔɹ ɪt.
+Ignore it.,ɪɡnɔɹ ɪt.
+Is Tom OK?,ɪz tɑm owkej?
+Is Tom in?,ɪz tɑm ɪn?
+Is it bad?,ɪz ɪt bæd?
+Is it bad?,ɪz ɪt bæd?
+Is it far?,ɪz ɪt fɑɹ?
+Is it you?,ɪz ɪt ju?
+It burned.,ɪt bɹ̩nd.
+It failed.,ɪt fejld.
+It failed.,ɪt fejld.
+It failed.,ɪt fejld.
+It is new.,ɪt ɪz nu.
+It rained.,ɪt ɹejnd.
+It snowed.,ɪt snowd.
+It stinks.,ɪt stɪŋks.
+It worked.,ɪt wɹ̩kt.
+It's 3:10.,ɪts 3:10.
+It's 8:30.,ɪts 8:30.
+It's a TV.,ɪts ə tv.
+It's cold.,ɪts kowld.
+It's cold.,ɪts kowld.
+It's cool.,ɪts kul.
+It's cool.,ɪts kul.
+It's dark.,ɪts dɑɹk.
+It's done!,ɪts dʌn!
+It's fine.,ɪts fajn.
+It's good.,ɪts ɡʊd.
+It's good.,ɪts ɡʊd.
+It's here.,ɪts hɪɹ.
+It's hers.,ɪts hɹ̩z.
+It's late.,ɪts lejt.
+It's mine.,ɪts majn.
+It's mine.,ɪts majn.
+It's nice.,ɪts najs.
+It's nice.,ɪts najs.
+It's okay.,ɪts owkej.
+It's okay.,ɪts owkej.
+It's ours.,ɪts awɹ̩z.
+It's ours.,ɪts awɹ̩z.
+It's over.,ɪts owvɹ̩.
+It's over.,ɪts owvɹ̩.
+It's time.,ɪts tajm.
+It's time.,ɪts tajm.
+It's time.,ɪts tajm.
+It's time.,ɪts tajm.
+It's true!,ɪts tɹu!
+It's true.,ɪts tɹu.
+It's work.,ɪts wɹ̩k.
+Keep them.,kip ðɛm.
+Keep them.,kip ðɛm.
+Keep them.,kip ðɛm.
+Keep this.,kip ðɪs.
+Keep this.,kip ðɪs.
+Keep this.,kip ðɪs.
+Keep this.,kip ðɪs.
+Keep this.,kip ðɪs.
+Keep warm.,kip wɔɹm.
+Keep warm.,kip wɔɹm.
+Keep warm.,kip wɔɹm.
+Keep warm.,kip wɔɹm.
+Keep warm.,kip wɔɹm.
+Keep warm.,kip wɔɹm.
+Keep warm.,kip wɔɹm.
+Keep warm.,kip wɔɹm.
+Leave Tom.,liv tɑm.
+Leave Tom.,liv tɑm.
+Leave Tom.,liv tɑm.
+Leave now.,liv naw.
+Leave now.,liv naw.
+Leave now.,liv naw.
+Let me go!,lɛt mi ɡow!
+Let me go!,lɛt mi ɡow!
+Let me go!,lɛt mi ɡow!
+Let me go.,lɛt mi ɡow.
+Let me in.,lɛt mi ɪn.
+Let me in.,lɛt mi ɪn.
+Let us in.,lɛt ʌs ɪn.
+Let's eat.,lɛtz it.
+Let's see.,lɛtz si.
+Let's try!,lɛtz tɹaj!
+Lie still.,laj stɪl.
+Lie still.,laj stɪl.
+Listen up.,lɪsən ʌp.
+Look away.,lʊk əwej.
+Look away.,lʊk əwej.
+Look away.,lʊk əwej.
+Look away.,lʊk əwej.
+Look back!,lʊk bæk!
+Look back.,lʊk bæk.
+Look back.,lʊk bæk.
+Look back.,lʊk bæk.
+Look here.,lʊk hɪɹ.
+Look here.,lʊk hɪɹ.
+Look here.,lʊk hɪɹ.
+Loosen it.,lusən ɪt.
+Loosen it.,lusən ɪt.
+Loosen it.,lusən ɪt.
+Loosen it.,lusən ɪt.
+Move over.,muv owvɹ̩.
+Move over.,muv owvɹ̩.
+Move over.,muv owvɹ̩.
+Move over.,muv owvɹ̩.
+Nice shot!,najs ʃɑt!
+Of course!,ʌv kɔɹs!
+Open fire!,owpən fajɹ̩!
+Open fire!,owpən fajɹ̩!
+Pardon me?,pɑɹdən mi?
+Please go.,pliz ɡow.
+Please go.,pliz ɡow.
+Please go.,pliz ɡow.
+Please go.,pliz ɡow.
+Put it on.,pʊt ɪt ɑn.
+Put it on.,pʊt ɪt ɑn.
+Read this.,ɹɛd ðɪs.
+Read this.,ɹɛd ðɪs.
+Read this.,ɹɛd ðɪs.
+Read this.,ɹɛd ðɪs.
+Say hello.,sej həlow.
+Search me.,sɹ̩t͡ʃ mi.
+See above.,si əbʌv.
+See above.,si əbʌv.
+Seize him!,siz hɪm!
+Seize him!,siz hɪm!
+Seize him!,siz hɪm!
+Seize him!,siz hɪm!
+Seize him!,siz hɪm!
+Seriously?,sɪɹiəsli?
+She tried.,ʃi tɹajd.
+She tried.,ʃi tɹajd.
+She walks.,ʃi wɔks.
+She walks.,ʃi wɔks.
+She walks.,ʃi wɔks.
+She's hot.,ʃɛz hɑt.
+She's hot.,ʃɛz hɑt.
+She's hot.,ʃɛz hɑt.
+She's hot.,ʃɛz hɑt.
+She's hot.,ʃɛz hɑt.
+Sign here.,sajn hɪɹ.
+Sign this.,sajn ðɪs.
+Sign this.,sajn ðɪs.
+Sign this.,sajn ðɪs.
+Sign this.,sajn ðɪs.
+Sign this.,sajn ðɪs.
+Sit still.,sɪt stɪl.
+Sit still.,sɪt stɪl.
+Sit there.,sɪt ðɛɹ.
+Sit tight.,sɪt tajt.
+Slow down.,slow dawn.
+Slow down.,slow dawn.
+Stay away.,stej əwej.
+Stay away.,stej əwej.
+Stay calm.,stej kɑm.
+Stay calm.,stej kɑm.
+Stay here.,stej hɪɹ.
+Step back.,stɛp bæk.
+Stop here.,stɑp hɪɹ.
+Stop here.,stɑp hɪɹ.
+Stop here.,stɑp hɪɹ.
+Stop here.,stɑp hɪɹ.
+Stop here.,stɑp hɪɹ.
+Stop here.,stɑp hɪɹ.
+Stop here.,stɑp hɪɹ.
+Stop that.,stɑp ðæt.
+Stop them.,stɑp ðɛm.
+Stop them.,stɑp ðɛm.
+Take care!,tejk kɛɹ!
+Take care!,tejk kɛɹ!
+Take care.,tejk kɛɹ.
+Take care.,tejk kɛɹ.
+Take mine.,tejk majn.
+Take mine.,tejk majn.
+Take over.,tejk owvɹ̩.
+Take over.,tejk owvɹ̩.
+Take over.,tejk owvɹ̩.
+Take this.,tejk ðɪs.
+Take this.,tejk ðɪs.
+Take this.,tejk ðɪs.
+Take this.,tejk ðɪs.
+Thank you.,θæŋk ju.
+Thank you.,θæŋk ju.
+Thank you.,θæŋk ju.
+That's it.,θætz ɪt.
+That's me.,θætz mi.
+That's me.,θætz mi.
+Then what?,ðɛn wʌt?
+They fell.,ðej fɛl.
+They left.,ðej lɛft.
+They left.,ðej lɛft.
+They left.,ðej lɛft.
+They lost.,ðej lɔst.
+Tom bowed.,tɑm bawd.
+Tom cared.,tɑm kɛɹd.
+Tom cared.,tɑm kɛɹd.
+Tom cared.,tɑm kɛɹd.
+Tom cares.,tɑm kɛɹz.
+Tom cares.,tɑm kɛɹz.
+Tom cried.,tɑm kɹajd.
+Tom dozed.,tɑm dowzd.
+Tom drove.,tɑm dɹowv.
+Tom is OK.,tɑm ɪz owkej.
+Tom is in.,tɑm ɪz ɪn.
+Tom is up.,tɑm ɪz ʌp.
+Tom knits.,tɑm nɪts.
+Tom knows.,tɑm nowz.
+Tom moved.,tɑm muvd.
+Tom moved.,tɑm muvd.
+Tom rocks.,tɑm ɹɑks.
+Tom stood.,tɑm stʊd.
+Tom swims.,tɑm swɪmz.
+Tom swore.,tɑm swɔɹ.
+Tom tried.,tɑm tɹajd.
+Tom tries.,tɑm tɹajz.
+Tom voted.,tɑm vowtəd.
+Tom walks.,tɑm wɔks.
+Tom waved.,tɑm wejvd.
+Tom works.,tɑm wɹ̩ks.
+Tom'll go.,tɑml ɡow.
+Tom's fat.,tɑmz fæt.
+Tom's mad.,tɑmz mæd.
+Tom's sad.,tɑmz sæd.
+Tom's shy.,tɑmz ʃaj.
+Trust Tom.,tɹʌst tɑm.
+Try again.,tɹaj əɡɛn.
+Try again.,tɹaj əɡɛn.
+Try again.,tɹaj əɡɛn.
+Try it on.,tɹaj ɪt ɑn.
+Wait here.,wejt hɪɹ.
+Wait here.,wejt hɪɹ.
+Wait here.,wejt hɪɹ.
+Wait here.,wejt hɪɹ.
+Watch Tom.,wɑt͡ʃ tɑm.
+Watch out!,wɑt͡ʃ awt!
+We can go.,wi kæn ɡow.
+We can go.,wi kæn ɡow.
+We can go.,wi kæn ɡow.
+We failed.,wi fejld.
+We failed.,wi fejld.
+We forgot.,wi fɹ̩ɡɑt.
+We saw it.,wi sɔ ɪt.
+We saw it.,wi sɔ ɪt.
+We talked.,wi tɔkt.
+We waited.,wi wejtəd.
+We'll see.,wɛl si.
+We'll see.,wɛl si.
+We'll try.,wɛl tɹaj.
+We've won!,wɛv wʌn!
+What's up?,wætz ʌp?
+Who cares?,hu kɛɹz?
+Who is he?,hu ɪz hi?
+Who is it?,hu ɪz ɪt?
+Who knows?,hu nowz?
+Who stood?,hu stʊd?
+Who'll go?,həl ɡow?
+Who's Tom?,hɑz tɑm?
+Write Tom.,ɹajt tɑm.
+You drive.,ju dɹajv.
+You start.,ju stɑɹt.
+You tried.,ju tɹajd.
+You're OK.,jɹ owkej.
+You're OK.,jɹ owkej.
+Aim higher.,ejm hajɹ̩.
+Aim higher.,ejm hajɹ̩.
+All aboard!,ɔl əbɔɹd!
+Am I right?,æm aj ɹajt?
+Am I wrong?,æm aj ɹɔŋ?
+Am I wrong?,æm aj ɹɔŋ?
+Answer Tom.,ænsɹ̩ tɑm.
+Answer Tom.,ænsɹ̩ tɑm.
+Answer Tom.,ænsɹ̩ tɑm.
+Are you 18?,ɑɹ ju 18?
+Are you 18?,ɑɹ ju 18?
+Are you OK?,ɑɹ ju owkej?
+Are you in?,ɑɹ ju ɪn?
+Are you in?,ɑɹ ju ɪn?
+Are you up?,ɑɹ ju ʌp?
+Ask anyone.,æsk ɛniwʌn.
+Ask anyone.,æsk ɛniwʌn.
+Ask anyone.,æsk ɛniwʌn.
+Ask anyone.,æsk ɛniwʌn.
+Ask around.,æsk ɹ̩awnd.
+Ask around.,æsk ɹ̩awnd.
+Be careful.,bi kɛɹfəl.
+Be careful.,bi kɛɹfəl.
+Be content.,bi kɑntɛnt.
+Be on time.,bi ɑn tajm.
+Be on time.,bi ɑn tajm.
+Be patient.,bi pejʃənt.
+Be serious.,bi sɪɹiəs.
+Birds sing.,bɹ̩dz sɪŋ.
+Birds sing.,bɹ̩dz sɪŋ.
+Bring food.,bɹɪŋ fud.
+Bring help.,bɹɪŋ hɛlp.
+Bring wine.,bɹɪŋ wajn.
+Can I come?,kæn aj kʌm?
+Can I come?,kæn aj kʌm?
+Can I come?,kæn aj kʌm?
+Can I help?,kæn aj hɛlp?
+Can I stay?,kæn aj stej?
+Carry this.,kæɹi ðɪs.
+Check that.,t͡ʃɛk ðæt.
+Check this.,t͡ʃɛk ðɪs.
+Choose one.,t͡ʃuz wʌn.
+Come again.,kʌm əɡɛn.
+Come alone.,kʌm əlown.
+Come along.,kʌm əlɔŋ.
+Come along.,kʌm əlɔŋ.
+Come early.,kʌm ɹ̩li.
+Come early.,kʌm ɹ̩li.
+Come early.,kʌm ɹ̩li.
+Come early.,kʌm ɹ̩li.
+Come on in!,kʌm ɑn ɪn!
+Come on in!,kʌm ɑn ɪn!
+Come on in!,kʌm ɑn ɪn!
+Come on in.,kʌm ɑn ɪn.
+Come on in.,kʌm ɑn ɪn.
+Come quick!,kʌm kwɪk!
+Come quick!,kʌm kwɪk!
+Come to me.,kʌm tə mi.
+Come to me.,kʌm tə mi.
+Come to us.,kʌm tə ʌs.
+Come to us.,kʌm tə ʌs.
+Cut it out!,kʌt ɪt awt!
+Did Tom go?,dɪd tɑm ɡow?
+Do come in!,du kʌm ɪn!
+Do come in!,du kʌm ɪn!
+Do come in!,du kʌm ɪn!
+Do come in.,du kʌm ɪn.
+Do come in.,du kʌm ɪn.
+Do come in.,du kʌm ɪn.
+Do come in.,du kʌm ɪn.
+Do men cry?,du mɛn kɹaj?
+Don't come.,dɑnt kʌm.
+Don't jump!,dɑnt d͡ʒʌmp!
+Don't look.,dɑnt lʊk.
+Don't move!,dɑnt muv!
+Don't move.,dɑnt muv.
+Don't move.,dɑnt muv.
+Don't move.,dɑnt muv.
+Don't sing.,dɑnt sɪŋ.
+Don't sing.,dɑnt sɪŋ.
+Don't stop.,dɑnt stɑp.
+Don't talk!,dɑnt tɔk!
+Don't talk.,dɑnt tɔk.
+Don't wait.,dɑnt wejt.
+Don't wait.,dɑnt wejt.
+Don't wait.,dɑnt wejt.
+Don't yell.,dɑnt jɛl.
+Eat slowly.,it slowli.
+Eat slowly.,it slowli.
+Fire burns.,fajɹ̩ bɹ̩nz.
+Follow Tom.,fɑlow tɑm.
+Follow Tom.,fɑlow tɑm.
+Follow Tom.,fɑlow tɑm.
+Follow Tom.,fɑlow tɑm.
+Follow him.,fɑlow hɪm.
+Follow him.,fɑlow hɪm.
+Forget Tom.,fɹ̩ɡɛt tɑm.
+Forget Tom.,fɹ̩ɡɛt tɑm.
+Forget Tom.,fɹ̩ɡɛt tɑm.
+Forget him.,fɹ̩ɡɛt hɪm.
+Forgive us.,fɹ̩ɡɪv ʌs.
+Forgive us.,fɹ̩ɡɪv ʌs.
+Get a life.,ɡɛt ə lajf.
+Get inside.,ɡɛt ɪnsajd.
+Get to bed.,ɡɛt tə bɛd.
+Give it up.,ɡɪv ɪt ʌp.
+Go on home.,ɡow ɑn howm.
+Go on home.,ɡow ɑn howm.
+Go see Tom.,ɡow si tɑm.
+Go to work.,ɡow tə wɹ̩k.
+God exists.,ɡɑd ɪɡzɪsts.
+Have faith.,hæv fejθ.
+Have faith.,hæv fejθ.
+Have faith.,hæv fejθ.
+Have faith.,hæv fejθ.
+He ate out.,hi ejt awt.
+He coughed.,hi kɔft.
+He gave in.,hi ɡejv ɪn.
+He gave up.,hi ɡejv ʌp.
+He gave up.,hi ɡejv ʌp.
+He gave up.,hi ɡejv ʌp.
+He gave up.,hi ɡejv ʌp.
+He hung up.,hi hʌŋ ʌp.

examples/piglattin/prepare_training_data.py

@@ -0,0 +1,69 @@
+alphabet = "abcdefghijklmnopqrstuvwxyz"
+vowels = "aeoiu"
+
+def english_to_piglattin( english ):
+
+    piglattin = ""
+
+    in_word = False
+    is_first = False
+    start = None
+    for char in english:
+        if not in_word:
+            if char in alphabet + alphabet.upper():
+                in_word = True
+
+                if char in vowels + vowels.upper():
+                    start = None
+                    piglattin += char
+                else:
+                    start = char
+                    is_first = True
+            else:
+                piglattin += char
+        else: #if in_word
+            if char in alphabet + alphabet.upper():
+                if is_first:
+                    is_first = False
+                    if start in alphabet.upper():
+                        piglattin += char.upper()
+                    else:
+                        piglattin += char
+                else:
+                    piglattin += char
+            else:
+                in_word = False
+                is_first = False
+                if start:
+                    piglattin += start.lower() + "ay" + char
+                else:
+                    piglattin += "yay" + char
+    
+    #end of sentance needs done as well.
+    if in_word:
+        if start:
+            piglattin += start.lower() + "ay"
+        else:
+            piglattin += "yay"
+    return piglattin
+
+
+def main():
+    with open( "spa.csv", "rt" ) as fin:
+        with open( "pig_lattin.csv", "wt" ) as f_out:
+            f_out.write( "English,Piglattin\n" )
+            for line in fin:
+                english = line.split( "\t" )[0]
+                english = english.replace( ",", " " )
+                piglattin = english_to_piglattin( english )
+
+                f_out.write( f"{english},{piglattin}\n" )
+
+if __name__ == '__main__':
+    main()
+
+    # print( english_to_piglattin( "I am not a potato." ) )
+    # print( english_to_piglattin( "I am not a potato" ) )
+    # print( english_to_piglattin( "I like chicken." ) )
+    # print( english_to_piglattin( "Do you know your a b c's?" ) )
+    # print( english_to_piglattin( "My name is Joshua." ) )

transmorgrify.py

@@ -0,0 +1,382 @@
+#!/usr/bin/env python3 
+import argparse
+import json
+import os
+import zipfile
+
+import pandas as pd
+from catboost import CatBoostClassifier, Pool
+
+MATCH = 0
+DELETE_FROM = 1
+INSERT_TO = 2
+START = 3
+
+
+
+def _list_trace( trace ):
+    if trace.parrent is None:
+        result = [trace]
+    else:
+        result = _list_trace( trace.parrent )
+        result.append( trace )
+    return result
+
+class _edit_trace_hop():
+    parrent = None
+    edit_distance = None
+    char = None
+    from_row_i = None
+    to_column_i = None
+    action = None
+
+    def __str__( self ):
+        if self.action == START:
+            return "<start>"
+        elif self.action == INSERT_TO:
+            return f"<ins> {self.char}"
+        elif self.action == DELETE_FROM:
+            return f"<del> {self.char}"
+        elif self.action == MATCH:
+            return f"<match> {self.char}"
+        return "eh?"
+
+    def __repr__( self ):
+        return self.__str__()
+
+def _trace_edits( from_sentance, to_sentance, print_debug=False ):
+    #iterating from will be the rows down the left side.
+    #iterating to will be the columns across the top.
+    #we will keep one row as we work on the next.
+
+    last_row = None
+    current_row = []
+
+    #the index handles one before the index in the string
+    #to handle the root cases across the top and down the left of the
+    #match matrix.
+    for from_row_i in range( len(from_sentance)+1 ):
+
+        for to_column_i in range( len(to_sentance )+1 ):
+
+            best_option = None
+
+            #root case.
+            if from_row_i == 0 and to_column_i == 0:
+                best_option = _edit_trace_hop()
+                best_option.parrent = None
+                best_option.edit_distance = 0
+                best_option.char = ""
+                best_option.from_row_i = from_row_i
+                best_option.to_column_i = to_column_i
+                best_option.action = START
+
+            #check left
+            if to_column_i > 0:
+                if best_option is None or current_row[to_column_i-1].edit_distance + 1 < best_option.edit_distance:
+                    best_option = _edit_trace_hop()
+                    best_option.parrent = current_row[to_column_i-1]
+                    best_option.edit_distance = best_option.parrent.edit_distance + 1
+                    best_option.char = to_sentance[to_column_i-1]
+                    best_option.from_row_i = from_row_i
+                    best_option.to_column_i = to_column_i
+                    best_option.action = INSERT_TO
+            
+            #check up
+            if from_row_i > 0:
+                if best_option is None or last_row[to_column_i].edit_distance + 1 < best_option.edit_distance:
+                    best_option = _edit_trace_hop()
+                    best_option.parrent = last_row[to_column_i]
+                    best_option.edit_distance = best_option.parrent.edit_distance + 1
+                    best_option.char = from_sentance[from_row_i-1]
+                    best_option.from_row_i = from_row_i
+                    best_option.to_column_i = to_column_i
+                    best_option.action = DELETE_FROM
+
+                #check match
+                if to_column_i > 0:
+                    if to_sentance[to_column_i-1] == from_sentance[from_row_i-1]:
+                        if best_option is None or last_row[to_column_i-1].edit_distance <= best_option.edit_distance: #prefer match so use <= than <
+                            best_option = _edit_trace_hop()
+                            best_option.parrent = last_row[to_column_i-1]
+                            best_option.edit_distance = best_option.parrent.edit_distance + 1
+                            best_option.char = from_sentance[from_row_i-1]
+                            best_option.from_row_i = from_row_i
+                            best_option.to_column_i = to_column_i
+                            best_option.action = MATCH
+
+            if best_option is None: raise Exception( "Shouldn't end up with best_option being None" )
+            current_row.append(best_option)
+
+        last_row = current_row
+        current_row = []
+
+    if print_debug:
+        def print_diffs( current_node ):
+            if current_node.parrent is not None:
+                print_diffs( current_node.parrent )
+            
+            if current_node.action == START:
+                print( "start" )
+            elif current_node.action == MATCH:
+                print( f"match {current_node.char}" )
+            elif current_node.action == INSERT_TO:
+                print( f"insert {current_node.char}" )
+            elif current_node.action == DELETE_FROM:
+                print( f"del {current_node.char}" )
+        print_diffs( last_row[-1] )
+    return last_row[-1]
+
+
+def _parse_single_for_training( from_sentance, to_sentance, num_pre_context_chars, num_post_context_chars ):
+    trace = _trace_edits( from_sentance, to_sentance )
+
+    #we will collect a snapshot at each step.
+    trace_list = _list_trace(trace)
+
+
+    training_collection = []
+
+    #execute these things on the from_sentance and see if we get the to_sentance.
+    working_from = from_sentance
+    working_to = ""
+    used_from = ""
+    continuous_added = 0
+    continuous_dropped = 0
+    for thing in trace_list:
+        #gather action and context for training
+        if thing.action != START:
+            from_context = (working_from + (" " * num_post_context_chars))[:num_post_context_chars]
+            to_context =   ((" " * num_pre_context_chars) + working_to )[-num_pre_context_chars:]
+            used_context = ((" " * num_pre_context_chars) + used_from  )[-num_pre_context_chars:]
+
+            training_collection.append({
+                "from_context": from_context,
+                "to_context": to_context,
+                "used_context": used_context,
+                "action": thing.action,
+                "continuous_added": continuous_added,
+                "continuous_dropped": continuous_dropped,
+                "char": thing.char if thing.action == INSERT_TO else ' ',
+            })
+
+        #now execute the action for the next step.
+        if thing.action == START:
+            pass
+        elif thing.action == INSERT_TO:
+            working_to += thing.char
+            continuous_added += 1
+            continuous_dropped = 0
+        elif thing.action == DELETE_FROM:
+            used_from += working_from[0]
+            working_from = working_from[1:]
+            continuous_added = 0
+            continuous_dropped += 1
+        elif thing.action == MATCH:
+            used_from += working_from[0]
+            working_to += working_from[0]
+            working_from = working_from[1:]
+            continuous_added = 0
+            continuous_dropped = 0
+
+    
+    if to_sentance != working_to:
+        print( "Replay failure" )
+
+    #so now I have training_collection which is a list of dictionaries where each dictionary is an action with a context.
+    #I need to change it into a dictionary of lists where each dictionary a column and the lists are the rows.
+    context_split_into_dict = {}
+
+    #first collect the from_context:
+    for i in range( num_post_context_chars ):
+        this_slice = []
+        for training in training_collection:
+            this_slice.append( training['from_context'][i] )
+        context_split_into_dict[ f"f{i}" ] = this_slice
+    
+    #now collect to_context:
+    for i in range( num_pre_context_chars ):
+        this_slice = []
+        for training in training_collection:
+            this_slice.append( training['to_context'][i] )
+        context_split_into_dict[ f"t{i}" ] = this_slice
+
+    #now collect used_context
+    for i in range( num_pre_context_chars ):
+        this_slice = []
+        for training in training_collection:
+            this_slice.append( training['used_context'][i] )
+        context_split_into_dict[ f"u{i}" ] = this_slice
+
+    
+    #now these two things.
+    context_split_into_dict["continuous_added"] = []
+    context_split_into_dict["continuous_dropped"] = []
+    for training in training_collection:
+        context_split_into_dict["continuous_added"].append( training["continuous_added"] )
+        context_split_into_dict["continuous_dropped"].append( training["continuous_dropped"] )
+
+    #now also collect the output answers.
+    result_split_into_dict = {}
+    action_slice = []
+    char_slice = []
+    for training in training_collection:
+        action_slice.append( training['action'] )
+        char_slice.append( training['char'] )
+    result_split_into_dict['action'] = action_slice
+    result_split_into_dict['char']   = char_slice
+        
+    #now return it as a dataframe.
+    return pd.DataFrame( context_split_into_dict ), pd.DataFrame( result_split_into_dict )
+
+
+def _parse_for_training( from_sentances, to_sentances, num_pre_context_chars, num_post_context_chars ):
+    out_observations_list = []
+    out_results_list = []
+
+    for index, (from_sentance, to_sentance) in enumerate(zip( from_sentances, to_sentances )):
+        if type(from_sentance) != float and type(to_sentance) != float: #bad lines are nan which are floats.
+            specific_observation, specific_result = _parse_single_for_training( from_sentance, to_sentance, num_pre_context_chars=num_pre_context_chars, num_post_context_chars=num_post_context_chars )
+
+            out_observations_list.append( specific_observation )
+            out_results_list.append( specific_result )
+        if index % 100 == 0:
+            print( f"parsing {index} of {len(from_sentances)}")
+
+    return pd.concat( out_observations_list ), pd.concat( out_results_list )
+
+def _train_catboost( X, y, iterations, device, verbose, model_piece, learning_rate = .07 ):
+
+    X = X.fillna( ' ' )
+    passed = False
+    while not passed:
+        train_pool = Pool(
+            data=X,
+            label=y,
+            cat_features=[i for i,x in enumerate(X.keys()) if len(x) == 2] #all cat keys are length 2
+        )
+        validation_pool = None #Can't use validation pool because it randomly has chars not in training.
+        model = CatBoostClassifier(
+            iterations = iterations,
+            learning_rate = learning_rate,
+            task_type="GPU" if device.lower() != 'cpu' else "CPU",
+            devices=device if device.lower() != 'cpu' else None
+        )
+        model.fit( train_pool, eval_set=validation_pool, verbose=True )
+        passed = True
+
+    if( verbose ): print( '{} is fitted: {}',format(model_piece,model.is_fitted()))
+    if( verbose ): print( '{} params:\n{}'.format(model_piece,model.get_params()))
+
+    return model
+
+def _train_reconstruct_models( from_sentances, to_sentances, iterations, device, num_pre_context_chars, num_post_context_chars, verbose ):
+  
+    X,Y = _parse_for_training( from_sentances, to_sentances, num_pre_context_chars=num_pre_context_chars, num_post_context_chars=num_post_context_chars )
+
+    #train and save the action_model
+    action_model = _train_catboost( X, Y['action'], iterations, verbose=verbose, device=device, model_piece='action' )
+
+    #and the char model
+    #slice through where only the action is insert.
+    insert_indexes = Y['action'] == INSERT_TO
+    char_model = _train_catboost( X[insert_indexes], Y['char'][insert_indexes], iterations, verbose=verbose, device=device, model_piece='char' )
+
+    return action_model, char_model
+
+def _mktemp():
+    #I know mktemp exists in the library but it has been depricated suggesting using
+    #mkstemp but catboost can't write to a filehandle yet, so I need an actual
+    #filename.
+    number = 0
+    while os.path.exists( f".temp_{number}~" ):
+        number += 1
+    return f".temp_{number}~"
+
+def train( in_csv, a_header, b_header, model, iterations, device, leading_context,trailing_context, train_percentage, verbose ):
+    if verbose: print( "loading csv" )
+    full_data = pd.read_csv( in_csv )
+
+    split_index = int( train_percentage/100*len(full_data) )
+    train_data = full_data.iloc[:split_index,:].reset_index(drop=True)
+
+    if verbose: print( "parcing data for training" )
+
+    action_model, char_model = _train_reconstruct_models( from_sentances=train_data[a_header], 
+            to_sentances=train_data[b_header], 
+            iterations = iterations,
+            device = device,
+            num_pre_context_chars = leading_context, 
+            num_post_context_chars = trailing_context,
+            verbose=verbose,
+                )
+
+    temp_action_filename = _mktemp()
+    action_model.save_model( temp_action_filename )
+    temp_char_filename = _mktemp()
+    char_model.save_model( temp_char_filename )
+
+    with zipfile.ZipFile( model, mode="w", compression=zipfile.ZIP_DEFLATED, compresslevel=9 ) as myzip:
+        with myzip.open( 'params.json', mode='w' ) as out:
+            out.write( json.dumps({
+                'version': 1,
+                'leading_context': leading_context,
+                'trailing_context': trailing_context,
+                'iterations': iterations,
+            }).encode())
+        myzip.write( temp_action_filename, "action.cb" )
+        myzip.write( temp_char_filename,   "char.cb" )
+
+    os.unlink( temp_action_filename )
+    os.unlink( temp_char_filename )
+    
+def main():
+    parser = argparse.ArgumentParser(
+                    prog = 'transmorgrify.py',
+                    description = 'Converts text from one to another according to a model.',
+                    epilog = '(C) Joshua Lansford')
+    parser.add_argument('-i', '--in_csv',  help='The csv to read training or input data from', required=True )     
+    parser.add_argument('-o', '--out_csv',  help='The csv to write conversion to', default='out.csv' )     
+    parser.add_argument('-a', '--a_header', help='The column header for training or transforming from', default="source" )
+    parser.add_argument('-b', '--b_header',   help='The column header for training the transformation to', default="target"  )
+    parser.add_argument('-m', '--model',help='The model file to create during training or use during transformation', default='model.tm' )
+    parser.add_argument('-n', '--iterations', help='The number of iterations to train', default=1000 )
+    parser.add_argument('-d', '--device',  help='Which device, i.e. if useing GPU', default='cpu' )
+    parser.add_argument('-x', '--context', help='The number of leading and trailing chars to use as context', default=7 )
+    parser.add_argument('-t', '--train', action='store_true', help='Train a model instead of executing a model')
+    parser.add_argument('-p', '--train_percentage', help="The percentage of data to train on, leaving the rest for testing.")
+    parser.add_argument('-e', '--execute', action='store_true', help='Use an existing trained model.')
+    parser.add_argument('-v', '--verbose', action='store_true', help='Talks alot?' )
+                        
+
+    args = parser.parse_args()
+
+    if not args.train and not args.execute: print( "Must include --execute and/or --train to do something." )
+
+    
+    if args.train:
+
+        train_percentage = args.train_percentage
+        if train_percentage is None:
+            if args.execute:
+                train_percentage = 50
+            else:
+                train_percentage = 100
+
+        train( in_csv=args.in_csv, 
+               a_header=args.a_header, 
+               b_header=args.b_header, 
+               model=args.model,
+               iterations=args.iterations,
+               device=args.device,
+               leading_context=args.context,
+               trailing_context=args.context,
+               train_percentage=train_percentage,
+               verbose=args.verbose,
+               )
+
+    #print(args)
+
+
+if __name__ == '__main__':
+    main()