Update file tokenizer.py
Browse files- tokenizer.py +10 -5
tokenizer.py
CHANGED
|
@@ -146,22 +146,27 @@ class Tokenizer:
|
|
| 146 |
|
| 147 |
|
| 148 |
|
| 149 |
-
|
|
|
|
| 150 |
import os
|
| 151 |
import subprocess
|
| 152 |
-
|
| 153 |
# Get the path of the current Python script
|
| 154 |
script_dir = os.path.dirname(__file__)
|
| 155 |
|
| 156 |
# Construct the path to the c_tokenizer directory
|
| 157 |
-
|
| 158 |
|
| 159 |
# Invoke Make on the c_tokenizer directory
|
| 160 |
-
subprocess.run(['make'], cwd=
|
|
|
|
| 161 |
|
| 162 |
|
| 163 |
def c_encode(self, text): #TODO: Implement
|
|
|
|
|
|
|
| 164 |
|
| 165 |
-
self.
|
|
|
|
| 166 |
|
| 167 |
return [1, 2, 3, 4]
|
|
|
|
| 146 |
|
| 147 |
|
| 148 |
|
| 149 |
+
|
| 150 |
+
def c_compile(self, c_dir):
|
| 151 |
import os
|
| 152 |
import subprocess
|
| 153 |
+
|
| 154 |
# Get the path of the current Python script
|
| 155 |
script_dir = os.path.dirname(__file__)
|
| 156 |
|
| 157 |
# Construct the path to the c_tokenizer directory
|
| 158 |
+
c_dir = os.path.join(script_dir, c_dir)
|
| 159 |
|
| 160 |
# Invoke Make on the c_tokenizer directory
|
| 161 |
+
subprocess.run(['make'], cwd=c_dir)
|
| 162 |
+
|
| 163 |
|
| 164 |
|
| 165 |
def c_encode(self, text): #TODO: Implement
|
| 166 |
+
c_dir = "c_tokenizer"
|
| 167 |
+
c_file = "tokenizer.bin"
|
| 168 |
|
| 169 |
+
self.to_file(self.c_dir + self.c_file)
|
| 170 |
+
self.c_compile(self.c_dir)
|
| 171 |
|
| 172 |
return [1, 2, 3, 4]
|