flpelerin commited on
Commit
d5d4a4a
·
1 Parent(s): d6ead5a

Update file tokenizer.py

Browse files
Files changed (1) hide show
  1. tokenizer.py +10 -5
tokenizer.py CHANGED
@@ -146,22 +146,27 @@ class Tokenizer:
146
 
147
 
148
 
149
- def c_compile(self):
 
150
  import os
151
  import subprocess
152
-
153
  # Get the path of the current Python script
154
  script_dir = os.path.dirname(__file__)
155
 
156
  # Construct the path to the c_tokenizer directory
157
- c_tokenizer_dir = os.path.join(script_dir, 'c_tokenizer')
158
 
159
  # Invoke Make on the c_tokenizer directory
160
- subprocess.run(['make'], cwd=c_tokenizer_dir)
 
161
 
162
 
163
  def c_encode(self, text): #TODO: Implement
 
 
164
 
165
- self.c_compile()
 
166
 
167
  return [1, 2, 3, 4]
 
146
 
147
 
148
 
149
+
150
+ def c_compile(self, c_dir):
151
  import os
152
  import subprocess
153
+
154
  # Get the path of the current Python script
155
  script_dir = os.path.dirname(__file__)
156
 
157
  # Construct the path to the c_tokenizer directory
158
+ c_dir = os.path.join(script_dir, c_dir)
159
 
160
  # Invoke Make on the c_tokenizer directory
161
+ subprocess.run(['make'], cwd=c_dir)
162
+
163
 
164
 
165
  def c_encode(self, text): #TODO: Implement
166
+ c_dir = "c_tokenizer"
167
+ c_file = "tokenizer.bin"
168
 
169
+ self.to_file(self.c_dir + self.c_file)
170
+ self.c_compile(self.c_dir)
171
 
172
  return [1, 2, 3, 4]