Add preprocess

infer.cpp

@@ -12,6 +12,28 @@ using namespace std;
 unordered_map<string, uint32_t> vocab;
 unordered_map<uint64_t, vector<uint32_t>> hasilToOutput;
 
+void preprocess(vector<string> &theString) {
+	vector<string> tmp;
+	for(auto s : theString) {
+		string tmpp;
+		for(auto c : s /*1.6 :P*/) {
+			if(!isalnum(c)) {
+				if(tmpp.length() == 0) {
+					tmp.push_back(string(1, c));
+				} else {
+					tmp.push_back(tmpp);
+					tmpp = "";
+					tmp.push_back(string(1, c));
+				}
+			} else {
+			tmpp += tolower(c);
+			}
+		}
+		if(tmpp != "") tmp.push_back(tmpp);
+	}
+	theString = tmp;
+}
+
 // === Fungsi: Load Model dari File ===
 void loadModel(const string &filename) {
   ifstream file(filename);
@@ -127,6 +149,7 @@ int main() {
   while (ss >> word) {
     context.push_back(word);
   }
+  preprocess(context);
   auto newContext = context;
 
   string prediction;