Upload folder using huggingface_hub

.gradio/flagged/dataset1.csv

@@ -0,0 +1,2 @@
+Shona Input,Max New Tokens,Temperature,output,timestamp
+"{""mhoro"":""urise"", ""ndinofara"":""iri bho"", ""ndipo marii"":}",125,0.9,"{""mhoro"":""urise"", ""ndinofara"":""iri bho"", ""ndipo marii"":} Akatsanangura kutenderedzwa kwemiti, uye akati akazvifambisa kubhawa. Mhedzisiro yacho yaiva yekuti kana vafambi vakabuda ndokubvunisana kuti: ""Izvi hazvisizvo uye zvinokwanisika"". Mumashoko, izvi zvaireva kuti Pauro aiva * napo* ndiye akakubvunza chero chinhu. Handisi kukuudzai kuti ndimuratidze kuti ndiri ani. Izvi zvingava zvisiri izvo. Ndeipi IMHO, zvinoita sokuti kwakanaka. Ndinokutendai!! Ndakanzwa kuti mitambo miviri haisi kuratidzwa pane yangu, asi ndakaiisa mumashoko mugore rokupedzisira rechi11, uye ndiri kuzviita mhando yemhando. Yeuka kuti hapana mumwe",2026-03-04 17:30:24.173349

app.py

@@ -1,42 +1,62 @@
 import gradio as gr
-from transformers import LlamaTokenizer
 import torch
 import sys
 import os
+import sentencepiece as spm
 
 # Add local path to import custom model
 sys.path.append(os.path.join(os.path.dirname(__file__), "model"))
 from modeling_ngwanda import NgwandaModel
 
-# Load the wrapped tokenizer
-tokenizer = LlamaTokenizer.from_pretrained("./ngwanda-tokenizer-hf")
-# Load your Ngwanda base model
-model = NgwandaModel.from_pretrained("./model")
+# Load the SentencePiece tokenizer directly
+sp = spm.SentencePieceProcessor(model_file="vocab_12k/corpus_tokenized_12000.model")
+bos_token = "<|startoftext|>"
+bos_id = sp.piece_to_id(bos_token)
+
+# Load your Ngwanda base model from the new HF repository
+model = NgwandaModel.from_pretrained("takuM23/ShonaTransformer-Basemodel")
 
 def predict(text, tokens, temp):
-    # Base models need the BOS token to start properly
-    if not text.startswith(tokenizer.bos_token):
-        text = tokenizer.bos_token + text
+    print(text)
+    
+    # encode as IDs
+    input_ids = sp.encode_as_ids(text)
+    if not text.startswith(bos_token):
+        input_ids = [bos_id] + input_ids
         
-    inputs = tokenizer(text, return_tensors="pt")
+    inputs_tensor = torch.tensor([input_ids])
+    print({'input_ids': inputs_tensor})
     
     with torch.no_grad():
         outputs = model.generate(
-            **inputs, 
-            max_new_tokens=int(tokens), 
+            input_ids=inputs_tensor,
+            max_new_tokens=int(tokens),
             temperature=float(temp),
-            do_sample=True
+            do_sample=True,
         )
+
+    out_ids = outputs[0].tolist()
     
+    # filter out special tokens manually
+    special_ids = {sp.bos_id(), sp.eos_id(), sp.pad_id(), sp.unk_id(), 
+                   sp.piece_to_id("<|startoftext|>"), sp.piece_to_id("<|endofturn|>"), 
+                   sp.piece_to_id("<|user|>"), sp.piece_to_id("<|agent|>")}
+    out_ids = [i for i in out_ids if i not in special_ids]
+
     # decode the original input plus newly generated
-    out_text = tokenizer.decode(outputs[0], skip_special_tokens=True)
+    out_text = sp.decode(out_ids)
+    print("Otput text ", out_text)
     return out_text
 
 # Gradio Interface (Text Completion style)
 gr.Interface(
     fn=predict,
-    inputs=[gr.Textbox(label="Shona Input"), gr.Slider(10, 200, value=50, step=1, label="Max New Tokens"), gr.Slider(0.1, 1.0, value=0.7, step=0.1, label="Temperature")],
+    inputs=[
+        gr.Textbox(label="Shona Input"),
+        gr.Slider(10, 200, value=50, step=1, label="Max New Tokens"),
+        gr.Slider(0.1, 1.0, value=0.7, step=0.1, label="Temperature"),
+    ],
     outputs="text",
-    title="Shona Base Model",
+    title="Ngwanda Base Model",
     description="A base language model for Shona."
 ).launch()