Change to Bahasa Indonesia

app.py

@@ -14,11 +14,9 @@ from openpyxl import load_workbook
 from typing import List, Dict, Any, Tuple
 from utils import *
 
-# # === [1] Model and Tokenizer Loading ===
 # base_model_id = "NousResearch/Nous-Hermes-2-Mistral-7B-DPO"
 # lora_path = "tat-llm-final-e4"
 
-# # Load base model and LoRA adapter
 # base_model = AutoModelForCausalLM.from_pretrained(base_model_id, torch_dtype=torch.float16)
 # model = PeftModel.from_pretrained(base_model, lora_path)
 
@@ -26,23 +24,19 @@ from utils import *
 # model = model.to(device)
 # model.eval()
 
-# # Load tokenizer
 # tokenizer = AutoTokenizer.from_pretrained(lora_path)
 
-# === Updated Generate Answer Function ===
 @spaces.GPU(duration=60)
 def generate_answer(json_data: Dict[str, Any], question: str) -> str:
     """
     Generate answer using the fine-tuned model.
     """
-        # === [1] Model and Tokenizer Loading ===
     base_model_id = "NousResearch/Nous-Hermes-2-Mistral-7B-DPO"
     lora_path = "tat-llm-final-e4"
     
     # Load base model and LoRA adapter
     base_model = AutoModelForCausalLM.from_pretrained(base_model_id, torch_dtype=torch.float16)
     model = PeftModel.from_pretrained(base_model, lora_path)
-    # Load tokenizer
     tokenizer = AutoTokenizer.from_pretrained(lora_path)
     device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
     model = model.to(device)
@@ -55,7 +49,6 @@ def generate_answer(json_data: Dict[str, Any], question: str) -> str:
     device = next(model.parameters()).device
     inputs = {k: v.to(device) for k, v in inputs.items()}
     
-    # Get input length to extract only generated text
     input_length = inputs["input_ids"].shape[1]
     
     with torch.no_grad():
@@ -67,13 +60,12 @@ def generate_answer(json_data: Dict[str, Any], question: str) -> str:
             pad_token_id=tokenizer.eos_token_id
         )
     
-    # Decode only the generated part
     generated_tokens = outputs[0][input_length:]
     answer = tokenizer.decode(generated_tokens, skip_special_tokens=True)
     
     return answer
 
-# === Gradio Interface Functions ===
+# Gradio interface functions
 def process_xlsx(file):
     """
     Process uploaded XLSX file and return JSON, JSONL, and Markdown.
@@ -82,10 +74,8 @@ def process_xlsx(file):
         return None, "", "", ""
     
     try:
-        # Convert XLSX to JSON
         json_data = xlsx_to_json(file.name)
-        
-        # Generate different formats
+
         json_str = json.dumps(json_data, indent=2, ensure_ascii=False)
         jsonl_str = json_to_jsonl(json_data)
         markdown_str = json_to_markdown(json_data)
@@ -110,7 +100,7 @@ def chat_interface(json_data, question, history):
     except Exception as e:
         return history + [[question, f"Error generating answer: {str(e)}"]]
 
-# === Gradio UI ===
+# Gradio UI
 with gr.Blocks(title="TAT-LLM: Semi-Tabular Data QA", theme=gr.themes.Soft()) as demo:
     gr.HTML("""
     <style>
@@ -132,12 +122,10 @@ with gr.Blocks(title="TAT-LLM: Semi-Tabular Data QA", theme=gr.themes.Soft()) as
     Sistem akan mengonversi berkas Anda ke format JSON dan menggunakan model TAT-LLM yang telah disempurnakan untuk menjawab pertanyaan.
     """)
     
-    # State to store JSON data
     json_data_state = gr.State()
     
     with gr.Row():
         with gr.Column(scale=1):
-            # File upload section
             file_input = gr.File(
                 label="Upload XLSX File",
                 file_types=[".xlsx"],
@@ -146,7 +134,6 @@ with gr.Blocks(title="TAT-LLM: Semi-Tabular Data QA", theme=gr.themes.Soft()) as
             
             process_btn = gr.Button("Process File", variant="primary")
             
-            # Format display tabs
             with gr.Tabs():
                 with gr.Tab("Markdown Preview"):
                     markdown_output = gr.Markdown(label="Markdown Preview")
@@ -166,7 +153,6 @@ with gr.Blocks(title="TAT-LLM: Semi-Tabular Data QA", theme=gr.themes.Soft()) as
                     )
         
         with gr.Column(scale=1):
-            # Chat interface
             gr.Markdown("### Ajukan Pertanyaan Mengenai Data Anda")
             chatbot = gr.Chatbot(height=400)
             msg = gr.Textbox(
@@ -179,7 +165,6 @@ with gr.Blocks(title="TAT-LLM: Semi-Tabular Data QA", theme=gr.themes.Soft()) as
                 submit_btn = gr.Button("Submit", variant="primary")
                 clear_btn = gr.Button("Clear Chat")
             
-            # Example questions
             gr.Examples(
                 examples=[
                     "Apa saja wawasan yang bisa kita ambil dari data ini?",
@@ -191,7 +176,6 @@ with gr.Blocks(title="TAT-LLM: Semi-Tabular Data QA", theme=gr.themes.Soft()) as
                 inputs=msg
             )
     
-    # Event handlers
     process_btn.click(
         fn=process_xlsx,
         inputs=[file_input],
@@ -221,6 +205,5 @@ with gr.Blocks(title="TAT-LLM: Semi-Tabular Data QA", theme=gr.themes.Soft()) as
         outputs=[chatbot]
     )
 
-# Launch the app
 if __name__ == "__main__":
     demo.queue().launch(share=True)

utils.py

@@ -3,7 +3,7 @@ import json
 from typing import List, Dict, Any, Tuple
 from openpyxl import load_workbook
 
-# === XLSX to JSON Conversion Functions ===
+# XLSX to JSON conversion functions
 def detect_table_and_paragraphs(worksheet) -> Tuple[List[List[str]], List[Dict[str, Any]]]:
     data = []
     max_col = worksheet.max_column
@@ -13,7 +13,6 @@ def detect_table_and_paragraphs(worksheet) -> Tuple[List[List[str]], List[Dict[s
         if any(cell is not None for cell in row):
             data.append([str(cell).strip() if cell is not None else "" for cell in row])
 
-    # Try detecting start of a table
     table_data = []
     paragraph_texts = []
     in_table = False
@@ -25,7 +24,6 @@ def detect_table_and_paragraphs(worksheet) -> Tuple[List[List[str]], List[Dict[s
             in_table = True
             table_data.append(row)
         elif in_table and len(non_empty) >= 2:
-            # Continue table (in case of header rows or descriptive rows)
             table_data.append(row)
         else:
             paragraph = " ".join(non_empty)
@@ -44,65 +42,48 @@ def detect_table_and_paragraphs(worksheet) -> Tuple[List[List[str]], List[Dict[s
     return table_data, paragraphs
 
 def xlsx_to_json(file_path) -> Dict[str, Any]:
-    """
-    Convert XLSX file to TAT-QA JSON format.
-    """
     workbook = load_workbook(file_path, data_only=True)
     worksheet = workbook.active
     
-    # Extract table and paragraphs
     table_data, paragraphs = detect_table_and_paragraphs(worksheet)
     
-    # Create JSON structure
     json_data = {
         "table": {
             "uid": str(uuid.uuid4()),
             "table": table_data
         },
         "paragraphs": paragraphs,
-        "questions": []  # Empty for user to fill later
+        "questions": []
     }
     
     return json_data
 
 def json_to_jsonl(json_data: Dict[str, Any]) -> str:
-    """
-    Convert JSON to JSONL format (one JSON object per line).
-    """
     return json.dumps(json_data, ensure_ascii=False)
 
 def json_to_markdown(json_data: Dict[str, Any]) -> str:
-    """
-    Convert JSON data to markdown format for display.
-    """
-    markdown_content = "## Table Data\n\n"
+    markdown_content = "## Data Tabel\n\n"
     
     # Convert table to markdown
     table = json_data["table"]["table"]
     if table:
-        # Create markdown table
         markdown_content += "| " + " | ".join(table[0]) + " |\n"
         markdown_content += "| " + " | ".join(["---"] * len(table[0])) + " |\n"
         for row in table[1:]:
             markdown_content += "| " + " | ".join(row) + " |\n"
     
     # Add paragraphs
-    markdown_content += "\n## Context/Paragraphs\n\n"
+    markdown_content += "\n## Konteks/Paragraf\n\n"
     for para in json_data["paragraphs"]:
         markdown_content += f"{para['order']}. {para['text']}\n\n"
     
     return markdown_content
 
-# === Updated Prompt Creation Function ===
+# Prompt creation function
 def create_prompt(table_data: Dict[str, Any], question: str) -> str:
-    """
-    Create prompt in the same format as training data.
-    """
-    # Convert table to markdown format
     table = table_data["table"]["table"]
     table_md = "\n".join(["| " + " | ".join(row) + " |" for row in table])
     
-    # Extract paragraph texts
     text_content = "\n".join([p["text"] for p in table_data["paragraphs"]])
     
     prompt = f"""### Instruction