Create app.py

app.py

@@ -0,0 +1,253 @@
+from transformers import AutoModelForCausalLM, AutoTokenizer
+import gradio as gr
+import json
+import torch
+import re
+import random
+
+class TableDataGenerator:
+    def __init__(self, model_name="Qwen/Qwen2.5-3B-Instruct"):
+        self.model_name = model_name
+        self.model = AutoModelForCausalLM.from_pretrained(
+            model_name,
+            torch_dtype="auto",
+            device_map="auto"
+        )
+        self.tokenizer = AutoTokenizer.from_pretrained(model_name)
+        
+    def generate_batch_data(self, llm_commands, num_rows=1000, batch_size=50):
+        """Generate table data in batches for better performance"""
+        all_rows = []
+        
+        # Create column headers description
+        columns_desc = ", ".join([f"Column {i+1}: {cmd}" for i, cmd in enumerate(llm_commands)])
+        
+        # Calculate number of batches
+        num_batches = (num_rows + batch_size - 1) // batch_size
+        
+        for batch_idx in range(num_batches):
+            current_batch_size = min(batch_size, num_rows - len(all_rows))
+            
+            # Create prompt for this batch
+            prompt = f"""Generate {current_batch_size} rows of realistic data for a table with these columns:
+{columns_desc}
+
+Requirements:
+- Each row should be different and realistic
+- Return ONLY a Python list format like: [['value1', 'value2'], ['value3', 'value4'], ...]
+- Make the data diverse and realistic
+- Use seed value {batch_idx + 1} for variety
+- No explanations, just the list
+
+Generate {current_batch_size} rows:"""
+
+            messages = [
+                {"role": "system", "content": "You are a data generator. Return only valid Python list format with realistic, diverse data."},
+                {"role": "user", "content": prompt}
+            ]
+            
+            # Generate response
+            response = self._generate_response(messages)
+            
+            # Parse the response to extract rows
+            batch_rows = self._parse_response(response, len(llm_commands))
+            
+            # Add to all rows
+            all_rows.extend(batch_rows)
+            
+            # Break if we have enough rows
+            if len(all_rows) >= num_rows:
+                break
+                
+        return all_rows[:num_rows]
+    
+    def _generate_response(self, messages):
+        """Generate response from the model"""
+        text = self.tokenizer.apply_chat_template(
+            messages,
+            tokenize=False,
+            add_generation_prompt=True
+        )
+        
+        model_inputs = self.tokenizer([text], return_tensors="pt").to(self.model.device)
+        
+        # Set random seed for variety
+        torch.manual_seed(random.randint(1, 10000))
+        
+        generated_ids = self.model.generate(
+            **model_inputs,
+            max_new_tokens=512,
+            temperature=0.8,
+            do_sample=True,
+            top_p=0.9
+        )
+        
+        generated_ids = [
+            output_ids[len(input_ids):] for input_ids, output_ids in zip(model_inputs.input_ids, generated_ids)
+        ]
+        
+        response = self.tokenizer.batch_decode(generated_ids, skip_special_tokens=True)[0]
+        return response
+    
+    def _parse_response(self, response, expected_columns):
+        """Parse the model response to extract table rows"""
+        rows = []
+        
+        try:
+            # Try to find list-like patterns in the response
+            # Look for patterns like [['value1', 'value2'], ['value3', 'value4']]
+            list_pattern = r'\[\s*\[.*?\]\s*\]'
+            matches = re.findall(list_pattern, response, re.DOTALL)
+            
+            if matches:
+                # Try to evaluate the largest match
+                largest_match = max(matches, key=len)
+                try:
+                    parsed_data = eval(largest_match)
+                    if isinstance(parsed_data, list):
+                        for row in parsed_data:
+                            if isinstance(row, list) and len(row) == expected_columns:
+                                rows.append([str(item) for item in row])
+                except:
+                    pass
+            
+            # If no valid list found, try to extract individual rows
+            if not rows:
+                # Look for individual row patterns like ['value1', 'value2']
+                row_pattern = r'\[([^\[\]]+)\]'
+                row_matches = re.findall(row_pattern, response)
+                
+                for match in row_matches:
+                    try:
+                        # Split by comma and clean up
+                        items = [item.strip().strip('"\'') for item in match.split(',')]
+                        if len(items) == expected_columns:
+                            rows.append(items)
+                    except:
+                        continue
+            
+        except Exception as e:
+            print(f"Error parsing response: {e}")
+        
+        return rows
+
+def generate_table_data(json_input, num_rows=1000):
+    """Main function to generate table data from JSON input"""
+    try:
+        # Parse JSON input
+        data = json.loads(json_input)
+        llm_commands = data.get('llm_commands', [])
+        
+        if not llm_commands:
+            return "Error: No llm_commands found in JSON input"
+        
+        # Initialize generator
+        generator = TableDataGenerator()
+        
+        # Generate data
+        rows = generator.generate_batch_data(llm_commands, num_rows)
+        
+        # Format output
+        result = f"Generated {len(rows)} rows:\n"
+        result += f"Columns: {llm_commands}\n\n"
+        
+        # Show first 10 rows as preview
+        result += "First 10 rows:\n"
+        for i, row in enumerate(rows[:10]):
+            result += f"{i+1}: {row}\n"
+            
+        if len(rows) > 10:
+            result += f"\n... and {len(rows) - 10} more rows"
+            
+        return result, rows
+        
+    except json.JSONDecodeError:
+        return "Error: Invalid JSON format", []
+    except Exception as e:
+        return f"Error: {str(e)}", []
+
+# Gradio Interface
+def process_json_input(json_input, num_rows):
+    """Process JSON input and return formatted results"""
+    result_text, rows = generate_table_data(json_input, int(num_rows))
+    
+    # Also return the raw data as a downloadable file
+    if rows:
+        csv_content = "\n".join([",".join(row) for row in rows])
+        return result_text, csv_content
+    else:
+        return result_text, ""
+
+# Create Gradio interface
+with gr.Blocks(title="Table Data Generator") as demo:
+    gr.Markdown("# Table Data Generator using LLM")
+    gr.Markdown("Generate realistic table data based on column descriptions")
+    
+    with gr.Row():
+        with gr.Column():
+            json_input = gr.Textbox(
+                label="JSON Input",
+                placeholder='{"llm_commands": ["ages between 1 to 20", "arabic name"]}',
+                lines=3,
+                value='{"llm_commands": ["ages between 1 to 20", "arabic name"]}'
+            )
+            num_rows = gr.Slider(
+                minimum=10,
+                maximum=2000,
+                value=100,
+                step=10,
+                label="Number of rows to generate"
+            )
+            generate_btn = gr.Button("Generate Data", variant="primary")
+            
+        with gr.Column():
+            output_text = gr.Textbox(
+                label="Generated Data Preview",
+                lines=15,
+                max_lines=20
+            )
+            download_csv = gr.File(
+                label="Download CSV",
+                visible=True
+            )
+    
+    def generate_and_save(json_input, num_rows):
+        result_text, csv_content = process_json_input(json_input, num_rows)
+        
+        if csv_content:
+            # Save to temporary file
+            import tempfile
+            import os
+            
+            with tempfile.NamedTemporaryFile(mode='w', suffix='.csv', delete=False) as f:
+                f.write(csv_content)
+                temp_path = f.name
+            
+            return result_text, temp_path
+        else:
+            return result_text, None
+    
+    generate_btn.click(
+        fn=generate_and_save,
+        inputs=[json_input, num_rows],
+        outputs=[output_text, download_csv]
+    )
+    
+    # Example inputs
+    gr.Examples(
+        examples=[
+            ['{"llm_commands": ["ages between 1 to 20", "arabic name"]}', 50],
+            ['{"llm_commands": ["random city", "population number", "country"]}', 100],
+            ['{"llm_commands": ["product name", "price in USD", "category"]}', 75],
+            ['{"llm_commands": ["email address", "phone number", "job title"]}', 60]
+        ],
+        inputs=[json_input, num_rows]
+    )
+
+if __name__ == "__main__":
+    demo.launch()
+
+# Example usage:
+# json_input = '{"llm_commands": ["ages between 1 to 20", "arabic name"]}'
+# result_text, rows = generate_table_data(json_input, 1000)
+# print(result_text)