Update app.py

app.py

@@ -123,13 +123,36 @@ def Dataset_Creator_Function(dataset_name: str, conversation_data: str) -> str:
                 # Process based on data structure
                 if isinstance(json_data, list) and all(isinstance(item, dict) for item in json_data):
                     log_text += f"Processing JSON array with {len(json_data)} items\n"
-                    df = pd.DataFrame(json_data)
-                    ds = Dataset.from_pandas(df)
+                    
+                    # Create a dataset with columns for all keys in the first item
+                    # This ensures the dataset structure is consistent
+                    first_item = json_data[0]
+                    columns = list(first_item.keys())
+                    log_text += f"Detected columns: {columns}\n"
+                    
+                    # Initialize data dictionary with empty lists for each column
+                    data_dict = {col: [] for col in columns}
+                    
+                    # Process each item
+                    for item in json_data:
+                        for col in columns:
+                            # Get the value for this column, or empty string if missing
+                            value = item.get(col, "")
+                            data_dict[col].append(value)
+                    
+                    # Debug output to verify data structure
+                    for col in columns:
+                        log_text += f"Column '{col}' has {len(data_dict[col])} entries\n"
+                        
+                    # Create dataset from dictionary
+                    ds = Dataset.from_dict(data_dict)
+                    log_text += f"Created dataset with {len(ds)} rows\n"
                     created_ds = DatasetDict({"train": ds})
                 elif isinstance(json_data, dict):
                     log_text += "Processing single JSON object\n"
-                    df = pd.DataFrame([json_data])
-                    ds = Dataset.from_pandas(df)
+                    # For a single object, create a dataset with one row
+                    data_dict = {k: [v] for k, v in json_data.items()}
+                    ds = Dataset.from_dict(data_dict)
                     created_ds = DatasetDict({"train": ds})
                 else:
                     raise ValueError("JSON not recognized as array or single object")
@@ -142,20 +165,29 @@ def Dataset_Creator_Function(dataset_name: str, conversation_data: str) -> str:
             if '|' in conversation_data and len(lines) > 1:
                 log_text += "Processing as pipe-separated data\n"
                 headers = [h.strip() for h in lines[0].split('|')]
-                data = []
+                log_text += f"Detected headers: {headers}\n"
+                
+                # Initialize data dictionary
+                data_dict = {header: [] for header in headers}
+                
+                # Process each data row
                 for i, line in enumerate(lines[1:], 1):
                     if not line.strip():
                         continue
                     values = [val.strip() for val in line.split('|')]
                     if len(values) == len(headers):
-                        data.append(dict(zip(headers, values)))
+                        for j, header in enumerate(headers):
+                            data_dict[header].append(values[j])
                     else:
                         log_text += f"Warning: Skipping row {i} (column count mismatch)\n"
-                if data:
-                    df = pd.DataFrame(data)
-                    ds = Dataset.from_pandas(df)
+                
+                # Create dataset from dictionary
+                if all(len(values) > 0 for values in data_dict.values()):
+                    ds = Dataset.from_dict(data_dict)
+                    log_text += f"Created dataset with {len(ds)} rows\n"
                     created_ds = DatasetDict({"train": ds})
                 else:
+                    log_text += "No valid rows found in pipe-separated data\n"
                     created_ds = DatasetDict({"train": Dataset.from_dict({"text": [conversation_data]})})
             else:
                 # Fallback for plain text