Update processor.py

processor.py

@@ -2,6 +2,7 @@ import json
 import logging
 import datasets
 from datasets import load_dataset, get_dataset_config_names, get_dataset_infos
+from huggingface_hub import HfApi, DatasetCard, DatasetCardData
 
 logging.basicConfig(level=logging.INFO)
 logger = logging.getLogger(__name__)
@@ -25,7 +26,27 @@ class DatasetCommandCenter:
                     splits = list(infos.values())[0].splits.keys()
             except:
                 splits = ['train', 'test', 'validation']
-            return {"status": "success", "configs": configs, "splits": list(splits)}
+            # license    
+            try:
+                configs = get_dataset_config_names(dataset_id, token=self.token)
+            except:
+                configs = ['default']
+            
+            license_name = "unknown"
+            try:
+                infos = get_dataset_infos(dataset_id, token=self.token)
+                # Try to grab license from first config
+                first = list(infos.values())[0]
+                license_name = first.license or "unknown"
+            except:
+                pass
+    
+            return {
+                "status": "success", 
+                "configs": configs, 
+                # We assume user will pick splits later, just return configs + license hint
+                "license_detected": license_name
+            }
         except Exception as e:
             return {"status": "error", "message": str(e)}
 
@@ -219,6 +240,59 @@ class DatasetCommandCenter:
         except:
             return False
 
+            
+    def _generate_card(self, source_id, target_id, recipe, license_name):
+        """
+        Generates a README.md with YAML metadata and a report of operations.
+        """
+        
+        # 1. YAML Metadata
+        card_data = DatasetCardData(
+            language="en",
+            license=license_name,
+            tags=["dataset-command-center", "etl", "generated-dataset"],
+            base_model=source_id, # Linking source
+        )
+        
+        # 2. Description & Recipe Table
+        content = f"""
+# {target_id.split('/')[-1]}
+
+This dataset is a transformation of [{source_id}](https://huggingface.co/datasets/{source_id}).
+It was generated using the **Hugging Face Dataset Command Center**.
+
+## Transformation Recipe
+
+The following operations were applied to the source data:
+
+| Target Column | Source | Type | Logic / Filter |
+|---------------|--------|------|----------------|
+"""
+        
+        for col in recipe['columns']:
+            c_type = col.get('type', 'simple')
+            c_name = col['name']
+            c_src = col.get('source', '-')
+            
+            if c_type == 'simple':
+                logic = "Direct Mapping"
+            elif c_type == 'list_search':
+                logic = f"Get `{col['target_key']}` where `{col['filter_key']} == {col['filter_val']}`"
+            elif c_type == 'python':
+                logic = f"`{col.get('expression')}`"
+            else:
+                logic = "-"
+            
+            content += f"| **{c_name}** | `{c_src}` | {c_type} | {logic} |\n"
+
+        if recipe.get('filter_rule'):
+            content += f"\n### Row Filtering\n**Filter Applied:** `{recipe['filter_rule']}`\n"
+
+        content += f"\n## Original License\nThis dataset inherits the license: `{license_name}` from the source."
+
+        card = DatasetCard.from_template(card_data, content=content)
+        return card
+
     def process_and_push(self, source_id, config, split, target_id, recipe, max_rows=None):
         logger.info(f"Job started: {source_id}")
         conf = config if config != 'default' else None
@@ -236,9 +310,17 @@ class DatasetCommandCenter:
         try:
             new_dataset = datasets.Dataset.from_generator(gen)
             new_dataset.push_to_hub(target_id, token=self.token)
+            # 2. GENERATE & PUSH CARD
+            try:
+                card = self._generate_card(source_id, target_id, recipe, new_license or "unknown")
+                card.push_to_hub(target_id, token=self.token)
+            except Exception as e:
+                logger.warning(f"Could not push dataset card: {e}")
+
             return {"status": "success", "rows_processed": len(new_dataset)}
         except Exception as e:
             return {"status": "error", "message": str(e)}
+
             
     def preview_transform(self, dataset_id, config, split, recipe):
         conf = config if config != 'default' else None