refactor(ingest): simplify HF dataset ingestion to core datasets npc-dialogue, fictional-characters, and tinystories

- Remove support for multi-character, system-chat, arxiv, prompt-report, novels, manuals, enterprise, portuguese-edu, and edustories datasets
- Replace removed transformers with SyntheticFictionalCharactersTransformer and TinyStoriesNarrativeTransformer
- Update CLI defaults and options accordingly
- Add kagglehub[hf-datasets]>=0.3.0 dependency to requirements.txt

This refactoring reduces complexity by focusing on fewer, targeted datasets for data preprocessing.

requirements.txt

@@ -27,6 +27,7 @@ requests>=2.32.0
 
 # Data Processing
 datasets>=3.1.0
+kagglehub[hf-datasets]>=0.3.0
 pyyaml>=6.0.2
 pdfplumber>=0.11.0
 

warbler_cda/utils/hf_warbler_ingest.py

@@ -14,15 +14,8 @@ import click
 
 from .transformers import (
     NPCDialogueTransformer,
-    MultiCharacterTransformer,
-    SystemChatTransformer,
-    ArxivTransformer,
-    PromptReportTransformer,
-    NovelsTransformer,
-    ManualsTransformer,
-    EnterpriseTransformer,
-    PortugueseEducationTransformer,
-    EdustoriesTransformer,
+    SyntheticFictionalCharactersTransformer,
+    TinyStoriesNarrativeTransformer,
     WarblerPackBuilder,
 )
 
@@ -47,25 +40,15 @@ def cli():
     type=click.Choice(
         [
             "npc-dialogue",
-            "multi-character",
-            "system-chat",
-            "arxiv",
-            "prompt-report",
-            "novels",
-            "manuals",
-            "enterprise",
-            "portuguese-edu",
-            "edustories",
+            "fictional-characters",
+            "tinystories",
             "all",
         ]
     ),
-    default=["arxiv"],
+    default=["npc-dialogue"],
     help="Datasets to ingest",
 )
 @click.option("--pack-prefix", "-p", default="warbler-pack-hf", help="Prefix for pack names")
-@click.option(
-    "--arxiv-limit", type=int, default=None, help="Limit number of arXiv papers to ingest (HARD LIMIT: 250,000 for 1GB storage compliance)"
-)
 @click.option(
     "--max-docs-per-chunk",
     type=int,
@@ -78,7 +61,7 @@ def cli():
     default=None,
     help="Maximum PDF pages to extract (default: None for unlimited)",
 )
-def ingest(datasets, pack_prefix, arxiv_limit, max_docs_per_chunk, max_pdf_pages):
+def ingest(datasets, pack_prefix, max_docs_per_chunk, max_pdf_pages):
     """Ingest HF datasets into Warbler packs."""
     PACKS_DIR.mkdir(exist_ok=True, parents=True)
     builder = WarblerPackBuilder(PACKS_DIR)
@@ -86,13 +69,9 @@ def ingest(datasets, pack_prefix, arxiv_limit, max_docs_per_chunk, max_pdf_pages
 
     if "all" in datasets:
         datasets = [
-            "arxiv",
-            "prompt-report",
-            "novels",
-            "manuals",
-            "enterprise",
-            "portuguese-edu",
-            "edustories",
+            "npc-dialogue",
+            "fictional-characters",
+            "tinystories",
         ]
 
     if max_docs_per_chunk > 0:
@@ -118,42 +97,14 @@ def ingest(datasets, pack_prefix, arxiv_limit, max_docs_per_chunk, max_pdf_pages
                 transformer = NPCDialogueTransformer(max_pdf_pages=max_pdf_pages)
                 docs = transformer.transform()
                 pack_name = f"{pack_prefix}-npc-dialogue"
-            elif dataset == "multi-character":
-                transformer = MultiCharacterTransformer(max_pdf_pages=max_pdf_pages)
-                docs = transformer.transform()
-                pack_name = f"{pack_prefix}-multi-character"
-            elif dataset == "system-chat":
-                transformer = SystemChatTransformer(max_pdf_pages=max_pdf_pages)
-                docs = transformer.transform()
-                pack_name = f"{pack_prefix}-system-chat"
-            elif dataset == "arxiv":
-                transformer = ArxivTransformer(max_pdf_pages=max_pdf_pages)
-                docs = transformer.transform(limit=arxiv_limit)
-                pack_name = f"{pack_prefix}-arxiv"
-            elif dataset == "prompt-report":
-                transformer = PromptReportTransformer(max_pdf_pages=max_pdf_pages)
-                docs = transformer.transform()
-                pack_name = f"{pack_prefix}-prompt-report"
-            elif dataset == "novels":
-                transformer = NovelsTransformer(max_pdf_pages=max_pdf_pages)
-                docs = transformer.transform()
-                pack_name = f"{pack_prefix}-novels"
-            elif dataset == "manuals":
-                transformer = ManualsTransformer(max_pdf_pages=max_pdf_pages)
-                docs = transformer.transform()
-                pack_name = f"{pack_prefix}-manuals"
-            elif dataset == "enterprise":
-                transformer = EnterpriseTransformer(max_pdf_pages=max_pdf_pages)
+            elif dataset == "fictional-characters":
+                transformer = SyntheticFictionalCharactersTransformer(max_pdf_pages=max_pdf_pages)
                 docs = transformer.transform()
-                pack_name = f"{pack_prefix}-enterprise"
-            elif dataset == "portuguese-edu":
-                transformer = PortugueseEducationTransformer(max_pdf_pages=max_pdf_pages)
+                pack_name = f"{pack_prefix}-fictional-characters"
+            elif dataset == "tinystories":
+                transformer = TinyStoriesNarrativeTransformer(max_pdf_pages=max_pdf_pages)
                 docs = transformer.transform()
-                pack_name = f"{pack_prefix}-portuguese-edu"
-            elif dataset == "edustories":
-                transformer = EdustoriesTransformer(max_pdf_pages=max_pdf_pages)
-                docs = transformer.transform()
-                pack_name = f"{pack_prefix}-edustories"
+                pack_name = f"{pack_prefix}-tinystories"
             else:
                 click.echo(f"[ERROR] Unknown dataset: {dataset}")
                 continue
@@ -214,42 +165,14 @@ class HFWarblerIngestor:
                 transformer = NPCDialogueTransformer(max_pdf_pages=max_pdf_pages)
                 docs = transformer.transform()
                 pack_name = f"{pack_prefix}-npc-dialogue"
-            elif dataset_name == "multi-character":
-                transformer = MultiCharacterTransformer(max_pdf_pages=max_pdf_pages)
-                docs = transformer.transform()
-                pack_name = f"{pack_prefix}-multi-character"
-            elif dataset_name == "system-chat":
-                transformer = SystemChatTransformer(max_pdf_pages=max_pdf_pages)
-                docs = transformer.transform()
-                pack_name = f"{pack_prefix}-system-chat"
-            elif dataset_name == "arxiv":
-                transformer = ArxivTransformer(max_pdf_pages=max_pdf_pages)
-                docs = transformer.transform(limit=arxiv_limit)
-                pack_name = f"{pack_prefix}-arxiv"
-            elif dataset_name == "prompt-report":
-                transformer = PromptReportTransformer(max_pdf_pages=max_pdf_pages)
-                docs = transformer.transform()
-                pack_name = f"{pack_prefix}-prompt-report"
-            elif dataset_name == "novels":
-                transformer = NovelsTransformer(max_pdf_pages=max_pdf_pages)
-                docs = transformer.transform()
-                pack_name = f"{pack_prefix}-novels"
-            elif dataset_name == "manuals":
-                transformer = ManualsTransformer(max_pdf_pages=max_pdf_pages)
-                docs = transformer.transform()
-                pack_name = f"{pack_prefix}-manuals"
-            elif dataset_name == "enterprise":
-                transformer = EnterpriseTransformer(max_pdf_pages=max_pdf_pages)
-                docs = transformer.transform()
-                pack_name = f"{pack_prefix}-enterprise"
-            elif dataset_name == "portuguese-edu":
-                transformer = PortugueseEducationTransformer(max_pdf_pages=max_pdf_pages)
+            elif dataset_name == "fictional-characters":
+                transformer = SyntheticFictionalCharactersTransformer(max_pdf_pages=max_pdf_pages)
                 docs = transformer.transform()
-                pack_name = f"{pack_prefix}-portuguese-edu"
-            elif dataset_name == "edustories":
-                transformer = EdustoriesTransformer(max_pdf_pages=max_pdf_pages)
+                pack_name = f"{pack_prefix}-fictional-characters"
+            elif dataset_name == "tinystories":
+                transformer = TinyStoriesNarrativeTransformer(max_pdf_pages=max_pdf_pages)
                 docs = transformer.transform()
-                pack_name = f"{pack_prefix}-edustories"
+                pack_name = f"{pack_prefix}-tinystories"
             else:
                 if self.verbose:
                     print(f"❌ Unknown dataset: {dataset_name}")

warbler_cda/utils/transformers/__init__.py

@@ -1,26 +1,12 @@
 from .base import BaseWarblerTransformer, WarblerPackBuilder
 from .npc_dialogue import NPCDialogueTransformer
-from .multi_character import MultiCharacterTransformer
-from .system_chat import SystemChatTransformer
-from .arxiv import ArxivTransformer
-from .prompt_report import PromptReportTransformer
-from .novels import NovelsTransformer
-from .manuals import ManualsTransformer
-from .enterprise import EnterpriseTransformer
-from .portuguese_education import PortugueseEducationTransformer
-from .edustories import EdustoriesTransformer
+from .synthetic_fictional_characters import SyntheticFictionalCharactersTransformer
+from .tiny_stories_narrative import TinyStoriesNarrativeTransformer
 
 __all__ = [
     "BaseWarblerTransformer",
     "WarblerPackBuilder",
     "NPCDialogueTransformer",
-    "MultiCharacterTransformer",
-    "SystemChatTransformer",
-    "ArxivTransformer",
-    "PromptReportTransformer",
-    "NovelsTransformer",
-    "ManualsTransformer",
-    "EnterpriseTransformer",
-    "PortugueseEducationTransformer",
-    "EdustoriesTransformer",
+    "SyntheticFictionalCharactersTransformer",
+    "TinyStoriesNarrativeTransformer",
 ]

warbler_cda/utils/transformers/synthetic_fictional_characters.py

@@ -0,0 +1,166 @@
+"""Synthetic Fictional Characters dataset transformer."""
+
+import logging
+from typing import List, Dict, Any
+
+import kagglehub
+from kagglehub import KaggleDatasetAdapter
+
+from .base import BaseWarblerTransformer
+
+
+logger = logging.getLogger(__name__)
+
+
+class SyntheticFictionalCharactersTransformer(BaseWarblerTransformer):
+    """Transform pratyushpuri/synthetic-fictional-characters-dataset."""
+
+    def transform(
+        self, dataset_name: str = "pratyushpuri/synthetic-fictional-characters-dataset",
+        file_path: str = ""
+    ) -> List[Dict[str, Any]]:
+        """
+        Transform synthetic fictional characters dataset.
+
+        Uses kagglehub with HF adapter to load the dataset, then transforms
+        character profiles into Warbler-compatible documents.
+
+        Fields include: Character Name, Media Type, Genre, Role, Personality Traits,
+        Backstory, Skills/Abilities, Appearance, Alignment, Relationships, etc.
+        """
+        logger.info(f"Loading {dataset_name}...")
+        try:
+            # Load using KaggleHub with HF adapter
+            hf_dataset = kagglehub.load_dataset(
+                KaggleDatasetAdapter.HUGGING_FACE,
+                dataset_name,
+                file_path,
+                # Provide any additional arguments like
+                # sql_query, hf_kwargs, or pandas_kwargs. See
+                # the documenation for more information:
+                # https://github.com/Kaggle/kagglehub/blob/main/README.md#kaggledatasetadapterhugging_face
+            )
+        except Exception as e:
+            logger.error(f"Failed to load {dataset_name}: {e}")
+            return []
+
+        warbler_docs = []
+
+        items = self.extract_dataset_items(hf_dataset)
+
+        for idx, item in enumerate(items):
+            if isinstance(item, dict):
+                try:
+                    doc = {
+                        "content_id": f"fictional-characters/{self._safe_slug(item.get('Character Name', f'character-{idx}'))}",
+                        "content": self._create_content(item),
+                        "metadata": {
+                            "pack": "warbler-pack-fictional-characters",
+                            "source_dataset": dataset_name,
+                            "character_name": item.get("Character Name", ""),
+                            "media_type": item.get("Media Type", ""),
+                            "media_source": item.get("Media Source", ""),
+                            "genre": item.get("Genre", ""),
+                            "role": item.get("Role", ""),
+                            "personality_traits": item.get("Personality Traits", ""),
+                            "skills_abilities": item.get("Skills/Abilities", ""),
+                            "alignment": item.get("Alignment", ""),
+                            "relationships": item.get("Relationships", ""),
+                            "significance_impact": item.get("Significance/Impact", ""),
+                            "realm_type": "character",
+                            "realm_label": "fictional_characters",
+                            "lifecycle_stage": "emergence",
+                            "activity_level": 0.85,
+                            "dialogue_type": "character_profile",
+                            "license": "MIT",
+                        },
+                    }
+                    warbler_docs.append(doc)
+                except Exception as e:
+                    logger.warning(f"Error processing character {idx}: {e}")
+                    continue
+
+        logger.info(f"✓ Transformed {len(warbler_docs)} fictional character profiles")
+        return warbler_docs
+
+    @staticmethod
+    def _safe_slug(text: str) -> str:
+        """Create a safe slug from text."""
+        if not text:
+            return "unknown"
+        return "".join(c for c in text.lower().replace(" ", "-") if c.isalnum() or c in "-").strip("-")
+
+    @staticmethod
+    def _create_content(item: Dict[str, Any]) -> str:
+        """Create content string for fictional character profile."""
+        character_name = item.get("Character Name", "Unknown Character")
+        media_type = item.get("Media Type", "")
+        media_source = item.get("Media Source", "")
+        genre = item.get("Genre", "")
+        role = item.get("Role", "")
+        personality_traits = item.get("Personality Traits", "")
+        backstory = item.get("Backstory", "")
+        skills_abilities = item.get("Skills/Abilities", "")
+        appearance_description = item.get("Appearance Description", "")
+        alignment = item.get("Alignment", "")
+        interests_hobbies = item.get("Interests/Hobbies", "")
+        relationships = item.get("Relationships", "")
+        significance_impact = item.get("Significance/Impact", "")
+        description = item.get("Description", "")
+        scenario_dialogue = item.get("Scenario/Dialogue Example", "")
+
+        content_parts = [
+            f"CHARACTER PROFILE: {character_name}",
+            ""
+        ]
+
+        if media_type and media_source:
+            content_parts.append(f"Source: {media_type} - {media_source}")
+        elif media_type:
+            content_parts.append(f"Media Type: {media_type}")
+
+        if genre:
+            content_parts.append(f"Genre: {genre}")
+
+        if role:
+            content_parts.append(f"Role: {role}")
+
+        content_parts.append("")
+
+        if personality_traits:
+            content_parts.append(f"Personality: {personality_traits}")
+
+        if appearance_description:
+            content_parts.append(f"Appearance: {appearance_description}")
+
+        if alignment:
+            content_parts.append(f"Alignment: {alignment}")
+
+        if skills_abilities:
+            content_parts.append(f"Skills & Abilities: {skills_abilities}")
+
+        if interests_hobbies:
+            content_parts.append(f"Interests & Hobbies: {interests_hobbies}")
+
+        content_parts.append("")
+
+        if backstory:
+            content_parts.append(f"Backstory: {backstory}")
+
+        if relationships:
+            content_parts.append(f"Relationships: {relationships}")
+
+        if significance_impact:
+            content_parts.append(f"Significance: {significance_impact}")
+
+        if description:
+            content_parts.append(f"Detailed Description: {description}")
+
+        if scenario_dialogue:
+            content_parts.append("")
+            content_parts.append(f"Example Scenario: {scenario_dialogue}")
+
+        content_parts.append("")
+        content_parts.append("This comprehensive character profile supports narrative development and character-driven storytelling.")
+
+        return "\n".join(content_parts)

warbler_cda/utils/transformers/tiny_stories_narrative.py

@@ -0,0 +1,141 @@
+"""Tiny Stories Narrative dataset transformer."""
+
+import logging
+from typing import List, Dict, Any
+
+import kagglehub
+from kagglehub import KaggleDatasetAdapter
+
+from .base import BaseWarblerTransformer
+
+
+logger = logging.getLogger(__name__)
+
+
+class TinyStoriesNarrativeTransformer(BaseWarblerTransformer):
+    """Transform thedevastator/tinystories-narrative-classification dataset."""
+
+    def transform(
+        self, dataset_name: str = "thedevastator/tinystories-narrative-classification",
+        file_path: str = ""
+    ) -> List[Dict[str, Any]]:
+        """
+        Transform TinyStories narrative classification dataset.
+
+        Uses kagglehub with HF adapter to load the dataset containing short
+        stories with characters, locations, and narrative elements.
+
+        The dataset contains story texts that demonstrate various narrative patterns,
+        character interactions, and storytelling techniques.
+        """
+        logger.info(f"Loading {dataset_name}...")
+        try:
+            # Load using KaggleHub with HF adapter
+            hf_dataset = kagglehub.load_dataset(
+                KaggleDatasetAdapter.HUGGING_FACE,
+                dataset_name,
+                file_path,
+                # Provide any additional arguments like
+                # sql_query, hf_kwargs, or pandas_kwargs. See
+                # the documenation for more information:
+                # https://github.com/Kaggle/kagglehub/blob/main/README.md#kaggledatasetadapterhugging_face
+            )
+        except Exception as e:
+            logger.error(f"Failed to load {dataset_name}: {e}")
+            return []
+
+        warbler_docs = []
+
+        items = self.extract_dataset_items(hf_dataset)
+
+        for idx, item in enumerate(items):
+            if isinstance(item, dict):
+                try:
+                    story_text = item.get("text", "").strip()
+                    if not story_text:
+                        continue
+
+                    # Extract basic narrative features for metadata
+                    narrative_features = self._analyze_narrative_features(story_text)
+
+                    doc = {
+                        "content_id": f"tinystories/{idx:06d}",
+                        "content": self._create_content(story_text),
+                        "metadata": {
+                            "pack": "warbler-pack-tinystories",
+                            "source_dataset": dataset_name,
+                            "story_length": len(story_text),
+                            "word_count": len(story_text.split()),
+                            "has_characters": narrative_features.get("has_characters", False),
+                            "has_location": narrative_features.get("has_location", False),
+                            "has_dialogue": narrative_features.get("has_dialogue", False),
+                            "narrative_elements": narrative_features.get("elements", []),
+                            "realm_type": "narrative",
+                            "realm_label": "tinystories",
+                            "lifecycle_stage": "emergence",
+                            "activity_level": 0.75,
+                            "dialogue_type": "story",
+                            "license": "CC0",
+                        },
+                    }
+                    warbler_docs.append(doc)
+                except Exception as e:
+                    logger.warning(f"Error processing story {idx}: {e}")
+                    continue
+
+        logger.info(f"✓ Transformed {len(warbler_docs)} tiny story narratives")
+        return warbler_docs
+
+    @staticmethod
+    def _analyze_narrative_features(story_text: str) -> Dict[str, Any]:
+        """Extract basic narrative features from story text."""
+        features = {
+            "has_characters": False,
+            "has_location": False,
+            "has_dialogue": False,
+            "elements": []
+        }
+
+        text_lower = story_text.lower()
+
+        # Check for character indicators
+        character_indicators = [
+            " once ", " there was ", " lived ", " the ", " said ", " asked ", " replied ",
+            " cried ", " shouted ", " whispered ", " thought ", " wanted ", " decided "
+        ]
+        if any(indicator in text_lower for indicator in character_indicators):
+            features["has_characters"] = True
+            features["elements"].append("characters")
+
+        # Check for location indicators
+        location_indicators = [
+            " house ", " home ", " village ", " town ", " forest ", " mountain ", " river ",
+            " garden ", " school ", " castle ", " kingdom ", " world ", " place "
+        ]
+        if any(indicator in text_lower for indicator in location_indicators):
+            features["has_location"] = True
+            features["elements"].append("locations")
+
+        # Check for dialogue indicators
+        if '"' in story_text or "'" in story_text or " said " in text_lower:
+            features["has_dialogue"] = True
+            features["elements"].append("dialogue")
+
+        # Additional narrative elements
+        if " and " in text_lower or " then " in text_lower:
+            features["elements"].append("sequencing")
+        if " happy " in text_lower or " sad " in text_lower or " angry " in text_lower:
+            features["elements"].append("emotions")
+        if any(word in text_lower for word in ["because", "so", "therefore"]):
+            features["elements"].append("causality")
+
+        return features
+
+    @staticmethod
+    def _create_content(story_text: str) -> str:
+        """Create content string for tiny story narrative."""
+        return f"""TINY STORY NARRATIVE
+
+{story_text}
+
+This short story demonstrates fundamental narrative patterns, character development, and storytelling techniques suitable for cognitive narrative analysis."""