core/dataset_builder.py

# core/dataset_builder.py
import json
import logging
from pathlib import Path
from typing import List, Dict

logger = logging.getLogger("lucie.dataset_builder")

class DatasetBuilder:
    def __init__(self, vector_memory, error_tree, validation_queue, data_dir="/data/datasets"):
        self.vector_memory = vector_memory
        self.error_tree = error_tree
        self.validation_queue = validation_queue
        self.data_dir = Path(data_dir)
        self.data_dir.mkdir(parents=True, exist_ok=True)

    def build_from_memory(self, limit: int = 100) -> List[Dict]:
        episodes = self.vector_memory.search("*", n=limit, min_score=0.0)
        dataset = []
        for ep in episodes:
            text = ep.get("text", "")
            score = ep.get("metadata", {}).get("score", 0.5)
            if score > 0.3:
                dataset.append({
                    "instruction": text[:200],
                    "response": text[200:800],
                    "score": score,
                    "source": "memory"
                })
        return dataset

    def build_from_errors(self, limit: int = 50) -> List[Dict]:
        dataset = []
        if hasattr(self.error_tree, "get_recent_errors"):
            errors = self.error_tree.get_recent_errors(limit=limit)
            for e in errors:
                dataset.append({
                    "instruction": f"Corrige l'erreur : {e.get('context', '')}",
                    "response": e.get('correction', 'Aucune correction'),
                    "score": 0.7,
                    "source": "error"
                })
        return dataset

    def build_from_validations(self, limit: int = 50) -> List[Dict]:
        dataset = []
        pending = self.validation_queue.get_pending()
        for prop in pending[:limit]:
            dataset.append({
                "instruction": f"Valide la proposition : {prop.description}",
                "response": prop.code,
                "score": prop.score,
                "source": "validation"
            })
        return dataset

    def build_full_dataset(self) -> List[Dict]:
        dataset = []
        dataset.extend(self.build_from_memory(100))
        dataset.extend(self.build_from_errors(50))
        dataset.extend(self.build_from_validations(30))
        seen = set()
        unique = []
        for item in dataset:
            key = item["instruction"][:50]
            if key not in seen:
                seen.add(key)
                unique.append(item)
        return unique

    def save_jsonl(self, dataset: List[Dict], filename: str = "dataset.jsonl") -> Path:
        path = self.data_dir / filename
        with open(path, "w") as f:
            for item in dataset:
                f.write(json.dumps(item) + "\n")
        return path