""" Perplexity analysis module for measuring model uncertainty. Mathematical formulation: PPL = exp(-1/N * Σ log p(w_i | w_{ Optional[float]: """ Compute perplexity of generated texts. PPL = exp(-1/N * Σ log p(w_i | w_{ 20: return float("inf") return float(math.exp(mean_loss)) def compute_cross_entropy(self, texts: List[str]) -> Optional[float]: """ Compute mean cross-entropy loss. Args: texts: List of generated texts Returns: Mean cross-entropy value """ clean = self._filter_valid_texts(texts) if not clean: return None device = next(self.model.parameters()).device losses = [] for text in clean: try: enc = self.tokenizer(text, return_tensors="pt", truncation=True, max_length=512) input_ids = enc["input_ids"].to(device) if input_ids.shape[1] < 2: continue with torch.no_grad(): outputs = self.model(input_ids=input_ids, labels=input_ids) if outputs.loss is not None: losses.append(float(outputs.loss.detach().cpu())) except Exception: continue return float(np.mean(losses)) if losses else None @staticmethod def _filter_valid_texts(texts: List[str]) -> List[str]: """Filter out empty or error texts.""" return [t.strip() for t in texts if t and not t.startswith("ERROR")] def compute_all(self, texts: List[str]) -> dict: """Compute both perplexity and cross-entropy.""" return { "Perplexity": self.compute_perplexity(texts), "Cross-Entropy": self.compute_cross_entropy(texts), }