Upload 4 files
Browse files- __init__.py +8 -0
- core.py +58 -18
- scoring.py +12 -19
__init__.py
ADDED
|
@@ -0,0 +1,8 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""
|
| 2 |
+
PRIVAL: Prompt Input Validation Toolkit
|
| 3 |
+
"""
|
| 4 |
+
|
| 5 |
+
# 对外暴露的核心接口
|
| 6 |
+
from .core import evaluate_prompt
|
| 7 |
+
|
| 8 |
+
__all__ = ["evaluate_prompt"]
|
core.py
CHANGED
|
@@ -1,21 +1,61 @@
|
|
| 1 |
-
|
| 2 |
import yaml
|
| 3 |
-
import concurrent.futures
|
| 4 |
-
from .detectors import clarity, ambiguity, step_guidance, verbosity, injection_risk, context_completeness, ethic_compliance, structural_cleanness, relevance, feasibility, grammar_spelling, length_appropriateness, diversity
|
| 5 |
|
| 6 |
-
|
| 7 |
-
|
| 8 |
-
|
| 9 |
-
|
| 10 |
-
|
| 11 |
-
|
| 12 |
-
|
| 13 |
-
|
| 14 |
-
|
| 15 |
-
|
| 16 |
-
|
| 17 |
-
|
| 18 |
-
|
| 19 |
-
|
| 20 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 21 |
}
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
import os
|
| 2 |
import yaml
|
|
|
|
|
|
|
| 3 |
|
| 4 |
+
from .detectors import (
|
| 5 |
+
clarity,
|
| 6 |
+
ambiguity,
|
| 7 |
+
step_guidance,
|
| 8 |
+
verbosity,
|
| 9 |
+
injection_risk,
|
| 10 |
+
context_completeness,
|
| 11 |
+
ethic_compliance,
|
| 12 |
+
structural_cleanness,
|
| 13 |
+
relevance,
|
| 14 |
+
feasibility,
|
| 15 |
+
grammar_spelling,
|
| 16 |
+
length_appropriateness,
|
| 17 |
+
diversity,
|
| 18 |
+
)
|
| 19 |
+
from .scoring import aggregate_scores
|
| 20 |
+
|
| 21 |
+
# Mapping from dimension name to detector function
|
| 22 |
+
_DETECTORS = {
|
| 23 |
+
'clarity': clarity.detect_clarity,
|
| 24 |
+
'ambiguity': ambiguity.detect_ambiguity,
|
| 25 |
+
'step_guidance': step_guidance.detect_step_guidance,
|
| 26 |
+
'verbosity': verbosity.detect_verbosity,
|
| 27 |
+
'injection_risk': injection_risk.detect_injection_risk,
|
| 28 |
+
'context_completeness': context_completeness.detect_context_completeness,
|
| 29 |
+
'ethic_compliance': ethic_compliance.detect_ethic_compliance,
|
| 30 |
+
'structural_cleanness': structural_cleanness.structural_cleanness,
|
| 31 |
+
'relevance': relevance.detect_relevance,
|
| 32 |
+
'feasibility': feasibility.detect_feasibility,
|
| 33 |
+
'grammar_spelling': grammar_spelling.grammar_spelling,
|
| 34 |
+
'length_appropriateness': length_appropriateness.detect_length_appropriateness,
|
| 35 |
+
'diversity': diversity.detect_diversity,
|
| 36 |
}
|
| 37 |
+
|
| 38 |
+
def evaluate_prompt(prompt: str) -> dict:
|
| 39 |
+
"""
|
| 40 |
+
Evaluate the given prompt across all enabled dimensions and return
|
| 41 |
+
a dictionary of results plus a total score.
|
| 42 |
+
"""
|
| 43 |
+
# Load configuration
|
| 44 |
+
here = os.path.dirname(__file__)
|
| 45 |
+
config_path = os.path.abspath(os.path.join(here, os.pardir, 'config.yaml'))
|
| 46 |
+
with open(config_path, encoding='utf-8') as cfg_file:
|
| 47 |
+
config = yaml.safe_load(cfg_file)
|
| 48 |
+
|
| 49 |
+
enabled = config.get('enabled_dimensions', [])
|
| 50 |
+
results = {}
|
| 51 |
+
for dim, func in _DETECTORS.items():
|
| 52 |
+
if dim in enabled:
|
| 53 |
+
try:
|
| 54 |
+
results[dim] = func(prompt)
|
| 55 |
+
except Exception as e:
|
| 56 |
+
results[dim] = {'score': None, 'suggestions': [f'Error in {dim}: {e}']}
|
| 57 |
+
|
| 58 |
+
# Compute total score
|
| 59 |
+
total = aggregate_scores(results, config)
|
| 60 |
+
results['total_score'] = total
|
| 61 |
+
return results
|
scoring.py
CHANGED
|
@@ -1,24 +1,17 @@
|
|
| 1 |
-
# scoring.py
|
| 2 |
"""
|
| 3 |
-
|
|
|
|
| 4 |
"""
|
| 5 |
|
| 6 |
-
def
|
| 7 |
-
"""按 weights 对 scores 中每个维度加权平均,返回总分(0.0–1.0)。"""
|
| 8 |
-
total_weight = sum(weights.values())
|
| 9 |
-
if total_weight == 0:
|
| 10 |
-
return 0.0
|
| 11 |
-
weighted_sum = sum(scores[dim] * weights.get(dim, 0) for dim in scores)
|
| 12 |
-
return round(weighted_sum / total_weight, 4)
|
| 13 |
-
|
| 14 |
-
|
| 15 |
-
def format_scores(scores: dict, suggestions: dict, overall: float) -> dict:
|
| 16 |
"""
|
| 17 |
-
|
| 18 |
-
返回格式:{"scores": {...}, "suggestions": {...}, "overall": float}
|
| 19 |
"""
|
| 20 |
-
|
| 21 |
-
|
| 22 |
-
|
| 23 |
-
|
| 24 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
"""
|
| 2 |
+
Scoring utilities for PRIVAL.
|
| 3 |
+
Aggregates individual dimension scores into a total score.
|
| 4 |
"""
|
| 5 |
|
| 6 |
+
def aggregate_scores(results: dict, config: dict) -> float:
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 7 |
"""
|
| 8 |
+
Compute the overall score as the simple average of available numeric scores.
|
|
|
|
| 9 |
"""
|
| 10 |
+
scores = []
|
| 11 |
+
for dim, res in results.items():
|
| 12 |
+
score = res.get('score')
|
| 13 |
+
if isinstance(score, (int, float)):
|
| 14 |
+
scores.append(score)
|
| 15 |
+
if not scores:
|
| 16 |
+
return None
|
| 17 |
+
return sum(scores) / len(scores)
|