File size: 1,929 Bytes
9f198fa | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 | {
"name": "RAG Evaluation Lab",
"problem": "RAG systems often ship without a stable regression set or failure taxonomy.",
"domain": "ai-evaluation",
"architecture": "classifier",
"hugging_face_tasks": [
"text-classification",
"question-answering",
"text-ranking",
"summarization"
],
"recommended_stack": [
"FastAPI for evaluation jobs and reports",
"Ragas-style retrieval and faithfulness metrics",
"MLflow for experiment and artifact tracking",
"PostgreSQL for versioned evaluation cases",
"OpenTelemetry plus Phoenix for trace inspection",
"GitHub Actions for threshold-based release gates"
],
"real_world_data_sources": [
{
"name": "GitHub REST API",
"url": "https://api.github.com/repos/huggingface/transformers/issues?state=open&per_page=5",
"purpose": "Real technical questions for evaluation-set construction"
},
{
"name": "arXiv API",
"url": "https://export.arxiv.org/api/query?search_query=all:retrieval%20augmented%20generation&start=0&max_results=5",
"purpose": "Public RAG literature for grounded-answer cases"
}
],
"job_description_skills": [
"LLM and RAG evaluation design",
"Golden datasets and failure taxonomies",
"Experiment tracking and model release gates",
"Trace-level diagnosis and prompt regression testing",
"Statistical comparison of AI system versions"
],
"impact_targets": [
"Detect 100% of seeded unsupported-answer regressions",
"Track retrieval, faithfulness, citation, latency, and cost metrics",
"Fail CI when any critical metric drops beyond tolerance",
"Produce comparable evaluation reports for every model version"
],
"baseline_evaluation": {
"test_examples": 4,
"accuracy": 0.75,
"synthetic_evaluation": true
},
"estimated_delivery": "8-12 weeks for one engineer",
"generated_baseline_is_production_ready": false
}
|