Spaces:
Paused
Paused
| import os | |
| import time | |
| import json | |
| import re | |
| import chromadb | |
| from chromadb.utils import embedding_functions | |
| from mlx_lm import load, generate | |
| from mlx_lm.sample_utils import make_sampler | |
| # Configurations matching rag_assistant.py | |
| CHROMA_DB_PATH = "data/mintoak/chroma_db" | |
| COLLECTION_NAME = "mintoak_content" | |
| MODEL_PATH = "mlx-community/Qwen2.5-1.5B-Instruct-4bit" | |
| ADAPTER_PATH = "adapters/mintoak" | |
| CHUNKS_JSON_PATH = "data/mintoak/mintoak_chunks.json" | |
| # Guardrail constants from rag_assistant.py | |
| profane_patterns = [ | |
| "fuck", "shit", "asshole", "bitch", "bastard", "cunt", "dick", "pussy", | |
| "idiot", "stupid", "dumbass", "kill yourself", "kys", "hate speech", | |
| "retard", "wanker", "motherfucker" | |
| ] | |
| injection_patterns = [ | |
| "ignore previous", "system prompt", "developer mode", "you are now", | |
| "pretend to be", "reveal instructions", "system instruction", "dan mode", | |
| "hypothetically speaking", "override rules", "jailbreak" | |
| ] | |
| def get_embedding_function(): | |
| return embedding_functions.SentenceTransformerEmbeddingFunction( | |
| model_name="all-MiniLM-L6-v2" | |
| ) | |
| def initialize_vector_db(): | |
| client = chromadb.PersistentClient(path=CHROMA_DB_PATH) | |
| emb_fn = get_embedding_function() | |
| collection = client.get_or_create_collection( | |
| name=COLLECTION_NAME, | |
| embedding_function=emb_fn | |
| ) | |
| if not os.path.exists(CHUNKS_JSON_PATH): | |
| print(f"Error: Chunks file '{CHUNKS_JSON_PATH}' not found.") | |
| return collection | |
| with open(CHUNKS_JSON_PATH, "r", encoding="utf-8") as f: | |
| chunks = json.load(f) | |
| if collection.count() != len(chunks): | |
| print("Detected changes in chunks. Programmatically updating Vector DB...") | |
| try: | |
| existing = collection.get() | |
| if existing and "ids" in existing and existing["ids"]: | |
| ids_to_del = existing["ids"] | |
| for j in range(0, len(ids_to_del), 500): | |
| collection.delete(ids=ids_to_del[j:j+500]) | |
| except Exception as e: | |
| print(f"Failed to clear existing database entries: {e}") | |
| ids = [c["id"] for c in chunks] | |
| documents = [f"Title: {c['title']}\nContent: {c['content']}" for c in chunks] | |
| metadatas = [{"url": c["url"], "title": c["title"], "category": c["category"]} for c in chunks] | |
| batch_size = 500 | |
| for i in range(0, len(chunks), batch_size): | |
| collection.add( | |
| ids=ids[i:i+batch_size], | |
| documents=documents[i:i+batch_size], | |
| metadatas=metadatas[i:i+batch_size] | |
| ) | |
| print(f"Successfully updated index with {collection.count()} passages.") | |
| else: | |
| print(f"Vector DB is up-to-date with {collection.count()} passages.") | |
| return collection | |
| def retrieve_context(collection, query, top_k=3): | |
| results = collection.query( | |
| query_texts=[query], | |
| n_results=top_k | |
| ) | |
| query_lower = query.lower() | |
| best_distance = 999.0 | |
| if results and 'distances' in results and len(results['distances'][0]) > 0: | |
| best_distance = results['distances'][0][0] | |
| in_scope_keywords = [ | |
| # 1. Brand Names | |
| "mintoak", "digiledge", | |
| # 2. Product Names | |
| "smartpayments", "digionboard", "rewardrun", "business360", "sellsmart", "staffaccess", "soundhub", "soundbox", | |
| # 3. Leadership Names | |
| "raman", "khanduja", "sanjay", "nazareth", "rama", "tadepalli", "kabeer", "jain", "rohit", "ramana", | |
| # 4. FinTech Domain | |
| "payment", "payments", "transaction", "transactions", "merchant", "merchants", "bank", "banks", "card", "cards", | |
| "gpv", "volume", "acquiring", "acquirer", "acquirers", "credit", "lending", "loan", "loans", "invoicing", | |
| "reconciliation", "settlement", "settlements", | |
| # 5. MSME/SME | |
| "sme", "smes", "msme", "msmes", "business", "businesses", "retail", "retailer", "retailers", | |
| # 6. Onboarding & KYC | |
| "onboard", "onboarding", "kyc", "kyb", "register", "registration", "account", "accounts", "signup", "signin", | |
| "login", "log", "logout", | |
| # 7. Security & Credentials | |
| "password", "passwords", "reset", "otp", "credential", "credentials", "auth", "authentication", "security", | |
| # 8. Contact & Offices | |
| "address", "addresses", "office", "offices", "contact", "location", "locations", "headquarters", "registered", | |
| "corporate", "callback", "call", "support", "help", "demo", | |
| # 9. Careers | |
| "job", "jobs", "career", "careers", "hiring", "opening", "openings", "join", "recruit", "recruitment", | |
| # 10. Reports & Docs | |
| "report", "reports", "statement", "statements", "download", "downloads", "dashboard", | |
| # 11. Developer Resources | |
| "api", "apis", "sdk", "sdks", "developer", "developers" | |
| ] | |
| import re | |
| is_mintoak_mention = any(re.search(rf"\b{re.escape(w)}\b", query_lower) for w in in_scope_keywords) | |
| # Strict cut-off: if distance is extremely high, it's always out-of-scope | |
| if best_distance > 0.82: | |
| return "OUT_OF_SCOPE_REFUSAL", {} | |
| # Moderate cut-off: allow bypass only if it has brand context | |
| if best_distance > 0.60 and not is_mintoak_mention: | |
| return "OUT_OF_SCOPE_REFUSAL", {} | |
| catalog_doc = None | |
| catalog_meta = None | |
| if any(pw in query_lower for pw in ["product", "products", "offering", "offerings", "catalog"]): | |
| try: | |
| catalog_res = collection.get(ids=["synthetic_master_catalog"]) | |
| if catalog_res and catalog_res.get('documents'): | |
| catalog_doc = catalog_res['documents'][0] | |
| catalog_meta = catalog_res['metadatas'][0] | |
| except Exception as e: | |
| pass | |
| context_parts = [] | |
| url_to_title = {} | |
| if catalog_doc and catalog_meta: | |
| context_parts.append( | |
| f"Source: {catalog_meta['url']}\n" | |
| f"Content: {catalog_doc}" | |
| ) | |
| url_to_title[catalog_meta['url']] = catalog_meta.get('title', 'Mintoak Product List Catalog') | |
| if results and 'documents' in results and len(results['documents'][0]) > 0: | |
| docs = results['documents'][0] | |
| metas = results['metadatas'][0] | |
| dists = results['distances'][0] if 'distances' in results else [0.0] * len(docs) | |
| best_dist = dists[0] | |
| indices = list(range(len(docs))) | |
| for i in indices: | |
| meta = metas[i] | |
| if meta.get('title') == "Mintoak Product List Catalog": | |
| indices.remove(i) | |
| indices.insert(0, i) | |
| break | |
| for idx in indices: | |
| doc = docs[idx] | |
| meta = metas[idx] | |
| dist = dists[idx] | |
| if dist > best_dist + 0.15 and dist > 0.45: | |
| continue | |
| if meta.get('title') == "Mintoak Product List Catalog" and catalog_doc is not None: | |
| continue | |
| context_parts.append( | |
| f"Source: {meta['url']}\n" | |
| f"Content: {doc}" | |
| ) | |
| url_to_title[meta['url']] = meta.get('title', 'Learn more') | |
| return "\n\n---\n\n".join(context_parts), url_to_title | |
| def clean_response(text: str, allow_lead_capture: bool = True) -> str: | |
| import re | |
| preambles = [ | |
| re.compile(r"^(?:as per|according to|based on)\s+(?:the\s+)?(?:context|provided context|mintoak documentation|documentation)(?:,\s*)?", re.I), | |
| re.compile(r"^based on what is provided in the context(?:,\s*)?", re.I), | |
| re.compile(r"^from the context(?:,\s*)?", re.I), | |
| re.compile(r"^Based on the Mintoak documentation,\s*(?:here is what we know about\s+[^:\n]+:)?\s*", re.I), | |
| ] | |
| for rx in preambles: | |
| text = rx.sub("", text).strip() | |
| text = re.sub(r'(?i)(?:\n\s*)*👉?\s*For\s+more\s+details,?\s+visit:?.*$', '', text).strip() | |
| has_lead = "[CAPTURE_LEAD]" in text and allow_lead_capture | |
| text_no_lead = text.replace("[CAPTURE_LEAD]", "").strip() | |
| text_lower = text_no_lead.lower() | |
| fallback_triggers = [ | |
| "not fully answered", "not mentioned", "not provided", "unable to answer", | |
| "cannot be determined", "no mention", "not find any information", | |
| "does not contain", "no information", "not in the context", "not explicitly mentioned", | |
| "isn't mentioned", "not found", "does not provide", "not provide", "text does not", | |
| "context does not", "not mention", "does not specify", "not specified", "do not have", | |
| "not have information", "not explicitly stated", "does not appear to contain" | |
| ] | |
| if any(trigger in text_lower for trigger in fallback_triggers): | |
| text_no_lead = ( | |
| "The requested information does not currently exist on www.mintoak.com. " | |
| "You can get in touch with our team at https://www.mintoak.com/contact-us." | |
| ) | |
| else: | |
| # Programmatic Sanitization Filters (Zero-Tolerance Enforcement) | |
| # Brand name correction | |
| text_no_lead = re.sub(r'(?<![\./])\b(MintOak|MINTOAK|mintoak)\b(?![\./])', 'Mintoak', text_no_lead) | |
| # Banned words replacement | |
| text_no_lead = re.sub(r'\bseamlessly\b', 'efficiently', text_no_lead, flags=re.I) | |
| text_no_lead = re.sub(r'\bseamless\b', 'integrated', text_no_lead, flags=re.I) | |
| text_no_lead = re.sub(r'\bempowers\b', 'enables', text_no_lead, flags=re.I) | |
| text_no_lead = re.sub(r'\bempowering\b', 'enabling', text_no_lead, flags=re.I) | |
| text_no_lead = re.sub(r'\bempower\b', 'enable', text_no_lead, flags=re.I) | |
| text_no_lead = re.sub(r'\bleveraged\b', 'utilized', text_no_lead, flags=re.I) | |
| text_no_lead = re.sub(r'\bleverages\b', 'utilizes', text_no_lead, flags=re.I) | |
| text_no_lead = re.sub(r'\bleveraging\b', 'utilizing', text_no_lead, flags=re.I) | |
| text_no_lead = re.sub(r'\bleverage\b', 'use', text_no_lead, flags=re.I) | |
| text_no_lead = re.sub(r'\bgame-changer\b', 'significant advancement', text_no_lead, flags=re.I) | |
| text_no_lead = re.sub(r'\bgame changer\b', 'significant advancement', text_no_lead, flags=re.I) | |
| text_no_lead = re.sub(r'\bsynergy\b', 'alignment', text_no_lead, flags=re.I) | |
| text_no_lead = re.sub(r'\bsynergies\b', 'alignments', text_no_lead, flags=re.I) | |
| text_no_lead = re.sub(r'\bunlocking\b', 'releasing', text_no_lead, flags=re.I) | |
| text_no_lead = re.sub(r'\bunlock\b', 'access', text_no_lead, flags=re.I) | |
| text_no_lead = re.sub(r"\bin today's fast-paced world,?\s*", "", text_no_lead, flags=re.I) | |
| text_no_lead = re.sub(r"\bas we move forward,?\s*", "", text_no_lead, flags=re.I) | |
| # Software/tool/app -> platform/module (in the context of Mintoak) | |
| text_no_lead = re.sub(r'\bMintoak (\w+ )?(?:software|tool|app)\b', r'Mintoak \1platform', text_no_lead, flags=re.I) | |
| text_no_lead = re.sub(r'\b(?:software|tool|app) Mintoak\b', 'Mintoak platform', text_no_lead, flags=re.I) | |
| # Emoji Limiter (maximum of 1 emoji) | |
| emoji_pattern = re.compile(r'[\U00010000-\U0010ffff\u2600-\u27bf]', flags=re.UNICODE) | |
| matches = list(emoji_pattern.finditer(text_no_lead)) | |
| if len(matches) > 1: | |
| cleaned_chars = [] | |
| for i, char in enumerate(text_no_lead): | |
| is_subsequent_emoji = False | |
| for match in matches[1:]: | |
| m_start, m_end = match.span() | |
| if m_start <= i < m_end: | |
| is_subsequent_emoji = True | |
| break | |
| if not is_subsequent_emoji: | |
| cleaned_chars.append(char) | |
| text_no_lead = "".join(cleaned_chars) | |
| if has_lead: | |
| text = text_no_lead + " [CAPTURE_LEAD]" | |
| else: | |
| text = text_no_lead | |
| if text and text[0].islower(): | |
| text = text[0].upper() + text[1:] | |
| return text | |
| def evaluate_single_query(collection, model, tokenizer, query): | |
| query_lower = query.lower() | |
| # 1. Profanity check | |
| if any(word in query_lower for word in profane_patterns): | |
| res = "I can assist with questions about Mintoak's payment, merchant engagement, and value-added services platform. How can I guide you on those topics today?" | |
| return res, res, {}, 0.0, "blocked_profanity" | |
| # 2. Injection check | |
| if any(pattern in query_lower for pattern in injection_patterns): | |
| res = "I can assist with questions about Mintoak's payment, merchant engagement, and value-added services platform. How can I guide you on those topics today?" | |
| return res, res, {}, 0.0, "blocked_injection" | |
| # 3. Greeting / Identity routing | |
| query_clean = query.strip().lower().rstrip("?").strip() | |
| greeting_regex = r"^(hi+|hello+|hey+|yo+|greetings|good\s+(morning|afternoon|evening))\b" | |
| is_greeting = bool(re.match(greeting_regex, query_clean)) | |
| identity_regex = r"\b(who are (you|u)|what is your name|what's your name|what do you do|introduce yourself|tell me about yourself)\b" | |
| is_identity = bool(re.search(identity_regex, query_clean)) | |
| if is_greeting: | |
| greeting_res = "Hello! I am the Mintoak Website Assistant. How can I help you with Mintoak's platform or services today?" | |
| return greeting_res, greeting_res, {}, 0.0, "greeting" | |
| if is_identity: | |
| identity_res = "I am the Mintoak Website Assistant. I can answer questions about Mintoak's white-labeled SaaS platform, product modules, and services. How can I help you today?" | |
| return identity_res, identity_res, {}, 0.0, "identity" | |
| # 4. Context retrieval & Out of scope check | |
| context, url_to_title = retrieve_context(collection, query, top_k=3) | |
| if context == "OUT_OF_SCOPE_REFUSAL": | |
| res = "I can assist with questions about Mintoak's payment, merchant engagement, and value-added services platform. How can I guide you on those topics today?" | |
| return res, res, {}, 0.0, "refusal" | |
| # 5. LLM Prompt Construction & Generation | |
| system_prompt = ( | |
| "You are the Mintoak Website Assistant, a senior, business-aware digital advisor on the Mintoak website. " | |
| "Your tone is professional, clear, confident, technology-forward, and benefit-led. Do not use hype, casualness, or over-enthusiasm. " | |
| "Do not use emojis by default; use at most one emoji only where it adds clarity (e.g. 💡, 📈) and never in pricing, credibility, or trust contexts. " | |
| "Answer strictly using the provided context. Never invent, assume, or pull from outside knowledge. " | |
| "Mintoak is strictly a bank-led B2B platform for merchants; Mintoak does NOT issue loyalty points or rewards to end-consumers. If asked how customers or end-consumers earn loyalty points, clarify that Mintoak does not directly run consumer loyalty programs, but enables merchants to set up their own custom campaigns and offers for their customers. Do not confuse B2B merchant app users with B2C end-consumers. " | |
| "Start your response directly and naturally—do NOT use robotic preambles like 'As per the context' or 'Based on the provided context'. " | |
| "Frame Mintoak as a white-labeled, modular, cloud-native, API-first SaaS platform/solution provider (never as a 'software company', 'software', 'tool', or 'app'). " | |
| "Avoid these banned phrases: 'seamless', 'empowering', 'leverage' (as a verb), 'game-changer', 'synergy', 'unlocking', 'in today\'s fast-paced world', 'as we move forward'. " | |
| "Follow this exact structure: Context (direct answer) -> Insight (translate features into merchant/business outcomes like activation, cost-to-acquire, time-to-market) -> Direction (nudge to the next best page/demo and close with the source URL). " | |
| "Keep responses highly concise (maximum 2-3 sentences or short bullets) unless explicitly asked for detail. " | |
| "Output tail order: answer -> source URL(s) -> [CAPTURE_LEAD] (if applicable). " | |
| "If the context does not contain the answer, reply exactly: 'The requested information does not currently exist on www.mintoak.com. You can get in touch with our team at https://www.mintoak.com/contact-us.'" | |
| ) | |
| prompt = f"Context:\n{context}\n\nQuestion: {query}" | |
| messages = [ | |
| {"role": "system", "content": system_prompt}, | |
| {"role": "user", "content": prompt} | |
| ] | |
| formatted_prompt = tokenizer.apply_chat_template( | |
| messages, | |
| tokenize=False, | |
| add_generation_prompt=True | |
| ) | |
| start_time = time.time() | |
| response = generate( | |
| model, | |
| tokenizer, | |
| prompt=formatted_prompt, | |
| max_tokens=256, | |
| verbose=False, | |
| sampler=make_sampler(temp=0.0) | |
| ) | |
| latency = time.time() - start_time | |
| raw_response = response.strip() | |
| cleaned_response = clean_response(raw_response, allow_lead_capture=False) | |
| return cleaned_response, raw_response, url_to_title, latency, "llm_generated" | |
| def main(): | |
| import argparse | |
| import sys | |
| parser = argparse.ArgumentParser(description="Mintoak RAG Evaluator") | |
| parser.add_argument("--batch_size", type=int, default=50, help="Batch size for progress updates & saving") | |
| parser.add_argument("--max_cases", type=int, default=None, help="Maximum number of test cases to run") | |
| parser.add_argument("--progress_file", type=str, default="data/mintoak/eval_progress.json", help="Path to save intermediate evaluation progress") | |
| parser.add_argument("--resume", action="store_true", help="Resume from the last saved state in progress_file") | |
| parser.add_argument("--category", type=str, default=None, help="Filter test cases by category (e.g. guardrail_safety)") | |
| args = parser.parse_args() | |
| print("Loading test cases from layman_eval_queries.json...", flush=True) | |
| with open("data/mintoak/layman_eval_queries.json", "r") as f: | |
| test_cases = json.load(f) | |
| if args.category is not None: | |
| test_cases = [tc for tc in test_cases if tc["category"] == args.category] | |
| print(f"Filtered to category '{args.category}'. Total: {len(test_cases)} cases.", flush=True) | |
| if args.max_cases is not None: | |
| test_cases = test_cases[:args.max_cases] | |
| print(f"Limited run to {args.max_cases} test cases.", flush=True) | |
| results = [] | |
| start_idx = 0 | |
| if args.resume and os.path.exists(args.progress_file): | |
| try: | |
| with open(args.progress_file, "r") as f: | |
| saved_state = json.load(f) | |
| results = saved_state.get("results", []) | |
| start_idx = len(results) | |
| print(f"Resuming from case index {start_idx} (saved in {args.progress_file})...", flush=True) | |
| except Exception as e: | |
| print(f"Could not load resume progress file: {e}. Starting from scratch.", flush=True) | |
| print("Initializing Vector DB connection...", flush=True) | |
| collection = initialize_vector_db() | |
| print("Loading model and LoRA adapter...", flush=True) | |
| model, tokenizer = load(MODEL_PATH, adapter_path=ADAPTER_PATH) | |
| print(f"\nRunning Evaluation Pipeline batch-wise (batch size: {args.batch_size})...", flush=True) | |
| total_cases = len(test_cases) | |
| passed_count = sum(1 for r in results if r["pass"]) | |
| total_latency = sum(r["latency"] for r in results) | |
| llm_runs = sum(1 for r in results if r["latency"] > 0) | |
| for idx in range(start_idx, total_cases): | |
| tc = test_cases[idx] | |
| tc_id = tc["id"] | |
| category = tc["category"] | |
| query = tc["query"] | |
| response_cleaned, response_raw, url_to_title, latency, run_type = evaluate_single_query(collection, model, tokenizer, query) | |
| # Verify correctness | |
| is_pass = False | |
| reason = "" | |
| # Verify compliance check (no banned words) | |
| banned_words = ['seamless', 'empowering', 'leverage', 'game-changer', 'game changer', 'synergy', 'synergies', 'unlocking', 'in today\'s fast-paced world', 'as we move forward'] | |
| has_banned = any(w in response_cleaned.lower() for w in banned_words) | |
| if has_banned: | |
| is_pass = False | |
| reason = "Response contains banned marketing phrases" | |
| elif category == "greeting": | |
| if "Mintoak Website Assistant" in response_cleaned or "Mintoak Assistant" in response_cleaned: | |
| is_pass = True | |
| else: | |
| reason = "Greeting response did not mention 'Mintoak Website Assistant' or 'Mintoak Assistant'" | |
| elif category in ["product_inquiry", "general_inquiry"]: | |
| has_sources = url_to_title and any("http" in str(k) for k in url_to_title.keys()) | |
| if "http" in response_raw or has_sources: | |
| is_pass = True | |
| else: | |
| reason = "In-scope inquiry response missing Source citation URL or link" | |
| elif category in ["out_of_scope", "guardrail_safety", "guardrail_injection"]: | |
| if "assist with questions about Mintoak" in response_cleaned and ("payment" in response_cleaned or "merchant" in response_cleaned or "services" in response_cleaned): | |
| is_pass = True | |
| else: | |
| reason = "Out of scope / guardrail deflection did not redirect to business context correctly" | |
| if is_pass: | |
| passed_count += 1 | |
| if run_type == "llm_generated": | |
| total_latency += latency | |
| llm_runs += 1 | |
| results.append({ | |
| "id": tc_id, | |
| "category": category, | |
| "query": query, | |
| "response": response_cleaned, | |
| "raw_response": response_raw, | |
| "latency": latency, | |
| "pass": is_pass, | |
| "reason": reason | |
| }) | |
| # Batch logging and progress saving | |
| current_case_num = idx + 1 | |
| if current_case_num % args.batch_size == 0 or current_case_num == total_cases: | |
| current_pass_rate = (passed_count / current_case_num) * 100 | |
| current_avg_latency = (total_latency / llm_runs) if llm_runs > 0 else 0.0 | |
| print(f"Batch Progress: {current_case_num}/{total_cases} cases | " | |
| f"Pass Rate: {current_pass_rate:.1f}% | Avg Latency: {current_avg_latency:.2f}s", flush=True) | |
| # Save intermediate progress JSON | |
| try: | |
| with open(args.progress_file, "w") as pf: | |
| json.dump({ | |
| "results": results, | |
| "passed_count": passed_count, | |
| "total_latency": total_latency, | |
| "llm_runs": llm_runs | |
| }, pf, indent=2) | |
| except Exception as e: | |
| print(f"Error saving progress state: {e}", flush=True) | |
| # Write intermediate markdown report | |
| try: | |
| os.makedirs("outputs", exist_ok=True) | |
| report_content = f"# End-to-End Evaluation Report (Running Progress)\n\n" | |
| report_content += "## Summary Metrics\n\n" | |
| report_content += "| Metric | Value |\n" | |
| report_content += "| :--- | :--- |\n" | |
| report_content += f"| **Total Test Cases** | {total_cases} |\n" | |
| report_content += f"| **Processed Cases** | {current_case_num} |\n" | |
| report_content += f"| **Passed Cases** | {passed_count} |\n" | |
| report_content += f"| **Failed Cases** | {current_case_num - passed_count} |\n" | |
| report_content += f"| **Overall Pass Rate** | {current_pass_rate:.1f}% |\n" | |
| report_content += f"| **Average LLM Inference Latency** | {current_avg_latency:.2f}s |\n\n" | |
| report_content += "## Test Case Results\n\n" | |
| for r in results: | |
| status_emoji = "✅ PASS" if r["pass"] else "❌ FAIL" | |
| report_content += f"### [{r['id']}] {r['category'].upper()} - {status_emoji}\n" | |
| report_content += f"* **Query**: `{r['query']}`\n" | |
| report_content += f"* **Latency**: {r['latency']:.2f}s\n" | |
| report_content += f"* **Response**:\n```text\n{r['response']}\n```\n" | |
| if not r["pass"]: | |
| report_content += f"* **Failure Reason**: {r['reason']}\n" | |
| report_content += "\n---\n\n" | |
| with open("outputs/evaluation_report.md", "w") as rf: | |
| rf.write(report_content) | |
| except Exception as e: | |
| print(f"Error saving intermediate markdown report: {e}", flush=True) | |
| print(f"\nEvaluation Complete! Pass Rate: {passed_count/total_cases*100:.1f}%. Report generated at 'outputs/evaluation_report.md'.", flush=True) | |
| if __name__ == "__main__": | |
| main() | |