Upload test_production_model_comprehensive.py with huggingface_hub

test_production_model_comprehensive.py

@@ -0,0 +1,160 @@
+# /// script
+# dependencies = ["transformers>=4.40.0", "peft>=0.7.0", "bitsandbytes>=0.41.0", "accelerate>=0.28.0"]
+# ///
+
+from transformers import AutoModelForCausalLM, AutoTokenizer, BitsAndBytesConfig
+from peft import PeftModel
+import torch
+
+print("=" * 80)
+print("COMPREHENSIVE BIOMEDICAL MODEL EVALUATION")
+print("=" * 80)
+print("\nModel: panikos/llama-biomedical-production-qlora")
+print("Training: 17,008 examples, 1 epoch, QLoRA")
+print("=" * 80)
+
+print("\n[1/3] Loading base model with 4-bit quantization...")
+bnb_config = BitsAndBytesConfig(
+    load_in_4bit=True,
+    bnb_4bit_use_double_quant=True,
+    bnb_4bit_quant_type="nf4",
+    bnb_4bit_compute_dtype=torch.bfloat16
+)
+
+base_model = AutoModelForCausalLM.from_pretrained(
+    "meta-llama/Llama-3.1-8B-Instruct",
+    quantization_config=bnb_config,
+    device_map="auto"
+)
+
+tokenizer = AutoTokenizer.from_pretrained("meta-llama/Llama-3.1-8B-Instruct")
+tokenizer.pad_token = tokenizer.eos_token
+print("  Base model loaded")
+
+print("\n[2/3] Loading production LoRA adapters...")
+model = PeftModel.from_pretrained(
+    base_model,
+    "panikos/llama-biomedical-production-qlora"
+)
+print("  LoRA adapters loaded")
+
+print("\n[3/3] Running comprehensive evaluation...")
+print("=" * 80)
+
+# Test cases covering various biomedical scenarios
+test_cases = [
+    {
+        "name": "Simple LOINC to SDTM Mapping",
+        "prompt": "Map the following LOINC code to CDISC SDTM domain:\n\nLOINC: 2339-0 (Glucose [Mass/volume] in Blood)",
+        "expected_keywords": ["LB", "Laboratory"]
+    },
+    {
+        "name": "Complex LOINC Panel Classification",
+        "prompt": """Analyze the following LOINC codes and classify each into the appropriate CDISC SDTM domain:
+
+1. 2339-0: Glucose [Mass/volume] in Blood
+2. 4548-4: Hemoglobin A1c/Hemoglobin.total in Blood
+3. 2160-0: Creatinine [Mass/volume] in Serum or Plasma
+
+Also identify if they form a panel and specify clinical significance.""",
+        "expected_keywords": ["LB", "Laboratory", "metabolic", "diabetes", "renal"]
+    },
+    {
+        "name": "CDISC Terminology Query",
+        "prompt": "What is the CDISC SDTM terminology for patient-reported adverse event severity?",
+        "expected_keywords": ["AESEV", "severity", "adverse event"]
+    },
+    {
+        "name": "Adverse Event Classification",
+        "prompt": "Classify the following observation into the appropriate SDTM domain:\n\nPatient reported experiencing headache with severity of moderate, lasting 2 hours after taking study medication.",
+        "expected_keywords": ["AE", "Adverse", "Event"]
+    },
+    {
+        "name": "SDTM vs ADaM Knowledge",
+        "prompt": "Explain the difference between SDTM and ADaM in CDISC standards.",
+        "expected_keywords": ["SDTM", "ADaM", "source", "analysis"]
+    },
+    {
+        "name": "Vital Signs Mapping",
+        "prompt": "Map the following LOINC code to CDISC SDTM domain:\n\nLOINC: 8867-4 (Heart rate)",
+        "expected_keywords": ["VS", "Vital", "Signs"]
+    }
+]
+
+results = []
+for i, test in enumerate(test_cases, 1):
+    print(f"\n{'='*80}")
+    print(f"TEST {i}/{len(test_cases)}: {test['name']}")
+    print(f"{'='*80}")
+    print(f"\nPrompt: {test['prompt'][:100]}...")
+
+    # Format prompt
+    messages = [{"role": "user", "content": test['prompt']}]
+
+    # Tokenize
+    input_ids = tokenizer.apply_chat_template(
+        messages,
+        add_generation_prompt=True,
+        return_tensors="pt"
+    ).to(model.device)
+
+    # Generate
+    print("\n>>> Model Response:")
+    print("-" * 80)
+    with torch.no_grad():
+        outputs = model.generate(
+            input_ids,
+            max_new_tokens=250,
+            temperature=0.7,
+            top_p=0.9,
+            do_sample=True,
+            pad_token_id=tokenizer.eos_token_id
+        )
+
+    # Decode
+    response = tokenizer.decode(outputs[0][input_ids.shape[1]:], skip_special_tokens=True)
+    print(response)
+    print("-" * 80)
+
+    # Evaluate
+    found_keywords = [kw for kw in test['expected_keywords'] if kw.lower() in response.lower()]
+    score = (len(found_keywords) / len(test['expected_keywords'])) * 100
+
+    print(f"\n>>> Evaluation:")
+    print(f"Expected keywords: {', '.join(test['expected_keywords'])}")
+    print(f"Found: {', '.join(found_keywords) if found_keywords else 'None'}")
+    print(f"Score: {score:.0f}% ({len(found_keywords)}/{len(test['expected_keywords'])})")
+
+    results.append({
+        'name': test['name'],
+        'score': score,
+        'found': len(found_keywords),
+        'total': len(test['expected_keywords'])
+    })
+
+# Overall evaluation
+print("\n" + "=" * 80)
+print("OVERALL EVALUATION SUMMARY")
+print("=" * 80)
+
+avg_score = sum(r['score'] for r in results) / len(results)
+total_found = sum(r['found'] for r in results)
+total_expected = sum(r['total'] for r in results)
+
+print(f"\nAverage Score: {avg_score:.1f}%")
+print(f"Total Keywords Found: {total_found}/{total_expected}")
+print(f"\nIndividual Test Results:")
+for r in results:
+    print(f"  {r['name']}: {r['score']:.0f}% ({r['found']}/{r['total']})")
+
+print("\n" + "=" * 80)
+if avg_score >= 70:
+    print("RESULT: EXCELLENT - Model shows strong biomedical understanding")
+    print("RECOMMENDATION: Model is ready for production use!")
+elif avg_score >= 50:
+    print("RESULT: GOOD - Model has solid biomedical knowledge")
+    print("RECOMMENDATION: Consider additional training for edge cases")
+else:
+    print("RESULT: NEEDS IMPROVEMENT - Consider additional training epochs")
+
+print("=" * 80)