| | |
| | |
| |
|
| | import json |
| | import os |
| | import numpy as np |
| | os.environ["CUDA_DEVICE_ORDER"] = "PCI_BUS_ID" |
| | os.environ["CUDA_VISIBLE_DEVICES"] = "2" |
| |
|
| | from datasets import load_dataset |
| | from transformers import AutoTokenizer, AutoModelForSequenceClassification |
| |
|
| | MODEL_DIR = "/home/mshahidul/readctrl_model/full_model/distilbert_classifier" |
| | TEST_DATA_PATH = "verified_combined_0-80_test.json" |
| | MAX_LENGTH = 512 |
| | ACCURACY_OUTPUT_PATH = "accuracy_results_distilbert.json" |
| |
|
| | LABELS = [ |
| | "low_health_literacy", |
| | "intermediate_health_literacy", |
| | "proficient_health_literacy", |
| | ] |
| | LABEL2ID = {label: idx for idx, label in enumerate(LABELS)} |
| | ID2LABEL = {idx: label for label, idx in LABEL2ID.items()} |
| |
|
| |
|
| | def build_input_text(fulltext: str, diff_label_texts: str) -> str: |
| | return ( |
| | "Classify the health literacy level of the rewritten text.\n\n" |
| | "Labels:\n" |
| | "- low_health_literacy: very simple, living-room language, minimal jargon.\n" |
| | "- intermediate_health_literacy: standard public-friendly language, limited jargon.\n" |
| | "- proficient_health_literacy: technical, clinical, or academic language.\n\n" |
| | f"Full Source Text:\n{fulltext}\n\n" |
| | f"Rewritten Text:\n{diff_label_texts}\n" |
| | ) |
| |
|
| |
|
| | def main(): |
| | tokenizer = AutoTokenizer.from_pretrained(MODEL_DIR, use_fast=True) |
| | model = AutoModelForSequenceClassification.from_pretrained(MODEL_DIR) |
| | model.eval() |
| | model.to("cuda") |
| |
|
| | dataset = load_dataset("json", data_files=TEST_DATA_PATH, split="train") |
| |
|
| | correct = 0 |
| | total = 0 |
| |
|
| | for example in dataset: |
| | text = build_input_text(example["fulltext"], example["diff_label_texts"]) |
| | inputs = tokenizer( |
| | text, |
| | max_length=MAX_LENGTH, |
| | truncation=True, |
| | return_tensors="pt", |
| | ) |
| | inputs = {k: v.to("cuda") for k, v in inputs.items()} |
| | with np.errstate(all="ignore"): |
| | outputs = model(**inputs) |
| | pred_id = int(outputs.logits.argmax(dim=-1).item()) |
| | pred_label = ID2LABEL.get(pred_id, "") |
| | print(f"Predicted: {pred_label}, Expected: {example['label']}") |
| | if pred_label == example["label"]: |
| | correct += 1 |
| | total += 1 |
| |
|
| | accuracy = (correct / total) if total else 0.0 |
| | results = { |
| | "accuracy": round(accuracy, 6), |
| | "correct": correct, |
| | "total": total, |
| | "model_dir": MODEL_DIR, |
| | "test_data_path": TEST_DATA_PATH, |
| | } |
| | with open(ACCURACY_OUTPUT_PATH, "w", encoding="utf-8") as handle: |
| | json.dump(results, handle, ensure_ascii=True) |
| | handle.write("\n") |
| | print(f"Accuracy: {accuracy:.4f} ({correct}/{total})") |
| | print(f"Saved accuracy info to {ACCURACY_OUTPUT_PATH}") |
| |
|
| |
|
| | if __name__ == "__main__": |
| | main() |
| |
|
