{ "cells": [ { "cell_type": "markdown", "metadata": {}, "source": [ "# SciHigh-2026 Subtask 1 \u2014 bart-large-cnn titled (run 1)\n", "\n", "Notebook version of `train_final.py` \u2014 run top to bottom.\n" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "\"\"\"\n", "SciHigh-2026 Subtask 1 (Research Highlight Generation) -- final training recipe.\n", "\n", "Fine-tunes facebook/bart-large-cnn on the expanded MixSub-SciHigh training pool\n", "to generate short research highlights from a paper's title and abstract.\n", "Inputs are ' | <abstract>' with truncated abstracts repaired via\n", "DOI-verified Semantic Scholar recovery (see final/ pipeline). This is the\n", "exact recipe that produced the submitted result:\n", "\n", " ROUGE-1=39.30% ROUGE-2=15.32% ROUGE-L=26.37%\n", " METEOR=36.17% BERTScore-F1=88.05%\n", " (vs. the FIRE-2025 winning baseline of 23.45% ROUGE-L)\n", "\n", "This script is a clean, standalone extraction of the winning configuration --\n", "it deliberately does NOT include the large space of experiments (backbone\n", "sweeps, input-feature engineering, decode-time reranking/logit-bias/checkpoint\n", "averaging, proxy-scale fast iteration mode, etc.) that were tried and used to\n", "arrive at this recipe. Those all live in the project's internal experiment\n", "history; none of them are part of what actually produced the result above.\n", "\n", "Usage:\n", " pip install torch transformers accelerate datasets evaluate rouge_score \\\n", " sentencepiece bert-score nltk\n", " python train_final.py --data_dir /path/to/data --output_dir ./output\n", "\n", "Expects --data_dir to contain:\n", " train_expanded_recovered_titled.csv (15,960 rows: the\n", " official 10,000-row MixSub-SciHigh train split plus\n", " 5,960 additional real pairs recovered from the\n", " dataset's original source release, leakage-checked\n", " against val/test by exact Abstract-text match)\n", " val_recovered_titled.csv (Filename, Abstract, Highlights -- 1,985 rows, the\n", " official held-out validation split, used only for\n", " per-epoch monitoring/checkpoint selection here)\n", " test_recovered_titled.csv (Filename, Abstract -- 1,840 rows, official masked\n", " test split, for the submission predictions)\n", "\"\"\"\n", "import argparse\n", "import json\n", "import os\n", "\n", "import evaluate\n", "import nltk\n", "import numpy as np\n", "import pandas as pd\n", "import torch\n", "from datasets import Dataset\n", "from transformers import (\n", " AutoModelForSeq2SeqLM,\n", " AutoTokenizer,\n", " DataCollatorForSeq2Seq,\n", " EarlyStoppingCallback,\n", " Seq2SeqTrainer,\n", " Seq2SeqTrainingArguments,\n", ")\n", "\n", "MODEL_NAME = \"facebook/bart-large-cnn\"\n", "MAX_INPUT_LEN = 512\n", "MAX_TARGET_LEN = 100 # matches the FIRE-2025 baseline recipe's output budget\n", "BATCH_SIZE = 2\n", "LEARNING_RATE = 2e-5\n", "NUM_BEAMS = 4\n", "\n", "# Epoch ceiling, not a fixed schedule: bart-large-cnn converges fast on this\n", "# task (a small-scale proxy run reached near-final quality in ~600-1,200\n", "# gradient steps, a fraction of one full epoch's ~7,980 steps at batch_size=2\n", "# over the 15,960-row pool). load_best_model_at_end + EarlyStoppingCallback\n", "# below let the run self-terminate rather than committing to a fixed count.\n", "# In the actual run that produced the reported numbers, training stopped\n", "# after epoch 3 (2 consecutive non-improving epochs), and the checkpoint from\n", "# epoch 1 -- the true best on val ROUGE-L -- was the one restored and\n", "# evaluated/submitted.\n", "EPOCH_CEILING = 4\n", "EARLY_STOPPING_PATIENCE = 2\n", "\n", "for _pkg in [\"wordnet\", \"punkt_tab\", \"omw-1.4\"]:\n", " try:\n", " nltk.download(_pkg, quiet=True)\n", " except Exception as e: # noqa: BLE001\n", " print(f\"warning: failed to download nltk resource '{_pkg}': {e}\")\n", "\n", "\n", "def parse_args():\n", " p = argparse.ArgumentParser()\n", " p.add_argument(\"--data_dir\", required=True)\n", " p.add_argument(\"--output_dir\", default=\"output\")\n", " p.add_argument(\"--epochs\", type=int, default=EPOCH_CEILING)\n", " p.add_argument(\"--skip_bertscore\", action=\"store_true\", help=\"BERTScore eval downloads its own scoring model; skip for a fast local check\")\n", " return p.parse_args()\n", "\n", "\n", "def to_hf_dataset(df, has_target):\n", " d = {\"Abstract\": df[\"Abstract\"].tolist()}\n", " if has_target:\n", " d[\"Highlights\"] = df[\"Highlights\"].tolist()\n", " return Dataset.from_dict(d)\n", "\n", "\n", "def make_preprocess_fn(tokenizer):\n", " def preprocess(batch):\n", " model_inputs = tokenizer(batch[\"Abstract\"], max_length=MAX_INPUT_LEN, truncation=True)\n", " labels = tokenizer(text_target=batch[\"Highlights\"], max_length=MAX_TARGET_LEN, truncation=True)\n", " model_inputs[\"labels\"] = labels[\"input_ids\"]\n", " return model_inputs\n", "\n", " return preprocess\n", "\n", "\n", "def main():\n", " args = parse_args()\n", " os.makedirs(args.output_dir, exist_ok=True)\n", " device = \"cuda\" if torch.cuda.is_available() else \"cpu\"\n", "\n", " train_df = pd.read_csv(os.path.join(args.data_dir, \"train_expanded_recovered_titled.csv\"))\n", " val_df = pd.read_csv(os.path.join(args.data_dir, \"val_recovered_titled.csv\"))\n", " test_df = pd.read_csv(os.path.join(args.data_dir, \"test_recovered_titled.csv\"))\n", " print(f\"[train_final] device={device} train={len(train_df)} val={len(val_df)} test={len(test_df)}\")\n", "\n", " tokenizer = AutoTokenizer.from_pretrained(MODEL_NAME)\n", " model = AutoModelForSeq2SeqLM.from_pretrained(MODEL_NAME).to(device)\n", "\n", " # Decode-time settings: bart-large-cnn already ships no_repeat_ngram_size=3\n", " # in its own generation_config, but it's set explicitly here rather than\n", " # left implicit, so the decision doesn't silently depend on that shipped\n", " # default surviving a future transformers/model-card change. Two other\n", " # settings that were tuned during small-scale proxy experiments\n", " # (repetition_penalty=1.5, min_new_tokens=15) were fixes for degenerate\n", " # repetition-collapse in a severely undertrained checkpoint -- this\n", " # full-scale, fully-converged model doesn't exhibit that failure mode, so\n", " # those are deliberately left untouched at bart-large-cnn's own defaults\n", " # (repetition_penalty=1.0, min_length=56) rather than carried over.\n", " model.generation_config.no_repeat_ngram_size = 3\n", " model.generation_config.max_length = MAX_TARGET_LEN\n", "\n", " train_ds = to_hf_dataset(train_df, has_target=True)\n", " val_ds = to_hf_dataset(val_df, has_target=True)\n", " preprocess = make_preprocess_fn(tokenizer)\n", " train_tok = train_ds.map(preprocess, batched=True, remove_columns=train_ds.column_names)\n", " val_tok = val_ds.map(preprocess, batched=True, remove_columns=val_ds.column_names)\n", " collator = DataCollatorForSeq2Seq(tokenizer, model=model)\n", "\n", " rouge = evaluate.load(\"rouge\")\n", " meteor = evaluate.load(\"meteor\")\n", "\n", " def compute_metrics(eval_preds):\n", " preds, labels = eval_preds\n", " if isinstance(preds, tuple):\n", " preds = preds[0]\n", " preds = np.where(preds != -100, preds, tokenizer.pad_token_id)\n", " decoded_preds = tokenizer.batch_decode(preds, skip_special_tokens=True)\n", " labels = np.where(labels != -100, labels, tokenizer.pad_token_id)\n", " decoded_labels = tokenizer.batch_decode(labels, skip_special_tokens=True)\n", "\n", " result = rouge.compute(predictions=decoded_preds, references=decoded_labels)\n", " result = {f\"rouge_{k}\": v for k, v in result.items()}\n", " result[\"meteor\"] = meteor.compute(predictions=decoded_preds, references=decoded_labels)[\"meteor\"]\n", " return result\n", "\n", " training_args = Seq2SeqTrainingArguments(\n", " output_dir=os.path.join(args.output_dir, \"checkpoints\"),\n", " num_train_epochs=args.epochs,\n", " per_device_train_batch_size=BATCH_SIZE,\n", " per_device_eval_batch_size=BATCH_SIZE,\n", " learning_rate=LEARNING_RATE,\n", " label_smoothing_factor=0.0,\n", " warmup_ratio=0.0,\n", " # Adafactor's factored second-moment estimates avoid the full-size\n", " # exp_avg/exp_avg_sq buffers that OOM'd a 16GB T4 with plain Adam; it's\n", " # also what the original PEGASUS/BART pretraining used.\n", " optim=\"adafactor\",\n", " predict_with_generate=True,\n", " generation_max_length=MAX_TARGET_LEN,\n", " generation_num_beams=NUM_BEAMS,\n", " eval_strategy=\"epoch\",\n", " save_strategy=\"epoch\",\n", " save_total_limit=1,\n", " load_best_model_at_end=True,\n", " metric_for_best_model=\"rouge_rougeL\",\n", " fp16=(device == \"cuda\"),\n", " logging_steps=50,\n", " report_to=[],\n", " )\n", "\n", " trainer = Seq2SeqTrainer(\n", " model=model,\n", " args=training_args,\n", " train_dataset=train_tok,\n", " eval_dataset=val_tok,\n", " data_collator=collator,\n", " compute_metrics=compute_metrics,\n", " callbacks=[EarlyStoppingCallback(early_stopping_patience=EARLY_STOPPING_PATIENCE)],\n", " )\n", "\n", " trainer.train()\n", "\n", " final_model_dir = os.path.join(args.output_dir, \"model\")\n", " trainer.save_model(final_model_dir)\n", " tokenizer.save_pretrained(final_model_dir)\n", "\n", " val_metrics = trainer.evaluate()\n", " print(\"[train_final] validation metrics:\", val_metrics)\n", "\n", " if not args.skip_bertscore:\n", " from bert_score import score as bertscore\n", "\n", " val_preds = trainer.predict(val_tok)\n", " preds = np.where(val_preds.predictions != -100, val_preds.predictions, tokenizer.pad_token_id)\n", " decoded_preds = tokenizer.batch_decode(preds, skip_special_tokens=True)\n", " _, _, f1 = bertscore(decoded_preds, val_df[\"Highlights\"].tolist(), lang=\"en\", verbose=False)\n", " val_metrics[\"bertscore_f1\"] = float(f1.mean())\n", " print(\"[train_final] bertscore_f1:\", val_metrics[\"bertscore_f1\"])\n", "\n", " with open(os.path.join(args.output_dir, \"val_metrics.json\"), \"w\") as f:\n", " json.dump(val_metrics, f, indent=2)\n", "\n", " # Generate the submission predictions on the masked test set.\n", " model.eval()\n", " gen_device = next(model.parameters()).device\n", " predictions = []\n", " batch_size = max(BATCH_SIZE, 8)\n", " abstracts = test_df[\"Abstract\"].tolist()\n", " for i in range(0, len(abstracts), batch_size):\n", " batch = abstracts[i : i + batch_size]\n", " inputs = tokenizer(batch, max_length=MAX_INPUT_LEN, truncation=True, padding=True, return_tensors=\"pt\").to(gen_device)\n", " with torch.no_grad():\n", " generated = model.generate(**inputs, max_length=MAX_TARGET_LEN, num_beams=NUM_BEAMS)\n", " predictions.extend(tokenizer.batch_decode(generated, skip_special_tokens=True))\n", "\n", " submission = pd.DataFrame({\n", " \"Filename\": test_df[\"Filename\"],\n", " \"Abstract\": test_df[\"Abstract\"],\n", " \"Predicted_Highlights\": predictions,\n", " })\n", " submission_path = os.path.join(args.output_dir, \"Yushkk99_Task1_run1.csv\")\n", " submission.to_csv(submission_path, index=False)\n", " print(f\"[train_final] wrote submission to {submission_path}\")\n", "\n", "\n", "if __name__ == \"__main__\":\n", " main()\n" ] } ], "metadata": { "kernelspec": { "display_name": "Python 3", "language": "python", "name": "python3" }, "language_info": { "name": "python", "version": "3.11" } }, "nbformat": 4, "nbformat_minor": 5 }