training completed

.gitignore

@@ -1,91 +1,84 @@
-# Environment files
-.env
-.env.*
-!.env.example
-.env.local
+# OS files
+.DS_Store
+Thumbs.db
 
 # Python
 __pycache__/
 *.py[cod]
-*.pyc
 *.pyo
 *.pyd
-.Python
-.venv/
-venv/
-env/
 .pytest_cache/
 .mypy_cache/
 .ruff_cache/
-.coverage
-htmlcov/
-dist/
-build/
-*.egg-info/
 
-# Node / Next.js
+# Virtual environments
+.venv/
+venv/
+env/
+ENV/
+
+# Jupyter
+.ipynb_checkpoints/
+
+# Environment variables and secrets
+.env
+.env.*
+!.env.example
+
+# Node / frontend
 node_modules/
 .next/
 out/
-coverage/
+dist/
+build/
 npm-debug.log*
 yarn-debug.log*
 yarn-error.log*
-pnpm-debug.log*
-*.tsbuildinfo
 
-# Colab / notebooks
-.ipynb_checkpoints/
-*.ipynb_checkpoints
-drive/
-wandb/
-lightning_logs/
-runs/
+# Training outputs and model artifacts
+training/outputs/
+training/output/
+training/checkpoints/
+training/training_model_files/
 checkpoints/
+runs/
+wandb/
+*.zip
+*.tar
+*.tar.gz
+*.safetensors
+*.bin
+*.pt
+*.pth
+*.ckpt
 
-# Genomics data and model artifacts
-data/raw/
-data/processed/
-*.vcf
-*.vcf.gz
-*.vcf.gz.tbi
-*.fa
-*.fasta
-*.fa.gz
-*.fasta.gz
-*.fai
-*.2bit
-*.bam
-*.bai
-*.cram
-*.crai
-*.bed
-*.jsonl
-*.parquet
-*.arrow
-checkpoints/
-models/
-model/
-outputs/
-artifacts/
-training/data/
-training/output/
-training/outputs/
-training/cache/
-training/data_cache/
+# Generated datasets
+training/csv_files/
+training/csv_files_alt/
 training/csv_files_large/
 training/csv_files_large_alt/
 training/csv_files_10k/
 training/csv_files_10k_alt/
 training/csv_files_20k/
 training/csv_files_20k_alt/
+training/data_cache/
+training/cache/
+data/raw/
+data/interim/
+data/processed/
+
+# HuggingFace / model cache
+.cache/
+hf_cache/
+huggingface_cache/
+
+# Local DNABERT generated patch/cache
 training/local_dnabert2_patch/
-*.zip
-*.safetensors
-*.bin
 
-# OS/editor
-.DS_Store
-.idea/
-.vscode/
-*.swp
+# Logs
+*.log
+logs/
+
+# Temporary files
+*.tmp
+*.temp

README.md

@@ -60,11 +60,21 @@ Open `http://localhost:3000`.
 
 Training is intended for Google Colab only. Do not train the model locally.
 
-1. Open `training/colab_dnabert2_clinvar_finetune.ipynb` in Google Colab.
+1. Open `training/colab_dnabert2_heavy_training.ipynb` in Google Colab for the current heavy-training workflow.
 2. Follow the notebook cells to install dependencies, download ClinVar GRCh38 data, prepare examples, and fine-tune DNABERT-2.
 3. Export the trained Hugging Face model directory.
 4. Point the backend `MODEL_DIR` environment variable to that exported model directory.
 
+## Data and Model Artifact Policy
+
+- Full datasets are not committed to GitHub.
+- Trained model weights are not committed to GitHub.
+- Use the training scripts or Colab notebook to regenerate datasets.
+- Store trained models in Google Drive or local storage.
+- For local evaluation, place the 20k alternate-sequence dataset in `training/csv_files_20k_alt/`.
+- For backend use, place the model folder in `backend/models/final_model/` or set the backend model path environment variable.
+- The local final model folder `training/training_model_files/` and the 20k CSV folder `training/csv_files_20k_alt/` are intentionally ignored by Git.
+
 ## Environment Files
 
 Each app folder includes a `.env.example`. Do not commit `.env`, `.env.local`, API keys, model checkpoints, or private data.

docs/MODEL_ARTIFACTS.md

@@ -0,0 +1,39 @@
+# Model Artifacts
+
+Large trained models are not stored in GitHub.
+
+The final cleaned model folder is kept locally at:
+
+```text
+training/training_model_files/
+```
+
+This folder is ignored by Git because it contains large model files such as
+`model.safetensors` and `training_args.bin`.
+
+The current best model is the 20k DNABERT-2 alternate-sequence model. Model
+artifacts should be stored in Google Drive or downloaded local storage rather
+than committed to the repository.
+
+For backend use, either:
+
+- place the final model folder at `backend/models/final_model/`, or
+- configure the backend model path environment variable to point to
+  `training/training_model_files/`.
+
+Keep these small metadata files with the model artifact:
+
+- `metrics.json`
+- `full_eval_metrics.json`
+
+The 20k CSV dataset is also ignored by Git:
+
+```text
+training/csv_files_20k_alt/
+```
+
+Store that dataset externally or regenerate it with the training scripts and
+Colab notebook.
+
+This project is for research and education only. It is not a clinical
+diagnostic tool.

docs/results/DNABERT2_20K_RESULTS.md

@@ -0,0 +1,16 @@
+# DNABERT-2 20k Result
+
+Final confirmed local evaluation on `training/csv_files_20k_alt/`:
+
+- Accuracy: 0.5537
+- Precision: 0.5384
+- Recall: 0.7533
+- F1: 0.6280
+- MCC: 0.1171
+- AUC ROC: 0.5928
+- Threshold: 0.1600
+
+The model was fine-tuned for binary ClinVar alternate-sequence classification
+using GRCh38 variant-centered sequences.
+
+This model is for research/demo use only and is not a clinical diagnostic tool.

docs/results/dnabert2_20k_full_eval_metrics.json

@@ -0,0 +1,67 @@
+{
+  "model_dir": "/Users/faisalimran/Desktop/tarns_project/training/training_model_files",
+  "dataset_dir": "/Users/faisalimran/Desktop/tarns_project/training/csv_files_20k_alt",
+  "device": "mps",
+  "max_length": 512,
+  "variant_center_index": 512,
+  "batch_size": 1,
+  "threshold_args": {
+    "tune_threshold": true,
+    "threshold": null,
+    "threshold_min": 0.1,
+    "threshold_max": 0.9,
+    "threshold_step": 0.01
+  },
+  "threshold": 0.16,
+  "selected_threshold": 0.16,
+  "threshold_selection": {
+    "threshold_min": 0.1,
+    "threshold_max": 0.9,
+    "threshold_step": 0.01,
+    "thresholds_tested": 81,
+    "best_threshold": 0.16,
+    "best_validation_mcc": 0.15644505036060144,
+    "mode": "full_validation_mcc",
+    "selected_threshold": 0.16
+  },
+  "validation_metrics": {
+    "threshold": 0.16,
+    "accuracy": 0.5723333333333334,
+    "precision": 0.552390149686142,
+    "recall": 0.7626666666666667,
+    "f1": 0.6407168860263232,
+    "mcc": 0.15644505036060144,
+    "auc_roc": 0.6018537777777778,
+    "confusion_matrix": [
+      [
+        573,
+        927
+      ],
+      [
+        356,
+        1144
+      ]
+    ],
+    "rows": 3000
+  },
+  "test_metrics": {
+    "threshold": 0.16,
+    "accuracy": 0.5536666666666666,
+    "precision": 0.5383515959980943,
+    "recall": 0.7533333333333333,
+    "f1": 0.6279522089469297,
+    "mcc": 0.1170731385436657,
+    "auc_roc": 0.5927693333333333,
+    "confusion_matrix": [
+      [
+        531,
+        969
+      ],
+      [
+        370,
+        1130
+      ]
+    ],
+    "rows": 3000
+  }
+}

docs/results/dnabert2_20k_metrics.json

@@ -0,0 +1,82 @@
+{
+  "model_name": "zhihan1996/DNABERT-2-117M",
+  "model_path": "/Users/faisalimran/Desktop/tarns_project/training/local_dnabert2_patch",
+  "train_csv": "/Users/faisalimran/Desktop/tarns_project/training/csv_files_large_alt/train_with_alt_sequences.csv",
+  "validation_csv": "/Users/faisalimran/Desktop/tarns_project/training/csv_files_large_alt/val_with_alt_sequences.csv",
+  "test_csv": "/Users/faisalimran/Desktop/tarns_project/training/csv_files_large_alt/test_with_alt_sequences.csv",
+  "dataset_dir": "/Users/faisalimran/Desktop/tarns_project/training/csv_files_large_alt",
+  "sequence_column": "sequence",
+  "is_alt_sequence_dataset": true,
+  "is_large_alt_sequence_dataset": true,
+  "device": "mps",
+  "freeze_encoder": true,
+  "unfreeze_last_n_layers": 2,
+  "freeze_embeddings": true,
+  "use_class_weights": true,
+  "class_weights": [
+    1.0,
+    1.0
+  ],
+  "center_crop": true,
+  "tune_threshold": true,
+  "save_eval_each_epoch": false,
+  "eval_accumulation_steps": 1,
+  "eval_subset_size": 750,
+  "selected_threshold": 0.41,
+  "best_validation_mcc_for_threshold": 0.12554169541533072,
+  "sample_size": 0,
+  "epochs": 5.0,
+  "max_length": 512,
+  "variant_center_index": 512,
+  "train_metrics": {
+    "train_runtime": 2529.8996,
+    "train_samples_per_second": 6.917,
+    "train_steps_per_second": 0.866,
+    "total_flos": 6124490695680000.0,
+    "train_loss": 0.6810980735848483,
+    "epoch": 5.0
+  },
+  "train_rows": 3500,
+  "validation_rows": 750,
+  "test_rows": 750,
+  "validation_eval_rows": 750,
+  "test_eval_rows": 750,
+  "validation_metrics": {
+    "selected_threshold": 0.41,
+    "accuracy": 0.54,
+    "precision": 0.5225903614457831,
+    "recall": 0.9253333333333333,
+    "f1": 0.6679499518768046,
+    "mcc": 0.12554169541533072,
+    "auc_roc": 0.572736,
+    "confusion_matrix": [
+      [
+        58,
+        317
+      ],
+      [
+        28,
+        347
+      ]
+    ]
+  },
+  "test_metrics": {
+    "selected_threshold": 0.41,
+    "accuracy": 0.512,
+    "precision": 0.5067873303167421,
+    "recall": 0.896,
+    "f1": 0.6473988439306358,
+    "mcc": 0.03747366058909428,
+    "auc_roc": 0.5497315555555555,
+    "confusion_matrix": [
+      [
+        48,
+        327
+      ],
+      [
+        39,
+        336
+      ]
+    ]
+  }
+}

training/COLAB_README.md

@@ -31,3 +31,13 @@ medical diagnosis and should not be used for clinical decisions.
 - Do not add API keys or secrets to the notebook.
 - Dataset preparation uses caching and progress files, so reruns can resume.
 - The training script uses CUDA when available, MPS on Mac, and CPU as a slow fallback.
+
+## Data and Model Artifact Policy
+
+- Full datasets are not committed to GitHub.
+- Trained model weights are not committed to GitHub.
+- Use the training scripts or Colab notebook to regenerate datasets.
+- Store trained models in Google Drive or local storage.
+- For local evaluation, place the 20k alternate-sequence dataset in `training/csv_files_20k_alt/`.
+- For backend use, place the model folder in `backend/models/final_model/` or set the backend model path environment variable.
+- Keep `metrics.json` and `full_eval_metrics.json` with the exported model artifact.

training/csv_files/README.md

@@ -1,11 +0,0 @@
-# Sequence CSV Files
-
-Place the sequence-enriched ClinVar split files here for Colab training:
-
-- `train_with_sequences.csv`
-- `val_with_sequences.csv`
-- `test_with_sequences.csv`
-
-These files are generated by `training/01_prepare_clinvar_dataset.ipynb`.
-
-Large generated CSVs should usually be stored outside Git unless you intentionally want to version the sample dataset.

training/evaluate_saved_model.py

@@ -45,6 +45,7 @@ from training.train_smoke_test import (  # noqa: E402
 
 OUTPUT_DIR = PROJECT_ROOT / "training" / "outputs" / "dnabert2_clinvar"
 MODEL_DIR = OUTPUT_DIR / "final_model"
+UPLOADED_MODEL_DIR = PROJECT_ROOT / "training" / "training_model_files"
 TRAINING_METRICS_PATH = OUTPUT_DIR / "metrics.json"
 FULL_EVAL_METRICS_PATH = OUTPUT_DIR / "full_eval_metrics.json"
 
@@ -120,6 +121,15 @@ def parse_args() -> argparse.Namespace:
         default=0.01,
         help="Threshold step size when tuning.",
     )
+    parser.add_argument(
+        "--model_dir",
+        type=Path,
+        default=None,
+        help=(
+            "Saved model folder to evaluate. Defaults to training/outputs/dnabert2_clinvar/final_model; "
+            "if that is missing, falls back to training/training_model_files."
+        ),
+    )
     return parser.parse_args()
 
 
@@ -140,6 +150,26 @@ def resolve_project_path(path: Path) -> Path:
     return PROJECT_ROOT / path
 
 
+def choose_model_dir(requested_model_dir: Path | None) -> Path:
+    if requested_model_dir is not None:
+        model_dir = resolve_project_path(requested_model_dir)
+        if not model_dir.exists():
+            raise FileNotFoundError(f"Requested saved model directory not found: {model_dir}")
+        return model_dir
+
+    if MODEL_DIR.exists():
+        return MODEL_DIR
+
+    if UPLOADED_MODEL_DIR.exists():
+        return UPLOADED_MODEL_DIR
+
+    raise FileNotFoundError(
+        "Saved model directory not found. Searched:\n"
+        f"{MODEL_DIR}\n"
+        f"{UPLOADED_MODEL_DIR}"
+    )
+
+
 def choose_device() -> str:
     if torch.cuda.is_available():
         return "cuda"
@@ -338,20 +368,20 @@ def load_saved_model_with_local_patch(model_dir: Path):
 
 
 def load_saved_model(model_dir: Path, device: str):
-    if device == "cuda":
-        print("CUDA detected. Trying to load saved model directly first.")
-        try:
-            model = AutoModelForSequenceClassification.from_pretrained(
-                str(model_dir),
-                trust_remote_code=True,
-                low_cpu_mem_usage=False,
-            )
-            print("Saved model loaded successfully without the local Mac patch.")
-            return model
-        except Exception as error:
-            print("Direct saved-model load failed.")
-            print(f"Direct load error: {error}")
-            print("Falling back to local no-Triton DNABERT-2 code.")
+    print(f"Trying to load saved model directly from: {model_dir}")
+    print(f"Selected device: {device}")
+    try:
+        model = AutoModelForSequenceClassification.from_pretrained(
+            str(model_dir),
+            trust_remote_code=True,
+            low_cpu_mem_usage=False,
+        )
+        print("Saved model loaded cleanly")
+        return model
+    except Exception as error:
+        print("Direct saved-model loading failed.")
+        print(f"Reason: {type(error).__name__}: {error}")
+        print("Falling back to training/local_dnabert2_patch.")
 
     return load_saved_model_with_local_patch(model_dir)
 
@@ -519,14 +549,12 @@ def main() -> None:
     args = parse_args()
     validate_args(args)
 
-    if not MODEL_DIR.exists():
-        raise FileNotFoundError(f"Saved model directory not found: {MODEL_DIR}")
-
+    model_dir = choose_model_dir(args.model_dir)
     dataset_dir = find_dataset_dir()
     device = choose_device()
 
     print("Memory-safe saved model evaluation")
-    print(f"Saved model directory: {MODEL_DIR}")
+    print(f"Saved model directory: {model_dir}")
     print(f"Selected dataset directory: {dataset_dir}")
     print(f"Selected device: {device}")
     print(f"Tune threshold on full validation set: {args.tune_threshold}")
@@ -542,11 +570,11 @@ def main() -> None:
     print()
 
     print("Loading tokenizer and model.")
-    tokenizer = AutoTokenizer.from_pretrained(str(MODEL_DIR), trust_remote_code=True)
-    model = load_saved_model(MODEL_DIR, device)
+    tokenizer = AutoTokenizer.from_pretrained(str(model_dir), trust_remote_code=True)
+    model = load_saved_model(model_dir, device)
 
     all_metrics = {
-        "model_dir": str(MODEL_DIR),
+        "model_dir": str(model_dir),
         "dataset_dir": str(dataset_dir),
         "device": device,
         "max_length": MAX_LENGTH,

training/train_local_dnabert2.py

@@ -13,6 +13,7 @@ import gc
 import inspect
 import json
 import os
+import shutil
 import sys
 import zipfile
 from dataclasses import dataclass
@@ -737,6 +738,45 @@ def create_patch_from_project_root() -> Path:
     return resolve_path(patch_dir)
 
 
+def ensure_local_patch_for_export() -> Path:
+    patch_dir = resolve_path(LOCAL_DNABERT2_PATCH_DIR)
+    if patch_dir.exists() and local_patch_is_ready(patch_dir):
+        return patch_dir
+
+    print("Creating local no-Triton DNABERT-2 patch for self-contained model export.")
+    return create_patch_from_project_root()
+
+
+def copy_dnabert2_custom_code_to_final_model(final_model_dir: Path) -> list[str]:
+    """Make final_model self-contained for trust_remote_code=True loading."""
+    patch_dir = ensure_local_patch_for_export()
+    final_model_dir.mkdir(parents=True, exist_ok=True)
+
+    py_files = sorted(patch_dir.glob("*.py"))
+    if not py_files:
+        raise FileNotFoundError(f"No DNABERT-2 Python code files found in local patch: {patch_dir}")
+
+    copied_files: list[str] = []
+    for source_path in py_files:
+        destination_path = final_model_dir / source_path.name
+        shutil.copy2(source_path, destination_path)
+        copied_files.append(source_path.name)
+
+    required_files = ["configuration_bert.py", "bert_layers.py", "bert_padding.py"]
+    missing_files = [filename for filename in required_files if not (final_model_dir / filename).exists()]
+    if missing_files:
+        raise FileNotFoundError(
+            "final_model is missing required DNABERT-2 custom code files after export: "
+            f"{missing_files}"
+        )
+
+    print("Copied DNABERT-2 custom code into final_model:")
+    for filename in copied_files:
+        print(f"  {filename}")
+    print()
+    return copied_files
+
+
 def load_model_from_source(model_source: str | Path):
     config = AutoConfig.from_pretrained(str(model_source), trust_remote_code=True)
     config = disable_flash_attention_on_config(config)
@@ -1157,6 +1197,7 @@ def main() -> None:
     final_model_dir.mkdir(parents=True, exist_ok=True)
     trainer.save_model(str(final_model_dir))
     tokenizer.save_pretrained(str(final_model_dir))
+    custom_code_files = copy_dnabert2_custom_code_to_final_model(final_model_dir)
 
     metrics = {
         "model_name": MODEL_NAME,
@@ -1196,6 +1237,7 @@ def main() -> None:
         "test_eval_rows": len(eval_test_df),
         "validation_metrics": validation_metrics,
         "test_metrics": test_metrics,
+        "final_model_custom_code_files": custom_code_files,
     }
     metrics_path = save_metrics(output_dir, metrics)
     zip_path = zip_final_model(output_dir, final_model_dir)