Upload 3 files

requirements.txt

@@ -0,0 +1,54 @@
+autopep8==2.0.2
+blinker==1.6.2
+certifi==2022.12.7
+charset-normalizer==3.1.0
+click==8.1.3
+cmake==3.26.3
+filelock==3.12.0
+Flask==2.3.1
+Flask-Cors==3.0.10
+fsspec==2023.4.0
+huggingface-hub==0.14.1
+idna==3.4
+importlib-metadata==6.6.0
+itsdangerous==2.1.2
+Jinja2==3.1.2
+lit==16.0.2
+MarkupSafe==2.1.2
+mpmath==1.3.0
+networkx==3.1
+numpy==1.24.3
+nvidia-cublas-cu11==11.10.3.66
+nvidia-cuda-cupti-cu11==11.7.101
+nvidia-cuda-nvrtc-cu11==11.7.99
+nvidia-cuda-runtime-cu11==11.7.99
+nvidia-cudnn-cu11==8.5.0.96
+nvidia-cufft-cu11==10.9.0.58
+nvidia-curand-cu11==10.2.10.91
+nvidia-cusolver-cu11==11.4.0.1
+nvidia-cusparse-cu11==11.7.4.91
+nvidia-nccl-cu11==2.14.3
+nvidia-nvtx-cu11==11.7.91
+packaging==23.1
+pycodestyle==2.10.0
+PyYAML==6.0
+regex==2023.3.23
+requests==2.28.2
+six==1.16.0
+sympy==1.11.1
+tokenizers==0.13.3
+tomli==2.0.1
+torch==2.0.0
+tqdm==4.65.0
+transformers==4.28.1
+triton==2.0.0
+typing_extensions==4.5.0
+urllib3==1.26.15
+Werkzeug==2.3.0
+zipp==3.15.0
+# Added for HF training and Spaces
+gradio==3.38.0
+datasets==2.13.1
+accelerate==0.20.3
+scikit-learn==1.2.2
+

run.py

@@ -0,0 +1,5 @@
+from toxic_comment.src.app import app
+
+if __name__ == '__main__':
+    app.run(debug=True)
+    

train.py

@@ -0,0 +1,104 @@
+"""Simple training script for a text toxicity classifier.
+
+Usage examples:
+  - Train from a CSV: python train.py --dataset_csv data/toxic_train.csv --text_col text --label_col label --output_dir ./outputs
+  - Push to Hub: python train.py --dataset_csv data/toxic_train.csv --output_dir ./outputs --push_to_hub --hub_model_id your-username/toxic-detector
+
+Expect CSV with columns: text, label (0/1) for single-label classification. For multi-label adjust the preprocessing.
+"""
+import argparse
+from pathlib import Path
+from datasets import load_dataset, Dataset
+from transformers import (
+    AutoTokenizer,
+    AutoModelForSequenceClassification,
+    TrainingArguments,
+    Trainer,
+    DataCollatorWithPadding,
+)
+import numpy as np
+import evaluate
+
+
+def parse_args():
+    p = argparse.ArgumentParser()
+    p.add_argument("--dataset_csv", type=str, default=None, help="Path to CSV dataset with text and label columns")
+    p.add_argument("--text_col", type=str, default="text")
+    p.add_argument("--label_col", type=str, default="label")
+    p.add_argument("--model_name_or_path", type=str, default="distilbert-base-uncased")
+    p.add_argument("--output_dir", type=str, default="./model_output")
+    p.add_argument("--push_to_hub", action="store_true")
+    p.add_argument("--hub_model_id", type=str, default=None)
+    p.add_argument("--num_train_epochs", type=int, default=1)
+    p.add_argument("--per_device_train_batch_size", type=int, default=16)
+    return p.parse_args()
+
+
+def main():
+    args = parse_args()
+
+    if args.dataset_csv:
+        ds = load_dataset("csv", data_files={"train": args.dataset_csv})
+        # if no validation split, take 10% for val
+        ds = ds["train"].train_test_split(test_size=0.1)
+        dataset = ds
+    else:
+        # small built-in fallback: use a tiny subset of imdb for demo (binary sentiment)
+        dataset = load_dataset("imdb", split={"train": "train[:2000]","test": "test[:500]"})
+        dataset = dataset
+
+    tokenizer = AutoTokenizer.from_pretrained(args.model_name_or_path)
+
+    def preprocess_function(examples):
+        texts = examples[args.text_col] if args.dataset_csv else examples["text"]
+        return tokenizer(texts, truncation=True)
+
+    if args.dataset_csv:
+        tokenized = dataset.map(preprocess_function, batched=True)
+    else:
+        # imdb default has 'text' and 'label'
+        tokenized = dataset.map(lambda x: tokenizer(x['text'], truncation=True), batched=True)
+
+    labels = tokenized["train"].features[args.label_col] if args.dataset_csv else None
+
+    num_labels = 2
+    model = AutoModelForSequenceClassification.from_pretrained(args.model_name_or_path, num_labels=num_labels)
+
+    metric_acc = evaluate.load("accuracy")
+
+    def compute_metrics(eval_pred):
+        logits, labels = eval_pred
+        preds = np.argmax(logits, axis=-1)
+        return metric_acc.compute(predictions=preds, references=labels)
+
+    data_collator = DataCollatorWithPadding(tokenizer=tokenizer)
+
+    training_args = TrainingArguments(
+        output_dir=args.output_dir,
+        evaluation_strategy="epoch",
+        num_train_epochs=args.num_train_epochs,
+        per_device_train_batch_size=args.per_device_train_batch_size,
+        save_total_limit=2,
+        push_to_hub=args.push_to_hub,
+        hub_model_id=args.hub_model_id,
+    )
+
+    trainer = Trainer(
+        model=model,
+        args=training_args,
+        train_dataset=tokenized["train"],
+        eval_dataset=tokenized.get("test", None),
+        tokenizer=tokenizer,
+        data_collator=data_collator,
+        compute_metrics=compute_metrics,
+    )
+
+    trainer.train()
+
+    trainer.save_model()
+    if args.push_to_hub and args.hub_model_id:
+        trainer.push_to_hub()
+
+
+if __name__ == "__main__":
+    main()