Spaces:
Sleeping
Sleeping
File size: 2,943 Bytes
3cdaafb |
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 |
import argparse
import json
import logging
import random
from pathlib import Path
from typing import Sequence
import numpy as np
import torch
from tqdm import tqdm
from .binoculars_detector import Binoculars
from ..utils import evaluate_metrics
logging.basicConfig(level=logging.INFO, format="%(asctime)s %(levelname)s %(message)s")
_LOG_PATH = Path(__file__).resolve().parents[3] / "runs" / "val-other_detector.txt"
_LOG_PATH.parent.mkdir(parents=True, exist_ok=True)
def load_jsonl(file_path):
out = []
with open(file_path, mode='r', encoding='utf-8') as jsonl_file:
for line in jsonl_file:
item = json.loads(line)
out.append(item)
print(f"Loaded {len(out)} examples from {file_path}")
return out
def dict2str(metrics):
out_str=''
for key in metrics.keys():
out_str+=f"{key}:{metrics[key]} "
return out_str
def experiment(args):
# Initialize Binoculars (experiments in paper use the "accuracy" mode threshold wherever applicable)
bino = Binoculars(mode="accuracy", max_token_observed=args.tokens_seen)
logging.info(f"Test in {args.test_data_path}")
test_data = load_jsonl(args.test_data_path)
random.seed(args.seed)
torch.manual_seed(args.seed)
np.random.seed(args.seed)
random.shuffle(test_data)
predictions = []
labels = []
for i, item in tqdm(enumerate(test_data), total=len(test_data)):
text = item["text"]
label = item["label"]
src = item["src"]
bino_score = -bino.compute_score(text)
if bino_score is None or np.isnan(bino_score) or np.isinf(bino_score):
bino_score = 0
if 'human' in src:
labels.append(0)
else:
labels.append(1)
predictions.append(bino_score)
metric = evaluate_metrics(labels, predictions)
print(dict2str(metric))
with _LOG_PATH.open("a+", encoding="utf-8") as f:
f.write(f"binoculars {args.test_data_path}\n")
f.write(f"{dict2str(metric)}\n")
def build_argument_parser() -> argparse.ArgumentParser:
parser = argparse.ArgumentParser()
parser.add_argument(
'--test_data_path',
type=str,
default='/path/to/RealBench/Deepfake/no_attack/test.jsonl',
help="Path to the test data. could be several files with ','. Note that the data should have been perturbed.",
)
parser.add_argument("--tokens_seen", type=int, default=512, help="Number of tokens seen by the model")
parser.add_argument('--DEVICE', default="cuda", type=str, required=False)
parser.add_argument('--seed', default=2023, type=int, required=False)
return parser
def main(argv: Sequence[str] | None = None) -> None:
parser = build_argument_parser()
args = parser.parse_args(argv)
experiment(args)
if __name__ == '__main__':
main()
|