File size: 2,883 Bytes
3dac39e | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 | #!/usr/bin/env python3
"""Merge cleaned training data with converted APTNER data for R7.
Produces:
- data/processed/r7_5class_train.jsonl (existing train + APTNER train new)
- data/processed/r7_5class_valid.jsonl (existing valid + APTNER dev new)
"""
import json
from collections import defaultdict
from pathlib import Path
def load_jsonl(path: Path):
records = []
with open(path) as f:
for line in f:
if line.strip():
records.append(json.loads(line))
return records
def count_entities(records):
counts = defaultdict(int)
total = 0
for r in records:
for key, positions in r["spans"].items():
label = key.split(":")[0]
counts[label] += len(positions)
total += len(positions)
return total, dict(counts)
def main():
base = Path("/home/ubuntu/alkyline/data/processed")
# Load sources
print("Loading existing cleaned data...")
train_existing = load_jsonl(base / "enriched_5class_train_cleaned.jsonl")
valid_existing = load_jsonl(base / "enriched_5class_valid_cleaned.jsonl")
print("Loading converted APTNER data...")
aptner_train = load_jsonl(base / "aptner_5class_train.jsonl")
aptner_dev = load_jsonl(base / "aptner_5class_dev.jsonl")
# Merge
train_merged = train_existing + aptner_train
valid_merged = valid_existing + aptner_dev
# Write
train_out = base / "r7_5class_train.jsonl"
valid_out = base / "r7_5class_valid.jsonl"
for path, records in [(train_out, train_merged), (valid_out, valid_merged)]:
with open(path, "w") as f:
for r in records:
f.write(json.dumps(r, ensure_ascii=False) + "\n")
# Statistics
print("\n=== R7 Dataset Statistics ===\n")
for name, existing, new, merged in [
("Train", train_existing, aptner_train, train_merged),
("Valid", valid_existing, aptner_dev, valid_merged),
]:
e_total, e_by_class = count_entities(existing)
n_total, n_by_class = count_entities(new)
m_total, m_by_class = count_entities(merged)
print(f"--- {name} ---")
print(f" Existing: {len(existing):>6} sentences, {e_total:>6} entities")
print(f" + APTNER: {len(new):>6} sentences, {n_total:>6} entities")
print(f" = Merged: {len(merged):>6} sentences, {m_total:>6} entities")
print(f"\n Per-class breakdown (merged):")
all_labels = sorted(set(list(e_by_class.keys()) + list(n_by_class.keys())))
for label in all_labels:
e = e_by_class.get(label, 0)
n = n_by_class.get(label, 0)
m = m_by_class.get(label, 0)
print(f" {label:<16} {e:>6} existing + {n:>5} new = {m:>6}")
print()
print(f"Written: {train_out}")
print(f"Written: {valid_out}")
if __name__ == "__main__":
main()
|