Spaces:
Running
Running
File size: 4,213 Bytes
fd179e3 5096233 fd179e3 5096233 fd179e3 5096233 fd179e3 5096233 fd179e3 5096233 fd179e3 5096233 fd179e3 5096233 fd179e3 5096233 fd179e3 5096233 fd179e3 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 | #!/usr/bin/env python3
"""Extract both corpora to JSON for the HTML viewer.
Uses the project's own load_units() pipeline so that the viewer
sees exactly the same normalised data as every other analysis.
The output uses a compact format:
- `texts`: dict of message_id → message_text (deduplicated)
- `records`: list of annotation records (without message_text, referencing texts via message_id)
"""
import json
import math
import sys
from pathlib import Path
# Ensure the project source is importable
project_root = Path(__file__).resolve().parent.parent
sys.path.insert(0, str(project_root / "src"))
from expression_emotionnelle.data.units import load_units # noqa: E402
def _serializable(value):
"""Convert value to JSON-serialisable form."""
if value is None:
return None
if isinstance(value, float) and math.isnan(value):
return None
if isinstance(value, (list, tuple)):
return [_serializable(v) for v in value]
if isinstance(value, bool):
return value
if isinstance(value, (int, float)):
return value
s = str(value).strip()
if s.lower() in ("nan", "none", "null", "<na>", ""):
return None
return s
def main():
output_path = Path(__file__).resolve().parent / "data.json"
print("Loading all units …")
units = load_units(source="all")
print(f" {len(units)} units loaded")
texts = {}
records = []
for _, row in units.iterrows():
msg_id = _serializable(row["message_id"])
msg_text = _serializable(row["message_text"])
if msg_id and msg_text and msg_id not in texts:
texts[msg_id] = msg_text
record = {
"corpus": _serializable(row["corpus"]),
"message_id": msg_id,
"unit_id": _serializable(row["unit_id"]),
"unit_type": _serializable(row["unit_type"]),
"segment_text": _serializable(row["segment_text"]),
"segment_offsets": _serializable(row["segment_offsets"]),
"declencheur_text": _serializable(row["declencheur_text"]),
"declencheur_offsets": _serializable(row["declencheur_offsets"]),
"mode": _serializable(row["mode"]),
"emotion1": _serializable(row["emotion1"]),
"emotion2": _serializable(row["emotion2"]),
"emotion3": _serializable(row["emotion3"]),
"nature_linguistique": _serializable(row["nature_linguistique"]),
"is_discontinuous": _serializable(row["is_discontinuous"]),
"source_file": _serializable(row["source_file"]),
}
records.append(record)
# Compute summary stats for the viewer sidebar
corpora = sorted(set(r["corpus"] for r in records if r["corpus"]))
emotions = sorted(set(
str(r[f"emotion{i}"])
for r in records
for i in (1, 2, 3)
if r[f"emotion{i}"] is not None and str(r[f"emotion{i}"]).strip() not in ("", "nan", "None")
))
modes = sorted(set(
str(r["mode"])
for r in records
if r["mode"] is not None and str(r["mode"]).strip() not in ("", "nan", "None")
))
natures = sorted(set(
str(r["nature_linguistique"])
for r in records
if r["nature_linguistique"] is not None
and str(r["nature_linguistique"]).strip() not in ("", "nan", "None")
))
unit_types = sorted(set(str(r["unit_type"]) for r in records if r["unit_type"]))
payload = {
"meta": {
"total": len(records),
"corpora": corpora,
"emotions": emotions,
"modes": modes,
"natures": natures,
"unit_types": unit_types,
},
"texts": texts,
"records": records,
}
output_path.write_text(
json.dumps(payload, ensure_ascii=False, indent=None),
encoding="utf-8",
)
size_mb = output_path.stat().st_size / 1024 / 1024
print(f"Wrote {len(records)} records + {len(texts)} unique texts to {output_path} ({size_mb:.1f} MB)")
print(f" Corpora: {corpora}")
print(f" Emotions: {emotions}")
print(f" Modes: {modes}")
print(f" Natures: {natures}")
if __name__ == "__main__":
main()
|