Spaces:
Running
Running
| #!/usr/bin/env python3 | |
| """Extract both corpora to JSON for the HTML viewer. | |
| Uses the project's own load_units() pipeline so that the viewer | |
| sees exactly the same normalised data as every other analysis. | |
| The output uses a compact format: | |
| - `texts`: dict of message_id → message_text (deduplicated) | |
| - `records`: list of annotation records (without message_text, referencing texts via message_id) | |
| """ | |
| import json | |
| import math | |
| import sys | |
| from pathlib import Path | |
| # Ensure the project source is importable | |
| project_root = Path(__file__).resolve().parent.parent | |
| sys.path.insert(0, str(project_root / "src")) | |
| from expression_emotionnelle.data.units import load_units # noqa: E402 | |
| def _serializable(value): | |
| """Convert value to JSON-serialisable form.""" | |
| if value is None: | |
| return None | |
| if isinstance(value, float) and math.isnan(value): | |
| return None | |
| if isinstance(value, (list, tuple)): | |
| return [_serializable(v) for v in value] | |
| if isinstance(value, bool): | |
| return value | |
| if isinstance(value, (int, float)): | |
| return value | |
| s = str(value).strip() | |
| if s.lower() in ("nan", "none", "null", "<na>", ""): | |
| return None | |
| return s | |
| def main(): | |
| output_path = Path(__file__).resolve().parent / "data.json" | |
| print("Loading all units …") | |
| units = load_units(source="all") | |
| print(f" {len(units)} units loaded") | |
| texts = {} | |
| records = [] | |
| for _, row in units.iterrows(): | |
| msg_id = _serializable(row["message_id"]) | |
| msg_text = _serializable(row["message_text"]) | |
| if msg_id and msg_text and msg_id not in texts: | |
| texts[msg_id] = msg_text | |
| record = { | |
| "corpus": _serializable(row["corpus"]), | |
| "message_id": msg_id, | |
| "unit_id": _serializable(row["unit_id"]), | |
| "unit_type": _serializable(row["unit_type"]), | |
| "segment_text": _serializable(row["segment_text"]), | |
| "segment_offsets": _serializable(row["segment_offsets"]), | |
| "declencheur_text": _serializable(row["declencheur_text"]), | |
| "declencheur_offsets": _serializable(row["declencheur_offsets"]), | |
| "mode": _serializable(row["mode"]), | |
| "emotion1": _serializable(row["emotion1"]), | |
| "emotion2": _serializable(row["emotion2"]), | |
| "emotion3": _serializable(row["emotion3"]), | |
| "nature_linguistique": _serializable(row["nature_linguistique"]), | |
| "is_discontinuous": _serializable(row["is_discontinuous"]), | |
| "source_file": _serializable(row["source_file"]), | |
| } | |
| records.append(record) | |
| # Compute summary stats for the viewer sidebar | |
| corpora = sorted(set(r["corpus"] for r in records if r["corpus"])) | |
| emotions = sorted(set( | |
| str(r[f"emotion{i}"]) | |
| for r in records | |
| for i in (1, 2, 3) | |
| if r[f"emotion{i}"] is not None and str(r[f"emotion{i}"]).strip() not in ("", "nan", "None") | |
| )) | |
| modes = sorted(set( | |
| str(r["mode"]) | |
| for r in records | |
| if r["mode"] is not None and str(r["mode"]).strip() not in ("", "nan", "None") | |
| )) | |
| natures = sorted(set( | |
| str(r["nature_linguistique"]) | |
| for r in records | |
| if r["nature_linguistique"] is not None | |
| and str(r["nature_linguistique"]).strip() not in ("", "nan", "None") | |
| )) | |
| unit_types = sorted(set(str(r["unit_type"]) for r in records if r["unit_type"])) | |
| payload = { | |
| "meta": { | |
| "total": len(records), | |
| "corpora": corpora, | |
| "emotions": emotions, | |
| "modes": modes, | |
| "natures": natures, | |
| "unit_types": unit_types, | |
| }, | |
| "texts": texts, | |
| "records": records, | |
| } | |
| output_path.write_text( | |
| json.dumps(payload, ensure_ascii=False, indent=None), | |
| encoding="utf-8", | |
| ) | |
| size_mb = output_path.stat().st_size / 1024 / 1024 | |
| print(f"Wrote {len(records)} records + {len(texts)} unique texts to {output_path} ({size_mb:.1f} MB)") | |
| print(f" Corpora: {corpora}") | |
| print(f" Emotions: {emotions}") | |
| print(f" Modes: {modes}") | |
| print(f" Natures: {natures}") | |
| if __name__ == "__main__": | |
| main() | |