cksajil commited on
Commit
eae9047
Β·
1 Parent(s): 191c3a1

fix(deps): Intel Mac compatibility fixes

Browse files

- KMP_DUPLICATE_LIB_OK env var for OpenMP conflict
- Pinned transformers==4.37.2, numpy==1.26.4, torch==2.2.2
- Switched to faster-whisper, removed openai-whisper
- Fixed pyproject.toml build backend
- Switched summarizer to sshleifer/distilbart-cnn-6-6 for CPU speed
- Added constraints.txt

.python-version ADDED
@@ -0,0 +1 @@
 
 
1
+ 3.11.9
.vscode/settings.json ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ {
2
+ "python-envs.defaultEnvManager": "ms-python.python:system"
3
+ }
omnisense/config.py CHANGED
@@ -28,7 +28,7 @@ DEVICE: str = os.getenv("DEVICE", "cpu") # "cuda" | "mps" | "cpu"
28
  # ── Model identifiers (single source of truth) ────────────────────────────────
29
  MODELS = {
30
  "whisper": "openai/whisper-base",
31
- "summarizer": "facebook/bart-large-cnn",
32
  "classifier": "facebook/bart-large-mnli",
33
  "ner": "dslim/bert-base-NER",
34
  "captioner": "Salesforce/blip-image-captioning-base",
 
28
  # ── Model identifiers (single source of truth) ────────────────────────────────
29
  MODELS = {
30
  "whisper": "openai/whisper-base",
31
+ "summarizer": "sshleifer/distilbart-cnn-6-6",
32
  "classifier": "facebook/bart-large-mnli",
33
  "ner": "dslim/bert-base-NER",
34
  "captioner": "Salesforce/blip-image-captioning-base",
omnisense/pipelines/audio.py CHANGED
@@ -1,11 +1,7 @@
1
  """
2
  Audio transcription pipeline.
3
-
4
- Uses OpenAI Whisper (via HuggingFace) to produce:
5
- - Full transcript text
6
- - Timestamped segments
7
- - NLP-ready text chunks
8
- - Detected language + confidence
9
  """
10
 
11
  from __future__ import annotations
@@ -13,8 +9,7 @@ from __future__ import annotations
13
  from pathlib import Path
14
  from typing import Any
15
 
16
- import torch
17
- import whisper
18
 
19
  from omnisense.config import CACHE_DIR, DEVICE, MAX_VIDEO_DURATION, MODELS
20
  from omnisense.pipelines.base import BasePipeline
@@ -25,86 +20,80 @@ from omnisense.utils.media import (
25
  get_audio_duration,
26
  )
27
 
28
- # Video file extensions we can handle
29
  VIDEO_EXTENSIONS = {".mp4", ".mov", ".avi", ".mkv", ".webm", ".m4v"}
30
  AUDIO_EXTENSIONS = {".mp3", ".wav", ".m4a", ".flac", ".ogg"}
31
 
32
 
33
  class AudioPipeline(BasePipeline):
34
  """
35
- Transcribes audio/video files using OpenAI Whisper.
36
-
37
- Usage:
38
- pipeline = AudioPipeline(device="cpu")
39
- result = pipeline("/path/to/file.mp4")
40
 
41
  Result shape:
42
  {
43
- "transcript": str, # full joined text
44
- "segments": list[dict], # raw Whisper segments
45
- "chunks": list[dict], # NLP-ready chunks with timestamps
46
- "language": str, # detected language code e.g. "en"
47
- "duration": float, # audio duration in seconds
48
- "model": str, # model identifier used
49
  }
50
  """
51
 
52
  def __init__(self, device: str = DEVICE) -> None:
53
  super().__init__(device=device)
54
- self._model: whisper.Whisper | None = None
55
- self._model_name = (
56
- MODELS["whisper"].split("/")[-1].replace("whisper-", "")
57
- ) # "base"
58
-
59
- # ── Lifecycle ─────────────────────────────────────────────────────────────
60
 
61
  def load(self) -> None:
62
- """Download and cache Whisper weights."""
63
- log.info(f"Loading Whisper model '{self._model_name}'…")
64
- self._model = whisper.load_model(
65
- self._model_name,
66
- device=self._device_for_whisper(),
 
67
  download_root=str(CACHE_DIR / "whisper"),
68
  )
69
- log.info("Whisper model loaded βœ“")
70
-
71
- # ── Core run ──────────────────────────────────────────────────────────────
72
 
73
  def run(self, media_path: str | Path) -> dict[str, Any]:
74
- """
75
- Transcribe an audio or video file.
76
-
77
- Args:
78
- media_path: Path to audio (.wav, .mp3, …) or video (.mp4, …).
79
-
80
- Returns:
81
- Structured result dict (see class docstring).
82
-
83
- Raises:
84
- FileNotFoundError: File does not exist.
85
- ValueError: Duration exceeds MAX_VIDEO_DURATION.
86
- RuntimeError: Whisper transcription fails.
87
- """
88
  media_path = Path(media_path)
89
  self._validate_file(media_path)
90
 
91
- # If it's a video, extract audio track first
92
  audio_path = self._resolve_audio(media_path)
93
 
94
- # Guard against runaway files
95
  duration = get_audio_duration(audio_path)
96
  if duration > MAX_VIDEO_DURATION:
97
  raise ValueError(
98
- f"File duration {duration:.0f}s exceeds limit "
99
- f"of {MAX_VIDEO_DURATION}s. Set MAX_VIDEO_DURATION_SECONDS in .env."
100
  )
101
 
102
  log.info(f"Transcribing {audio_path.name} ({duration:.1f}s)…")
103
- raw = self._transcribe(audio_path)
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
104
 
105
- segments: list[dict] = raw.get("segments", [])
106
- transcript: str = raw.get("text", "").strip()
107
- language: str = raw.get("language", "unknown")
108
  chunks = chunk_transcript(segments)
109
 
110
  log.info(
@@ -121,8 +110,6 @@ class AudioPipeline(BasePipeline):
121
  "model": MODELS["whisper"],
122
  }
123
 
124
- # ── Private helpers ───────────────────────────────────────────────────────
125
-
126
  def _validate_file(self, path: Path) -> None:
127
  if not path.exists():
128
  raise FileNotFoundError(f"Media file not found: {path}")
@@ -134,30 +121,6 @@ class AudioPipeline(BasePipeline):
134
  )
135
 
136
  def _resolve_audio(self, path: Path) -> Path:
137
- """Return audio path β€” extract from video if needed."""
138
  if path.suffix.lower() in VIDEO_EXTENSIONS:
139
  return extract_audio_from_video(path, output_dir=CACHE_DIR / "audio")
140
  return path
141
-
142
- def _transcribe(self, audio_path: Path) -> dict:
143
- """Run Whisper inference with verbose logging disabled."""
144
- if self._model is None:
145
- raise RuntimeError("Model not loaded. Call load() first.")
146
- try:
147
- return self._model.transcribe(
148
- str(audio_path),
149
- verbose=False,
150
- fp16=self.device == "cuda",
151
- word_timestamps=True,
152
- )
153
- except Exception as exc:
154
- raise RuntimeError(f"Whisper transcription failed: {exc}") from exc
155
-
156
- def _device_for_whisper(self) -> str:
157
- """
158
- Whisper uses its own device strings.
159
- MPS (Apple Silicon) falls back to CPU β€” Whisper doesn't support MPS yet.
160
- """
161
- if self.device == "cuda" and torch.cuda.is_available():
162
- return "cuda"
163
- return "cpu"
 
1
  """
2
  Audio transcription pipeline.
3
+ Uses faster-whisper β€” CTranslate2 backend, no numba/llvmlite dependency,
4
+ 2-4x faster than openai-whisper with identical accuracy.
 
 
 
 
5
  """
6
 
7
  from __future__ import annotations
 
9
  from pathlib import Path
10
  from typing import Any
11
 
12
+ from faster_whisper import WhisperModel
 
13
 
14
  from omnisense.config import CACHE_DIR, DEVICE, MAX_VIDEO_DURATION, MODELS
15
  from omnisense.pipelines.base import BasePipeline
 
20
  get_audio_duration,
21
  )
22
 
 
23
  VIDEO_EXTENSIONS = {".mp4", ".mov", ".avi", ".mkv", ".webm", ".m4v"}
24
  AUDIO_EXTENSIONS = {".mp3", ".wav", ".m4a", ".flac", ".ogg"}
25
 
26
 
27
  class AudioPipeline(BasePipeline):
28
  """
29
+ Transcribes audio/video files using faster-whisper.
 
 
 
 
30
 
31
  Result shape:
32
  {
33
+ "transcript": str,
34
+ "segments": list[dict],
35
+ "chunks": list[dict],
36
+ "language": str,
37
+ "duration": float,
38
+ "model": str,
39
  }
40
  """
41
 
42
  def __init__(self, device: str = DEVICE) -> None:
43
  super().__init__(device=device)
44
+ self._model: WhisperModel | None = None
45
+ self._model_size = MODELS["whisper"].split("/")[-1].replace("whisper-", "")
46
+ self._fw_device = "cuda" if device == "cuda" else "cpu"
 
 
 
47
 
48
  def load(self) -> None:
49
+ """Download and cache faster-whisper weights."""
50
+ log.info(f"Loading faster-whisper model '{self._model_size}'…")
51
+ self._model = WhisperModel(
52
+ self._model_size,
53
+ device=self._fw_device,
54
+ compute_type="int8",
55
  download_root=str(CACHE_DIR / "whisper"),
56
  )
57
+ log.info("faster-whisper model loaded βœ“")
 
 
58
 
59
  def run(self, media_path: str | Path) -> dict[str, Any]:
60
+ """Transcribe an audio or video file."""
 
 
 
 
 
 
 
 
 
 
 
 
 
61
  media_path = Path(media_path)
62
  self._validate_file(media_path)
63
 
 
64
  audio_path = self._resolve_audio(media_path)
65
 
 
66
  duration = get_audio_duration(audio_path)
67
  if duration > MAX_VIDEO_DURATION:
68
  raise ValueError(
69
+ f"Duration {duration:.0f}s exceeds limit of {MAX_VIDEO_DURATION}s."
 
70
  )
71
 
72
  log.info(f"Transcribing {audio_path.name} ({duration:.1f}s)…")
73
+ segments_raw, info = self._model.transcribe(
74
+ str(audio_path),
75
+ beam_size=5,
76
+ word_timestamps=True,
77
+ )
78
+
79
+ segments: list[dict] = []
80
+ transcript_parts: list[str] = []
81
+ for seg in segments_raw:
82
+ segments.append(
83
+ {
84
+ "text": seg.text.strip(),
85
+ "start": seg.start,
86
+ "end": seg.end,
87
+ "words": [
88
+ {"word": w.word, "start": w.start, "end": w.end}
89
+ for w in (seg.words or [])
90
+ ],
91
+ }
92
+ )
93
+ transcript_parts.append(seg.text.strip())
94
 
95
+ transcript = " ".join(transcript_parts)
96
+ language = info.language
 
97
  chunks = chunk_transcript(segments)
98
 
99
  log.info(
 
110
  "model": MODELS["whisper"],
111
  }
112
 
 
 
113
  def _validate_file(self, path: Path) -> None:
114
  if not path.exists():
115
  raise FileNotFoundError(f"Media file not found: {path}")
 
121
  )
122
 
123
  def _resolve_audio(self, path: Path) -> Path:
 
124
  if path.suffix.lower() in VIDEO_EXTENSIONS:
125
  return extract_audio_from_video(path, output_dir=CACHE_DIR / "audio")
126
  return path
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
omnisense/pipelines/nlp.py CHANGED
@@ -1,13 +1,283 @@
1
- """NLP analysis pipeline β€” summarization, NER, zero-shot classification."""
 
2
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
3
  from omnisense.pipelines.base import BasePipeline
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
4
 
5
 
6
  class NLPPipeline(BasePipeline):
7
- """Phase 3 implementation."""
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
8
 
9
- def load(self) -> None: # noqa: D102
10
- pass
 
 
 
 
 
 
 
 
11
 
12
- def run(self, text: str) -> dict: # noqa: D102
13
- return {}
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """
2
+ NLP analysis pipeline.
3
 
4
+ Runs three models over transcript chunks produced by AudioPipeline:
5
+ 1. facebook/bart-large-cnn β€” extractive summarisation
6
+ 2. dslim/bert-base-NER β€” named entity recognition
7
+ 3. facebook/bart-large-mnli β€” zero-shot topic classification
8
+
9
+ Designed to receive the output of AudioPipeline.run() directly.
10
+ """
11
+
12
+ from __future__ import annotations
13
+
14
+ from typing import Any
15
+
16
+ import torch
17
+ from transformers import (
18
+ AutoModelForSeq2SeqLM,
19
+ AutoTokenizer,
20
+ )
21
+ from transformers import (
22
+ pipeline as hf_pipeline,
23
+ )
24
+
25
+ from omnisense.config import DEVICE, MODELS
26
  from omnisense.pipelines.base import BasePipeline
27
+ from omnisense.utils.logger import log
28
+ from omnisense.utils.text import (
29
+ aggregate_summaries,
30
+ clean_text,
31
+ merge_ner_entities,
32
+ top_entities,
33
+ )
34
+
35
+ DEFAULT_TOPICS = [
36
+ "technology",
37
+ "politics",
38
+ "sports",
39
+ "science",
40
+ "business",
41
+ "health",
42
+ "entertainment",
43
+ "education",
44
+ "environment",
45
+ "travel",
46
+ ]
47
 
48
 
49
  class NLPPipeline(BasePipeline):
50
+ """
51
+ Runs summarisation, NER, and zero-shot classification on transcript text.
52
+
53
+ Usage:
54
+ audio_result = audio_pipeline("file.mp4")
55
+ nlp = NLPPipeline()
56
+ result = nlp(audio_result)
57
+
58
+ Result shape:
59
+ {
60
+ "summary": str,
61
+ "chunk_summaries": list[str],
62
+ "entities": list[dict],
63
+ "topics": list[dict],
64
+ "top_topic": str,
65
+ "word_count": int,
66
+ "models": dict,
67
+ }
68
+ """
69
+
70
+ def __init__(self, device: str = DEVICE) -> None:
71
+ super().__init__(device=device)
72
+ self._summarizer_model = None
73
+ self._summarizer_tokenizer = None
74
+ self._ner = None
75
+ self._classifier = None
76
+ self._hf_device = 0 if device == "cuda" else -1
77
+ self._torch_device = torch.device("cuda" if device == "cuda" else "cpu")
78
+
79
+ # ── Lifecycle ─────────────────────────────────────────────────────────────
80
+
81
+ def load(self) -> None:
82
+ """Load all three models. Downloads weights on first run (~1.5 GB)."""
83
+ log.info("Loading NLP models β€” this may take a few minutes on first run…")
84
+
85
+ log.info(f" [1/3] Loading summarizer: {MODELS['summarizer']}")
86
+ self._summarizer_tokenizer = AutoTokenizer.from_pretrained(MODELS["summarizer"])
87
+ self._summarizer_model = AutoModelForSeq2SeqLM.from_pretrained(
88
+ MODELS["summarizer"]
89
+ ).to(self._torch_device)
90
+ self._summarizer_model.eval()
91
+
92
+ log.info(f" [2/3] Loading NER model: {MODELS['ner']}")
93
+ self._ner = hf_pipeline(
94
+ "token-classification",
95
+ model=MODELS["ner"],
96
+ aggregation_strategy="simple",
97
+ device=self._hf_device,
98
+ )
99
+
100
+ log.info(f" [3/3] Loading zero-shot classifier: {MODELS['classifier']}")
101
+ self._classifier = hf_pipeline(
102
+ "zero-shot-classification",
103
+ model=MODELS["classifier"],
104
+ device=self._hf_device,
105
+ )
106
+
107
+ log.info("All NLP models loaded βœ“")
108
+
109
+ # ── Core run ──────────────────────────────────────────────────────────────
110
+
111
+ def run(
112
+ self,
113
+ audio_result: dict[str, Any],
114
+ topics: list[str] | None = None,
115
+ max_summary_length: int = 150,
116
+ min_summary_length: int = 40,
117
+ ) -> dict[str, Any]:
118
+ """
119
+ Run the full NLP analysis on transcript data.
120
+
121
+ Args:
122
+ audio_result: Output dict from AudioPipeline.run().
123
+ topics: Custom topic labels for zero-shot classification.
124
+ max_summary_length: Max tokens per chunk summary.
125
+ min_summary_length: Min tokens per chunk summary.
126
+
127
+ Returns:
128
+ Structured NLP result dict.
129
+ """
130
+ self._validate_input(audio_result)
131
+
132
+ transcript: str = clean_text(audio_result["transcript"])
133
+ chunks: list[dict] = audio_result.get("chunks", [])
134
+ topics = topics or DEFAULT_TOPICS
135
+
136
+ if not transcript.strip():
137
+ log.warning("Empty transcript received β€” returning empty NLP result")
138
+ return self._empty_result()
139
+
140
+ text_chunks = [c["text"] for c in chunks] if chunks else [transcript]
141
+
142
+ log.info(f"Running NLP on {len(text_chunks)} chunk(s)…")
143
+
144
+ # 1. Summarisation
145
+ chunk_summaries = self._summarise_chunks(
146
+ text_chunks, max_summary_length, min_summary_length
147
+ )
148
+ summary = aggregate_summaries(chunk_summaries)
149
+ log.info(f"Summarisation complete β€” {len(chunk_summaries)} chunk summaries")
150
+
151
+ # 2. NER
152
+ raw_entities = self._extract_entities(transcript)
153
+ entities = top_entities(merge_ner_entities(raw_entities))
154
+ log.info(f"NER complete β€” {len(entities)} unique entities found")
155
+
156
+ # 3. Zero-shot classification
157
+ topic_result = self._classify_topics(summary or transcript[:512], topics)
158
+ log.info(f"Classification complete β€” top topic: {topic_result[0]['label']}")
159
+
160
+ return {
161
+ "summary": summary,
162
+ "chunk_summaries": chunk_summaries,
163
+ "entities": entities,
164
+ "topics": topic_result,
165
+ "top_topic": topic_result[0]["label"] if topic_result else "unknown",
166
+ "word_count": len(transcript.split()),
167
+ "models": {
168
+ "summarizer": MODELS["summarizer"],
169
+ "ner": MODELS["ner"],
170
+ "classifier": MODELS["classifier"],
171
+ },
172
+ }
173
+
174
+ # ── Private helpers ───────────────────────────────────────────────────────
175
+
176
+ def _summarise_chunks(
177
+ self,
178
+ chunks: list[str],
179
+ max_length: int,
180
+ min_length: int,
181
+ ) -> list[str]:
182
+ """Summarise each chunk using BART directly via AutoModel."""
183
+ summaries = []
184
+ for i, chunk in enumerate(chunks):
185
+ word_count = len(chunk.split())
186
+ if word_count < 30:
187
+ log.debug(f"Chunk {i} too short ({word_count} words) β€” using as-is")
188
+ summaries.append(chunk.strip())
189
+ continue
190
+ try:
191
+ inputs = self._summarizer_tokenizer(
192
+ chunk,
193
+ return_tensors="pt",
194
+ max_length=1024,
195
+ truncation=True,
196
+ ).to(self._torch_device)
197
+
198
+ with torch.no_grad():
199
+ output_ids = self._summarizer_model.generate(
200
+ **inputs,
201
+ max_new_tokens=max_length,
202
+ min_new_tokens=min(min_length, word_count // 2),
203
+ num_beams=4,
204
+ early_stopping=True,
205
+ )
206
+
207
+ summary = self._summarizer_tokenizer.decode(
208
+ output_ids[0], skip_special_tokens=True
209
+ )
210
+ summaries.append(summary)
211
+
212
+ except Exception as exc:
213
+ log.warning(f"Summarisation failed for chunk {i}: {exc}")
214
+ sentences = chunk.split(". ")[:2]
215
+ summaries.append(". ".join(sentences))
216
+
217
+ return summaries
218
+
219
+ def _extract_entities(self, text: str) -> list[dict]:
220
+ """Run token-classification NER over text."""
221
+ max_chars = 512 * 4
222
+ if len(text) <= max_chars:
223
+ try:
224
+ return self._ner(text)
225
+ except Exception as exc:
226
+ log.warning(f"NER failed: {exc}")
227
+ return []
228
+
229
+ all_entities: list[dict] = []
230
+ for start in range(0, len(text), max_chars):
231
+ window = text[start : start + max_chars]
232
+ try:
233
+ entities = self._ner(window)
234
+ for ent in entities:
235
+ ent["start"] = ent.get("start", 0) + start
236
+ ent["end"] = ent.get("end", 0) + start
237
+ all_entities.extend(entities)
238
+ except Exception as exc:
239
+ log.warning(f"NER failed on window at {start}: {exc}")
240
+
241
+ return all_entities
242
+
243
+ def _classify_topics(self, text: str, topics: list[str]) -> list[dict]:
244
+ """Run zero-shot classification."""
245
+ try:
246
+ result = self._classifier(
247
+ text[:1024],
248
+ candidate_labels=topics,
249
+ multi_label=False,
250
+ )
251
+ return [
252
+ {"label": label, "score": round(score, 4)}
253
+ for label, score in zip(result["labels"], result["scores"])
254
+ ]
255
+ except Exception as exc:
256
+ log.warning(f"Zero-shot classification failed: {exc}")
257
+ return [{"label": "unknown", "score": 0.0}]
258
 
259
+ def _validate_input(self, audio_result: dict) -> None:
260
+ if not isinstance(audio_result, dict):
261
+ raise ValueError(
262
+ f"audio_result must be a dict, got {type(audio_result).__name__}"
263
+ )
264
+ if "transcript" not in audio_result:
265
+ raise ValueError(
266
+ "audio_result must contain 'transcript' key. "
267
+ "Pass the direct output of AudioPipeline.run()."
268
+ )
269
 
270
+ def _empty_result(self) -> dict:
271
+ return {
272
+ "summary": "",
273
+ "chunk_summaries": [],
274
+ "entities": [],
275
+ "topics": [{"label": "unknown", "score": 0.0}],
276
+ "top_topic": "unknown",
277
+ "word_count": 0,
278
+ "models": {
279
+ "summarizer": MODELS["summarizer"],
280
+ "ner": MODELS["ner"],
281
+ "classifier": MODELS["classifier"],
282
+ },
283
+ }
omnisense/utils/text.py ADDED
@@ -0,0 +1,147 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """
2
+ Text processing utilities for the NLP pipeline.
3
+ Handles cleaning, merging, and post-processing of model outputs.
4
+ """
5
+
6
+ from __future__ import annotations
7
+
8
+ import re
9
+
10
+
11
+ def clean_text(text: str) -> str:
12
+ """
13
+ Normalise raw transcript text before passing to NLP models.
14
+
15
+ - Collapses multiple spaces/newlines
16
+ - Strips leading/trailing whitespace
17
+ - Removes filler words common in ASR output
18
+
19
+ Args:
20
+ text: Raw transcript string.
21
+
22
+ Returns:
23
+ Cleaned string.
24
+ """
25
+ # Collapse whitespace
26
+ text = re.sub(r"\s+", " ", text).strip()
27
+ # Remove common ASR artifacts
28
+ text = re.sub(r"\b(um|uh|hmm|mhm|uh-huh)\b", "", text, flags=re.IGNORECASE)
29
+ # Clean up any double spaces left behind
30
+ text = re.sub(r"\s{2,}", " ", text).strip()
31
+ return text
32
+
33
+
34
+ def merge_ner_entities(raw_entities: list[dict]) -> list[dict]:
35
+ """
36
+ Merge consecutive subword/word tokens from HuggingFace NER output
37
+ into clean, readable entity spans.
38
+
39
+ HuggingFace NER returns one dict per token β€” e.g. "New", "York" as
40
+ separate B-LOC / I-LOC tokens. This function joins them into
41
+ {"text": "New York", "label": "LOC", "score": 0.97}.
42
+
43
+ Args:
44
+ raw_entities: List of raw HF NER dicts with keys:
45
+ word, entity_group, score, start, end.
46
+
47
+ Returns:
48
+ Deduplicated list of merged entity dicts.
49
+ """
50
+ if not raw_entities:
51
+ return []
52
+
53
+ merged: list[dict] = []
54
+ current: dict | None = None
55
+
56
+ for ent in raw_entities:
57
+ word = ent.get("word", "").replace("##", "") # strip BERT subword prefix
58
+ group = ent.get("entity_group", ent.get("entity", ""))
59
+ score = ent.get("score", 0.0)
60
+ start = ent.get("start", 0)
61
+ end = ent.get("end", 0)
62
+
63
+ # Strip B- / I- prefixes if aggregation wasn't done upstream
64
+ label = re.sub(r"^[BI]-", "", group)
65
+
66
+ if current is None:
67
+ current = {
68
+ "text": word,
69
+ "label": label,
70
+ "score": score,
71
+ "start": start,
72
+ "end": end,
73
+ }
74
+ elif label == current["label"] and start <= current["end"] + 2:
75
+ # Continuation β€” extend current entity
76
+ current["text"] = f"{current['text']} {word}".strip()
77
+ current["end"] = end
78
+ current["score"] = (current["score"] + score) / 2
79
+ else:
80
+ merged.append(current)
81
+ current = {
82
+ "text": word,
83
+ "label": label,
84
+ "score": score,
85
+ "start": start,
86
+ "end": end,
87
+ }
88
+
89
+ if current:
90
+ merged.append(current)
91
+
92
+ # Deduplicate by (text, label) keeping highest score
93
+ seen: dict[tuple, dict] = {}
94
+ for ent in merged:
95
+ key = (ent["text"].lower(), ent["label"])
96
+ if key not in seen or ent["score"] > seen[key]["score"]:
97
+ seen[key] = ent
98
+
99
+ return sorted(seen.values(), key=lambda e: e["score"], reverse=True)
100
+
101
+
102
+ def aggregate_summaries(summaries: list[str]) -> str:
103
+ """
104
+ Combine per-chunk summaries into a single coherent summary.
105
+
106
+ For short inputs (1–2 chunks) just joins with a space.
107
+ For longer inputs, deduplicates repeated sentences that
108
+ BART tends to produce across overlapping chunks.
109
+
110
+ Args:
111
+ summaries: List of summary strings, one per chunk.
112
+
113
+ Returns:
114
+ Single aggregated summary string.
115
+ """
116
+ if not summaries:
117
+ return ""
118
+ if len(summaries) == 1:
119
+ return summaries[0]
120
+
121
+ # Split into sentences, deduplicate while preserving order
122
+ seen_sentences: set[str] = set()
123
+ final_sentences: list[str] = []
124
+
125
+ for summary in summaries:
126
+ sentences = re.split(r"(?<=[.!?])\s+", summary.strip())
127
+ for sent in sentences:
128
+ normalised = sent.lower().strip()
129
+ if normalised and normalised not in seen_sentences:
130
+ seen_sentences.add(normalised)
131
+ final_sentences.append(sent.strip())
132
+
133
+ return " ".join(final_sentences)
134
+
135
+
136
+ def top_entities(entities: list[dict], top_n: int = 10) -> list[dict]:
137
+ """
138
+ Return the top N most confident entities, one per unique text span.
139
+
140
+ Args:
141
+ entities: Merged entity list from merge_ner_entities().
142
+ top_n: Maximum number of entities to return.
143
+
144
+ Returns:
145
+ Top N entities sorted by score descending.
146
+ """
147
+ return entities[:top_n]
pyproject.toml CHANGED
@@ -1,6 +1,6 @@
1
  [build-system]
2
  requires = ["setuptools>=68", "wheel"]
3
- build-backend = "setuptools.backends.legacy:build"
4
 
5
  [project]
6
  name = "omnisense"
 
1
  [build-system]
2
  requires = ["setuptools>=68", "wheel"]
3
+ build-backend = "setuptools.build_meta"
4
 
5
  [project]
6
  name = "omnisense"
tests/test_nlp_pipeline.py ADDED
@@ -0,0 +1,268 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """
2
+ Tests for the NLP pipeline and text utilities.
3
+ All model calls are mocked β€” no weights downloaded during testing.
4
+ """
5
+
6
+ from __future__ import annotations
7
+
8
+ from unittest.mock import MagicMock
9
+
10
+ import pytest
11
+
12
+ from omnisense.pipelines.nlp import NLPPipeline
13
+ from omnisense.utils.text import (
14
+ aggregate_summaries,
15
+ clean_text,
16
+ merge_ner_entities,
17
+ top_entities,
18
+ )
19
+
20
+ # ── Text utilities ────────────────────────────────────────────────────────────
21
+
22
+
23
+ class TestCleanText:
24
+ def test_collapses_whitespace(self):
25
+ assert clean_text("hello world") == "hello world"
26
+
27
+ def test_strips_filler_words(self):
28
+ result = clean_text("um so uh this is hmm a test")
29
+ assert "um" not in result
30
+ assert "uh" not in result
31
+ assert "hmm" not in result
32
+
33
+ def test_preserves_meaningful_content(self):
34
+ text = "The quick brown fox jumps over the lazy dog."
35
+ assert clean_text(text) == text
36
+
37
+ def test_handles_empty_string(self):
38
+ assert clean_text("") == ""
39
+
40
+
41
+ class TestMergeNerEntities:
42
+ def test_empty_input(self):
43
+ assert merge_ner_entities([]) == []
44
+
45
+ def test_merges_consecutive_tokens(self):
46
+ raw = [
47
+ {"word": "New", "entity_group": "LOC", "score": 0.99, "start": 0, "end": 3},
48
+ {
49
+ "word": "York",
50
+ "entity_group": "LOC",
51
+ "score": 0.98,
52
+ "start": 4,
53
+ "end": 8,
54
+ },
55
+ ]
56
+ merged = merge_ner_entities(raw)
57
+ assert len(merged) == 1
58
+ assert merged[0]["text"] == "New York"
59
+ assert merged[0]["label"] == "LOC"
60
+
61
+ def test_separates_different_entity_types(self):
62
+ raw = [
63
+ {
64
+ "word": "Apple",
65
+ "entity_group": "ORG",
66
+ "score": 0.95,
67
+ "start": 0,
68
+ "end": 5,
69
+ },
70
+ {
71
+ "word": "London",
72
+ "entity_group": "LOC",
73
+ "score": 0.97,
74
+ "start": 10,
75
+ "end": 16,
76
+ },
77
+ ]
78
+ merged = merge_ner_entities(raw)
79
+ assert len(merged) == 2
80
+
81
+ def test_deduplicates_same_entity(self):
82
+ raw = [
83
+ {
84
+ "word": "Apple",
85
+ "entity_group": "ORG",
86
+ "score": 0.95,
87
+ "start": 0,
88
+ "end": 5,
89
+ },
90
+ {
91
+ "word": "Apple",
92
+ "entity_group": "ORG",
93
+ "score": 0.90,
94
+ "start": 20,
95
+ "end": 25,
96
+ },
97
+ ]
98
+ merged = merge_ner_entities(raw)
99
+ assert len(merged) == 1
100
+ assert merged[0]["score"] == 0.95 # keeps highest score
101
+
102
+ def test_strips_bi_prefixes(self):
103
+ raw = [
104
+ {
105
+ "word": "Paris",
106
+ "entity_group": "B-LOC",
107
+ "score": 0.99,
108
+ "start": 0,
109
+ "end": 5,
110
+ },
111
+ ]
112
+ merged = merge_ner_entities(raw)
113
+ assert merged[0]["label"] == "LOC"
114
+
115
+
116
+ class TestAggregateSummaries:
117
+ def test_empty_returns_empty_string(self):
118
+ assert aggregate_summaries([]) == ""
119
+
120
+ def test_single_summary_returned_as_is(self):
121
+ assert aggregate_summaries(["Hello world."]) == "Hello world."
122
+
123
+ def test_deduplicates_repeated_sentences(self):
124
+ s = "The cat sat on the mat."
125
+ result = aggregate_summaries([s, s])
126
+ assert result.count("The cat sat on the mat.") == 1
127
+
128
+ def test_joins_unique_sentences(self):
129
+ result = aggregate_summaries(["First sentence.", "Second sentence."])
130
+ assert "First sentence." in result
131
+ assert "Second sentence." in result
132
+
133
+
134
+ class TestTopEntities:
135
+ def test_limits_to_top_n(self):
136
+ entities = [
137
+ {"text": f"Entity{i}", "label": "ORG", "score": float(i) / 10}
138
+ for i in range(20)
139
+ ]
140
+ assert len(top_entities(entities, top_n=5)) == 5
141
+
142
+ def test_preserves_order(self):
143
+ entities = [
144
+ {"text": "A", "label": "ORG", "score": 0.9},
145
+ {"text": "B", "label": "ORG", "score": 0.8},
146
+ ]
147
+ result = top_entities(entities)
148
+ assert result[0]["text"] == "A"
149
+
150
+
151
+ # ── NLPPipeline ───────────────────────────────────────────────────────────────
152
+
153
+
154
+ def make_audio_result(
155
+ transcript: str = "This is a test transcript about technology.",
156
+ ) -> dict:
157
+ """Helper β€” minimal valid AudioPipeline output."""
158
+ return {
159
+ "transcript": transcript,
160
+ "chunks": [{"text": transcript, "start": 0.0, "end": 5.0, "segment_ids": [0]}],
161
+ "language": "en",
162
+ "duration": 5.0,
163
+ "model": "openai/whisper-base",
164
+ }
165
+
166
+
167
+ class TestNLPPipeline:
168
+ def _make_loaded_pipeline(self) -> NLPPipeline:
169
+ """Return a pipeline with mocked models β€” no weights downloaded."""
170
+ pipeline = NLPPipeline(device="cpu")
171
+ pipeline._loaded = True
172
+
173
+ pipeline._summarizer = MagicMock(
174
+ return_value=[{"summary_text": "A test summary about technology."}]
175
+ )
176
+ pipeline._ner = MagicMock(
177
+ return_value=[
178
+ {
179
+ "word": "OpenAI",
180
+ "entity_group": "ORG",
181
+ "score": 0.99,
182
+ "start": 0,
183
+ "end": 6,
184
+ },
185
+ ]
186
+ )
187
+ pipeline._classifier = MagicMock(
188
+ return_value={
189
+ "labels": ["technology", "science", "business"],
190
+ "scores": [0.85, 0.10, 0.05],
191
+ }
192
+ )
193
+ return pipeline
194
+
195
+ def test_raises_on_missing_transcript_key(self):
196
+ pipeline = self._make_loaded_pipeline()
197
+ with pytest.raises(ValueError, match="transcript"):
198
+ pipeline.run({"chunks": []})
199
+
200
+ def test_raises_on_non_dict_input(self):
201
+ pipeline = self._make_loaded_pipeline()
202
+ with pytest.raises(ValueError):
203
+ pipeline.run("raw string input")
204
+
205
+ def test_result_has_expected_keys(self):
206
+ pipeline = self._make_loaded_pipeline()
207
+ result = pipeline.run(make_audio_result())
208
+ expected = {
209
+ "summary",
210
+ "chunk_summaries",
211
+ "entities",
212
+ "topics",
213
+ "top_topic",
214
+ "word_count",
215
+ "models",
216
+ }
217
+ assert set(result.keys()) == expected
218
+
219
+ def test_empty_transcript_returns_empty_result(self):
220
+ pipeline = self._make_loaded_pipeline()
221
+ result = pipeline.run(make_audio_result(transcript=""))
222
+ assert result["summary"] == ""
223
+ assert result["entities"] == []
224
+ assert result["top_topic"] == "unknown"
225
+
226
+ def test_top_topic_matches_highest_score(self):
227
+ pipeline = self._make_loaded_pipeline()
228
+ result = pipeline.run(make_audio_result())
229
+ assert result["top_topic"] == "technology"
230
+
231
+ def test_word_count_is_accurate(self):
232
+ transcript = "one two three four five"
233
+ pipeline = self._make_loaded_pipeline()
234
+ result = pipeline.run(make_audio_result(transcript=transcript))
235
+ assert result["word_count"] == 5
236
+
237
+ def test_models_dict_contains_all_three(self):
238
+ pipeline = self._make_loaded_pipeline()
239
+ result = pipeline.run(make_audio_result())
240
+ assert "summarizer" in result["models"]
241
+ assert "ner" in result["models"]
242
+ assert "classifier" in result["models"]
243
+
244
+ def test_custom_topics_are_used(self):
245
+ pipeline = self._make_loaded_pipeline()
246
+ custom_topics = ["finance", "crypto", "real estate"]
247
+ pipeline._classifier = MagicMock(
248
+ return_value={
249
+ "labels": ["finance", "crypto", "real estate"],
250
+ "scores": [0.7, 0.2, 0.1],
251
+ }
252
+ )
253
+ result = pipeline.run(make_audio_result(), topics=custom_topics)
254
+ assert result["top_topic"] == "finance"
255
+
256
+ def test_graceful_degradation_on_summarizer_failure(self):
257
+ pipeline = self._make_loaded_pipeline()
258
+ pipeline._summarizer = MagicMock(side_effect=RuntimeError("model error"))
259
+ # Should not raise β€” falls back to first 2 sentences
260
+ result = pipeline.run(make_audio_result())
261
+ assert isinstance(result["summary"], str)
262
+
263
+ def test_no_chunks_falls_back_to_full_transcript(self):
264
+ pipeline = self._make_loaded_pipeline()
265
+ audio_result = make_audio_result()
266
+ audio_result["chunks"] = [] # no chunks
267
+ result = pipeline.run(audio_result)
268
+ assert result["summary"] != ""