cksajil commited on
Commit Β·
eae9047
1
Parent(s): 191c3a1
fix(deps): Intel Mac compatibility fixes
Browse files- KMP_DUPLICATE_LIB_OK env var for OpenMP conflict
- Pinned transformers==4.37.2, numpy==1.26.4, torch==2.2.2
- Switched to faster-whisper, removed openai-whisper
- Fixed pyproject.toml build backend
- Switched summarizer to sshleifer/distilbart-cnn-6-6 for CPU speed
- Added constraints.txt
- .python-version +1 -0
- .vscode/settings.json +3 -0
- omnisense/config.py +1 -1
- omnisense/pipelines/audio.py +45 -82
- omnisense/pipelines/nlp.py +276 -6
- omnisense/utils/text.py +147 -0
- pyproject.toml +1 -1
- tests/test_nlp_pipeline.py +268 -0
.python-version
ADDED
|
@@ -0,0 +1 @@
|
|
|
|
|
|
|
| 1 |
+
3.11.9
|
.vscode/settings.json
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"python-envs.defaultEnvManager": "ms-python.python:system"
|
| 3 |
+
}
|
omnisense/config.py
CHANGED
|
@@ -28,7 +28,7 @@ DEVICE: str = os.getenv("DEVICE", "cpu") # "cuda" | "mps" | "cpu"
|
|
| 28 |
# ββ Model identifiers (single source of truth) ββββββββββββββββββββββββββββββββ
|
| 29 |
MODELS = {
|
| 30 |
"whisper": "openai/whisper-base",
|
| 31 |
-
"summarizer": "
|
| 32 |
"classifier": "facebook/bart-large-mnli",
|
| 33 |
"ner": "dslim/bert-base-NER",
|
| 34 |
"captioner": "Salesforce/blip-image-captioning-base",
|
|
|
|
| 28 |
# ββ Model identifiers (single source of truth) ββββββββββββββββββββββββββββββββ
|
| 29 |
MODELS = {
|
| 30 |
"whisper": "openai/whisper-base",
|
| 31 |
+
"summarizer": "sshleifer/distilbart-cnn-6-6",
|
| 32 |
"classifier": "facebook/bart-large-mnli",
|
| 33 |
"ner": "dslim/bert-base-NER",
|
| 34 |
"captioner": "Salesforce/blip-image-captioning-base",
|
omnisense/pipelines/audio.py
CHANGED
|
@@ -1,11 +1,7 @@
|
|
| 1 |
"""
|
| 2 |
Audio transcription pipeline.
|
| 3 |
-
|
| 4 |
-
|
| 5 |
-
- Full transcript text
|
| 6 |
-
- Timestamped segments
|
| 7 |
-
- NLP-ready text chunks
|
| 8 |
-
- Detected language + confidence
|
| 9 |
"""
|
| 10 |
|
| 11 |
from __future__ import annotations
|
|
@@ -13,8 +9,7 @@ from __future__ import annotations
|
|
| 13 |
from pathlib import Path
|
| 14 |
from typing import Any
|
| 15 |
|
| 16 |
-
import
|
| 17 |
-
import whisper
|
| 18 |
|
| 19 |
from omnisense.config import CACHE_DIR, DEVICE, MAX_VIDEO_DURATION, MODELS
|
| 20 |
from omnisense.pipelines.base import BasePipeline
|
|
@@ -25,86 +20,80 @@ from omnisense.utils.media import (
|
|
| 25 |
get_audio_duration,
|
| 26 |
)
|
| 27 |
|
| 28 |
-
# Video file extensions we can handle
|
| 29 |
VIDEO_EXTENSIONS = {".mp4", ".mov", ".avi", ".mkv", ".webm", ".m4v"}
|
| 30 |
AUDIO_EXTENSIONS = {".mp3", ".wav", ".m4a", ".flac", ".ogg"}
|
| 31 |
|
| 32 |
|
| 33 |
class AudioPipeline(BasePipeline):
|
| 34 |
"""
|
| 35 |
-
Transcribes audio/video files using
|
| 36 |
-
|
| 37 |
-
Usage:
|
| 38 |
-
pipeline = AudioPipeline(device="cpu")
|
| 39 |
-
result = pipeline("/path/to/file.mp4")
|
| 40 |
|
| 41 |
Result shape:
|
| 42 |
{
|
| 43 |
-
"transcript": str,
|
| 44 |
-
"segments": list[dict],
|
| 45 |
-
"chunks": list[dict],
|
| 46 |
-
"language": str,
|
| 47 |
-
"duration": float,
|
| 48 |
-
"model": str,
|
| 49 |
}
|
| 50 |
"""
|
| 51 |
|
| 52 |
def __init__(self, device: str = DEVICE) -> None:
|
| 53 |
super().__init__(device=device)
|
| 54 |
-
self._model:
|
| 55 |
-
self.
|
| 56 |
-
|
| 57 |
-
) # "base"
|
| 58 |
-
|
| 59 |
-
# ββ Lifecycle βββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββ
|
| 60 |
|
| 61 |
def load(self) -> None:
|
| 62 |
-
"""Download and cache
|
| 63 |
-
log.info(f"Loading
|
| 64 |
-
self._model =
|
| 65 |
-
self.
|
| 66 |
-
device=self.
|
|
|
|
| 67 |
download_root=str(CACHE_DIR / "whisper"),
|
| 68 |
)
|
| 69 |
-
log.info("
|
| 70 |
-
|
| 71 |
-
# ββ Core run ββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββ
|
| 72 |
|
| 73 |
def run(self, media_path: str | Path) -> dict[str, Any]:
|
| 74 |
-
"""
|
| 75 |
-
Transcribe an audio or video file.
|
| 76 |
-
|
| 77 |
-
Args:
|
| 78 |
-
media_path: Path to audio (.wav, .mp3, β¦) or video (.mp4, β¦).
|
| 79 |
-
|
| 80 |
-
Returns:
|
| 81 |
-
Structured result dict (see class docstring).
|
| 82 |
-
|
| 83 |
-
Raises:
|
| 84 |
-
FileNotFoundError: File does not exist.
|
| 85 |
-
ValueError: Duration exceeds MAX_VIDEO_DURATION.
|
| 86 |
-
RuntimeError: Whisper transcription fails.
|
| 87 |
-
"""
|
| 88 |
media_path = Path(media_path)
|
| 89 |
self._validate_file(media_path)
|
| 90 |
|
| 91 |
-
# If it's a video, extract audio track first
|
| 92 |
audio_path = self._resolve_audio(media_path)
|
| 93 |
|
| 94 |
-
# Guard against runaway files
|
| 95 |
duration = get_audio_duration(audio_path)
|
| 96 |
if duration > MAX_VIDEO_DURATION:
|
| 97 |
raise ValueError(
|
| 98 |
-
f"
|
| 99 |
-
f"of {MAX_VIDEO_DURATION}s. Set MAX_VIDEO_DURATION_SECONDS in .env."
|
| 100 |
)
|
| 101 |
|
| 102 |
log.info(f"Transcribing {audio_path.name} ({duration:.1f}s)β¦")
|
| 103 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 104 |
|
| 105 |
-
|
| 106 |
-
|
| 107 |
-
language: str = raw.get("language", "unknown")
|
| 108 |
chunks = chunk_transcript(segments)
|
| 109 |
|
| 110 |
log.info(
|
|
@@ -121,8 +110,6 @@ class AudioPipeline(BasePipeline):
|
|
| 121 |
"model": MODELS["whisper"],
|
| 122 |
}
|
| 123 |
|
| 124 |
-
# ββ Private helpers βββββββββββββββββββββββββββββββββββββββββββββββββββββββ
|
| 125 |
-
|
| 126 |
def _validate_file(self, path: Path) -> None:
|
| 127 |
if not path.exists():
|
| 128 |
raise FileNotFoundError(f"Media file not found: {path}")
|
|
@@ -134,30 +121,6 @@ class AudioPipeline(BasePipeline):
|
|
| 134 |
)
|
| 135 |
|
| 136 |
def _resolve_audio(self, path: Path) -> Path:
|
| 137 |
-
"""Return audio path β extract from video if needed."""
|
| 138 |
if path.suffix.lower() in VIDEO_EXTENSIONS:
|
| 139 |
return extract_audio_from_video(path, output_dir=CACHE_DIR / "audio")
|
| 140 |
return path
|
| 141 |
-
|
| 142 |
-
def _transcribe(self, audio_path: Path) -> dict:
|
| 143 |
-
"""Run Whisper inference with verbose logging disabled."""
|
| 144 |
-
if self._model is None:
|
| 145 |
-
raise RuntimeError("Model not loaded. Call load() first.")
|
| 146 |
-
try:
|
| 147 |
-
return self._model.transcribe(
|
| 148 |
-
str(audio_path),
|
| 149 |
-
verbose=False,
|
| 150 |
-
fp16=self.device == "cuda",
|
| 151 |
-
word_timestamps=True,
|
| 152 |
-
)
|
| 153 |
-
except Exception as exc:
|
| 154 |
-
raise RuntimeError(f"Whisper transcription failed: {exc}") from exc
|
| 155 |
-
|
| 156 |
-
def _device_for_whisper(self) -> str:
|
| 157 |
-
"""
|
| 158 |
-
Whisper uses its own device strings.
|
| 159 |
-
MPS (Apple Silicon) falls back to CPU β Whisper doesn't support MPS yet.
|
| 160 |
-
"""
|
| 161 |
-
if self.device == "cuda" and torch.cuda.is_available():
|
| 162 |
-
return "cuda"
|
| 163 |
-
return "cpu"
|
|
|
|
| 1 |
"""
|
| 2 |
Audio transcription pipeline.
|
| 3 |
+
Uses faster-whisper β CTranslate2 backend, no numba/llvmlite dependency,
|
| 4 |
+
2-4x faster than openai-whisper with identical accuracy.
|
|
|
|
|
|
|
|
|
|
|
|
|
| 5 |
"""
|
| 6 |
|
| 7 |
from __future__ import annotations
|
|
|
|
| 9 |
from pathlib import Path
|
| 10 |
from typing import Any
|
| 11 |
|
| 12 |
+
from faster_whisper import WhisperModel
|
|
|
|
| 13 |
|
| 14 |
from omnisense.config import CACHE_DIR, DEVICE, MAX_VIDEO_DURATION, MODELS
|
| 15 |
from omnisense.pipelines.base import BasePipeline
|
|
|
|
| 20 |
get_audio_duration,
|
| 21 |
)
|
| 22 |
|
|
|
|
| 23 |
VIDEO_EXTENSIONS = {".mp4", ".mov", ".avi", ".mkv", ".webm", ".m4v"}
|
| 24 |
AUDIO_EXTENSIONS = {".mp3", ".wav", ".m4a", ".flac", ".ogg"}
|
| 25 |
|
| 26 |
|
| 27 |
class AudioPipeline(BasePipeline):
|
| 28 |
"""
|
| 29 |
+
Transcribes audio/video files using faster-whisper.
|
|
|
|
|
|
|
|
|
|
|
|
|
| 30 |
|
| 31 |
Result shape:
|
| 32 |
{
|
| 33 |
+
"transcript": str,
|
| 34 |
+
"segments": list[dict],
|
| 35 |
+
"chunks": list[dict],
|
| 36 |
+
"language": str,
|
| 37 |
+
"duration": float,
|
| 38 |
+
"model": str,
|
| 39 |
}
|
| 40 |
"""
|
| 41 |
|
| 42 |
def __init__(self, device: str = DEVICE) -> None:
|
| 43 |
super().__init__(device=device)
|
| 44 |
+
self._model: WhisperModel | None = None
|
| 45 |
+
self._model_size = MODELS["whisper"].split("/")[-1].replace("whisper-", "")
|
| 46 |
+
self._fw_device = "cuda" if device == "cuda" else "cpu"
|
|
|
|
|
|
|
|
|
|
| 47 |
|
| 48 |
def load(self) -> None:
|
| 49 |
+
"""Download and cache faster-whisper weights."""
|
| 50 |
+
log.info(f"Loading faster-whisper model '{self._model_size}'β¦")
|
| 51 |
+
self._model = WhisperModel(
|
| 52 |
+
self._model_size,
|
| 53 |
+
device=self._fw_device,
|
| 54 |
+
compute_type="int8",
|
| 55 |
download_root=str(CACHE_DIR / "whisper"),
|
| 56 |
)
|
| 57 |
+
log.info("faster-whisper model loaded β")
|
|
|
|
|
|
|
| 58 |
|
| 59 |
def run(self, media_path: str | Path) -> dict[str, Any]:
|
| 60 |
+
"""Transcribe an audio or video file."""
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 61 |
media_path = Path(media_path)
|
| 62 |
self._validate_file(media_path)
|
| 63 |
|
|
|
|
| 64 |
audio_path = self._resolve_audio(media_path)
|
| 65 |
|
|
|
|
| 66 |
duration = get_audio_duration(audio_path)
|
| 67 |
if duration > MAX_VIDEO_DURATION:
|
| 68 |
raise ValueError(
|
| 69 |
+
f"Duration {duration:.0f}s exceeds limit of {MAX_VIDEO_DURATION}s."
|
|
|
|
| 70 |
)
|
| 71 |
|
| 72 |
log.info(f"Transcribing {audio_path.name} ({duration:.1f}s)β¦")
|
| 73 |
+
segments_raw, info = self._model.transcribe(
|
| 74 |
+
str(audio_path),
|
| 75 |
+
beam_size=5,
|
| 76 |
+
word_timestamps=True,
|
| 77 |
+
)
|
| 78 |
+
|
| 79 |
+
segments: list[dict] = []
|
| 80 |
+
transcript_parts: list[str] = []
|
| 81 |
+
for seg in segments_raw:
|
| 82 |
+
segments.append(
|
| 83 |
+
{
|
| 84 |
+
"text": seg.text.strip(),
|
| 85 |
+
"start": seg.start,
|
| 86 |
+
"end": seg.end,
|
| 87 |
+
"words": [
|
| 88 |
+
{"word": w.word, "start": w.start, "end": w.end}
|
| 89 |
+
for w in (seg.words or [])
|
| 90 |
+
],
|
| 91 |
+
}
|
| 92 |
+
)
|
| 93 |
+
transcript_parts.append(seg.text.strip())
|
| 94 |
|
| 95 |
+
transcript = " ".join(transcript_parts)
|
| 96 |
+
language = info.language
|
|
|
|
| 97 |
chunks = chunk_transcript(segments)
|
| 98 |
|
| 99 |
log.info(
|
|
|
|
| 110 |
"model": MODELS["whisper"],
|
| 111 |
}
|
| 112 |
|
|
|
|
|
|
|
| 113 |
def _validate_file(self, path: Path) -> None:
|
| 114 |
if not path.exists():
|
| 115 |
raise FileNotFoundError(f"Media file not found: {path}")
|
|
|
|
| 121 |
)
|
| 122 |
|
| 123 |
def _resolve_audio(self, path: Path) -> Path:
|
|
|
|
| 124 |
if path.suffix.lower() in VIDEO_EXTENSIONS:
|
| 125 |
return extract_audio_from_video(path, output_dir=CACHE_DIR / "audio")
|
| 126 |
return path
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
omnisense/pipelines/nlp.py
CHANGED
|
@@ -1,13 +1,283 @@
|
|
| 1 |
-
"""
|
|
|
|
| 2 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 3 |
from omnisense.pipelines.base import BasePipeline
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 4 |
|
| 5 |
|
| 6 |
class NLPPipeline(BasePipeline):
|
| 7 |
-
"""
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 8 |
|
| 9 |
-
def
|
| 10 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 11 |
|
| 12 |
-
def
|
| 13 |
-
return {
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""
|
| 2 |
+
NLP analysis pipeline.
|
| 3 |
|
| 4 |
+
Runs three models over transcript chunks produced by AudioPipeline:
|
| 5 |
+
1. facebook/bart-large-cnn β extractive summarisation
|
| 6 |
+
2. dslim/bert-base-NER β named entity recognition
|
| 7 |
+
3. facebook/bart-large-mnli β zero-shot topic classification
|
| 8 |
+
|
| 9 |
+
Designed to receive the output of AudioPipeline.run() directly.
|
| 10 |
+
"""
|
| 11 |
+
|
| 12 |
+
from __future__ import annotations
|
| 13 |
+
|
| 14 |
+
from typing import Any
|
| 15 |
+
|
| 16 |
+
import torch
|
| 17 |
+
from transformers import (
|
| 18 |
+
AutoModelForSeq2SeqLM,
|
| 19 |
+
AutoTokenizer,
|
| 20 |
+
)
|
| 21 |
+
from transformers import (
|
| 22 |
+
pipeline as hf_pipeline,
|
| 23 |
+
)
|
| 24 |
+
|
| 25 |
+
from omnisense.config import DEVICE, MODELS
|
| 26 |
from omnisense.pipelines.base import BasePipeline
|
| 27 |
+
from omnisense.utils.logger import log
|
| 28 |
+
from omnisense.utils.text import (
|
| 29 |
+
aggregate_summaries,
|
| 30 |
+
clean_text,
|
| 31 |
+
merge_ner_entities,
|
| 32 |
+
top_entities,
|
| 33 |
+
)
|
| 34 |
+
|
| 35 |
+
DEFAULT_TOPICS = [
|
| 36 |
+
"technology",
|
| 37 |
+
"politics",
|
| 38 |
+
"sports",
|
| 39 |
+
"science",
|
| 40 |
+
"business",
|
| 41 |
+
"health",
|
| 42 |
+
"entertainment",
|
| 43 |
+
"education",
|
| 44 |
+
"environment",
|
| 45 |
+
"travel",
|
| 46 |
+
]
|
| 47 |
|
| 48 |
|
| 49 |
class NLPPipeline(BasePipeline):
|
| 50 |
+
"""
|
| 51 |
+
Runs summarisation, NER, and zero-shot classification on transcript text.
|
| 52 |
+
|
| 53 |
+
Usage:
|
| 54 |
+
audio_result = audio_pipeline("file.mp4")
|
| 55 |
+
nlp = NLPPipeline()
|
| 56 |
+
result = nlp(audio_result)
|
| 57 |
+
|
| 58 |
+
Result shape:
|
| 59 |
+
{
|
| 60 |
+
"summary": str,
|
| 61 |
+
"chunk_summaries": list[str],
|
| 62 |
+
"entities": list[dict],
|
| 63 |
+
"topics": list[dict],
|
| 64 |
+
"top_topic": str,
|
| 65 |
+
"word_count": int,
|
| 66 |
+
"models": dict,
|
| 67 |
+
}
|
| 68 |
+
"""
|
| 69 |
+
|
| 70 |
+
def __init__(self, device: str = DEVICE) -> None:
|
| 71 |
+
super().__init__(device=device)
|
| 72 |
+
self._summarizer_model = None
|
| 73 |
+
self._summarizer_tokenizer = None
|
| 74 |
+
self._ner = None
|
| 75 |
+
self._classifier = None
|
| 76 |
+
self._hf_device = 0 if device == "cuda" else -1
|
| 77 |
+
self._torch_device = torch.device("cuda" if device == "cuda" else "cpu")
|
| 78 |
+
|
| 79 |
+
# ββ Lifecycle βββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββ
|
| 80 |
+
|
| 81 |
+
def load(self) -> None:
|
| 82 |
+
"""Load all three models. Downloads weights on first run (~1.5 GB)."""
|
| 83 |
+
log.info("Loading NLP models β this may take a few minutes on first runβ¦")
|
| 84 |
+
|
| 85 |
+
log.info(f" [1/3] Loading summarizer: {MODELS['summarizer']}")
|
| 86 |
+
self._summarizer_tokenizer = AutoTokenizer.from_pretrained(MODELS["summarizer"])
|
| 87 |
+
self._summarizer_model = AutoModelForSeq2SeqLM.from_pretrained(
|
| 88 |
+
MODELS["summarizer"]
|
| 89 |
+
).to(self._torch_device)
|
| 90 |
+
self._summarizer_model.eval()
|
| 91 |
+
|
| 92 |
+
log.info(f" [2/3] Loading NER model: {MODELS['ner']}")
|
| 93 |
+
self._ner = hf_pipeline(
|
| 94 |
+
"token-classification",
|
| 95 |
+
model=MODELS["ner"],
|
| 96 |
+
aggregation_strategy="simple",
|
| 97 |
+
device=self._hf_device,
|
| 98 |
+
)
|
| 99 |
+
|
| 100 |
+
log.info(f" [3/3] Loading zero-shot classifier: {MODELS['classifier']}")
|
| 101 |
+
self._classifier = hf_pipeline(
|
| 102 |
+
"zero-shot-classification",
|
| 103 |
+
model=MODELS["classifier"],
|
| 104 |
+
device=self._hf_device,
|
| 105 |
+
)
|
| 106 |
+
|
| 107 |
+
log.info("All NLP models loaded β")
|
| 108 |
+
|
| 109 |
+
# ββ Core run ββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββ
|
| 110 |
+
|
| 111 |
+
def run(
|
| 112 |
+
self,
|
| 113 |
+
audio_result: dict[str, Any],
|
| 114 |
+
topics: list[str] | None = None,
|
| 115 |
+
max_summary_length: int = 150,
|
| 116 |
+
min_summary_length: int = 40,
|
| 117 |
+
) -> dict[str, Any]:
|
| 118 |
+
"""
|
| 119 |
+
Run the full NLP analysis on transcript data.
|
| 120 |
+
|
| 121 |
+
Args:
|
| 122 |
+
audio_result: Output dict from AudioPipeline.run().
|
| 123 |
+
topics: Custom topic labels for zero-shot classification.
|
| 124 |
+
max_summary_length: Max tokens per chunk summary.
|
| 125 |
+
min_summary_length: Min tokens per chunk summary.
|
| 126 |
+
|
| 127 |
+
Returns:
|
| 128 |
+
Structured NLP result dict.
|
| 129 |
+
"""
|
| 130 |
+
self._validate_input(audio_result)
|
| 131 |
+
|
| 132 |
+
transcript: str = clean_text(audio_result["transcript"])
|
| 133 |
+
chunks: list[dict] = audio_result.get("chunks", [])
|
| 134 |
+
topics = topics or DEFAULT_TOPICS
|
| 135 |
+
|
| 136 |
+
if not transcript.strip():
|
| 137 |
+
log.warning("Empty transcript received β returning empty NLP result")
|
| 138 |
+
return self._empty_result()
|
| 139 |
+
|
| 140 |
+
text_chunks = [c["text"] for c in chunks] if chunks else [transcript]
|
| 141 |
+
|
| 142 |
+
log.info(f"Running NLP on {len(text_chunks)} chunk(s)β¦")
|
| 143 |
+
|
| 144 |
+
# 1. Summarisation
|
| 145 |
+
chunk_summaries = self._summarise_chunks(
|
| 146 |
+
text_chunks, max_summary_length, min_summary_length
|
| 147 |
+
)
|
| 148 |
+
summary = aggregate_summaries(chunk_summaries)
|
| 149 |
+
log.info(f"Summarisation complete β {len(chunk_summaries)} chunk summaries")
|
| 150 |
+
|
| 151 |
+
# 2. NER
|
| 152 |
+
raw_entities = self._extract_entities(transcript)
|
| 153 |
+
entities = top_entities(merge_ner_entities(raw_entities))
|
| 154 |
+
log.info(f"NER complete β {len(entities)} unique entities found")
|
| 155 |
+
|
| 156 |
+
# 3. Zero-shot classification
|
| 157 |
+
topic_result = self._classify_topics(summary or transcript[:512], topics)
|
| 158 |
+
log.info(f"Classification complete β top topic: {topic_result[0]['label']}")
|
| 159 |
+
|
| 160 |
+
return {
|
| 161 |
+
"summary": summary,
|
| 162 |
+
"chunk_summaries": chunk_summaries,
|
| 163 |
+
"entities": entities,
|
| 164 |
+
"topics": topic_result,
|
| 165 |
+
"top_topic": topic_result[0]["label"] if topic_result else "unknown",
|
| 166 |
+
"word_count": len(transcript.split()),
|
| 167 |
+
"models": {
|
| 168 |
+
"summarizer": MODELS["summarizer"],
|
| 169 |
+
"ner": MODELS["ner"],
|
| 170 |
+
"classifier": MODELS["classifier"],
|
| 171 |
+
},
|
| 172 |
+
}
|
| 173 |
+
|
| 174 |
+
# ββ Private helpers βββββββββββββββββββββββββββββββββββββββββββββββββββββββ
|
| 175 |
+
|
| 176 |
+
def _summarise_chunks(
|
| 177 |
+
self,
|
| 178 |
+
chunks: list[str],
|
| 179 |
+
max_length: int,
|
| 180 |
+
min_length: int,
|
| 181 |
+
) -> list[str]:
|
| 182 |
+
"""Summarise each chunk using BART directly via AutoModel."""
|
| 183 |
+
summaries = []
|
| 184 |
+
for i, chunk in enumerate(chunks):
|
| 185 |
+
word_count = len(chunk.split())
|
| 186 |
+
if word_count < 30:
|
| 187 |
+
log.debug(f"Chunk {i} too short ({word_count} words) β using as-is")
|
| 188 |
+
summaries.append(chunk.strip())
|
| 189 |
+
continue
|
| 190 |
+
try:
|
| 191 |
+
inputs = self._summarizer_tokenizer(
|
| 192 |
+
chunk,
|
| 193 |
+
return_tensors="pt",
|
| 194 |
+
max_length=1024,
|
| 195 |
+
truncation=True,
|
| 196 |
+
).to(self._torch_device)
|
| 197 |
+
|
| 198 |
+
with torch.no_grad():
|
| 199 |
+
output_ids = self._summarizer_model.generate(
|
| 200 |
+
**inputs,
|
| 201 |
+
max_new_tokens=max_length,
|
| 202 |
+
min_new_tokens=min(min_length, word_count // 2),
|
| 203 |
+
num_beams=4,
|
| 204 |
+
early_stopping=True,
|
| 205 |
+
)
|
| 206 |
+
|
| 207 |
+
summary = self._summarizer_tokenizer.decode(
|
| 208 |
+
output_ids[0], skip_special_tokens=True
|
| 209 |
+
)
|
| 210 |
+
summaries.append(summary)
|
| 211 |
+
|
| 212 |
+
except Exception as exc:
|
| 213 |
+
log.warning(f"Summarisation failed for chunk {i}: {exc}")
|
| 214 |
+
sentences = chunk.split(". ")[:2]
|
| 215 |
+
summaries.append(". ".join(sentences))
|
| 216 |
+
|
| 217 |
+
return summaries
|
| 218 |
+
|
| 219 |
+
def _extract_entities(self, text: str) -> list[dict]:
|
| 220 |
+
"""Run token-classification NER over text."""
|
| 221 |
+
max_chars = 512 * 4
|
| 222 |
+
if len(text) <= max_chars:
|
| 223 |
+
try:
|
| 224 |
+
return self._ner(text)
|
| 225 |
+
except Exception as exc:
|
| 226 |
+
log.warning(f"NER failed: {exc}")
|
| 227 |
+
return []
|
| 228 |
+
|
| 229 |
+
all_entities: list[dict] = []
|
| 230 |
+
for start in range(0, len(text), max_chars):
|
| 231 |
+
window = text[start : start + max_chars]
|
| 232 |
+
try:
|
| 233 |
+
entities = self._ner(window)
|
| 234 |
+
for ent in entities:
|
| 235 |
+
ent["start"] = ent.get("start", 0) + start
|
| 236 |
+
ent["end"] = ent.get("end", 0) + start
|
| 237 |
+
all_entities.extend(entities)
|
| 238 |
+
except Exception as exc:
|
| 239 |
+
log.warning(f"NER failed on window at {start}: {exc}")
|
| 240 |
+
|
| 241 |
+
return all_entities
|
| 242 |
+
|
| 243 |
+
def _classify_topics(self, text: str, topics: list[str]) -> list[dict]:
|
| 244 |
+
"""Run zero-shot classification."""
|
| 245 |
+
try:
|
| 246 |
+
result = self._classifier(
|
| 247 |
+
text[:1024],
|
| 248 |
+
candidate_labels=topics,
|
| 249 |
+
multi_label=False,
|
| 250 |
+
)
|
| 251 |
+
return [
|
| 252 |
+
{"label": label, "score": round(score, 4)}
|
| 253 |
+
for label, score in zip(result["labels"], result["scores"])
|
| 254 |
+
]
|
| 255 |
+
except Exception as exc:
|
| 256 |
+
log.warning(f"Zero-shot classification failed: {exc}")
|
| 257 |
+
return [{"label": "unknown", "score": 0.0}]
|
| 258 |
|
| 259 |
+
def _validate_input(self, audio_result: dict) -> None:
|
| 260 |
+
if not isinstance(audio_result, dict):
|
| 261 |
+
raise ValueError(
|
| 262 |
+
f"audio_result must be a dict, got {type(audio_result).__name__}"
|
| 263 |
+
)
|
| 264 |
+
if "transcript" not in audio_result:
|
| 265 |
+
raise ValueError(
|
| 266 |
+
"audio_result must contain 'transcript' key. "
|
| 267 |
+
"Pass the direct output of AudioPipeline.run()."
|
| 268 |
+
)
|
| 269 |
|
| 270 |
+
def _empty_result(self) -> dict:
|
| 271 |
+
return {
|
| 272 |
+
"summary": "",
|
| 273 |
+
"chunk_summaries": [],
|
| 274 |
+
"entities": [],
|
| 275 |
+
"topics": [{"label": "unknown", "score": 0.0}],
|
| 276 |
+
"top_topic": "unknown",
|
| 277 |
+
"word_count": 0,
|
| 278 |
+
"models": {
|
| 279 |
+
"summarizer": MODELS["summarizer"],
|
| 280 |
+
"ner": MODELS["ner"],
|
| 281 |
+
"classifier": MODELS["classifier"],
|
| 282 |
+
},
|
| 283 |
+
}
|
omnisense/utils/text.py
ADDED
|
@@ -0,0 +1,147 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""
|
| 2 |
+
Text processing utilities for the NLP pipeline.
|
| 3 |
+
Handles cleaning, merging, and post-processing of model outputs.
|
| 4 |
+
"""
|
| 5 |
+
|
| 6 |
+
from __future__ import annotations
|
| 7 |
+
|
| 8 |
+
import re
|
| 9 |
+
|
| 10 |
+
|
| 11 |
+
def clean_text(text: str) -> str:
|
| 12 |
+
"""
|
| 13 |
+
Normalise raw transcript text before passing to NLP models.
|
| 14 |
+
|
| 15 |
+
- Collapses multiple spaces/newlines
|
| 16 |
+
- Strips leading/trailing whitespace
|
| 17 |
+
- Removes filler words common in ASR output
|
| 18 |
+
|
| 19 |
+
Args:
|
| 20 |
+
text: Raw transcript string.
|
| 21 |
+
|
| 22 |
+
Returns:
|
| 23 |
+
Cleaned string.
|
| 24 |
+
"""
|
| 25 |
+
# Collapse whitespace
|
| 26 |
+
text = re.sub(r"\s+", " ", text).strip()
|
| 27 |
+
# Remove common ASR artifacts
|
| 28 |
+
text = re.sub(r"\b(um|uh|hmm|mhm|uh-huh)\b", "", text, flags=re.IGNORECASE)
|
| 29 |
+
# Clean up any double spaces left behind
|
| 30 |
+
text = re.sub(r"\s{2,}", " ", text).strip()
|
| 31 |
+
return text
|
| 32 |
+
|
| 33 |
+
|
| 34 |
+
def merge_ner_entities(raw_entities: list[dict]) -> list[dict]:
|
| 35 |
+
"""
|
| 36 |
+
Merge consecutive subword/word tokens from HuggingFace NER output
|
| 37 |
+
into clean, readable entity spans.
|
| 38 |
+
|
| 39 |
+
HuggingFace NER returns one dict per token β e.g. "New", "York" as
|
| 40 |
+
separate B-LOC / I-LOC tokens. This function joins them into
|
| 41 |
+
{"text": "New York", "label": "LOC", "score": 0.97}.
|
| 42 |
+
|
| 43 |
+
Args:
|
| 44 |
+
raw_entities: List of raw HF NER dicts with keys:
|
| 45 |
+
word, entity_group, score, start, end.
|
| 46 |
+
|
| 47 |
+
Returns:
|
| 48 |
+
Deduplicated list of merged entity dicts.
|
| 49 |
+
"""
|
| 50 |
+
if not raw_entities:
|
| 51 |
+
return []
|
| 52 |
+
|
| 53 |
+
merged: list[dict] = []
|
| 54 |
+
current: dict | None = None
|
| 55 |
+
|
| 56 |
+
for ent in raw_entities:
|
| 57 |
+
word = ent.get("word", "").replace("##", "") # strip BERT subword prefix
|
| 58 |
+
group = ent.get("entity_group", ent.get("entity", ""))
|
| 59 |
+
score = ent.get("score", 0.0)
|
| 60 |
+
start = ent.get("start", 0)
|
| 61 |
+
end = ent.get("end", 0)
|
| 62 |
+
|
| 63 |
+
# Strip B- / I- prefixes if aggregation wasn't done upstream
|
| 64 |
+
label = re.sub(r"^[BI]-", "", group)
|
| 65 |
+
|
| 66 |
+
if current is None:
|
| 67 |
+
current = {
|
| 68 |
+
"text": word,
|
| 69 |
+
"label": label,
|
| 70 |
+
"score": score,
|
| 71 |
+
"start": start,
|
| 72 |
+
"end": end,
|
| 73 |
+
}
|
| 74 |
+
elif label == current["label"] and start <= current["end"] + 2:
|
| 75 |
+
# Continuation β extend current entity
|
| 76 |
+
current["text"] = f"{current['text']} {word}".strip()
|
| 77 |
+
current["end"] = end
|
| 78 |
+
current["score"] = (current["score"] + score) / 2
|
| 79 |
+
else:
|
| 80 |
+
merged.append(current)
|
| 81 |
+
current = {
|
| 82 |
+
"text": word,
|
| 83 |
+
"label": label,
|
| 84 |
+
"score": score,
|
| 85 |
+
"start": start,
|
| 86 |
+
"end": end,
|
| 87 |
+
}
|
| 88 |
+
|
| 89 |
+
if current:
|
| 90 |
+
merged.append(current)
|
| 91 |
+
|
| 92 |
+
# Deduplicate by (text, label) keeping highest score
|
| 93 |
+
seen: dict[tuple, dict] = {}
|
| 94 |
+
for ent in merged:
|
| 95 |
+
key = (ent["text"].lower(), ent["label"])
|
| 96 |
+
if key not in seen or ent["score"] > seen[key]["score"]:
|
| 97 |
+
seen[key] = ent
|
| 98 |
+
|
| 99 |
+
return sorted(seen.values(), key=lambda e: e["score"], reverse=True)
|
| 100 |
+
|
| 101 |
+
|
| 102 |
+
def aggregate_summaries(summaries: list[str]) -> str:
|
| 103 |
+
"""
|
| 104 |
+
Combine per-chunk summaries into a single coherent summary.
|
| 105 |
+
|
| 106 |
+
For short inputs (1β2 chunks) just joins with a space.
|
| 107 |
+
For longer inputs, deduplicates repeated sentences that
|
| 108 |
+
BART tends to produce across overlapping chunks.
|
| 109 |
+
|
| 110 |
+
Args:
|
| 111 |
+
summaries: List of summary strings, one per chunk.
|
| 112 |
+
|
| 113 |
+
Returns:
|
| 114 |
+
Single aggregated summary string.
|
| 115 |
+
"""
|
| 116 |
+
if not summaries:
|
| 117 |
+
return ""
|
| 118 |
+
if len(summaries) == 1:
|
| 119 |
+
return summaries[0]
|
| 120 |
+
|
| 121 |
+
# Split into sentences, deduplicate while preserving order
|
| 122 |
+
seen_sentences: set[str] = set()
|
| 123 |
+
final_sentences: list[str] = []
|
| 124 |
+
|
| 125 |
+
for summary in summaries:
|
| 126 |
+
sentences = re.split(r"(?<=[.!?])\s+", summary.strip())
|
| 127 |
+
for sent in sentences:
|
| 128 |
+
normalised = sent.lower().strip()
|
| 129 |
+
if normalised and normalised not in seen_sentences:
|
| 130 |
+
seen_sentences.add(normalised)
|
| 131 |
+
final_sentences.append(sent.strip())
|
| 132 |
+
|
| 133 |
+
return " ".join(final_sentences)
|
| 134 |
+
|
| 135 |
+
|
| 136 |
+
def top_entities(entities: list[dict], top_n: int = 10) -> list[dict]:
|
| 137 |
+
"""
|
| 138 |
+
Return the top N most confident entities, one per unique text span.
|
| 139 |
+
|
| 140 |
+
Args:
|
| 141 |
+
entities: Merged entity list from merge_ner_entities().
|
| 142 |
+
top_n: Maximum number of entities to return.
|
| 143 |
+
|
| 144 |
+
Returns:
|
| 145 |
+
Top N entities sorted by score descending.
|
| 146 |
+
"""
|
| 147 |
+
return entities[:top_n]
|
pyproject.toml
CHANGED
|
@@ -1,6 +1,6 @@
|
|
| 1 |
[build-system]
|
| 2 |
requires = ["setuptools>=68", "wheel"]
|
| 3 |
-
build-backend = "setuptools.
|
| 4 |
|
| 5 |
[project]
|
| 6 |
name = "omnisense"
|
|
|
|
| 1 |
[build-system]
|
| 2 |
requires = ["setuptools>=68", "wheel"]
|
| 3 |
+
build-backend = "setuptools.build_meta"
|
| 4 |
|
| 5 |
[project]
|
| 6 |
name = "omnisense"
|
tests/test_nlp_pipeline.py
ADDED
|
@@ -0,0 +1,268 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""
|
| 2 |
+
Tests for the NLP pipeline and text utilities.
|
| 3 |
+
All model calls are mocked β no weights downloaded during testing.
|
| 4 |
+
"""
|
| 5 |
+
|
| 6 |
+
from __future__ import annotations
|
| 7 |
+
|
| 8 |
+
from unittest.mock import MagicMock
|
| 9 |
+
|
| 10 |
+
import pytest
|
| 11 |
+
|
| 12 |
+
from omnisense.pipelines.nlp import NLPPipeline
|
| 13 |
+
from omnisense.utils.text import (
|
| 14 |
+
aggregate_summaries,
|
| 15 |
+
clean_text,
|
| 16 |
+
merge_ner_entities,
|
| 17 |
+
top_entities,
|
| 18 |
+
)
|
| 19 |
+
|
| 20 |
+
# ββ Text utilities ββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββ
|
| 21 |
+
|
| 22 |
+
|
| 23 |
+
class TestCleanText:
|
| 24 |
+
def test_collapses_whitespace(self):
|
| 25 |
+
assert clean_text("hello world") == "hello world"
|
| 26 |
+
|
| 27 |
+
def test_strips_filler_words(self):
|
| 28 |
+
result = clean_text("um so uh this is hmm a test")
|
| 29 |
+
assert "um" not in result
|
| 30 |
+
assert "uh" not in result
|
| 31 |
+
assert "hmm" not in result
|
| 32 |
+
|
| 33 |
+
def test_preserves_meaningful_content(self):
|
| 34 |
+
text = "The quick brown fox jumps over the lazy dog."
|
| 35 |
+
assert clean_text(text) == text
|
| 36 |
+
|
| 37 |
+
def test_handles_empty_string(self):
|
| 38 |
+
assert clean_text("") == ""
|
| 39 |
+
|
| 40 |
+
|
| 41 |
+
class TestMergeNerEntities:
|
| 42 |
+
def test_empty_input(self):
|
| 43 |
+
assert merge_ner_entities([]) == []
|
| 44 |
+
|
| 45 |
+
def test_merges_consecutive_tokens(self):
|
| 46 |
+
raw = [
|
| 47 |
+
{"word": "New", "entity_group": "LOC", "score": 0.99, "start": 0, "end": 3},
|
| 48 |
+
{
|
| 49 |
+
"word": "York",
|
| 50 |
+
"entity_group": "LOC",
|
| 51 |
+
"score": 0.98,
|
| 52 |
+
"start": 4,
|
| 53 |
+
"end": 8,
|
| 54 |
+
},
|
| 55 |
+
]
|
| 56 |
+
merged = merge_ner_entities(raw)
|
| 57 |
+
assert len(merged) == 1
|
| 58 |
+
assert merged[0]["text"] == "New York"
|
| 59 |
+
assert merged[0]["label"] == "LOC"
|
| 60 |
+
|
| 61 |
+
def test_separates_different_entity_types(self):
|
| 62 |
+
raw = [
|
| 63 |
+
{
|
| 64 |
+
"word": "Apple",
|
| 65 |
+
"entity_group": "ORG",
|
| 66 |
+
"score": 0.95,
|
| 67 |
+
"start": 0,
|
| 68 |
+
"end": 5,
|
| 69 |
+
},
|
| 70 |
+
{
|
| 71 |
+
"word": "London",
|
| 72 |
+
"entity_group": "LOC",
|
| 73 |
+
"score": 0.97,
|
| 74 |
+
"start": 10,
|
| 75 |
+
"end": 16,
|
| 76 |
+
},
|
| 77 |
+
]
|
| 78 |
+
merged = merge_ner_entities(raw)
|
| 79 |
+
assert len(merged) == 2
|
| 80 |
+
|
| 81 |
+
def test_deduplicates_same_entity(self):
|
| 82 |
+
raw = [
|
| 83 |
+
{
|
| 84 |
+
"word": "Apple",
|
| 85 |
+
"entity_group": "ORG",
|
| 86 |
+
"score": 0.95,
|
| 87 |
+
"start": 0,
|
| 88 |
+
"end": 5,
|
| 89 |
+
},
|
| 90 |
+
{
|
| 91 |
+
"word": "Apple",
|
| 92 |
+
"entity_group": "ORG",
|
| 93 |
+
"score": 0.90,
|
| 94 |
+
"start": 20,
|
| 95 |
+
"end": 25,
|
| 96 |
+
},
|
| 97 |
+
]
|
| 98 |
+
merged = merge_ner_entities(raw)
|
| 99 |
+
assert len(merged) == 1
|
| 100 |
+
assert merged[0]["score"] == 0.95 # keeps highest score
|
| 101 |
+
|
| 102 |
+
def test_strips_bi_prefixes(self):
|
| 103 |
+
raw = [
|
| 104 |
+
{
|
| 105 |
+
"word": "Paris",
|
| 106 |
+
"entity_group": "B-LOC",
|
| 107 |
+
"score": 0.99,
|
| 108 |
+
"start": 0,
|
| 109 |
+
"end": 5,
|
| 110 |
+
},
|
| 111 |
+
]
|
| 112 |
+
merged = merge_ner_entities(raw)
|
| 113 |
+
assert merged[0]["label"] == "LOC"
|
| 114 |
+
|
| 115 |
+
|
| 116 |
+
class TestAggregateSummaries:
|
| 117 |
+
def test_empty_returns_empty_string(self):
|
| 118 |
+
assert aggregate_summaries([]) == ""
|
| 119 |
+
|
| 120 |
+
def test_single_summary_returned_as_is(self):
|
| 121 |
+
assert aggregate_summaries(["Hello world."]) == "Hello world."
|
| 122 |
+
|
| 123 |
+
def test_deduplicates_repeated_sentences(self):
|
| 124 |
+
s = "The cat sat on the mat."
|
| 125 |
+
result = aggregate_summaries([s, s])
|
| 126 |
+
assert result.count("The cat sat on the mat.") == 1
|
| 127 |
+
|
| 128 |
+
def test_joins_unique_sentences(self):
|
| 129 |
+
result = aggregate_summaries(["First sentence.", "Second sentence."])
|
| 130 |
+
assert "First sentence." in result
|
| 131 |
+
assert "Second sentence." in result
|
| 132 |
+
|
| 133 |
+
|
| 134 |
+
class TestTopEntities:
|
| 135 |
+
def test_limits_to_top_n(self):
|
| 136 |
+
entities = [
|
| 137 |
+
{"text": f"Entity{i}", "label": "ORG", "score": float(i) / 10}
|
| 138 |
+
for i in range(20)
|
| 139 |
+
]
|
| 140 |
+
assert len(top_entities(entities, top_n=5)) == 5
|
| 141 |
+
|
| 142 |
+
def test_preserves_order(self):
|
| 143 |
+
entities = [
|
| 144 |
+
{"text": "A", "label": "ORG", "score": 0.9},
|
| 145 |
+
{"text": "B", "label": "ORG", "score": 0.8},
|
| 146 |
+
]
|
| 147 |
+
result = top_entities(entities)
|
| 148 |
+
assert result[0]["text"] == "A"
|
| 149 |
+
|
| 150 |
+
|
| 151 |
+
# ββ NLPPipeline βββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββ
|
| 152 |
+
|
| 153 |
+
|
| 154 |
+
def make_audio_result(
|
| 155 |
+
transcript: str = "This is a test transcript about technology.",
|
| 156 |
+
) -> dict:
|
| 157 |
+
"""Helper β minimal valid AudioPipeline output."""
|
| 158 |
+
return {
|
| 159 |
+
"transcript": transcript,
|
| 160 |
+
"chunks": [{"text": transcript, "start": 0.0, "end": 5.0, "segment_ids": [0]}],
|
| 161 |
+
"language": "en",
|
| 162 |
+
"duration": 5.0,
|
| 163 |
+
"model": "openai/whisper-base",
|
| 164 |
+
}
|
| 165 |
+
|
| 166 |
+
|
| 167 |
+
class TestNLPPipeline:
|
| 168 |
+
def _make_loaded_pipeline(self) -> NLPPipeline:
|
| 169 |
+
"""Return a pipeline with mocked models β no weights downloaded."""
|
| 170 |
+
pipeline = NLPPipeline(device="cpu")
|
| 171 |
+
pipeline._loaded = True
|
| 172 |
+
|
| 173 |
+
pipeline._summarizer = MagicMock(
|
| 174 |
+
return_value=[{"summary_text": "A test summary about technology."}]
|
| 175 |
+
)
|
| 176 |
+
pipeline._ner = MagicMock(
|
| 177 |
+
return_value=[
|
| 178 |
+
{
|
| 179 |
+
"word": "OpenAI",
|
| 180 |
+
"entity_group": "ORG",
|
| 181 |
+
"score": 0.99,
|
| 182 |
+
"start": 0,
|
| 183 |
+
"end": 6,
|
| 184 |
+
},
|
| 185 |
+
]
|
| 186 |
+
)
|
| 187 |
+
pipeline._classifier = MagicMock(
|
| 188 |
+
return_value={
|
| 189 |
+
"labels": ["technology", "science", "business"],
|
| 190 |
+
"scores": [0.85, 0.10, 0.05],
|
| 191 |
+
}
|
| 192 |
+
)
|
| 193 |
+
return pipeline
|
| 194 |
+
|
| 195 |
+
def test_raises_on_missing_transcript_key(self):
|
| 196 |
+
pipeline = self._make_loaded_pipeline()
|
| 197 |
+
with pytest.raises(ValueError, match="transcript"):
|
| 198 |
+
pipeline.run({"chunks": []})
|
| 199 |
+
|
| 200 |
+
def test_raises_on_non_dict_input(self):
|
| 201 |
+
pipeline = self._make_loaded_pipeline()
|
| 202 |
+
with pytest.raises(ValueError):
|
| 203 |
+
pipeline.run("raw string input")
|
| 204 |
+
|
| 205 |
+
def test_result_has_expected_keys(self):
|
| 206 |
+
pipeline = self._make_loaded_pipeline()
|
| 207 |
+
result = pipeline.run(make_audio_result())
|
| 208 |
+
expected = {
|
| 209 |
+
"summary",
|
| 210 |
+
"chunk_summaries",
|
| 211 |
+
"entities",
|
| 212 |
+
"topics",
|
| 213 |
+
"top_topic",
|
| 214 |
+
"word_count",
|
| 215 |
+
"models",
|
| 216 |
+
}
|
| 217 |
+
assert set(result.keys()) == expected
|
| 218 |
+
|
| 219 |
+
def test_empty_transcript_returns_empty_result(self):
|
| 220 |
+
pipeline = self._make_loaded_pipeline()
|
| 221 |
+
result = pipeline.run(make_audio_result(transcript=""))
|
| 222 |
+
assert result["summary"] == ""
|
| 223 |
+
assert result["entities"] == []
|
| 224 |
+
assert result["top_topic"] == "unknown"
|
| 225 |
+
|
| 226 |
+
def test_top_topic_matches_highest_score(self):
|
| 227 |
+
pipeline = self._make_loaded_pipeline()
|
| 228 |
+
result = pipeline.run(make_audio_result())
|
| 229 |
+
assert result["top_topic"] == "technology"
|
| 230 |
+
|
| 231 |
+
def test_word_count_is_accurate(self):
|
| 232 |
+
transcript = "one two three four five"
|
| 233 |
+
pipeline = self._make_loaded_pipeline()
|
| 234 |
+
result = pipeline.run(make_audio_result(transcript=transcript))
|
| 235 |
+
assert result["word_count"] == 5
|
| 236 |
+
|
| 237 |
+
def test_models_dict_contains_all_three(self):
|
| 238 |
+
pipeline = self._make_loaded_pipeline()
|
| 239 |
+
result = pipeline.run(make_audio_result())
|
| 240 |
+
assert "summarizer" in result["models"]
|
| 241 |
+
assert "ner" in result["models"]
|
| 242 |
+
assert "classifier" in result["models"]
|
| 243 |
+
|
| 244 |
+
def test_custom_topics_are_used(self):
|
| 245 |
+
pipeline = self._make_loaded_pipeline()
|
| 246 |
+
custom_topics = ["finance", "crypto", "real estate"]
|
| 247 |
+
pipeline._classifier = MagicMock(
|
| 248 |
+
return_value={
|
| 249 |
+
"labels": ["finance", "crypto", "real estate"],
|
| 250 |
+
"scores": [0.7, 0.2, 0.1],
|
| 251 |
+
}
|
| 252 |
+
)
|
| 253 |
+
result = pipeline.run(make_audio_result(), topics=custom_topics)
|
| 254 |
+
assert result["top_topic"] == "finance"
|
| 255 |
+
|
| 256 |
+
def test_graceful_degradation_on_summarizer_failure(self):
|
| 257 |
+
pipeline = self._make_loaded_pipeline()
|
| 258 |
+
pipeline._summarizer = MagicMock(side_effect=RuntimeError("model error"))
|
| 259 |
+
# Should not raise β falls back to first 2 sentences
|
| 260 |
+
result = pipeline.run(make_audio_result())
|
| 261 |
+
assert isinstance(result["summary"], str)
|
| 262 |
+
|
| 263 |
+
def test_no_chunks_falls_back_to_full_transcript(self):
|
| 264 |
+
pipeline = self._make_loaded_pipeline()
|
| 265 |
+
audio_result = make_audio_result()
|
| 266 |
+
audio_result["chunks"] = [] # no chunks
|
| 267 |
+
result = pipeline.run(audio_result)
|
| 268 |
+
assert result["summary"] != ""
|