""" Word Highlighting - Correct Approach Keep slide text as-is, highlight based on audio transcript analysis by LLM Fixed: No duplicates, no stop words, only important unique words """ import re import json from typing import List, Dict from openai import OpenAI import logging logger = logging.getLogger(__name__) def get_word_timings_from_whisper(audio_path: str, client: OpenAI) -> List[Dict]: """Get word-level timestamps from audio using Whisper API.""" try: with open(audio_path, 'rb') as audio_file: transcript = client.audio.transcriptions.create( model="whisper-1", file=audio_file, response_format="verbose_json", timestamp_granularities=["word"] ) word_timings = [] if hasattr(transcript, 'words') and transcript.words: logger.debug(f"Type of transcript.words: {type(transcript.words)}") if isinstance(transcript.words, list) and len(transcript.words) > 0: logger.debug(f"First 5 words: {[w.word for w in transcript.words[:5]]}") elif not isinstance(transcript.words, list): logger.debug(f"transcript.words is not a list. Content: {transcript.words}") for word_data in transcript.words: word_timings.append({ 'word': word_data.word.strip(), 'start': word_data.start, 'end': word_data.end }) logger.info(f"Extracted {len(word_timings)} word timings from Whisper") return word_timings except Exception as e: logger.error(f"Error getting word timings from Whisper: {e}") return [] def identify_highlight_words_with_llm(slide_title: str, slide_text: str, audio_transcript: str, word_timings: List[Dict], client: OpenAI) -> List[str]: """Identifies important keywords from slide text and audio transcript using an LLM, avoiding words in the title.""" try: prompt = f"""You are an expert linguistic analyst. Your task is to identify the most important keywords from a presentation slide's narration for visual highlighting. SLIDE TITLE: --- {slide_title} --- SLIDE TEXT (the content on the screen): --- {slide_text} --- AUDIO TRANSCRIPT (the spoken words): --- {audio_transcript} --- TASK: 1. Identify the 4-6 MOST IMPORTANT keywords from the audio transcript to highlight. 2. Focus on words that add new, specific information beyond the slide title. 3. Prioritize technical terms, action verbs, and concepts that are crucial for understanding the details. 4. EXCLUDE common stop words (e.g., 'the', 'a', 'is', 'in', 'of') and filler words. 5. Avoid selecting words that are already in the SLIDE TITLE, unless they are critical to the meaning of the sentence. For example, if the title is "Understanding APIs", you can still highlight "API" or "APIs" if they are the main subject of a bullet point.6. Return ONLY UNIQUE, SINGLE words. Do not return phrases. If a word appears multiple times, include it only once. CRITICAL INSTRUCTIONS: - The output MUST be a valid JSON array of strings. - Do not include any explanation or introductory text, only the JSON array itself. EXAMPLE (for a slide titled "Introduction to Python"): [ "Variables", "Functions", "Looping", "Syntax" ] Now, analyze the provided title, text, and transcript and return the JSON array of keywords.""" completion = client.chat.completions.create( model="gpt-3.5-turbo", messages=[{"role": "user", "content": prompt}], temperature=0.2, max_tokens=1000 ) response = completion.choices[0].message.content.strip() try: highlight_words = json.loads(response) if not isinstance(highlight_words, list) or not all(isinstance(i, str) for i in highlight_words): logger.warning(f"LLM response was not a JSON array of strings: {response}") return [] seen_words = set() unique_highlights = [] for word in highlight_words: cleaned_word = word.strip().lower() if cleaned_word and cleaned_word not in seen_words: seen_words.add(cleaned_word) unique_highlights.append(word) logger.debug(f"LLM identified {len(unique_highlights)} unique keywords to highlight: {unique_highlights}") return unique_highlights except json.JSONDecodeError: logger.error(f"Failed to parse LLM response as JSON: {response}") return [] except Exception as e: logger.error(f"Error in identify_highlight_words_with_llm: {e}", exc_info=True) return [] def process_all_bullets_for_highlighting(slide_title: str, bullets: List[str], audio_path: str, audio_duration: float, client: OpenAI, base_delay: float = 2.8, bullet_spacing: float = 1.7) -> List[Dict]: """ Processes bullets to identify and time word highlights, correctly handling multiple instances of the same word. """ logger.debug("\n===== WORD HIGHLIGHTING V2 (INSTANCE-AWARE) ======") # Step 1: Get Whisper transcript with word-level timings. word_timings = get_word_timings_from_whisper(audio_path, client) if not word_timings: logger.warning("Whisper returned no word timings. Cannot perform highlighting.") return [{'text': bullet, 'word_data': [{'word': w, 'start': 0, 'end': 0, 'highlight': False} for w in re.findall(r'\b[\w\']+\b', bullet)]} for bullet in bullets] full_transcript = ' '.join([w['word'] for w in word_timings]) slide_text = '\n'.join(bullets) # Step 2: Ask LLM for a list of unique keywords to highlight. highlight_keywords = identify_highlight_words_with_llm( slide_title=slide_title, slide_text=slide_text, audio_transcript=full_transcript, word_timings=word_timings, client=client ) if not highlight_keywords: logger.warning("LLM returned no keywords to highlight. Skipping word highlighting.") return [{'text': bullet, 'word_data': [{'word': w, 'start': 0, 'end': 0, 'highlight': False} for w in re.findall(r'\b[\w\']+\b', bullet)]} for bullet in bullets] highlight_keywords_lower = {k.lower().strip(".,?!'\"") for k in highlight_keywords} # Step 3: Create a dictionary mapping each keyword to a list of its timed occurrences from Whisper. keyword_timings = {key: [] for key in highlight_keywords_lower} for wt in word_timings: word_lower = wt['word'].lower().strip(".,?!'\"") if word_lower in highlight_keywords_lower: keyword_timings[word_lower].append(wt) # Step 4: Process bullets, assigning the correct timing to each keyword instance. keyword_usage_index = {key: 0 for key in highlight_keywords_lower} processed_bullets = [] HIGHLIGHT_DELAY_S = 0.1 # Small delay to sync with animation for bullet_text in bullets: if not bullet_text.strip(): processed_bullets.append({'text': bullet_text, 'word_data': []}) continue bullet_words = re.findall(r"\b[\w']+\b", bullet_text) word_data = [] for word in bullet_words: should_highlight = False timing_info = None word_lower = word.lower().strip(".,?!'\"") if word_lower in keyword_timings: usage_idx = keyword_usage_index.get(word_lower, 0) if usage_idx < len(keyword_timings[word_lower]): should_highlight = True timing_info = keyword_timings[word_lower][usage_idx] keyword_usage_index[word_lower] = usage_idx + 1 word_data.append({ 'word': word, 'start': timing_info['start'] + HIGHLIGHT_DELAY_S if timing_info else 0.0, 'end': timing_info['end'] + HIGHLIGHT_DELAY_S if timing_info else 0.5, 'highlight': should_highlight }) highlighted_count = sum(1 for w in word_data if w['highlight']) logger.debug(f"Bullet: '{bullet_text[:60]}...' - Highlighted words: {highlighted_count}") processed_bullets.append({'text': bullet_text, 'word_data': word_data}) total_highlighted = sum(v for v in keyword_usage_index.values()) logger.debug(f"\nTotal keyword instances highlighted: {total_highlighted}\n") return processed_bullets