summarization script

summarize_transcript.py

@@ -0,0 +1,109 @@
+#!/usr/bin/env python3
+"""
+Script to summarize transcript using Falcon-H1-Tiny-Multilingual model with SYCL acceleration.
+"""
+
+import os
+from llama_cpp import Llama
+from huggingface_hub import hf_hub_download
+
+def load_model():
+    """Load the model from Hugging Face Hub."""
+        
+    # Initialize the model with SYCL support
+    llm = Llama.from_pretrained(
+        repo_id="Luigi/Falcon-H1-Tiny-Multilingual-100M-Instruct-GGUF",
+        filename="*IQ4_NL.gguf",
+        n_gpu_layers=-1,  # Use all layers on GPU
+        seed=1337,
+        n_ctx=32768,       # Context size
+        verbose=True,
+        n_batch=1024,
+        n_ubatch=512,
+        v_type=2,
+        k_type=2
+    )
+    
+    return llm
+
+def read_transcript(file_path):
+    """Read the transcript file."""
+    with open(file_path, 'r', encoding='utf-8') as f:
+        content = f.read()
+    return content
+
+def summarize_transcript(llm, transcript, language='zh-TW'):
+    """Summarize the transcript using the loaded model."""
+    # Truncate the transcript to fit within the context window
+    # Account for the prompt tokens as well
+    max_transcript_length = 1000  # Leave room for prompt and response
+
+    if len(transcript) > max_transcript_length:
+        transcript = transcript[:max_transcript_length]
+        print(f"Transcript truncated to {max_transcript_length} characters to fit context window.")
+
+    # Use the model's chat format based on its template
+    if language == 'en':
+        messages = [
+            {"role": "system", "content": "You are a helpful assistant that summarizes transcripts."},
+            {"role": "user", "content": f"Please summarize the following transcript:\n\n{transcript}"}
+        ]
+    else:  # Default to zh-TW
+        messages = [
+            {"role": "system", "content": "你是一個有助的助手，負責總結轉錄內容。"},
+            {"role": "user", "content": f"請總結以下內容：\n\n{transcript}"}
+        ]
+
+    # Generate the summary using chat completion
+    output = llm.create_chat_completion(
+        messages=messages,
+        max_tokens=512,
+        temperature=0.3,
+        top_p=0.9,
+        repeat_penalty=1.1,
+        stop=["<|end_of_text|>", "<|eot_id|>", "<|eom_id|>"]
+    )
+
+    llm.reset()
+    
+    return output['choices'][0]['message']['content'].strip()
+
+def main():
+    print("Loading Falcon-H1-Tiny-Multilingual model with SYCL acceleration...")
+    
+    # Load the model
+    llm = load_model()
+    
+    # Read the transcript
+    transcript_path = "/home/luigi/tiny-scribe/transcripts/short.txt"
+    transcript = read_transcript(transcript_path)
+    
+    print("\nOriginal Transcript:")
+    print(transcript[:500] + "..." if len(transcript) > 500 else transcript)
+    
+    # Summarize in Chinese (zh-TW)
+    print("\nGenerating Chinese (zh-TW) summary...")
+    chinese_summary = summarize_transcript(llm, transcript, language='zh-TW')
+    print("Chinese Summary:")
+    print(chinese_summary)
+    
+    # Summarize in English
+    print("\nGenerating English summary...")
+    english_summary = summarize_transcript(llm, transcript, language='en')
+    print("English Summary:")
+    print(english_summary)
+    
+    # Save summaries to files
+    with open("/home/luigi/tiny-scribe/chinese_summary.txt", 'w', encoding='utf-8') as f:
+        f.write(chinese_summary)
+    
+    with open("/home/luigi/tiny-scribe/english_summary.txt", 'w', encoding='utf-8') as f:
+        f.write(english_summary)
+    
+    print("\nSummaries saved to files.")
+    
+    # Clean up
+    del llm
+
+if __name__ == "__main__":
+    main()