Upload 3 files

.gitattributes

@@ -33,3 +33,4 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
 *.zip filter=lfs diff=lfs merge=lfs -text
 *.zst filter=lfs diff=lfs merge=lfs -text
 *tfevents* filter=lfs diff=lfs merge=lfs -text
+input.mp4 filter=lfs diff=lfs merge=lfs -text

app.py

@@ -0,0 +1,116 @@
+import streamlit as st
+import torch
+from transformers import AutoProcessor, AutoModelForSpeechSeq2Seq, AutoTokenizer, AutoModelForSeq2SeqLM
+from pydub import AudioSegment
+import librosa
+import ffmpeg
+import os
+import re
+import tempfile
+
+@st.cache_resource
+def load_model():
+    device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
+    processor = AutoProcessor.from_pretrained("openai/whisper-medium")
+    model = AutoModelForSpeechSeq2Seq.from_pretrained("openai/whisper-medium").to(device).half()
+    summarizer_tokenizer = AutoTokenizer.from_pretrained("facebook/bart-large-cnn")
+    summarizer_model = AutoModelForSeq2SeqLM.from_pretrained("facebook/bart-large-cnn").to(device)
+    return processor, model, summarizer_tokenizer, summarizer_model, device
+
+def extract_audio(video_path, output_audio_path):
+    if not os.path.exists(video_path):
+        raise FileNotFoundError(f"Video file not found: {video_path}")
+    try:
+        (ffmpeg.input(video_path).output(output_audio_path, ac=1, ar=16000, preset="ultrafast").overwrite_output().run(quiet=True))
+    except ffmpeg.Error as e:
+        raise RuntimeError(f"FFmpeg error: {e.stderr.decode()}")
+
+def split_audio(audio_path, chunk_duration_ms=5000):
+    audio = AudioSegment.from_file(audio_path)
+    chunks = [audio[i:i + chunk_duration_ms] for i in range(0, len(audio), chunk_duration_ms)]
+    return chunks
+
+def transcribe_in_batches(chunks, processor, model, device, progress_bar, batch_size=4):
+    transcriptions = []
+    forced_decoder_ids = processor.get_decoder_prompt_ids(language="en", task="transcribe")
+    total_batches = len(range(0, len(chunks), batch_size))
+    for i in range(0, len(chunks), batch_size):
+        batch = chunks[i:i + batch_size]
+        batch_features = []
+        temp_files = []
+        for idx, chunk in enumerate(batch):
+            temp_audio_path = f"temp_chunk_{i+idx}.wav"
+            chunk.export(temp_audio_path, format="wav")
+            temp_files.append(temp_audio_path)
+            audio, sr = librosa.load(temp_audio_path, sr=16000)
+            inputs = processor(audio, sampling_rate=16000, return_tensors="pt")
+            input_features = inputs.input_features.to(device).half()
+            batch_features.append(input_features)
+        input_features = torch.cat(batch_features).to(device)
+        with torch.no_grad():
+            generated_ids = model.generate(input_features, forced_decoder_ids=forced_decoder_ids)
+            transcriptions += processor.batch_decode(generated_ids, skip_special_tokens=True)
+        for file in temp_files:
+            os.remove(file)
+        progress_bar.progress((i + batch_size) / len(chunks))
+    return transcriptions
+
+def combine_transcriptions(transcriptions):
+    return "\n".join(transcriptions)
+
+def remove_timecodes(text):
+    return re.sub(r'\[.*?\]', '', text)
+
+def summarize_text(text, tokenizer, model, device):
+    text = text.encode('utf-8', 'ignore').decode()
+    inputs = tokenizer(text, return_tensors="pt", max_length=1024, truncation=True, padding=True).to(device)
+    with torch.no_grad():
+        summary_ids = model.generate(inputs['input_ids'], num_beams=4, min_length=50, max_length=200, early_stopping=True)
+    return tokenizer.decode(summary_ids[0], skip_special_tokens=True)
+
+st.title("Video Transcription and Summarization")
+st.write("Upload a video file to generate transcription and summary")
+uploaded_file = st.file_uploader("Choose a video file", type=['mp4', 'avi', 'mov'])
+
+if uploaded_file is not None:
+    with tempfile.NamedTemporaryFile(delete=False, suffix='.mp4') as tmp_video:
+        tmp_video.write(uploaded_file.read())
+        video_path = tmp_video.name
+    
+    with tempfile.NamedTemporaryFile(delete=False, suffix='.wav') as tmp_audio:
+        audio_path = tmp_audio.name
+    
+    try:
+        with st.spinner("Loading models..."):
+            processor, model, summarizer_tokenizer, summarizer_model, device = load_model()
+        
+        with st.spinner("Extracting audio..."):
+            extract_audio(video_path, audio_path)
+        
+        chunks = split_audio(audio_path)
+        progress_bar = st.progress(0)
+        st.write("Transcribing audio...")
+        transcriptions = transcribe_in_batches(chunks, processor, model, device, progress_bar)
+        full_transcription = combine_transcriptions(transcriptions)
+        
+        st.subheader("Transcription")
+        st.text_area("Full transcription", full_transcription, height=200)
+        
+        clean_transcription = remove_timecodes(full_transcription)
+        with st.spinner("Generating summary..."):
+            summary = summarize_text(clean_transcription, summarizer_tokenizer, summarizer_model, device)
+        
+        st.subheader("Summary")
+        st.text_area("Text summary", summary, height=100)
+        
+        col1, col2 = st.columns(2)
+        with col1:
+            st.download_button("Download Transcription", full_transcription, "transcription.txt")
+        with col2:
+            st.download_button("Download Summary", summary, "summary.txt")
+    
+    except Exception as e:
+        st.error(f"An error occurred: {e}")
+    finally:
+        os.unlink(video_path)
+        os.unlink(audio_path)

input.mp4

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:bba3d6bd0287fcfcdf98fd007226ddb006f67c5e8f44197964322aab49a089eb
+size 749762

requirements.txt

@@ -0,0 +1,6 @@
+pydub
+moviepy
+transformers
+librosa
+ffmpeg-python
+langdetect