Spaces:
Paused
Paused
File size: 3,099 Bytes
36b0811 7c91632 36b0811 7c91632 26a6e19 36b0811 7c91632 36b0811 7c91632 26a6e19 7c91632 36b0811 26a6e19 7c91632 36b0811 7c91632 36b0811 7c91632 36b0811 7c91632 36b0811 |
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 |
import os
import re
import pickle
import webvtt
import pandas as pd
from sklearn.feature_extraction.text import TfidfVectorizer
# 1) Video ID
video_id_map = {
'machine learning what is machine learning introduction to machine learning 2024 simplilearn': 'ukzFI9rgwfU',
'introduction to artificial intelligence what is ai artificial intelligence tutorial simplilearn': 'SSE4M0gcmvE',
'what is deep learning introduction to deep learning deep learning tutorial simplilearn': 'FbxTVRfQFuI'
}
# 2) File list
videos = [
(
'Machine Learning What Is Machine Learning Introduction To Machine Learning 2024 Simplilearn',
'Machine Learning.vtt'
),
(
'Introduction To Artificial Intelligence What Is AI Artificial Intelligence Tutorial Simplilearn',
'Artificial Intelligence.vtt'
),
(
'What is Deep Learning Introduction to Deep Learning Deep Learning Tutorial Simplilearn',
'Deep Learning.vtt'
)
]
_space_re = re.compile(r'\s+')
def normalize_minimal(text: str) -> str:
if not isinstance(text, str):
return ''
t = text.lower()
t = _space_re.sub(' ', t).strip()
return t
# DF (with YT ID)
all_subtitles = []
for raw_title, file_path in videos:
cleaned_title = re.sub(r'\s+', ' ', raw_title).strip().lower()
youtube_id = video_id_map.get(cleaned_title, '')
if not youtube_id:
print(f"⚠️ YouTube ID not found for title: {cleaned_title}")
continue
for caption in webvtt.read(file_path):
start = caption.start
end = caption.end
text_raw = caption.text or ''
all_subtitles.append([start, end, text_raw, cleaned_title, youtube_id])
# DF with RAW text
df = pd.DataFrame(
all_subtitles,
columns=['Start Time', 'End Time', 'Subtitle Text Raw', 'Video Title', 'YouTube ID']
)
# CLEAN column for indexing
df['Subtitle Text Clean'] = df['Subtitle Text Raw'].apply(normalize_minimal)
df['Subtitle Text'] = df['Subtitle Text Clean']
before_drop = len(df)
df = df[df['Subtitle Text Clean'].str.len() > 0].copy()
after_drop = len(df)
df.sort_values(by=['YouTube ID', 'Start Time'], inplace=True, ignore_index=True)
df.to_csv('cleaned_subtitles.csv', index=False)
# 5) TF-IDF fit
vectorizer = TfidfVectorizer(
ngram_range=(1, 2),
sublinear_tf=True,
lowercase=True,
min_df=1
)
tfidf_matrix = vectorizer.fit_transform(df['Subtitle Text Clean'])
with open('tfidf_vectorizer.pkl', 'wb') as f:
pickle.dump(vectorizer, f)
with open('tfidf_matrix.pkl', 'wb') as f:
pickle.dump(tfidf_matrix, f)
# 6) Export metadata
metadata = df[['Subtitle Text Raw', 'Subtitle Text Clean', 'Subtitle Text',
'Start Time', 'End Time', 'Video Title', 'YouTube ID']]
metadata.to_csv('indexed_metadata.csv', index=False)
print(" Files saved.")
print(f" Rows read: {len(all_subtitles)}")
print(f" Rows after clean: {after_drop} (dropped {before_drop - after_drop} empty rows)")
print(f" Unique videos: {df['YouTube ID'].nunique()}")
print(f" TF-IDF shape: {tfidf_matrix.shape}")
|