Update app.py

app.py

@@ -14,18 +14,11 @@ from sentence_transformers import SentenceTransformer
 from bertopic import BERTopic
 import faiss
 import numpy as np
-from googleapiclient.discovery import build
-from youtube_transcript_api import YouTubeTranscriptApi
+from datasets import load_dataset, Features, Value
 
 # Initialize FastAPI app
 app = FastAPI()
 
-# YouTube Data API setup
-API_KEY = "AIzaSyDBdxA6KdOwtaaTgt26EBYRyvknOObmgAc"
-YOUTUBE_API_SERVICE_NAME = "youtube"
-YOUTUBE_API_VERSION = "v3"
-youtube = build(YOUTUBE_API_SERVICE_NAME, YOUTUBE_API_VERSION, developerKey=API_KEY)
-
 # Preprocessing function
 def preprocess_text(text):
     """
@@ -116,8 +109,6 @@ class SearchEngine:
         """
         Searches the index for the top_k most relevant documents.
         """
-        if self.index is None:
-            raise ValueError("Index not initialized. Call build_index() first.")
         query_embedding = self.model.encode(query, convert_to_tensor=True)
         distances, indices = self.index.search(query_embedding.cpu().detach().numpy().reshape(1, -1), top_k)
         return [(self.documents[i], distances[0][i]) for i in indices[0]]
@@ -155,58 +146,39 @@ documents = [
 ]
 search_engine.build_index(documents)
 
-# Fetch video metadata using YouTube Data API
-def fetch_video_metadata(video_id):
-    request = youtube.videos().list(
-        part="snippet,statistics",
-        id=video_id
+# Define the schema
+features = Features({
+    "video_id": Value("string"),
+    "video_link": Value("string"),
+    "title": Value("string"),
+    "text": Value("string"),
+    "channel": Value("string"),
+    "channel_id": Value("string"),
+    "date": Value("string"),
+    "license": Value("string"),
+    "original_language": Value("string"),
+    "source_language": Value("string"),
+    "transcription_language": Value("string"),
+    "word_count": Value("int64"),
+    "character_count": Value("int64"),
+})
+
+# Load the dataset from Hugging Face Hub
+try:
+    dataset = load_dataset(
+        "PleIAs/YouTube-Commons",
+        features=features,
+        streaming=True,
     )
-    response = request.execute()
-    return response["items"][0] if response["items"] else None
-
-
-# Fetch video transcript using youtube-transcript-api
-def fetch_video_transcript(video_id):
-    try:
-        transcript = YouTubeTranscriptApi.get_transcript(video_id)
-        return " ".join([entry["text"] for entry in transcript])
-    except Exception as e:
-        print(f"Error fetching transcript: {e}")
-        return None
-
-
-# Fetch and preprocess video data
-def fetch_and_preprocess_video_data(video_id):
-    metadata = fetch_video_metadata(video_id)
-    if not metadata:
-        return None
-
-    transcript = fetch_video_transcript(video_id)
-
-    # Preprocess the data
-    video_data = {
-        "video_id": video_id,
-        "video_link": f"https://www.youtube.com/watch?v={video_id}",
-        "title": metadata["snippet"]["title"],
-        "text": transcript if transcript else metadata["snippet"]["description"],
-        "channel": metadata["snippet"]["channelTitle"],
-        "channel_id": metadata["snippet"]["channelId"],
-        "date": metadata["snippet"]["publishedAt"],
-        "license": "Unknown",
-        "original_language": "Unknown",
-        "source_language": "Unknown",
-        "transcription_language": "Unknown",
-        "word_count": len(metadata["snippet"]["description"].split()),
-        "character_count": len(metadata["snippet"]["description"]),
-    }
-    return video_data
 
+    # Process the dataset
+    for example in dataset["train"]:
+        print(example)  # Process each example
+        break  # Stop after the first example for demonstration
+except Exception as e:
+    print(f"Error loading dataset: {e}")
 
 # Pydantic models for request validation
-class VideoRequest(BaseModel):
-    video_id: str
-
-
 class TextRequest(BaseModel):
     text: str
 
@@ -221,35 +193,32 @@ class PromptRequest(BaseModel):
 
 # API Endpoints
 @app.post("/classify")
-async def classify(request: VideoRequest):
-    video_id = request.video_id
-    video_data = fetch_and_preprocess_video_data(video_id)
-    if not video_data:
-        raise HTTPException(status_code=400, detail="Failed to fetch video data")
+async def classify(request: TextRequest):
+    text = request.text
+    if not text:
+        raise HTTPException(status_code=400, detail="No text provided")
 
-    result = classifier.classify(video_data["text"])
+    result = classifier.classify(text)
     return {"result": result}
 
 
 @app.post("/relevance")
-async def relevance(request: VideoRequest):
-    video_id = request.video_id
-    video_data = fetch_and_preprocess_video_data(video_id)
-    if not video_data:
-        raise HTTPException(status_code=400, detail="Failed to fetch video data")
+async def relevance(request: TextRequest):
+    text = request.text
+    if not text:
+        raise HTTPException(status_code=400, detail="No text provided")
 
-    relevant = relevance_detector.detect_relevance(video_data["text"])
+    relevant = relevance_detector.detect_relevance(text)
     return {"relevant": relevant}
 
 
 @app.post("/summarize")
-async def summarize(request: VideoRequest):
-    video_id = request.video_id
-    video_data = fetch_and_preprocess_video_data(video_id)
-    if not video_data:
-        raise HTTPException(status_code=400, detail="Failed to fetch video data")
+async def summarize(request: TextRequest):
+    text = request.text
+    if not text:
+        raise HTTPException(status_code=400, detail="No text provided")
 
-    summary = summarizer.summarize(video_data["text"])
+    summary = summarizer.summarize(text)
     return {"summary": summary}
 
 
@@ -259,11 +228,8 @@ async def search(request: QueryRequest):
     if not query:
         raise HTTPException(status_code=400, detail="No query provided")
 
-    try:
-        results = search_engine.search(query)
-        return {"results": results}
-    except ValueError as e:
-        raise HTTPException(status_code=500, detail=str(e))
+    results = search_engine.search(query)
+    return {"results": results}
 
 
 @app.post("/topics")