Update app.py

app.py

@@ -20,15 +20,26 @@ LANGUAGES = {
     "Tamil": "ta"
 }
 
-# Function to scrape content from a URL
-def scrape_content(url):
+# Function to scrape and summarize content from a URL
+def scrape_and_summarize(url):
     try:
         response = requests.get(url)
         soup = BeautifulSoup(response.content, 'html.parser')
         title = soup.find('title').get_text() if soup.find('title') else "No Title"
         paragraphs = soup.find_all('p')
         content = '\n'.join([para.get_text() for para in paragraphs])
-        return title, content
+        
+        # Summarize the content using OpenAI
+        prompt = f"Summarize the following content:\n\n{content}"
+        response = openai.chat.completions.create(
+            model="gpt-4o-mini",
+            messages=[
+                {"role": "system", "content": "You are an expert summarizer."},
+                {"role": "user", "content": prompt}
+            ]
+        )
+        summary = response.choices[0].message.content.strip()
+        return title, summary
     except Exception as e:
         return "Error", f"Failed to scrape content from {url}: {str(e)}"
 
@@ -50,9 +61,9 @@ def translate_content(content, target_language):
 # Function to create newsletter
 def create_newsletter(contents, language):
     prompt = "Create a newsletter with the following content:\n\n"
-    for title, body, url in contents:
-        translated_body = translate_content(body, LANGUAGES[language])
-        prompt += f"Title: {title}\nURL: {url}\n\n{translated_body}\n\n"
+    for title, summary, url in contents:
+        translated_summary = translate_content(summary, LANGUAGES[language])
+        prompt += f"Title: {title}\nURL: {url}\n\n{translated_summary}\n\n"
     
     response = openai.chat.completions.create(
         model="gpt-4o-mini",
@@ -75,8 +86,8 @@ def process_urls(url1, url2, url3, url4, url5, language):
     
     contents = []
     for url in urls:
-        title, content = scrape_content(url)
-        contents.append((title, content, url))
+        title, summary = scrape_and_summarize(url)
+        contents.append((title, summary, url))
     
     with gr.update() as progress:
         progress(0.6)
@@ -106,7 +117,7 @@ iface = gr.Interface(
     ],
     outputs=["html", gr.File(label="Download Newsletter")],
     title="Multilingual AI Newsletter Generator",
-    description="Enter up to 5 URLs to generate a newsletter in your preferred language. Copy and paste the output into your CMS tool for further editing. A progress indicator will show the process, and you can download the newsletter as a text file."
+    description="Enter up to 5 URLs to generate a summarized newsletter in your preferred language. Copy and paste the output into your CMS tool for further editing. A progress indicator will show the process, and you can download the newsletter as a text file."
 )
 
-iface.launch()
+iface.launch()