.

.gitignore

@@ -0,0 +1,3 @@
+.env
+.cache
+.__pycache__

app.py

@@ -13,19 +13,15 @@ import autogen
 from autogen import Agent, AssistantAgent, UserProxyAgent, config_list_from_json
 import chainlit as cl
 from chainlit.client.base import ConversationDict
-import requests
-import json
-from dotenv import load_dotenv
+from webscrape import grab_articles
+from dotenv import load_dotenv, find_dotenv
 import os
 
-load_dotenv()
-
-API_KEY = os.getenv('NEWS_API_KEY')
-BASE_URL = 'https://newsapi.org/v2/everything'
+load_dotenv(find_dotenv())
 
 WELCOME_MESSAGE = f"""Soap Opera Team 👾
 \n\n
-Let's make some soap opera! What topic do you want to search? (Just reply the keyword)
+Let's make some soap opera! What topic do you want to search? 
 """
 
 # Agents
@@ -33,54 +29,6 @@ USER_PROXY_NAME = "Query Agent"
 PROOF_READER = "Proofreader"
 WRITER = "Writer"
 ARTICLES = None
-
-def get_top_articles(query, n):
-    # Prepare the parameters for the API call
-    params = {
-        'q': query,  # search query
-        'apiKey': API_KEY,
-        'pageSize': n,  # number of articles to return
-        'language': 'en',  # get English articles
-    }
-    
-    # Make the API request
-    response = requests.get(BASE_URL, params=params)
-    
-    # Check if the request was successful
-    if response.status_code == 200:
-        return response.json()['articles']
-    else:
-        raise Exception(f"Error: {response.status_code}, {response.text}")
-
-def grab_articles(query, num):
-    try:
-        # Get top N articles
-        articles = get_top_articles(query, num)
-        # Save articles to JSON file
-        return(articles)
-    except Exception as e:
-        print(str(e))
-
-def extract_article_info_from_list(article_list):   
-    try:
-        # Initialize a string to store the extracted information
-        extracted_info = ""
-
-        # Extract and store the desired information for each article
-        for article in article_list:
-            description = article.get("description", "N/A")
-            title = article.get("title", "N/A")
-            author = article.get("author", "N/A")
-            content = article.get("content", "N/A")
-
-            # Append the information to the string
-            extracted_info += f"Title: {title}\nAuthor: {author}\nDescription: {description}\nContent: {content}\n\n"
-
-        return extracted_info
-
-    except Exception as e:
-        # Handle any exceptions
-        return f"An error occurred: {str(e)}"
     
 async def ask_helper(func, **kwargs):
     res = await func(**kwargs).send()
@@ -111,7 +59,6 @@ class ChainlitAssistantAgent(AssistantAgent):
             request_reply=request_reply,
             silent=silent,
         )
-
 class ChainlitUserProxyAgent(UserProxyAgent):
     """
     Wrapper for AutoGens UserProxy Agent. Simplifies the UI by adding CL Actions. 
@@ -177,13 +124,29 @@ class ChainlitUserProxyAgent(UserProxyAgent):
 #     cl.user_session.get("chat_profile")
 #     # Or just pass if you do not need to run any custom logic
 #     pass
+@cl.action_callback("confirm_action")
+async def on_action(action: cl.Action):
+    if action.value == "everything":
+        content = "everything"
+    elif action.value == "top-headlines":
+        content = "top_headlines"
+    else:
+        await cl.ErrorMessage(content="Invalid action").send()
+        return
+
+    prev_msg = cl.user_session.get("url_actions")  # type: cl.Message
+    if prev_msg:
+        await prev_msg.remove_actions()
+        cl.user_session.set("url_actions", None)
 
+    await cl.Message(content=content).send()
+    
 @cl.on_chat_start
 async def on_chat_start():
+
   try:
     # app_user = cl.user_session.get("user")
     # await cl.Message(f"Hello {app_user.username}").send()
-    
     config_list = config_list_from_json(env_or_file="OAI_CONFIG_LIST")
     proof_reader = ChainlitAssistantAgent(
         name="Proof_Reader", llm_config={"config_list": config_list},
@@ -209,27 +172,69 @@ async def on_chat_start():
     cl.user_session.set(WRITER, writer)
     
     # WEB-SCRAPING LOGIC
-    QUERY = None
-    NUM_ARTICLES = 5
-    while QUERY is None:
-        QUERY = await cl.AskUserMessage(content=WELCOME_MESSAGE, timeout=180, author="User_Proxy").send()
+    URL = None
+    URL_option = None
+    BASE_URL_EVERYTHING = 'https://newsapi.org/v2/everything?'
+    BASE_URL_TOP_HEADLINES = 'https://newsapi.org/v2/top-headlines?'
+    TOPIC = None # Topic
+    SORTBY = 'relevancy' # relevancy, popularity, publishedAt
+    COUNTRY = 'gb' # Country you want to use
+    CATEGORY = None # 'business' # Options: business, entertainment, general health, science, sports, technology
+    API_KEY = os.getenv('NEWS_API_KEY')
     
-    print("QUERY: ", QUERY)
-    articles = grab_articles(QUERY['content'], NUM_ARTICLES)
-    processed_articles = extract_article_info_from_list(articles)
+    everything = cl.Action( name="everything", value="everything", label="Everything" )
+    top_headlines = cl.Action( name="top-headlines",value="top-headlines", label="TopHeadlines")
+    business = cl.Action( name="business", value="business", label="Business" )
+    entertainment = cl.Action( name="entertainment", value="entertainment", label="Entertainment" )
+    science = cl.Action( name="science", value="science", label="Science" )
+    sports = cl.Action( name="sports", value="sports", label="Sports" )
+    technology = cl.Action( name="technology", value="technology", label="Technology" ) 
+     
+    url_actions = [everything, top_headlines]
+    category_actions = [business, entertainment, science, sports, technology]
     
-    msg = cl.Message(content=f"Processing `{QUERY['content']}`...", disable_human_feedback=True, author="User_Proxy")
-    await msg.send()
-    msg = cl.Message(content=f"Articles grabbed: `{articles}`...", disable_human_feedback=True, author="User_Proxy")
+    URL_option = cl.AskActionMessage(
+        content="Would you like to grab sources from Everything or from Top Headlines?",
+        actions=url_actions,
+    )
+    
+    await URL_option.send()
+    
+    URL_choice = URL_option.content.split()[-1]
+    if URL_choice == "Everything":
+        while TOPIC is None:
+            TOPIC = await cl.AskUserMessage(content="What topic would you like to make a Soap Opera about?", timeout=180, author="User_Proxy").send()
+        topic = TOPIC['content']
+        URL = f'{BASE_URL_EVERYTHING}q={topic}&sortBy={SORTBY}&pageSize=5&apiKey={API_KEY}'      
+        print("URL: ", URL) 
+    if URL_choice == "TopHeadlines":
+        while TOPIC is None:
+            TOPIC = await cl.AskUserMessage(content="What topic would you like to make a Soap Opera about?", 
+                                            timeout=180, 
+                                            author="User_Proxy").send()
+        category = cl.AskActionMessage(
+            content="What category would you like to use?",
+            actions=category_actions,
+            )
+        await category.send()
+        CATEGORY = category.content.split()[-1]
+        topic = TOPIC['content']
+        URL = f'{BASE_URL_TOP_HEADLINES}topic={topic}&country={COUNTRY}&category={CATEGORY}&pageSize=5&apiKey={API_KEY}'
+        print("URL: ", URL)
+
+    articles = grab_articles(URL)
+    print(articles)
+    print("Articles grabbed.")
+    
+    msg = cl.Message(content=f"Processing `{articles}`...", disable_human_feedback=True, author="User_Proxy")
     await msg.send()
     
-    cl.user_session.set(ARTICLES, processed_articles)
+    cl.user_session.set(ARTICLES, articles)
     print("Articles set...")
     
-    msg = cl.Message(content=f"""This is the Soap Opera Team, please give instructions on how the Soap Opera should be structured and made. \n\n
-                     Sample input: "Create a captivating soap opera scene inspired by the content of the articles. Feature characters 
-                     who are entangled in a complex web of emotions, ambitions, and conflicts. Craft dialogue and actions that convey the 
-                     essence of the articles, infusing drama, suspense, and emotion. Leave the audience eagerly anticipating the next twist in this gripping narrative."
+    msg = cl.Message(content=f"""This is the Soap Opera Team, please give instructions on how the Soap Opera should be structured and made. 
+                     \nSample input: 
+                     "Create a captivating soap opera scene inspired by the content of the articles. Feature characters who are entangled in a complex web of emotions, ambitions, and conflicts. Craft dialogue and actions that convey the essence of the articles, infusing drama, suspense, and emotion. Leave the audience eagerly anticipating the next twist in this gripping narrative."
                      """, 
                      disable_human_feedback=True, 
                      author="User_Proxy")

chainlit.md

@@ -0,0 +1,14 @@
+# Welcome to Chainlit! 🚀🤖
+
+Hi there, Developer! 👋 We're excited to have you on board. Chainlit is a powerful tool designed to help you prototype, debug and share applications built on top of LLMs.
+
+## Useful Links 🔗
+
+- **Documentation:** Get started with our comprehensive [Chainlit Documentation](https://docs.chainlit.io) 📚
+- **Discord Community:** Join our friendly [Chainlit Discord](https://discord.gg/k73SQ3FyUh) to ask questions, share your projects, and connect with other developers! 💬
+
+We can't wait to see what you create with Chainlit! Happy coding! 💻😊
+
+## Welcome screen
+
+To modify the welcome screen, edit the `chainlit.md` file at the root of your project. If you do not want a welcome screen, just leave this file empty.

webscrape.py

@@ -0,0 +1,59 @@
+import requests
+import json
+from dotenv import load_dotenv
+import os
+
+load_dotenv()
+
+API_KEY = os.getenv('NEWS_API_KEY')
+
+# Functions -------------------------------------------------------------------------------------------------------------> 
+def extract_article_info_from_list(article_list):   
+    try:
+        # Initialize a string to store the extracted information
+        extracted_info = ""
+
+        # Extract and store the desired information for each article
+        for article in article_list:
+            description = article.get("description", "N/A")
+            title = article.get("title", "N/A")
+            author = article.get("author", "N/A")
+            content = article.get("content", "N/A")
+
+            # Append the information to the string
+            extracted_info += f"Title: {title}\nAuthor: {author}\nDescription: {description}\nContent: {content}\n\n"
+
+        return extracted_info
+
+    except Exception as e:
+        # Handle any exceptions
+        return f"An error occurred: {str(e)}"
+      
+def grab_articles(url, **kwargs):
+    response = requests.get(url, 5)
+    
+    if response.status_code == 200:
+        extracted_article = extract_article_info_from_list(response.json()['articles'])
+        print(extracted_article)
+        return extracted_article
+    else:
+        raise Exception(f"Error: {response.status_code}, {response.text}")
+
+# URL Configuration Here -------------------------------------------------------------------------------------------------------------> 
+
+BASE_URL_EVERYTHING = 'https://newsapi.org/v2/everything?'
+BASE_URL_TOP_HEADLINES = 'https://newsapi.org/v2/top-headlines?'
+TOPIC = 'bitcoin'
+FROM = '2023-01-08' # Starting date
+TO = '2023-11-08' # Ending date
+SORTBY = 'popularity' # Relevancy, Popularity, PublishedAt
+DOMAINS = 'techcrunch.com, thenextweb.com' # Domains you want to use. 
+COUNTRY = 'gb&' # Options: US, AU, JP, PH etc. 
+SOURCES = 'bbc-news' 
+CATEGORY = 'business' # Options: business, entertainment, general health, science, sports, technology
+
+# Everything URL list
+urls_everything = f"{BASE_URL_EVERYTHING}q={TOPIC}&domains={DOMAINS}&sortBy={SORTBY}&apiKey={API_KEY}" 
+
+# Top Headlines URL list
+urls_top_headlines = f"{BASE_URL_TOP_HEADLINES}topic={TOPIC}&country={COUNTRY}&category={CATEGORY}&sources={SOURCES}&apiKey={API_KEY}"