Refactored to suggest articles based on user insights

- Added callback functions to the task to capture required info.

.gitignore

@@ -1,5 +1,9 @@
 __pycache__/
-privacy_policy.pkl.tar.gz
 .env
 .DS_Store
 venv
+article_suggestions.json
+evaluated_articles.json
+final_articles.json
+learning_profile.json
+pitched_articles.json

agents/learning_profiler.py

@@ -6,7 +6,6 @@ learning_profiler = Agent(
     role="Personal Learning Profiler",
     goal="Make sure to create an excellent learning profile of the user based on his interests and previous reading history.",
     verbose=True,
-    # tools=[scrape_tool],
     backstory=(
       "As a Personal Learning Profiler, you excel at building a learning profile of a user. "
       "The profile you build gives a high level overview of what interests that the user has. "

app.py

@@ -2,6 +2,10 @@ from dotenv import load_dotenv
 load_dotenv()
 
 from crew.article_suggestion import article_recommendation_crew
+import utils.settings as settings
+from utils.write_to_json import write_dict_to_json as write_dict_to_json
+
+settings.init()
 
 result = article_recommendation_crew.kickoff(inputs={
   "interests": "Ruby On Rails, Architecture, Go Lang",
@@ -23,3 +27,5 @@ result = article_recommendation_crew.kickoff(inputs={
 
 print(result)
 print("Usage Metrics:\n", article_recommendation_crew.usage_metrics)
+
+write_dict_to_json(settings.articles, filename="final_articles.json")

crew/article_suggestion.py

@@ -23,12 +23,4 @@ article_recommendation_crew = Crew(
                     "model": 'text-embedding-3-small'
         }
     }
-    # embedder={
-    #         "provider": "google",
-    #         "config":{
-    #             "model": 'models/embedding-001',
-    #             "task_type": "retrieval_document",
-    #             "title": "Embeddings for Embedchain"
-    #         }
-    #     }
 )

tasks/create_article_pitch.py

@@ -1,20 +1,45 @@
 from crewai import Task
 from agents.curiosity_catalyst import curiosity_catalyst
 from tools.scrape_website import scrape_tool
-from utils.recommended_article import RecommendedArticles
+from crewai.tasks.task_output import TaskOutput
+import utils.settings as settings
+from pydantic import BaseModel
+from typing import List
+import json
+
+
+class PitchedArticle(BaseModel):
+    title: str
+    url: str
+    pitch: str
+
+
+class PitchedArticles(BaseModel):
+    articles: List[PitchedArticle]
+
+
+def callback_function(output: TaskOutput):
+    evaluated_articles = json.loads(output.exported_output)['articles']
+
+    for article in evaluated_articles:
+        settings.articles[article['url']]['pitch'] = article['pitch']
+
 
 article_pitch_task = Task(
-  description=(
-    "Create a pitch only for the articles that have passed evaluation and none other links. "
-    "Craft the pitch so to that it teases the article's most intriguing aspects, "
-    "by posing questions that the article might answer or "
-    "highlighting surprising facts to pique the user's curiosity "
-    " to read the article for incremental learning."
-  ),
-  expected_output=(
-    "List of all the artilces that have passed evaluation phase."
-  ),
-  output_json=RecommendedArticles,
-  tools=[scrape_tool],
-  agent=curiosity_catalyst
+    description=(
+        "Create a pitch only for the articles that have passed evaluation and no other links. "
+        "Craft the pitch so to that it teases the article's most intriguing aspects, "
+        "by posing questions that the article might answer or "
+        "highlighting surprising facts to pique the user's curiosity "
+        " to read the article for incremental learning."
+    ),
+    expected_output=(
+        "List of all the artilces that have passed evaluation phase along with their url and pitch statement."
+    ),
+    output_json=PitchedArticles,
+    output_file="pitched_articles.json",
+    tools=[scrape_tool],
+    agent=curiosity_catalyst,
+    async_execution=False,
+    callback=callback_function,
 )

tasks/create_learning_profile.py

@@ -1,25 +1,31 @@
 from crewai import Task
 from agents.learning_profiler import learning_profiler
-from utils.learning_profile import LearningProfile
+from pydantic import BaseModel
+from typing import List
+
+
+class Topic(BaseModel):
+    name: str
+    insights: List[str]
+
+
+class LearningProfile(BaseModel):
+    topics_of_interests: List[str]
+    learnings: List[Topic]
+
 
 learning_profile_task = Task(
     description=(
-        # "Create a Learning profile of the user based on his following interests {interests} "
-        # "and based on the following articles and insights he has read in the past: \n"
-        # "{previous_article_insights}"
         "Create a Learning profile of the user based on "
         "the following articles and insights he has read in the past: \n"
         "{previous_article_insights}"
     ),
     expected_output=(
         "A structured learning profile of the user with his interests, topics he has read about "
-        "and any other information you feel is relavant. Be precise."
-        # "A structured learning profile. "
-        # "The learning profile should only contain a section called 'Topics of interest' and 'Insights'. "
-        # "It should not have any other sections such as summary or conclusion. "
-        # "'Insights' section should contain the insights captured by the user and they should be grouped by topic."
+        "and insights he has captured on the topics."
     ),
     agent=learning_profiler,
     output_json=LearningProfile,
+    output_file="learning_profile.json",
     async_execution=False
 )

tasks/evaluate_articles.py

@@ -1,15 +1,48 @@
 from crewai import Task
 from agents.article_evaluator import article_evaluator
+from crewai.tasks.task_output import TaskOutput
+import utils.settings as settings
+from pydantic import BaseModel
+from typing import List
+import json
+
+
+class EvaluatedArticle(BaseModel):
+    title: str
+    url: str
+    evaluation_score: int
+    evaluation_reason: str
+
+
+class EvaluatedArticles(BaseModel):
+    articles: List[EvaluatedArticle]
+
+
+def callback_function(output: TaskOutput):
+    evaluated_articles = json.loads(output.exported_output)['articles']
+
+    for article in evaluated_articles:
+        settings.articles[article['url']
+                          ]['evaluation_score'] = article['evaluation_score']
+        settings.articles[article['url']
+                          ]['evaluation_reason'] = article['evaluation_reason']
+
 
 evaluation_task = Task(
-  description=(
-    "Evaluate articles to ensure that they provide incremental learning for the user "
-    "based on his interests and previous article summaries he has captured."
-  ),
-  expected_output=(
-    "List of article titles along with their links and evaluation reason of why you "
-    "think the article is a good recommendation for the user."
-  ),
-  agent=article_evaluator,
-  async_execution=False
+    description=(
+        "Evaluate artilces based on the metric does the articles provide incremenrtal "
+        "learning w.r.t the insights captured by the user. "
+        "Score the articles on the scale of 1 to 10, "
+        "1 being doesn't provide incremental learning and "
+        "10 being provides incremental learning to the user."
+    ),
+    expected_output=(
+        "List of article titles along with their links, evaluation score and "
+        "evaluation reason w.r.t to insights captured by the user."
+    ),
+    output_json=EvaluatedArticles,
+    output_file="evaluated_articles.json",
+    agent=article_evaluator,
+    async_execution=False,
+    callback=callback_function,
 )

tasks/new_article_suggestion.py

@@ -1,14 +1,40 @@
 from crewai import Task
 from agents.learning_curator import learning_curator
+from crewai.tasks.task_output import TaskOutput
+import utils.settings as settings
+from pydantic import BaseModel
+from typing import List
+import json
+
+
+class SuggestedArticle(BaseModel):
+    title: str
+    url: str
+    reason_for_recommendation: str
+
+
+class SuggestedArticles(BaseModel):
+    articles: List[SuggestedArticle]
+
+
+def callback_function(output: TaskOutput):
+    suggested_articles = json.loads(output.exported_output)['articles']
+
+    for article in suggested_articles:
+        settings.articles[article['url']] = article
+
 
 article_suggestion_task = Task(
-  description=(
-    "Suggest 5 recent articles (i.e published in last 10 days ) to the user based on his learning profile. "
-    "The articles should provide incremental learning to the user."
-  ),
-  expected_output=(
-    "List of article titles along with their links"
-  ),
-  agent=learning_curator,
-  async_execution=False
+    description=(
+        "Find 5 articles from the past 10 days that align with the user's learning interests. "
+        "The articles should provide incremental learning to the user based on their insights."
+    ),
+    expected_output=(
+        "List of article titles along with their links. "
+    ),
+    output_json=SuggestedArticles,
+    output_file="article_suggestions.json",
+    agent=learning_curator,
+    async_execution=False,
+    callback=callback_function,
 )

utils/learning_profile.py

@@ -1,12 +0,0 @@
-from pydantic import BaseModel
-from typing import List
-
-
-class Topic(BaseModel):
-    name: str
-    insights: List[str]
-
-
-class LearningProfile(BaseModel):
-    topics_of_interests: List[str]
-    learnings: List[Topic]

utils/recommended_article.py

@@ -1,11 +0,0 @@
-from pydantic import BaseModel
-from typing import List
-
-class RecommendedArticle(BaseModel):
-  title: str
-  url: str
-  pitch: str
-  reason_for_recommendation: str
-
-class RecommendedArticles(BaseModel):
-  articles: List[RecommendedArticle]

utils/settings.py

@@ -0,0 +1,3 @@
+def init():
+    global articles
+    articles = {}

utils/write_to_json.py

@@ -0,0 +1,9 @@
+import json
+
+def write_dict_to_json(data, filename="data.json", indent=4):
+    try:
+        with open(filename, 'w') as json_file:
+            json.dump(data, json_file, indent=indent)
+        print(f"Successfully wrote dictionary to {filename}")
+    except (IOError, json.JSONDecodeError) as e:
+        print(f"Error writing to JSON file: {e}")