File size: 4,093 Bytes
325b94c | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 | from crewai import Agent, Task, Crew, Process, LLM
import os
from modules import llm_oss
from crewai_tools import SerperDevTool
from schemas import SearchEngineOutput
from tools import URLValidatorTool
# --- Tool ---
search_tool = SerperDevTool()
url_validate_tool = URLValidatorTool(name="url_validator")
# --- Agent Definition ---
search_engine_agent = Agent(
role="Search and Validation agent",
goal="\n".join(
[
"Search for diverse, high-quality, and reliable information related to {topic} using the suggested search queries.",
"Prioritize **scholarly and educational resources**: academic journals, books, research articles, and trusted institutional websites and our trusted sites we know: {TRUSTED_SITES}.",
"Include Arabic sources when contextually valuable, but prioritize English academic sources for broader coverage.",
"Ensure all results are concise, relevant, and aligned with the outline headings to directly support course design and educational content.",
"For each query, return at most {no_links} results with structured metadata (title, url, content summary, score, search_query).",
"Validate URLs before retrieving them and if it is not valid dont retieve",
]
),
backstory=(
"This agent acts like a scientific researcher and educational content curator."
"It executes the generated queries and gathers high-value, trustworthy information."
"The agent is trained to filter out irrelevant, commercial, or low-quality content, ensuring that "
"only academic and pedagogically useful resources are kept."
"for building interactive e-learning content are retrieved."
"It balances global knowledge with local cultural enrichment by including both English and Arabic sources."
"Uses Serper search and validates each found URL."
),
verbose=True,
llm=llm_oss,
tools=[search_tool, url_validate_tool],
allow_delegation=False,
)
# --- Task Definition ---
search_engine_task = Task(
name="Academic Web Search",
description="\n".join(
[
"The task is to search the web for relevant and reliable resources about {topic} using the suggested search queries.",
"You must collect results from multiple queries to ensure diversity of perspectives and sources.",
"Here are the queries you can use to search:",
"{queries}",
"use unit title: {unit_title} and subtopic title: {subtopic_title} to complete the output and search correctly for each query.",
"For each query, retrieve at least 1 link and a maximum of {no_links} links only.",
"Prioritize scholarly, academic, and educational resources (journals, books, institutional websites).",
"Filter out irrelevant, suspicious, or low-quality results (e.g., personal blogs, spam, commercial ads).",
"Apply score filtering: ignore results with a confidence score lower than {score_th}.",
"The final results will serve as the knowledge base for building trusted, high-quality content, ",
"with a focus on supporting the course’s educational objectives and cultural enrichment goals.",
"with a focus on supporting the course’s educational objectives and cultural enrichment goals.",
"Get sites that newer than 2021.",
"Validate URLs before retrieving them and if it is not valid dont retieve",
]
),
expected_output=(
"Return ONLY a valid UTF-8 safe Python dictionary (no markdown, no code blocks). "
"It must be directly parsable with ast.literal_eval. Use this format:\n\n"
"{\n"
' "unit_title": "...",\n'
' "subtopic_title": "...",\n'
' "query": "...",\n'
' "results": [\n'
' {"url": "...", "title": "...", "content": "...", "score": 0.91}\n'
" ]\n"
"}"
),
output_json=SearchEngineOutput,
agent=search_engine_agent,
output_file="search_results.json",
)
|