Upload 3 files

app.py

@@ -0,0 +1,113 @@
+import gradio as gr
+import asyncio
+from crawl4ai import AsyncWebCrawler
+from urllib.parse import urlparse
+from langchain_community.document_loaders import TextLoader
+from langchain.text_splitter import CharacterTextSplitter
+from langchain_huggingface import HuggingFaceEmbeddings
+from langchain_community.vectorstores import FAISS
+from langchain.prompts import PromptTemplate
+from langchain.schema.runnable import RunnableMap, RunnablePassthrough
+from langchain.schema.output_parser import StrOutputParser
+from langchain_groq import ChatGroq
+import re
+import os
+from dotenv import load_dotenv
+load_dotenv()
+
+
+GROQ_API_KEY=os.getenv("GROQ_API_KEY")
+
+qa_chain = None
+scraped_file = None
+
+# Clean LLM output
+class StrictOutputParser(StrOutputParser):
+    def parse(self, text: str) -> str:
+        text = re.sub(r'<think>.*?</think>', '', text, flags=re.DOTALL)
+        text = re.sub(r'^(Reasoning|Thought|Analysis):.*?\n', '', text, flags=re.IGNORECASE)
+        return text.strip()
+
+# Async crawl function
+async def crawl_site(url):
+    async with AsyncWebCrawler() as crawler:
+        result = await crawler.arun(url=url)
+        return result.markdown
+
+# UI-triggered scraper
+def scrape_website(url):
+    global scraped_file
+    markdown = asyncio.run(crawl_site(url))
+    domain = urlparse(url).netloc.replace("www.", "")
+    filename = f"{domain}.txt"
+    with open(filename, "w", encoding="utf-8") as f:
+        f.write(markdown)
+    scraped_file = filename
+    return filename, markdown
+
+# Query setup
+def setup_qa():
+    global qa_chain
+    loader = TextLoader(scraped_file, encoding="utf-8")
+    docs = CharacterTextSplitter(chunk_size=1000, chunk_overlap=100).split_documents(loader.load())
+    vectorstore = FAISS.from_documents(docs, HuggingFaceEmbeddings(model_name="sentence-transformers/all-MiniLM-L6-v2"))
+    retriever = vectorstore.as_retriever()
+    prompt = PromptTemplate.from_template("""
+You are an AI assistant. Return ONLY the final answer.
+
+**Rules (MUST follow):**
+1. NO <think>, reasoning, or explanations.
+2. NO markdown/formatting tags.
+3. Answer in 3-4 concise sentences.
+
+Context:
+{context}
+
+Question:
+{question}
+
+Answer (direct and short):""")
+
+    llm = ChatGroq(
+        api_key=GROQ_API_KEY,  # Use environment variable for security
+        model="deepseek-r1-distill-llama-70b",
+        temperature=0.0
+    )
+
+    qa_chain = (
+        RunnableMap({
+            "context": retriever,
+            "question": RunnablePassthrough()
+        }) | prompt | llm | StrictOutputParser()
+    )
+    return "✅ Query system ready!"
+
+# Handle questions
+def ask_question(query):
+    if not qa_chain:
+        return "❗ Please set up the QA system first."
+    return qa_chain.invoke(query)
+
+# Gradio interface
+with gr.Blocks(title="Web Scraping AI Agent") as demo:
+    gr.Markdown("## 🌐 Website Scraper AI Agent")
+
+    url_input = gr.Textbox(label="Enter Website URL")
+    scrape_btn = gr.Button("🔍 Scrape Website")
+    download_output = gr.File(label="📄 Download Scraped File")
+    markdown_box = gr.Textbox(label="Scraped Text", lines=10)
+    
+    setup_btn = gr.Button("💬 Query This Website")
+    setup_status = gr.Textbox(label="Status")
+
+    query_input = gr.Textbox(label="Ask a Question")
+    query_btn = gr.Button("Ask")
+    query_output = gr.Textbox(label="Answer")
+
+    # Wire components
+    scrape_btn.click(fn=scrape_website, inputs=[url_input], outputs=[download_output, markdown_box])
+    setup_btn.click(fn=setup_qa, outputs=setup_status)
+    query_btn.click(fn=ask_question, inputs=[query_input], outputs=[query_output])
+
+# Run
+demo.launch(share=True)

main.py

@@ -0,0 +1,22 @@
+import asyncio
+from crawl4ai import *
+from urllib.parse import urlparse
+
+
+url =  url = input("Enter the website URL: ").strip()
+async def main():
+    async with AsyncWebCrawler() as crawler:
+        result = await crawler.arun(
+            url=url,
+        )
+        print(result.markdown)
+        
+    domain = urlparse(url).netloc.replace("www.", "")
+    filename = f"{domain}.txt"
+    with open(filename, "w", encoding="utf-8") as f:
+            f.write(result.markdown)
+
+    print(f"\n✅ Scraped content saved to '{filename}'")
+
+if __name__ == "__main__":
+    asyncio.run(main())

requirement.txt

@@ -0,0 +1,15 @@
+gradio
+asyncio
+crawl4ai
+urllib3
+langchain
+langchain-core
+langchain-community
+langchain-huggingface
+langchain-groq
+huggingface-hub
+sentence-transformers
+faiss-cpu
+python-dotenv
+aiohttp
+re