Upload 5 files

.gitattributes

@@ -33,3 +33,4 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
 *.zip filter=lfs diff=lfs merge=lfs -text
 *.zst filter=lfs diff=lfs merge=lfs -text
 *tfevents* filter=lfs diff=lfs merge=lfs -text
+chromedriver filter=lfs diff=lfs merge=lfs -text

chromedriver

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:d2ff8d0f3628d5684b8ce86ec32790ebf13fecca004a938270b119a93608ef09
+size 19025744

main.py

@@ -0,0 +1,85 @@
+import streamlit as st
+from scrape import scrape_website, extract_body_content, clean_body_content, split_dom_content
+from parse import parse_with_groq
+
+# Streamlit UI with sidebar
+st.set_page_config(page_title="Web Scraping App 🧠", page_icon="🌐")
+
+st.sidebar.title("🚀 Model Selection")
+selected_model = st.sidebar.selectbox(
+    "Choose a Model for Parsing:",
+    [
+        "llama3-8b-8192",
+        "distil-whisper-large-v3-en",
+        "llama3-groq-70b-8192-tool-use-preview",
+        "llama-3.1-8b-instant",
+        "llava-v1.5-7b-4096-preview",
+        "mixtral-8x7b-32768",
+    ]
+)
+
+# Application title
+st.title("AI Web Scraper App 🌐")
+st.write("Easily scrape and analyze web content using advanced AI models. 🌟")
+
+# Input for website URL
+url = st.text_input("Enter Website URL 🔗")
+
+# Step 1: Scrape the Website
+if st.button("Scrape Website"):
+    if url:
+        st.write("🕵️‍♂️ Scraping the website...")
+
+        # Scrape the website
+        dom_content = scrape_website(url)
+        body_content = extract_body_content(dom_content)
+        cleaned_content = clean_body_content(body_content)
+
+        # Store the DOM content in Streamlit session state
+        st.session_state.dom_content = cleaned_content
+
+        # Display the DOM content in an expandable text box
+        with st.expander("View DOM Content"):
+            st.text_area("DOM Content", cleaned_content, height=300)
+
+# Step 2: Parse the Content
+if "dom_content" in st.session_state:
+    parse_description = st.text_area("Describe what you want to parse 📝")
+
+    if st.button("Parse Content"):
+        if parse_description:
+            st.write(f"🤖 Parsing the content with {selected_model}...")
+
+            # Parse content using Groq
+            dom_chunks = split_dom_content(st.session_state.dom_content)
+            parsed_result = parse_with_groq(dom_chunks, parse_description, model=selected_model)
+            st.write(parsed_result)
+
+# CSS for footer at the bottom of the sidebar
+st.markdown(
+    """
+    <style>
+    .footer {
+        position: fixed;
+        bottom: 0;
+        left: 0;
+        width: 100%;
+        background-color: #272432;  /* Dark background for visibility */
+        color: white;
+        text-align: center;
+        padding: 10px;
+        font-size: 14px;
+    }
+    .sidebar .footer {
+        position: fixed;
+        bottom: 0;
+    }
+    </style>
+
+    <div class="footer">
+    Made with ❤️ by Usman Yousaf 🚀<br>
+    Feel free to improve and expand this app for more powerful insights! 🔥
+    </div>
+    """,
+    unsafe_allow_html=True
+)

parse.py

@@ -0,0 +1,35 @@
+from groq import Groq
+
+# Initialize Groq client with the API key directly
+client = Groq(api_key="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")
+
+# Define the template for parsing
+template = (
+    "You are tasked with extracting specific information from the following text content: {dom_content}. "
+    "Please follow these instructions carefully: \n\n"
+    "1. **Extract Information:** Only extract the information that directly matches the provided description: {parse_description}. "
+    "2. **No Extra Content:** Do not include any additional text, comments, or explanations in your response. "
+    "3. **Empty Response:** If no information matches the description, return an empty string ('')."
+    "4. **Direct Data Only:** Your output should contain only the data that is explicitly requested, with no other text."
+)
+
+def parse_with_groq(dom_chunks, parse_description, model="llama3-8b-8192"):
+    parsed_results = []
+
+    for i, chunk in enumerate(dom_chunks, start=1):
+        # Prepare the prompt
+        prompt = template.format(dom_content=chunk, parse_description=parse_description)
+
+        # Send prompt to Groq for processing, specifying the model
+        response = client.chat.completions.create(
+            messages=[
+                {"role": "user", "content": prompt}
+            ],
+            model=model  # Specify the model
+        )
+        
+        # Print status and store result
+        print(f"Parsed batch: {i} of {len(dom_chunks)}")
+        parsed_results.append(response.choices[0].message.content)  # Access the content
+
+    return "\n".join(parsed_results)

requirements.txt

@@ -0,0 +1,5 @@
+streamlit
+selenium
+beautifulsoup4
+groq
+python-dotenv

scrape.py

@@ -0,0 +1,60 @@
+from selenium import webdriver 
+from selenium.webdriver.chrome.service import Service 
+from selenium.webdriver.chrome.options import Options 
+from bs4 import BeautifulSoup 
+from dotenv import load_dotenv 
+import os
+import time
+
+load_dotenv()
+
+CHROME_DRIVER_PATH = os.getenv("./chromedriver")
+
+def scrape_website(website):
+    print("Connecting to Chrome Browser...")
+    
+    # Setup ChromeDriver service and options
+    service = Service(CHROME_DRIVER_PATH)
+    options = Options()
+    driver = webdriver.Chrome(service=service, options=options)
+    
+    try:
+        driver.get(website)
+        print("Waiting for CAPTCHA to be solved manually (if present)...")
+        
+        # Optional waiting loop for manual CAPTCHA solving
+        while "captcha" in driver.page_source.lower():
+            print("CAPTCHA detected, waiting...")
+            time.sleep(5)
+
+        print("CAPTCHA solved or not present. Scraping page content...")
+        html = driver.page_source
+        return html
+    
+    finally:
+        driver.quit()
+
+def extract_body_content(html_content):
+    soup = BeautifulSoup(html_content, "html.parser")
+    body_content = soup.body
+    if body_content:
+        return str(body_content)
+    return ""
+
+def clean_body_content(body_content):
+    soup = BeautifulSoup(body_content, "html.parser")
+
+    for script_or_style in soup(["script", "style"]):
+        script_or_style.extract()
+
+    cleaned_content = soup.get_text(separator="\n")
+    cleaned_content = "\n".join(
+        line.strip() for line in cleaned_content.splitlines() if line.strip()
+    )
+
+    return cleaned_content
+
+def split_dom_content(dom_content, max_length=6000):
+    return [
+        dom_content[i : i + max_length] for i in range(0, len(dom_content), max_length)
+    ]