Upload 5 files

.gitattributes

@@ -33,3 +33,4 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
 *.zip filter=lfs diff=lfs merge=lfs -text
 *.zst filter=lfs diff=lfs merge=lfs -text
 *tfevents* filter=lfs diff=lfs merge=lfs -text
+chromedriver.exe filter=lfs diff=lfs merge=lfs -text

chromedriver.exe

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:55e67707e3ca5b68b16d1bf436c3c7cdd845977846bda16fb89308a6994e6006
+size 17792000

main.py

@@ -0,0 +1,27 @@
+import streamlit as st
+from scapping import scrape_website, split_dom_content, clean_body_content, extract_body_content
+from parse import parse_with_gemini
+
+st.title("Web Scrapper")
+url = st.text_input("Enter the Web URL:", placeholder="URL")
+
+if st.button("Start Scraping"):
+    st.write("Scrapping...")
+    result = scrape_website(url)
+
+    body_content = extract_body_content(result)
+    cleaned_content = clean_body_content(body_content)
+
+    st.session_state.dom_content = cleaned_content
+    with st.expander("View DOM Content"):
+        st.text_area("DOM Content", cleaned_content, height=300)
+
+if "dom_content" in st.session_state:
+    parse_description = st.text_area("Describe what you want to parse?")
+
+    if st.button("Parse Content"):
+        if parse_description:
+            st.write("Parsing the content...")
+            dom_chunks = split_dom_content(st.session_state.dom_content)
+            result = parse_with_gemini(dom_chunks, parse_description)
+            st.write(result)

parse.py

@@ -0,0 +1,29 @@
+import os
+from langchain_google_genai import GoogleGenerativeAI
+from langchain_core.prompts import ChatPromptTemplate
+from dotenv import load_dotenv
+load_dotenv()
+
+
+llm = GoogleGenerativeAI(model="gemini-pro")
+
+template = (
+    "You are tasked with extracting specific information from the following text content: {dom_content}. "
+    "Please follow these instructions carefully: \n\n"
+    "1. **Extract Information:** Only extract the information that directly matches the provided description: {parse_description}. "
+    "2. **No Extra Content:** Do not include any additional text, comments, or explanations in your response. "
+    "3. **Empty Response:** If no information matches the description, return an empty string ('')."
+    "4. **Direct Data Only:** Your output should contain only the data that is explicitly requested, with no other text."
+)
+
+def parse_with_gemini(dom_chunks, parse_description):
+    prompt = ChatPromptTemplate.from_template(template)
+    chain = prompt | llm
+
+    parsed_results = []
+    for i, chunk in enumerate(dom_chunks, start=1):
+        response = chain.invoke({"dom_content": chunk, "parse_description": parse_description})
+        print(f"Parsed batch {i} of {len(dom_chunks)}")
+        parsed_results.append(response)
+
+    return "\n".join(parsed_results)

requirements.txt

@@ -0,0 +1,8 @@
+streamlit
+langchain
+langchain_ollama
+selenium
+beautifulsoup4
+lxml
+html5lib
+python-dotenv

scapping.py

@@ -0,0 +1,46 @@
+import selenium.webdriver as webdriver
+from selenium.webdriver.chrome.service import Service
+import time
+from bs4 import BeautifulSoup
+
+def scrape_website(website):
+    print("launching chrome browser...")
+
+    chrome_driver_path = "chromedriver.exe"
+    options = webdriver.ChromeOptions()
+    driver = webdriver.Chrome(service=Service(chrome_driver_path), options=options)
+
+    try:
+        driver.get(website)
+        print('page loaded..,')
+        html = driver.page_source
+        time.sleep(10)
+
+        return html
+    finally:
+        driver.quit()
+
+def extract_body_content(html_content):
+    soup = BeautifulSoup(html_content, "html.parser")
+    body_content = soup.body
+    if body_content:
+        return str(body_content)
+    return ""
+
+def clean_body_content(body_content):
+    soup = BeautifulSoup(body_content, "html.parser")
+
+    for script_or_style in soup({"script", "style"}):
+        script_or_style.extract()
+
+    cleaned_content = soup.get_text(separator="\n")
+    cleaned_content = "\n".join(
+        line.strip() for line in cleaned_content.splitlines() if line.strip()
+    )
+
+    return cleaned_content
+
+def split_dom_content(dom_content,max_length=6000):
+    return [
+        dom_content[i: i+max_length] for i in range(0, len(dom_content), max_length)
+    ]