Update app.py

app.py

@@ -6,8 +6,6 @@ from deep_translator import GoogleTranslator
 import pandas as pd
 from tqdm import tqdm
 import urllib
-import aiohttp
-import asyncio
 from bs4 import BeautifulSoup
 
 # Configure logging to write messages to a file
@@ -17,14 +15,14 @@ logging.basicConfig(filename='app.log', level=logging.ERROR)
 max_seq_length = 2048
 dtype = None  # Auto detection of dtype
 load_in_4bit = True  # Use 4-bit quantization to reduce memory usage
-peft_model_name = "limitedonly41/website_qwen2_7b_2"
+
+peft_model_name = "limitedonly41/website_mistral7b_v02_1200_finetuned_7"
 
 # Initialize model and tokenizer variables
 model = None
 tokenizer = None
 
-# Async function to fetch data
-async def fetch_data(url):
+def fetch_data(url):
     headers = {
         'Accept': '*/*',
         'Accept-Language': 'ru-RU,ru;q=0.9,en-US;q=0.8,en;q=0.7',
@@ -33,7 +31,7 @@ async def fetch_data(url):
         'Sec-Fetch-Dest': 'empty',
         'Sec-Fetch-Mode': 'cors',
         'Sec-Fetch-Site': 'cross-site',
-        'User-Agent': 'Mozilla/5.0',
+        'User-Agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/125.0.0.0 Safari/537.36',
         'sec-ch-ua': '"Google Chrome";v="125", "Chromium";v="125", "Not.A/Brand";v="24"',
         'sec-ch-ua-mobile': '?0',
         'sec-ch-ua-platform': '"macOS"',
@@ -41,20 +39,15 @@ async def fetch_data(url):
 
     encoding = 'utf-8'
     timeout = 10  # Set your desired timeout value in seconds
-
     try:
-        # Function to make the request using urllib
-        def get_content():
-            req = urllib.request.Request(url, headers=headers)
-            with urllib.request.urlopen(req, timeout=timeout) as response:
-                return response.read()
-
-        # Async task using executor for blocking I/O
-        loop = asyncio.get_event_loop()
-        response_content = await loop.run_in_executor(None, get_content)
+        # Make the request using urllib
+        req = urllib.request.Request(url, headers=headers)
+        with urllib.request.urlopen(req, timeout=timeout) as response:
+            response_content = response.read()
 
         soup = BeautifulSoup(response_content, 'html.parser', from_encoding=encoding)
-        title = soup.find('title').text if soup.find('title') else ""
+
+        title = soup.find('title').text
         description = soup.find('meta', attrs={'name': 'description'})
         description = description.get("content") if description and "content" in description.attrs else ""
 
@@ -66,8 +59,12 @@ async def fetch_data(url):
         h2_all = ". ".join(h.text for h in soup.find_all('h2'))
         h3_all = ". ".join(h.text for h in soup.find_all('h3'))
 
-        allthecontent = f"{title} {description} {h1_all} {h2_all} {h3_all} {paragraphs_all}"
-        allthecontent = allthecontent[:4999]
+        allthecontent = f"{title} {description} {h1_all} {h2_all} {h3_all} {paragraphs_all}"[:4999]
+
+        # Clean up the text
+        h1_all = h1_all.replace(r'\xa0', ' ').replace('\n', ' ').replace('\t', ' ')
+        h2_all = h2_all.replace(r'\xa0', ' ').replace('\n', ' ').replace('\t', ' ')
+        h3_all = h3_all.replace(r'\xa0', ' ').replace('\n', ' ').replace('\t', ' ')
 
         return {
             'url': url,
@@ -80,9 +77,8 @@ async def fetch_data(url):
             'paragraphs': paragraphs_all,
             'text': allthecontent
         }
-
     except Exception as e:
-        logging.exception(f"Error fetching data for {url}: {e}")
+        print(url, e)
         return {
             'url': url,
             'title': None,
@@ -95,25 +91,19 @@ async def fetch_data(url):
             'text': None
         }
 
-# Main async function to process multiple URLs
-async def main(urls):
-    tasks = [fetch_data(url) for url in urls]
+def main(urls):
     results = []
-    for future in tqdm(asyncio.as_completed(tasks), total=len(tasks)):
-        result = await future
+    for url in tqdm(urls):
+        result = fetch_data(url)
         results.append(result)
     return results
 
 @spaces.GPU()
 def classify_website(url):
-    global model, tokenizer
+    global model, tokenizer  # Declare model and tokenizer as global variables
 
     urls = [url]
-
-    # Start asyncio loop for fetching data
-    loop = asyncio.new_event_loop()
-    asyncio.set_event_loop(loop)
-    results_shop = loop.run_until_complete(main(urls))  # Correctly use asyncio loop
+    results_shop = main(urls)
 
     # Convert results to DataFrame
     df_result_train_more = pd.DataFrame(results_shop)
@@ -121,17 +111,18 @@ def classify_website(url):
     translated = GoogleTranslator(source='auto', target='en').translate(text[:4990])
 
     try:
-        # Load the model and tokenizer if not already loaded
+        # Load the model and tokenizer if they are not already loaded
         if model is None or tokenizer is None:
             from unsloth import FastLanguageModel
 
+            # Load the model and tokenizer
             model, tokenizer = FastLanguageModel.from_pretrained(
-                model_name=peft_model_name,
+                model_name=peft_model_name,  # Model used for training
                 max_seq_length=max_seq_length,
                 dtype=dtype,
                 load_in_4bit=load_in_4bit,
             )
-            FastLanguageModel.for_inference(model)
+            FastLanguageModel.for_inference(model)  # Enable native 2x faster inference
 
         prompt = f"""Below is an instruction that describes a task, paired with an input that provides further context. Write a response that appropriately completes the request.
 
@@ -173,6 +164,7 @@ iface = gr.Interface(
     title="Website Categorization",
     description="Categorize a website into one of the 3 categories: OTHER, NEWS/BLOG, or E-commerce."
 )
-iface.queue()  # Enable queue with default settings
-iface.launch()
+iface.queue()  # Sets up a queue with default parameters
 
+# Launch the interface
+iface.launch()