Upload 6 files

Dockerfile

@@ -0,0 +1,66 @@
+FROM python:3.10
+
+WORKDIR /home/
+
+COPY ./requirements.txt /home/requirements.txt
+
+RUN apt update && \
+    apt install -y --no-install-recommends \
+        curl \
+        git \
+        git-lfs \
+        libatomic1 \
+        locales \
+        man \
+        nano \
+        net-tools \
+        openssh-client \
+        python3 \
+        python3-pip \
+        python3-venv \
+        sudo \
+        vim \
+        wget \
+        zsh \
+        zip \
+        unzip \
+        ffmpeg \
+        imagemagick \
+    && git lfs install \
+    && rm -rf /var/lib/apt/lists/*
+ENV USERNAME=user \
+    USER_UID=1000 \
+    USER_GID=1000 \
+    LANG=C.UTF-8 \
+    LC_ALL=C.UTF-8 \
+    NVIDIA_VISIBLE_DEVICES=all \
+    NVIDIA_DRIVER_CAPABILITIES=all \
+    EDITOR=code \
+    VISUAL=code \
+    GIT_EDITOR="code --wait" \
+    OPENVSCODE_SERVER_ROOT=/home/.vscode \
+    OPENVSCODE=/home/.vscode/bin/openvscode-server
+ENV DEBIAN_FRONTEND=dialog
+RUN wget http://archive.ubuntu.com/ubuntu/pool/main/o/openssl/libssl1.1_1.1.1f-1ubuntu2_amd64.deb
+RUN sudo dpkg -i libssl1.1_1.1.1f-1ubuntu2_amd64.deb
+RUN wget https://repo.mongodb.org/apt/ubuntu/dists/focal/mongodb-org/8.0/multiverse/binary-amd64/mongodb-org-server_8.0.0_amd64.deb
+RUN sudo apt install ./mongodb-org-server_8.0.0_amd64.deb
+
+RUN pip install --no-cache-dir --upgrade -r /home/requirements.txt
+
+COPY . .
+WORKDIR /home/user/
+
+# Creating the user and usergroup
+RUN groupadd --gid ${USER_GID} ${USERNAME} \
+    && useradd --uid ${USER_UID} --gid ${USERNAME} -m -s /bin/bash ${USERNAME} \
+    && echo ${USERNAME} ALL=\(root\) NOPASSWD:ALL > /etc/sudoers.d/${USERNAME} \
+    && chmod 0440 /etc/sudoers.d/${USERNAME}
+
+RUN chmod g+rw /home && \
+    chown -R ${USERNAME}:${USERNAME} /home/${USERNAME}
+RUN mkdir -p /db && chown -R ${USERNAME}:${USERNAME} /db
+USER $USERNAME
+COPY . .
+# Install oh-my-zsh & Init# Start MongoDB and Flask
+CMD ["sh", "-c", "Malloc=system mongod --dbpath /db --logpath mongod.log --fork && flask run --host=0.0.0.0 --port=7860"]

app.py

@@ -0,0 +1,192 @@
+import time
+import copy
+import os
+from flask import Flask, render_template, request
+import json
+from main import bing_serach, extract_web
+import asyncio
+import requests
+from langchain_text_splitters import RecursiveCharacterTextSplitter
+import numpy as np
+from pymongo import MongoClient
+client = MongoClient('mongodb://localhost:27017/')  # Replace with your MongoDB URI
+
+    # Create or access a database
+db = client['webdata']  # Replace 'my_database' with your database name
+
+collection = db['data']
+
+def cosine_similarity(vec1, vec2):
+    # Compute the dot product between the two vectors
+    dot_product = np.dot(vec1, vec2)
+
+    # Compute the magnitude (norm) of the vectors
+    norm_vec1 = np.linalg.norm(vec1)
+    norm_vec2 = np.linalg.norm(vec2)
+
+    # Compute cosine similarity
+    similarity = dot_product / (norm_vec1 * norm_vec2)
+
+    return similarity
+
+def percentage_similarity(vec1, vec2):
+    # Get cosine similarity
+
+    try:
+      cosine_sim = cosine_similarity(vec1, vec2)
+
+      # Convert cosine similarity to percentage similarity
+      percentage_sim = (cosine_sim + 1) / 2 * 100  # Shift range from [-1,1] to [0,100]
+
+      return percentage_sim
+    except:
+      return 0
+text_splitter = RecursiveCharacterTextSplitter(
+    # Set a really small chunk size, just to show.
+    chunk_size=2000,
+    chunk_overlap=100,
+    length_function=len,
+    is_separator_regex=False,
+)
+app = Flask(__name__)
+@app.route("/status", methods=['GET'])
+def status():
+    return "OK"
+
+@app.route("/", methods=['GET','POST'])
+def websearch():
+    try:
+        if request.args.get('q'):
+            query = request.args.get('q')
+            ifextract =  request.args.get('ifextract')
+            try:
+                start = int(request.args.get('start'))
+            except:
+                start = 0
+            if ifextract == '1':
+                return asyncio.run(bing_serach(query,collection,ifextract=True,start=start))
+            elif ifextract == '0':
+                return asyncio.run(bing_serach(query,collection,ifextract=False,start=start))
+            else:
+                return '<h1>Invalid Value of ifextract</h1><br>it can Two Value either 0 or 1<br> for 1 it will provide Webpage Extracted'
+        else:
+            return '<h1>Enter Valid Query</h1> <br> GET parameters<br>1. q(query) = Search query in quote_plus ex: Is+Mango+Sweet<br>1. ifextract(ifextract) = 0,1 for 1 it will provide extracted webpage for suitable websites<br>2. startIndex(start) =Optional Ender the start index of search query'
+    except Exception as e:
+        return {'type':'error','message':'Unexpected Error',"detail":str(e)}
+@app.route("/adv",methods=["POST","GET"])
+def adv_make():
+    global collection
+    args = request.get_json()
+    if all(key in args for key in ['long_query', 'short_query']):
+        short_query = args["short_query"]
+        dataz = asyncio.run(bing_serach(short_query, collection, ifextract=True))
+        data = dataz['result']
+        with open("r.json",'w') as f:
+            f.write(json.dumps(data,indent=4))
+        toembed = [z['webpage'] for z in data if "embedding_data" not in z and z['webpage'] != "Some Error while Extracting"]
+
+# Split these documents into chunks
+        toemb = [text_splitter.create_documents([z]) for z in toembed]
+
+        # Flatten the document chunks
+        toembz = [sublist.page_content for z in toemb for sublist in z]
+        print("Length of Documents")
+        print(len(toembz))
+        if(len(toembz) > 0):
+            data_to_send = {
+                "text":toembz
+            }
+            embedding = requests.post("https://mangoman7002-flash-embedding.hf.space",json=data_to_send)
+            if(embedding.status_code != 200):
+                return json.dumps({"type":"error","message":f"error With API {str(embedding.status_code)}"},indent=4)
+            embedding = embedding.json()
+        else:
+            embedding = {'result':[]}
+        data_to_send = {
+            "text":[args['long_query']]
+        }
+        query_embedding = requests.post("https://mangoman7002-flash-embedding.hf.space",json=data_to_send)
+        if(query_embedding.status_code != 200):
+            return json.dumps({"type":"error","message":f"error With API {str(embedding.status_code)}"},indent=4)
+        query_embedding = query_embedding.json()
+        results = embedding['result']
+        current_index=0
+        embedding_index = 0
+        for index,value in enumerate(dataz['result']):
+            if("embedding_data" in dataz['result'][index] and dataz['result'][index]['webpage'] != "Some Error while Extracting"):
+                pass
+            elif(dataz['result'][index]['webpage'] != "Some Error while Extracting"):
+                em_vector = results[embedding_index:embedding_index+len(toemb[current_index])]
+                embedding_index+=len(toemb[current_index])
+                dataz['result'][index]['embedding_data'] = em_vector
+                current_index+=1
+            else:
+                pass
+        final_results = []
+        for z in range(len(dataz['result'])):
+            thisdata = copy.deepcopy(dataz['result'][z])
+            # data['result'][z].pop("embedding")
+            collection.update_one({"URL":thisdata['URL']},{"$set":thisdata})
+        for z in copy.deepcopy(dataz['result']):
+            try:
+                for a in copy.deepcopy(z['embedding_data']):
+                    results.append(a)
+            except:
+                pass
+        results = copy.deepcopy(results)
+        for thisr in results:
+            thisr['similairy'] = percentage_similarity(thisr['embedding'],query_embedding['result'][0]['embedding'])
+            final_results.append(thisr)
+        final_results = [z for z in final_results if z['similairy'] > 80]
+        final_results = sorted(final_results,key=lambda x:x['similairy'],reverse=True )
+        remove_embedding = [z.pop("embedding") for z in final_results]
+        dataz['extracts'] = final_results
+        return dataz
+    
+    else:
+        return(json.dumps({"type":'error','message':"long_query and short_query is not in request"},indent=4))
+
+@app.route("/webpage",methods=["POST","GET"])
+def webpage():
+    global collection
+    args = request.get_json()
+    url = args.get("url",None)
+    if(url == None):
+        return(json.dumps({'type':'error','message':'url is not provided'},indent=4))
+    else:
+        previous_data = collection.find_one({"URL":url})
+        if(previous_data is None):
+            result = {}
+            result['URL'] = url
+            result['time'] = time.time()
+            result['webpage'] = asyncio.run(extract_web(result))
+        else:
+            time_change = time.time() - previous_data['time']
+            if(time_change < 86400):
+                result = previous_data
+            else:
+                result = {}
+                result['time'] = time.time()
+                result['URL'] = url
+                result['webpage'] = asyncio.run(extract_web(result))
+
+        
+        if("embedding_data" not in result and result['webpage'] != "Some Error while Extracting"):
+            toemb = text_splitter.create_documents([result['webpage']])
+            toembz = [z.page_content for z in toemb]
+            data_to_send = {
+                "text":toembz
+            }
+
+            embedding = requests.post("https://mangoman7002-flash-embedding.hf.space",json=data_to_send)
+            if(embedding.status_code != 200):
+                return json.dumps({"type":"error","message":f"error With API {str(embedding.status_code)}"},indent=4)
+            embedding = embedding.json()
+            result['embedding_data'] = embedding['result']
+        try:
+            result.pop("_id")
+        except:
+            pass
+    return(json.dumps(result))
+if __name__ == '__main__':
+    app.run(debug=False)

main.py

@@ -0,0 +1,104 @@
+from fake_useragent import UserAgent
+import copy
+import time
+import asyncio
+import urllib.parse
+import requests
+import json
+import re
+from bs4 import BeautifulSoup as bs
+import urllib
+from concurrent.futures import ThreadPoolExecutor
+from patterns import patterns
+from urllib.parse import urlparse, parse_qs
+ua = UserAgent()
+patterns = patterns()
+async def bing_serach(query, collection, ifextract=False,start:int=0):
+    count = "none"
+    r = requests.get(f"https://www.google.com/search?q={urllib.parse.quote_plus(query)}")
+    r.status_code
+    # print(results)
+    soup  = bs(r.text,'html.parser')
+    main = soup.find_all("div", id='main')[0]
+    l = [z for z in main.children if z.name == "div"]
+    results = []
+    for z in l:
+        kCrYT = z.find_all("div",class_="kCrYT")
+        if(len(kCrYT) > 0):
+            try:
+                if(len(kCrYT[0].find_all("a")[0].find_all(class_="DnJfK"))>0):
+                    abstract = kCrYT[1].text
+                    title = kCrYT[0].find_all("a")[0].find_all(class_="DnJfK")[0].find_all("h3")[0].text
+                    url = kCrYT[0].find_all("a")[0].get('href')
+                    results.append({
+                        'Abstract':abstract,
+                        'Title':title,
+                        'URL':parse_qs(urlparse(url).query).get('q',[""])[0]
+                    })
+            except Exception as e:
+                try:
+                    if(len(kCrYT[1].find_all("a")[0].find_all("span")) > 1):
+                        abstract = kCrYT[0].text
+                        title = kCrYT[1].find_all("a")[0].find_all("span")[0].text
+                        url = kCrYT[1].find_all("a")[0].get('href')
+                        results.append({
+                            'Abstract':abstract,
+                            'Title':title,
+                            'URL':parse_qs(urlparse(url).query).get('q',[""])[0]
+                        })
+                    else:
+                        print("Method 2 Failed")
+                except:
+                 print(e)
+    if ifextract:
+        for i, result in enumerate(results):
+            previous_data = collection.find_one({"URL":result['URL']})
+            if(previous_data is None):
+                result['webpage'] = asyncio.create_task(extract_web(result))
+                result['time'] = time.time()
+                
+            else:
+                print(f"This is Taken from cache {result['URL']}\n\n")
+                result['webpage'] = previous_data['webpage']
+                try:
+                    result['embedding_data'] = previous_data['embedding_data']
+                except:
+                    print(f"embedding_data not exist in {result['URL']}")
+                result['from'] = "cache"
+        for result in results:
+            try:
+                result['webpage'] =await result['webpage']
+            except:
+                pass
+            dummy_result = copy.deepcopy(result)  # Creates a completely independent copy
+            collection.insert_one(dummy_result) 
+        # print({'count':count,'result':results})
+        return {'count':count,'result':results}
+    else:
+        return {'count':count,'result':results}
+
+
+
+
+async def extract_web(result):
+    try:
+        try:
+            headers = {
+                'User-Agent': ua.random
+            }
+            content = requests.get(result['URL'],headers=headers,verify=False)
+        except:
+            print("some Error While Initial Request")
+        print(content.status_code)
+        for pattern in patterns:
+            if(re.match(pattern['recode'],result['URL'])):
+                thisr = pattern['function'](content)
+                result['webpage'] = thisr
+                result['time'] = time.time()
+                return thisr
+    except Exception as e:
+        print(str(e))
+        return 'Some Error while Extracting'
+                # return ('There is some error with This Pattern\n','Pattern Name',pattern['Title'],'\nPattern Id',pattern['id'])
+    
+    

pattern_functions.py

@@ -0,0 +1,28 @@
+from bs4 import BeautifulSoup as bs
+import re
+def extract_1(content):
+    finalcontent = ''
+    toremove = ['link','script','style','iframe','object','noscript','param','embed','meta','base','canvas','svg']
+    content_soup = bs(content.text, 'html.parser')
+    for soup_body in content_soup.find_all('body'):
+        for remove_tag in toremove:
+            for trash_tag in soup_body.find_all(remove_tag):
+                trash_tag.decompose()
+        thisbody = soup_body.get_text()
+        thisbody = thisbody.replace("\t",'')
+        thisbody = re.sub(r"\n\w\n",'\n',thisbody)
+        while True:
+            old_body = thisbody
+            thisbody = thisbody.replace('  ', ' ')
+            if old_body == thisbody:
+                break
+
+        while True:
+            old_body = thisbody
+            thisbody = thisbody.replace('\n\n', '\n')
+            if old_body == thisbody:
+                break
+
+        finalcontent = finalcontent + thisbody
+    print('content Extracted')
+    return "\n".join([z.strip() for z in finalcontent.split("\n") if len(z.strip().split(" ")) > 3])

patterns.py

@@ -0,0 +1,9 @@
+from pattern_functions import *
+def patterns():
+    patterns = [{
+            'Title':'All Extractor',
+            'id':'extract_1',
+            'recode':'^https://.*$',
+            'function':extract_1
+    }]
+    return patterns

requirements.txt

@@ -0,0 +1,9 @@
+urllib3
+bs4
+requests
+flask
+protobuf
+langchain_text_splitters
+numpy
+pymongo
+fake_useragent