Upload 2 files

app.py

@@ -0,0 +1,157 @@
+# -*- coding: utf-8 -*-
+"""
+Created on Fri May 19 10:37:00 2023
+
+@author: Goutam
+"""
+
+from langchain.embeddings.openai import OpenAIEmbeddings
+from langchain.vectorstores import Chroma
+from langchain.text_splitter import CharacterTextSplitter
+from langchain.llms import OpenAI
+from langchain.chains import RetrievalQA
+from langchain.document_loaders import PyPDFLoader
+from langchain.chat_models import ChatOpenAI
+from googletrans import Translator
+import re
+import os
+import gradio as gr
+
+#from langchain.document_loaders import TextLoader
+#Other loaders PyPDFLoader,PyPDFDirectoryLoader
+#from langchain.document_loaders import UnstructuredFileLoader
+
+os.environ['OPENAI_API_KEY']='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'
+def translate(in_str):
+  URL_COM = 'translate.google.com'
+  LANG = "zh-CN" #hi is for Hindi, en for English, zh or zh-CN for chinese simplified,zh-TW for traditional chinese
+  translator = Translator(service_urls=[URL_COM])
+  translation = translator.translate(in_str, dest=LANG)
+  #print(translation)
+  translation_str = str(translation)
+  answer_group = re.search('text=(.*)pronunciation=', translation_str)
+  answer_chinese=""
+  if answer_group is not None:
+      answer_chinese = answer_group.group(1)
+      print("Group not none-",answer_chinese)
+  else:
+      answer_chinese = translation_str
+      print("Group is None")
+  #print("Answer in Chinese-", answer_chinese)
+  return answer_chinese  
+def cccs_demo(question):
+    embeddings = OpenAIEmbeddings()
+    docsearch = Chroma(persist_directory="ChinaDB/", embedding_function=embeddings)
+#Custom prompt
+    from langchain.prompts import PromptTemplate
+
+    prompt_template = """Use the documents uploaded on China, to answer the question at the end. If you don't know the answer'
+
+{context}
+Question: {question}
+
+Answer:"""
+    PROMPT = PromptTemplate(
+    template=prompt_template, input_variables=["context", "question"]
+    )
+    chain_type_kwargs = {"prompt": PROMPT}
+    llm = OpenAI(temperature=0)
+#Have commented the original below
+    qa = RetrievalQA.from_chain_type(llm=llm, chain_type="stuff", retriever=docsearch.as_retriever(), return_source_documents=True,chain_type_kwargs=chain_type_kwargs)
+#new function
+#qa = VectorDBQA.from_chain_type(llm=OpenAI(), chain_type="stuff", vectorstore=vectordb)
+#qa = RetrievalQA.from_chain_type(llm=OpenAI(), chain_type="stuff", retriever=docsearch.as_retriever())
+    query = question
+#answer = qa.run(query)
+    answer = qa({"query": query})
+    full_result = answer['result']
+    print("Answer ",full_result)
+    chinese_result = ""
+    chinese_result = translate(full_result)
+    final_result = full_result + '\n' + chinese_result
+    print("Final result-",final_result)
+    
+    """
+    english_group = re.search('ENGLISH=(.*)CHINESE=', full_result)
+    english_answer = english_group.group(1)
+    chinese_group = re.search('CHINESE=(.*)END', full_result)
+    chinese_answer = chinese_group.group(1)
+    print("English Answer-",english_answer)
+    print("Chinese Answer-",chinese_answer)
+    """
+    source_docs = answer['source_documents']
+    print("Number of sources ",len(source_docs))
+    print("Source docs-",source_docs)
+    # Add Doc summary
+    chain_type_kwargs = {"prompt": PROMPT}
+#Have commented the original below
+    qa = RetrievalQA.from_chain_type(llm=llm, chain_type="stuff", retriever=docsearch.as_retriever(), return_source_documents=True,chain_type_kwargs=chain_type_kwargs)
+#new function
+#qa = VectorDBQA.from_chain_type(llm=OpenAI(), chain_type="stuff", vectorstore=vectordb)
+#qa = RetrievalQA.from_chain_type(llm=OpenAI(), chain_type="stuff", retriever=docsearch.as_retriever())
+    query = "Please give summary of contents all the documents."
+#answer = qa.run(query)
+    answer = qa({"query": query})
+    doc_summary = answer['result']
+    print("Document Summary-",doc_summary)
+    chinese_summary =""
+    chinese_summary = translate(doc_summary)
+    final_summary = doc_summary+ '\n' + chinese_summary
+    print("Final result-",final_summary)
+# End doc summary
+    ref_str = "None.txt"
+    source_len = len(source_docs)
+    if (source_len > 0):
+        with open("Referenced.txt",'w',encoding='utf-8') as f:
+            for i in range(len(source_docs)):
+    #print("Referred source-",i+1,answer['source_documents'][i])
+                source_string = str(source_docs[i])
+                page_content = re.search('page_content=(.*)metadata=', source_string)
+                source = page_content.group(1)
+                source = source.replace('\\n','\n')
+                source = source.replace('\\uf07d',' ')
+                source = source.replace('\\xa0',' ')
+                page_str = "Page Content"+'\n'
+                print("Page Content",'\n')
+                print(source)
+                f.write(page_str)
+                f.write(source)
+                f.write('\n')
+                meta_data_group = re.search('metadata={(.*)}',source_string)
+                meta_data = meta_data_group.group(1)
+                meta_str = "Meta Data-"
+                print("Meta Data-",'\n')
+                print(meta_data)
+                f.write(meta_str)
+                f.write(meta_data)
+                f.write('\n\n')
+            ref_str = 'Referenced.txt'
+    else:
+        with open("None.txt",'w',encoding='utf-8') as f:
+            none_str = "No directy sources found"
+            f.write(none_str)
+    return final_result,final_summary,ref_str
+"""
+URL_COM = 'translate.google.com'
+URL_HI = 'translate.google.hi'
+LANG = "hi" #hi is for Hindi, en for English, zh or zh-CN for chinese simplified,zh-TW for traditional chinese
+translator = Translator(service_urls=[URL_COM])
+translation = translator.translate(answer['result'], dest=LANG)
+#print(translation)
+translation_str = str(translation)
+answer_group = re.search('text=(.*)pronunciation=', translation_str)
+answer_hindi = answer_group.group(1)
+print("Answer in Hindi-", answer_hindi)
+"""
+title = "Zero2AI CCCS Demo"
+description = "Demonstration of multi-document and multi-lingual Q&A on China."
+demo = gr.Interface(cccs_demo, [gr.Textbox(label="Question")],[gr.Textbox(label="Answer"),gr.Textbox(label="Repository Summary"),gr.File(label="Reference Details")], title=title, description=description,theme=gr.themes.Glass(primary_hue="indigo", secondary_hue="purple"),allow_flagging='never')
+demo.launch()
+
+#formatted_source = source_docs.replace('\\n', '\n').replace('\\t', '\t')
+#print("Source Documents ",formatted_source)
+#To check source in RetrievalQA
+#qa = RetrievalQA.from_chain_type(llm=OpenAI(), chain_type="stuff", retriever=docsearch.as_retriever(), chain_type_kwargs=chain_type_kwargs,return_source_documents=True)
+#result
+#answer['result']
+#answer['source_documents']

requirements.txt

@@ -0,0 +1,6 @@
+openai == 0.27.8
+langchain == 0.0.205
+chromadb == 0.3.23
+pypdf == 3.9.1
+PyPDF2 == 3.0.1
+googletrans==4.0.0rc1