[update][abaoxomtieu]: remove unnecessary componenents

apis/V1/prompts/document_type_clf.py

@@ -1,78 +0,0 @@
-from langchain.prompts import PromptTemplate, ChatPromptTemplate
-from langchain_core.output_parsers import JsonOutputParser
-from langchain_core.messages import HumanMessage, SystemMessage
-from ..configs.llm_config import llm
-from ..interfaces.llm_interface import ClassifyDocumentOutput
-
-# System message for initial context and instructions
-system_instructions = SystemMessage(
-    content="""You are an expert in classifying document types. Identify the type of document based on short content as Business or RnD."""
-)
-
-# Example system message to clarify format with examples
-system_examples = SystemMessage(
-    content="""
-Examples:
-
-1.
-Document: 
-"A central goal of machine learning is the development of systems that can solve many problems in as many data domains as possible. Current architectures, however, cannot be applied beyond a small set of stereotyped settings, as they bake in domain & task assumptions or scale poorly to large inputs or outputs. In this work, we propose Perceiver IO, a general-purpose architecture that handles data from arbitrary settings while scaling linearly with the size of inputs and outputs."
-Type: RnD
-
-2.
-Document:
-"The global economy is projected to grow at a slower pace this year, as rising inflation and supply chain disruptions continue to impact markets worldwide. Central banks are adjusting their monetary policies in response to these challenges."
-Type: Business
-
-3.
-Document:
-"The company’s latest financial report shows a significant increase in net income, driven by cost reduction strategies and increased sales in emerging markets. Investors are optimistic about the future performance given these strong quarterly results."
-Type: Business
-
-4.
-Document:
-"Our research introduces a novel approach to quantum computing, focusing on error correction and scalability. The proposed methods significantly improve the stability of qubits, which is crucial for the practical implementation of quantum algorithms."
-Type: RnD
-
-5.
-Document:
-"An analysis of the stock market reveals that technology stocks have outperformed other sectors over the past quarter. This trend is attributed to the rapid digital transformation across industries and the increasing demand for tech solutions."
-Type: Business
-
-6.
-Document:
-"The startup has launched an innovative mobile app designed to streamline project management for remote teams. The app includes features such as real-time collaboration, task tracking, and performance analytics, making it a comprehensive tool for businesses."
-Type: Business
-"""
-)
-
-# Function to create the prompt template dynamically based on input
-def create_prompt(document):
-    user_message = HumanMessage(
-        content=f"""
-Document: {document}
-Output format: 
-"type": type of document, 
-
-If the document is Business, please type "Business" else type "RnD".
-{{format_instructions}}
-"""
-    )
-    # Combine all messages into a conversation
-    messages = [system_instructions, system_examples, user_message]
-    chat_prompt = ChatPromptTemplate(messages=messages)
-
-    return chat_prompt.format()
-
-# Use the JSON output parser
-parser = JsonOutputParser(pydantic_object=ClassifyDocumentOutput)
-
-# Create the prompt template
-prompt = PromptTemplate(
-    input_variables=["document"],
-    template=create_prompt("{document}"),
-    partial_variables={"format_instructions": parser.get_format_instructions()},
-)
-
-# Chain the prompt with the LLM and parser
-chain_type_classify = prompt | llm | parser

apis/V1/prompts/grammarly.py

@@ -1,82 +0,0 @@
-from langchain.prompts import PromptTemplate, ChatPromptTemplate
-from langchain_core.output_parsers import JsonOutputParser
-from langchain_core.messages import AIMessage, HumanMessage, SystemMessage
-from ..configs.llm_config import llm
-from ..interfaces.llm_interface import GrammarlyOutput
-
-# System message for initial context and instructions
-system_instructions = SystemMessage(
-    content="""You are an expert in checking grammar errors. Identify the errors and provide corrections."""
-)
-
-# Example system message to clarify format with examples
-system_examples = SystemMessage(
-    content="""
-Examples:
-
-1.
-Sentence: "She don't know nothing about the new project, and there is many details that needs to be explained."
-Incorrect: [don't know nothing, is, needs]
-Correct: [doesn't know anything, are, need]
-Corrected sentence: "She doesn't know anything about the new project, and there are many details that need to be explained."
-
-2.
-Sentence: "He go to the store every day, but he never buy nothing."
-Incorrect: [go, buy nothing]
-Correct: [goes, buys anything]
-Corrected sentence: "He goes to the store every day, but he never buys anything."
-
-3.
-Sentence: "She quickly ran to the store, bought some groceries, and returning home."
-Incorrect: [returning]
-Correct: [returned]
-Corrected sentence: "She quickly ran to the store, bought some groceries, and returned home."
-
-4.
-Sentence: "Each of the students have completed their assignments, but the teacher is not satisfied with their works."
-Incorrect: [have, works]
-Correct: [has, work]
-Corrected sentence: "Each of the students has completed their assignments, but the teacher is not satisfied with their work."
-
-5.
-Sentence: "If he was more careful, he will not make so many mistakes, which is causing problems for the team."
-Incorrect: [was, will, is]
-Correct: [were, would, are]
-Corrected sentence: "If he were more careful, he would not make so many mistakes, which are causing problems for the team."
-"""
-)
-
-
-# Function to create the prompt template dynamically based on input
-def create_prompt(sentence):
-    user_message = HumanMessage(
-        content=f"""
-My sentence: {sentence}
-Output format: 
-"sentence": my sentence, 
-"incorrect": [list of incorrect words or phrases], 
-"correct": [list of correct words or phrases], 
-"corrected_sentence": corrected sentence
-
-If no corrections are needed, return the original sentence as the corrected sentence and empty lists for incorrect and correct.
-{{format_instructions}}
-"""
-    )
-    # Combine all messages into a conversation
-    messages = [system_instructions, system_examples, user_message]
-    chat_prompt = ChatPromptTemplate(messages=messages)
-    return chat_prompt.format()
-
-
-# Use the JSON output parser
-parser = JsonOutputParser(pydantic_object=GrammarlyOutput)
-
-# Create the prompt template
-prompt = PromptTemplate(
-    input_variables=["sentence"],
-    template=create_prompt("{sentence}"),
-    partial_variables={"format_instructions": parser.get_format_instructions()},
-)
-
-# Chain the prompt with the LLM and parser
-chain_grammarly = prompt | llm | parser

apis/V1/prompts/summarize_doc.py

@@ -1,19 +0,0 @@
-from ..configs.llm_config import llm
-from langchain_core.output_parsers import JsonOutputParser
-
-from langchain_core.prompts import PromptTemplate
-from ..interfaces.llm_interface import ClassifyAndSummarizeOutput
-
-# Define prompt
-prompt_template = """Write a concise summary of the following:
-"{text}" and classify the document type as Business or RnD.
-Return the answer as: \n{format_instructions}.
-"""
-parser = JsonOutputParser(pydantic_object=ClassifyAndSummarizeOutput)
-prompt = PromptTemplate.from_template(
-    template=prompt_template,
-    partial_variables={"format_instructions": parser.get_format_instructions()},
-)
-
-# Define LLM chain
-chain_summarize = prompt | llm | parser

apis/V1/prompts/translate.py

@@ -1,60 +0,0 @@
-from langchain.prompts import PromptTemplate, ChatPromptTemplate
-from langchain_core.output_parsers import JsonOutputParser
-from langchain_core.messages import AIMessage, HumanMessage, SystemMessage
-from ..configs.llm_config import llm
-from ..interfaces.llm_interface import TranslateOutput
-
-# System message for initial context and instructions
-system_instructions = SystemMessage(
-    content="""You are a master in translation. You are given a sentence and a word. 
-    Translate the word to Vietnamese based on the context of the sentence. """
-)
-
-# Example system message to clarify format with examples
-system_examples = SystemMessage(
-    content="""
-Example 1:
-    My sentence: "outperforms a Transformer-based BERT baseline on the GLUE language benchmark"
-    My word: "outperforms"
-    Answer: "vượt trội"
-Example 2:
-    My sentence: "I love my dog so much, I was interested in him the first time I saw him"
-    My word: "so much"
-    Answer: "rất nhiều"
-"""
-)
-
-
-# Function to create the prompt template dynamically based on input
-def create_prompt(sentence, word):
-    user_message = HumanMessage(
-        content=f"""
-My sentence: {sentence}
-My word: {word}
-        
-If this word is not in dictionary, return "Không tìm thấy từ này trong từ điển".
-Else, Translate the word to Vietnamese based on the context of the sentence. Return the answer as the translated word.
-I want output to have the format: "answer": translated word, "options": list of options. 
-Options is a list of words consisting of the answer and 3 random words which can be used to create flashcards for learning.
-"""
-    )
-    ai_response = AIMessage(content="Answer: [Your AI will provide the answer here]")
-    # Combine all messages into a conversation
-    messages = [system_instructions, system_examples, user_message, ai_response]
-    chat_prompt = ChatPromptTemplate(messages=messages)
-
-    return chat_prompt.format()
-
-
-# Use the JSON output parser
-parser = JsonOutputParser(pydantic_object=TranslateOutput)
-
-# Create the prompt template
-prompt = PromptTemplate(
-    input_variables=["sentence", "word"],
-    template=create_prompt("{sentence}", "{word}"),
-    partial_variables={"format_instructions": parser.get_format_instructions()},
-)
-
-# Chain the prompt with the LLM and parser
-chain_translate = prompt | llm | parser

apis/V1/prompts/translate_abstract_BIZ.py

@@ -1,74 +0,0 @@
-from langchain.prompts import PromptTemplate, ChatPromptTemplate
-from langchain_core.output_parsers import JsonOutputParser
-from langchain_core.messages import HumanMessage, SystemMessage
-from ..configs.llm_config import llm
-from ..interfaces.llm_interface import TranslateOutput
-
-# System message for initial context and instructions
-system_instructions = SystemMessage(
-    content="""You are an expert in translating English into Vietnamese for Finance and Economics topics. Translate the given word or phrase based on the context of the abstract and sentence.
-1. Read the abstract to understand the context.
-2. Translate the given word or phrase into Vietnamese using the context.
-3. If the word or phrase is not in the dictionary, return "Không tìm thấy từ này trong từ điển".
-4. If the word or phrase is an acronym, expand it and provide the translation.
-5. Output must be in Vietnamese.
-"""
-)
-
-# Example system message to clarify format with examples
-system_examples = SystemMessage(
-    content="""
-**Examples:**
-
-Abstract: "The efficient market hypothesis (EMH) suggests that financial markets are informationally efficient, meaning that asset prices fully reflect all available information."
-Sentence: "The efficient market hypothesis (EMH) suggests that financial markets are informationally efficient."
-Word: "efficient"
-hiệu quả
-
-Abstract: "Quantitative easing (QE) is a monetary policy whereby a central bank purchases government securities or other securities from the market in order to increase the money supply and encourage lending and investment."
-Sentence: "Quantitative easing (QE) is a monetary policy whereby a central bank purchases government securities."
-Word: "Quantitative easing"
-Nới lỏng định lượng
-
-Abstract: "Inflation is the rate at which the general level of prices for goods and services rises, eroding purchasing power."
-Sentence: "Inflation is the rate at which the general level of prices for goods and services rises."
-Word: "Inflation"
-lạm phát
-
-Abstract: "GDP, or Gross Domestic Product, measures the total value of all goods and services produced within a country in a given period."
-Sentence: "GDP, or Gross Domestic Product, measures the total value of all goods and services produced within a country."
-Word: "GDP"
-Tổng sản phẩm quốc nội
-"""
-)
-
-
-# Function to create the prompt template dynamically based on input
-def create_prompt(abstract, sentence, word):
-    user_message = HumanMessage(
-        content=f"""
-Abstract: {abstract}
-Sentence: {sentence}
-Word: {word}
-
-Translate the word to Vietnamese based on the context of the sentence and abstract.
-If the word is not in the dictionary, return "Không tìm thấy từ này trong từ điển".
-If it is an acronym, expand and translate it.
-"""
-    )
-    messages = [system_instructions, system_examples, user_message]
-    chat_prompt = ChatPromptTemplate(messages=messages)
-
-    return chat_prompt.format()
-
-
-# Use the JSON output parser
-
-# Create the prompt template
-prompt = PromptTemplate(
-    input_variables=["abstract", "sentence", "word"],
-    template=create_prompt("{abstract}", "{sentence}", "{word}"),
-)
-
-# Chain the prompt with the LLM and parser
-chain_BIZ = prompt | llm

apis/V1/prompts/translate_abstract_RnD.py

@@ -1,71 +0,0 @@
-from langchain.prompts import PromptTemplate, ChatPromptTemplate
-from langchain_core.output_parsers import JsonOutputParser
-from langchain_core.messages import HumanMessage, SystemMessage
-from ..configs.llm_config import llm
-from ..interfaces.llm_interface import TranslateOutput
-
-system_instructions = SystemMessage(
-    content="""You are an expert in translating English into Vietnamese for R&D topics. Translate the given word or phrase based on the context of the abstract and sentence.
-1. Read the abstract to understand the context.
-2. Translate the given word or phrase into Vietnamese using the context.
-3. If the word or phrase is not in the dictionary, return "Không tìm thấy từ này trong từ điển".
-4. If the word or phrase is an acronym, expand it and provide the translation.
-5. Output must be in Vietnamese.
-"""
-)
-
-system_examples = SystemMessage(
-    content="""
-**Examples:**
-
-Abstract: "In recent years, there has been significant progress in the development of Transformer-based models for natural language processing tasks. These models, such as BERT, have set new benchmarks in various language understanding tasks."
-Sentence: "outperforms a Transformer-based BERT baseline on the GLUE language benchmark"
-Word: "outperforms"
-vượt trội
-
-Abstract: "This paper explores the use of reinforcement learning in optimizing neural network architectures. We demonstrate that our approach achieves state-of-the-art results on several benchmark datasets."
-Sentence: "Our approach achieves state-of-the-art results on several benchmark datasets."
-Word: "approach"
-phương pháp
-
-Abstract: "The integration of machine learning techniques in medical diagnosis has shown promising results. Our study focuses on the application of deep learning to detect early signs of diseases."
-Sentence: "The integration of machine learning techniques in medical diagnosis has shown promising results."
-Word: "integration"
-tích hợp
-
-Abstract: "Transformer-based models like BERT have transformed NLP tasks by achieving new state-of-the-art results."
-Sentence: "Transformer-based models like BERT have transformed NLP tasks."
-Word: "BERT"
-Bidirectional Encoder Representations from Transformers
-"""
-)
-
-
-def create_prompt(abstract, sentence, word):
-    user_message = HumanMessage(
-        content=f"""
-Abstract: {abstract}
-Sentence: {sentence}
-Word: {word}
-
-Translate the word to Vietnamese based on the context of the sentence and abstract.
-If the word is not in the dictionary, return "Không tìm thấy từ này trong từ điển".
-If it is an acronym, expand and translate it.
-"""
-    )
-    messages = [system_instructions, system_examples, user_message]
-    chat_prompt = ChatPromptTemplate(messages=messages)
-
-    return chat_prompt.format()
-
-
-# Use the JSON output parser
-
-# Create the prompt template
-prompt = PromptTemplate(
-    input_variables=["abstract", "sentence", "word"],
-    template=create_prompt("{abstract}", "{sentence}", "{word}"),
-)
-
-# Chain the prompt with the LLM and parser
-chain_RnD = prompt | llm

apis/V1/prompts/translate_detail.py

@@ -23,7 +23,7 @@ Explanation: "Thuật toán gradient descent là một phương pháp tối ưu
 For the word "overfitting":
 Explanation: "Overfitting là hiện tượng một mô hình học máy biểu hiện quá mức các dữ liệu huấn luyện, làm giảm khả năng dự đoán chính xác dữ liệu mới. Điều này thường xảy ra khi mô hình quá phức tạp so với dữ liệu."
 
-Now, follow the steps and provide the explanation in Vietnamese for the word "{word}".
+Now, follow the steps and provide the explanation in Vietnamese for the word: "{word}".
 Explanation:
 """
 

apis/V1/prompts/translate_test.py

@@ -1,72 +0,0 @@
-from langchain.prompts import PromptTemplate, ChatPromptTemplate
-from langchain_core.output_parsers import JsonOutputParser
-from langchain_core.messages import HumanMessage, SystemMessage
-from ..configs.llm_config import llm
-from pydantic import BaseModel, Field
-
-class TranslateOutput(BaseModel):
-    answer: str = Field(description="translated word")
-    word: str = Field(description="word to be translated")
-
-system_instructions = SystemMessage(
-    content="""You are a master at translating English into Vietnamese. You are provided with an abstract of a paper, a sentence containing the word you want to translate, and the word itself. Translate the word into Vietnamese based on the context of the document and the sentence.
-1. Read the abstract to understand the context.
-2. Use the context to translate the given word into Vietnamese.
-3. If the word is not in the dictionary, return "Không tìm thấy từ này trong từ điển".
-4. If the word is an acronym, expand it and provide the translation.
-"""
-)
-
-system_examples = SystemMessage(
-    content="""
-**Examples:**
-
-Abstract: "In recent years, there has been significant progress in the development of Transformer-based models for natural language processing tasks. These models, such as BERT, have set new benchmarks in various language understanding tasks."
-Sentence: "outperforms a Transformer-based BERT baseline on the GLUE language benchmark"
-Word: "outperforms"
-Answer: "vượt trội"
-
-Abstract: "This paper explores the use of reinforcement learning in optimizing neural network architectures. We demonstrate that our approach achieves state-of-the-art results on several benchmark datasets."
-Sentence: "Our approach achieves state-of-the-art results on several benchmark datasets."
-Word: "approach"
-Answer: "phương pháp"
-
-Abstract: "The integration of machine learning techniques in medical diagnosis has shown promising results. Our study focuses on the application of deep learning to detect early signs of diseases."
-Sentence: "The integration of machine learning techniques in medical diagnosis has shown promising results."
-Word: "integration"
-Answer: "tích hợp"
-
-Abstract: "Transformer-based models like BERT have transformed NLP tasks by achieving new state-of-the-art results."
-Sentence: "Transformer-based models like BERT have transformed NLP tasks."
-Word: "BERT"
-Answer: "Bidirectional Encoder Representations from Transformers"
-"""
-)
-
-def create_prompt(abstract, sentence, word):
-    user_message = HumanMessage(
-        content=f"""
-Abstract: {abstract}
-Sentence: {sentence}
-Word: {word}
-
-Translate the word to Vietnamese based on the context of the sentence and abstract.
-If the word is not in the dictionary, return "Không tìm thấy từ này trong từ điển".
-If it is an acronym, expand and translate it.
-Return the answer as: "answer": Vietnamese word, "word": input word (English).
-"""
-    )
-    messages = [system_instructions, system_examples, user_message]
-    chat_prompt = ChatPromptTemplate(messages=messages)
-
-    return chat_prompt.format()
-
-parser = JsonOutputParser(pydantic_object=TranslateOutput)
-
-prompt = PromptTemplate(
-    input_variables=["abstract", "sentence", "word"],
-    template=create_prompt("{abstract}", "{sentence}", "{word}"),
-    partial_variables={"format_instructions": parser.get_format_instructions()},
-)
-
-chain_test_RnD = prompt | llm | parser

apis/V1/providers/__init__.py

@@ -1,5 +1,3 @@
 from .jwt_provider import JWTProvider
-from .llm_provider import chain
 
 jwt = JWTProvider()
-llm = chain

apis/V1/providers/llm_provider.py

@@ -1,60 +0,0 @@
-from langchain.prompts import PromptTemplate, ChatPromptTemplate
-from langchain_core.output_parsers import JsonOutputParser
-from langchain_core.messages import AIMessage, HumanMessage, SystemMessage
-from ..configs.llm_config import llm
-from ..interfaces.llm_interface import TranslateOutput
-
-# System message for initial context and instructions
-system_instructions = SystemMessage(
-    content="""You are a master in translation. You are given a sentence and a word. 
-    Translate the word to Vietnamese based on the context of the sentence. """
-)
-
-# Example system message to clarify format with examples
-system_examples = SystemMessage(
-    content="""
-Example 1:
-    My sentence: "outperforms a Transformer-based BERT baseline on the GLUE language benchmark"
-    My word: "outperforms"
-    Answer: "vượt trội"
-Example 2:
-    My sentence: "I love my dog so much, I was interested in him the first time I saw him"
-    My word: "so much"
-    Answer: "rất nhiều"
-"""
-)
-
-
-# Function to create the prompt template dynamically based on input
-def create_prompt(sentence, word):
-    user_message = HumanMessage(
-        content=f"""
-My sentence: {sentence}
-My word: {word}
-        
-If this word is not in dictionary, return "Không tìm thấy từ này trong từ điển".
-Else, Translate the word to Vietnamese based on the context of the sentence. Return the answer as the translated word.
-I want output to have the format: "word": input word ,"answer": translated word, "options": list of options. 
-Options is a list of words consisting of the answer and 3 random words which can be used to create flashcards for learning.
-"""
-    )
-    ai_response = AIMessage(content="Answer: [Your AI will provide the answer here]")
-    # Combine all messages into a conversation
-    messages = [system_instructions, system_examples, user_message, ai_response]
-    chat_prompt = ChatPromptTemplate(messages=messages)
-
-    return chat_prompt.format()
-
-
-# Use the JSON output parser
-parser = JsonOutputParser(pydantic_object=TranslateOutput)
-
-# Create the prompt template
-prompt = PromptTemplate(
-    input_variables=["sentence", "word"],
-    template=create_prompt("{sentence}", "{word}"),
-    partial_variables={"format_instructions": parser.get_format_instructions()},
-)
-
-# Chain the prompt with the LLM and parser
-chain = prompt | llm | parser