Update app.py

app.py

@@ -1,7 +1,145 @@
-import gradio as gr
+/*import gradio as gr
 
 def greet(name):
     return "Hello " + name + "!!"
 
 demo = gr.Interface(fn=greet, inputs="text", outputs="text")
-demo.launch()
+demo.launch()*/
+
+from datasets import load_dataset
+from transformers import AutoTokenizer, AutoModel
+from transformers import T5ForConditionalGeneration, T5Tokenizer
+
+import faiss
+import numpy as np
+import os
+
+
+# Define the path to the docs folder
+docs_path = "docs"
+
+# Load documents from the folder
+def load_documents(docs_path):
+    data_files = {"train": [os.path.join(docs_path, file) for file in os.listdir(docs_path) if file.endswith(".txt")]}
+    dataset = load_dataset("text", data_files=data_files)
+    return dataset["train"]
+
+documents_dataset = load_documents(docs_path)
+
+
+
+
+# Load model and tokenizer
+tokenizer = AutoTokenizer.from_pretrained("distilbert-base-uncased")
+model = AutoModel.from_pretrained("distilbert-base-uncased")
+
+# Preprocess and encode documents
+def encode_documents(documents):
+    inputs = tokenizer(documents["text"], return_tensors='pt', padding=True, truncation=True)
+    outputs = model(**inputs)
+    embeddings = outputs.last_hidden_state.mean(dim=1).detach().numpy()
+    return embeddings
+
+embeddings = encode_documents(documents_dataset)
+
+# Index embeddings
+index = faiss.IndexFlatL2(embeddings.shape[1])
+index.add(embeddings)
+
+
+# Load T5 model and tokenizer
+t5_tokenizer = T5Tokenizer.from_pretrained("t5-small")
+t5_model = T5ForConditionalGeneration.from_pretrained("t5-small")
+
+def generate_questions(text):
+    input_text = f"generate question: {text}"
+    input_ids = t5_tokenizer.encode(input_text, return_tensors="pt")
+    outputs = t5_model.generate(input_ids)
+    question = t5_tokenizer.decode(outputs[0], skip_special_tokens=True)
+    return question
+
+def retrieve_documents(query, index, documents):
+    query_embedding = encode_documents({"text": [query]})
+    D, I = index.search(query_embedding, k=5)
+    return [documents[i] for i in I[0]]
+
+# Answer Generation
+from transformers import GPT2LMHeadModel, GPT2Tokenizer
+
+# Load GPT-2 model and tokenizer
+gpt2_tokenizer = GPT2Tokenizer.from_pretrained("gpt2")
+gpt2_model = GPT2LMHeadModel.from_pretrained("gpt2")
+
+def generate_answer(question, context):
+    input_text = f"Question: {question}\nContext: {context}\nAnswer:"
+    input_ids = gpt2_tokenizer.encode(input_text, return_tensors="pt")
+    outputs = gpt2_model.generate(input_ids, max_length=150)
+    answer = gpt2_tokenizer.decode(outputs[0], skip_special_tokens=True)
+    return answer
+
+#fine tuning the language model
+
+from transformers import GPT2LMHeadModel, GPT2Tokenizer, Trainer, TrainingArguments
+
+# Load dataset
+dataset = load_dataset("text", data_files={"train": "docs/*.txt"})
+
+# Load model and tokenizer
+model_name = "gpt2"
+tokenizer = GPT2Tokenizer.from_pretrained(model_name)
+model = GPT2LMHeadModel.from_pretrained(model_name)
+
+# Tokenize the dataset
+def tokenize_function(examples):
+    return tokenizer(examples["text"], padding="max_length", truncation=True, max_length=512)
+
+tokenized_datasets = dataset.map(tokenize_function, batched=True)
+
+# Set training arguments
+training_args = TrainingArguments(
+    output_dir="./results",
+    evaluation_strategy="epoch",
+    learning_rate=2e-5,
+    per_device_train_batch_size=2,
+    num_train_epochs=3,
+    weight_decay=0.01,
+)
+
+# Initialize Trainer
+trainer = Trainer(
+    model=model,
+    args=training_args,
+    train_dataset=tokenized_datasets["train"],
+)
+
+# Fine-tune the model
+trainer.train()
+
+def generate_document(prompt):
+    input_ids = tokenizer.encode(prompt, return_tensors="pt")
+    outputs = model.generate(input_ids, max_length=512, num_return_sequences=1)
+    document = tokenizer.decode(outputs[0], skip_special_tokens=True)
+    return document
+
+# Gradio
+import gradio as gr
+
+def rag_pipeline(query):
+    question = generate_questions(query)
+    retrieved_docs = retrieve_documents(question, index, documents_dataset["text"])
+    context = " ".join(retrieved_docs)
+    answer = generate_answer(question, context)
+    return question, context, answer
+
+def generate_support_document(prompt):
+    return generate_document(prompt)
+
+iface = gr.Interface(
+    fn=[rag_pipeline, generate_support_document],
+    inputs=["text", "text"],
+    outputs=["text", "text", "text", "text"],
+    title="RAG Pipeline for Product Support",
+    description="Ask a question about product support and get a detailed answer. Generate new support documents based on prompts."
+)
+
+iface.launch()