Update README.md

README.md

@@ -29,42 +29,57 @@ pass
 ## Inference
 ```python
 
-max_seq_length = 2048 # Choose any! We auto support RoPE Scaling internally!
-dtype = None # None for auto detection. Float16 for Tesla T4, V100, Bfloat16 for Ampere+
-load_in_4bit = True # Use 4bit quantization to reduce memory usage. Can be False.
 import torch
-from peft import AutoPeftModelForCausalLM
 from transformers import AutoTokenizer
-model_id="DisgustingOzil/MIstral_Pak_Law"
+from peft import AutoPeftModelForCausalLM
+import gradio as gr
+
+# Load the model and tokenizer
+model_id = "DisgustingOzil/MIstral_Pak_Law"
+dtype = torch.float16  # Adjust as necessary
+load_in_4bit = True
+
 model = AutoPeftModelForCausalLM.from_pretrained(
-        model_id, # YOUR MODEL YOU USED FOR TRAINING
-        load_in_4bit = load_in_4bit,
-        torch_dtype = torch.float16,
-        
-    )
+    model_id,
+    load_in_4bit=load_in_4bit,
+    torch_dtype=dtype,
+).to("cuda")
 tokenizer = AutoTokenizer.from_pretrained(model_id)
-# FastLanguageModel.for_inference(model) # Enable native 2x faster inference
-alpaca_prompt = """Given the context and a specific question, generate a comprehensive and detailed response that accurately addresses the query.
+
+def generate_response(context, question):
+    alpaca_prompt = f"""Given the context and a specific question, generate a comprehensive and detailed response that accurately addresses the query.
 
 ### Context:
-{}
+{context}
 
 ### Question:
-{}
+{question}
 
 ### Answer:
-{}"""
-inputs = tokenizer(
-[
-    alpaca_prompt.format(
-        """1 Abandoned Properties (Taking over and Management) Act 1975 ACT XX OF 1975 ABANDONED PROPERTIES (TAKING OVER AND MANAGEMENT) ACT, 1975 An Act to provide for the taking over and management of certain properties [Gazette of Pakistan, Extraordinary, Part I, 17th February 1975] The following Act of Parliament received the assent of the President on the 12th February 1975, and is hereby published for general information :- Whereas it is expedient to provide for the taking over and management of certain properties, and for matters connected therewith ; And whereas the Proclamation of Emergency referred to in Article 280 of the Constitution is still in force; It is hereby enacted as follows:- ACT XX OF 1975 ABANDONED PROPERTIES (TAKING OVER AND MANAGEMENT) ACT, 1975 An Act to provide for the taking over and management of certain properties [Gazette of Pakistan, Extraordinary, Part I, 17th February 1975] The following Act of Parliament received the assent of the President on the 12th February 1975, and is hereby published for general information :- 1. Short title, extent and commencement.-(1) This Act may be called the Abandoned Properties (Taking Over and Management) Act, 1975. (2) It extends to the whole of Pakistan. (3) It""", # instruction
-        """Could you please explain the process defined in the Abandoned Properties (Taking Over and Management) Act 1975 for determining if a property is considered abandoned?""", # input
-        "", # output - leave this blank for generation!
-    )
-], return_tensors = "pt").to("cuda")
-
-outputs = model.generate(**inputs, max_new_tokens = 64, use_cache = True)
-tokenizer.batch_decode(outputs)
+"""
+
+    inputs = tokenizer(
+        [alpaca_prompt], return_tensors="pt", padding=True, truncation=True, max_length=2048
+    ).to("cuda")
+
+    outputs = model.generate(**inputs, max_new_tokens=64, use_cache=True)
+    response = tokenizer.batch_decode(outputs, skip_special_tokens=True)
+    summary_start_index = response[0].find("### Answer:")
+    summary_text = response[0][summary_start_index:].replace("### Answer:", "").strip()
+    return summary_text
+    # return response[0]
+
+# Define the Gradio interface
+iface = gr.Interface(
+    fn=generate_response,
+    inputs=[gr.Textbox(label="Context"), gr.Textbox(label="Question")],
+    outputs=gr.Textbox(label="Answer"),
+    title="Abandoned Properties Act 1975 Query",
+    description="Enter the context and a specific question to generate a response based on the Abandoned Properties (Taking Over and Management) Act 1975.",
+)
+
+# Run the app
+iface.launch(debug=True)