zeeshan4801 commited on
Commit
d9576ee
Β·
verified Β·
1 Parent(s): 3945001

Update app.py

Browse files
Files changed (1) hide show
  1. app.py +77 -35
app.py CHANGED
@@ -1,43 +1,48 @@
1
  # ============================================
2
- # Civil Engineering RAG (ASTM) - Uploadable Version
3
  # ============================================
4
  import os
5
- import fitz # PyMuPDF
6
  import faiss
7
  import numpy as np
8
  import gradio as gr
 
9
  from typing import List
10
  from groq import Groq
11
  from sentence_transformers import SentenceTransformer
12
- import tempfile
13
 
14
  # --------------------------
15
- # Config
16
  # --------------------------
17
- GROQ_API_KEY = os.environ.get("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", "")
18
  if not GROQ_API_KEY:
19
- raise RuntimeError("❌ Missing GROQ_API_KEY. Add it in Hugging Face β†’ Settings β†’ Secrets.")
20
 
 
21
  client = Groq(api_key=GROQ_API_KEY)
22
- embedder = SentenceTransformer("sentence-transformers/all-MiniLM-L6-v2")
23
 
24
  INDEX, CORPUS = None, []
25
 
26
  # --------------------------
27
- # PDF text extraction
28
  # --------------------------
29
  def extract_text_from_pdf(file_path: str) -> str:
30
- text = []
31
- with fitz.open(file_path) as doc:
32
- for page in doc:
33
- text.append(page.get_text("text"))
34
- return "\n".join(text)
 
 
 
35
 
36
  # --------------------------
37
- # Chunking helper
38
  # --------------------------
39
  def chunk_text(text: str, chunk_size: int = 800, overlap: int = 120) -> List[str]:
40
- chunks, start, n = [], 0, len(text)
 
41
  while start < n:
42
  end = min(start + chunk_size, n)
43
  chunk = text[start:end].strip()
@@ -49,44 +54,72 @@ def chunk_text(text: str, chunk_size: int = 800, overlap: int = 120) -> List[str
49
  return chunks
50
 
51
  # --------------------------
52
- # Build FAISS index from uploaded PDFs
53
  # --------------------------
54
  def build_faiss_index(paths: List[str]):
55
  texts, vectors = [], []
56
  for p in paths:
57
- raw = extract_text_from_pdf(p)
58
- chunks = chunk_text(raw)
 
 
 
 
59
  embs = embedder.encode(chunks, convert_to_numpy=True, show_progress_bar=False)
60
  texts.extend(chunks)
61
  vectors.append(embs.astype("float32"))
 
 
 
 
62
  vectors = np.vstack(vectors).astype("float32")
63
  index = faiss.IndexFlatL2(vectors.shape[1])
64
  index.add(vectors)
65
  return index, texts
66
 
 
 
 
67
  def rebuild_index_from_upload(files):
68
  if not files:
69
  return "⚠️ Please upload at least one PDF."
 
70
  paths = []
71
  for f in files:
72
- temp_path = os.path.join(tempfile.gettempdir(), f.name)
73
- f.save(temp_path)
74
- paths.append(temp_path)
75
- global INDEX, CORPUS
76
- INDEX, CORPUS = build_faiss_index(paths)
77
- return f"βœ… Indexed {len(paths)} file(s). You can now ask questions!"
 
 
 
 
 
 
 
 
 
 
 
 
 
78
 
79
  # --------------------------
80
- # Retrieval + Groq LLM
81
  # --------------------------
82
  def retrieve_context(query: str, top_k: int = 4) -> str:
83
  if INDEX is None:
84
- return "⚠️ Please upload PDFs first."
85
  q_emb = embedder.encode([query], convert_to_numpy=True).astype("float32")
86
  distances, indices = INDEX.search(q_emb, top_k)
87
  selected = [CORPUS[i] for i in indices[0] if 0 <= i < len(CORPUS)]
88
  return "\n\n---\n\n".join(selected)
89
 
 
 
 
90
  SYSTEM_PROMPT = (
91
  "You are a helpful Civil Engineering assistant. "
92
  "Use ONLY the provided ASTM or uploaded document context to answer. "
@@ -96,7 +129,11 @@ SYSTEM_PROMPT = (
96
  def ask_groq(query: str, top_k: int = 4, model: str = "llama-3.3-70b-versatile") -> str:
97
  if INDEX is None:
98
  return "⚠️ Please upload PDFs first."
 
99
  context = retrieve_context(query, top_k)
 
 
 
100
  prompt = f"""{SYSTEM_PROMPT}
101
 
102
  Context:
@@ -105,15 +142,19 @@ Context:
105
  Question:
106
  {query}
107
  """
108
- completion = client.chat.completions.create(
109
- model=model,
110
- messages=[{"role": "user", "content": prompt}],
111
- temperature=0.2,
112
- )
113
- return completion.choices[0].message.content
 
 
 
 
114
 
115
  # --------------------------
116
- # Gradio UI
117
  # --------------------------
118
  def ui_ask(query: str, top_k: int):
119
  try:
@@ -123,13 +164,14 @@ def ui_ask(query: str, top_k: int):
123
 
124
  with gr.Blocks(title="Civil Engineering RAG (ASTM)") as demo:
125
  gr.Markdown("## πŸ—οΈ Civil Engineering RAG\nUpload ASTM or civil-engineering PDFs, build an index, and ask questions.")
 
126
  with gr.Row():
127
- uploader = gr.File(label="Upload PDFs", file_count="multiple", file_types=[".pdf"])
128
  status = gr.Textbox(label="Status", interactive=False)
129
  uploader.upload(rebuild_index_from_upload, uploader, status)
130
 
131
  gr.Markdown("---")
132
- inp = gr.Textbox(label="Your Question", placeholder="e.g., What is the standard curing time for concrete?")
133
  k = gr.Slider(1, 10, value=4, step=1, label="Top-K passages")
134
  out = gr.Textbox(label="Answer")
135
  btn = gr.Button("Ask")
 
1
  # ============================================
2
+ # Civil Engineering RAG (ASTM) - Hugging Face Version
3
  # ============================================
4
  import os
5
+ import fitz # PyMuPDF
6
  import faiss
7
  import numpy as np
8
  import gradio as gr
9
+ import tempfile
10
  from typing import List
11
  from groq import Groq
12
  from sentence_transformers import SentenceTransformer
 
13
 
14
  # --------------------------
15
+ # πŸ”‘ API Key
16
  # --------------------------
17
+ GROQ_API_KEY = os.environ.get("GROQ_API_KEY", "")
18
  if not GROQ_API_KEY:
19
+ raise RuntimeError("❌ Missing GROQ_API_KEY. Please add it in Hugging Face β†’ Settings β†’ Secrets.")
20
 
21
+ # Initialize Groq client and embedding model
22
  client = Groq(api_key=GROQ_API_KEY)
23
+ embedder = SentenceTransformer("all-MiniLM-L6-v2")
24
 
25
  INDEX, CORPUS = None, []
26
 
27
  # --------------------------
28
+ # πŸ“„ Safe PDF Text Extraction
29
  # --------------------------
30
  def extract_text_from_pdf(file_path: str) -> str:
31
+ try:
32
+ text = ""
33
+ with fitz.open(file_path) as doc:
34
+ for page in doc:
35
+ text += page.get_text("text")
36
+ return text
37
+ except Exception as e:
38
+ return f"Error extracting text from {file_path}: {e}"
39
 
40
  # --------------------------
41
+ # βœ‚οΈ Chunking Function
42
  # --------------------------
43
  def chunk_text(text: str, chunk_size: int = 800, overlap: int = 120) -> List[str]:
44
+ chunks = []
45
+ start, n = 0, len(text)
46
  while start < n:
47
  end = min(start + chunk_size, n)
48
  chunk = text[start:end].strip()
 
54
  return chunks
55
 
56
  # --------------------------
57
+ # πŸ”’ Build FAISS Index
58
  # --------------------------
59
  def build_faiss_index(paths: List[str]):
60
  texts, vectors = [], []
61
  for p in paths:
62
+ text = extract_text_from_pdf(p)
63
+ if text.startswith("Error extracting text"):
64
+ raise RuntimeError(text)
65
+ chunks = chunk_text(text)
66
+ if not chunks:
67
+ continue
68
  embs = embedder.encode(chunks, convert_to_numpy=True, show_progress_bar=False)
69
  texts.extend(chunks)
70
  vectors.append(embs.astype("float32"))
71
+
72
+ if not texts:
73
+ raise RuntimeError("❌ No valid text extracted from PDFs.")
74
+
75
  vectors = np.vstack(vectors).astype("float32")
76
  index = faiss.IndexFlatL2(vectors.shape[1])
77
  index.add(vectors)
78
  return index, texts
79
 
80
+ # --------------------------
81
+ # πŸ“€ Rebuild Index from Upload
82
+ # --------------------------
83
  def rebuild_index_from_upload(files):
84
  if not files:
85
  return "⚠️ Please upload at least one PDF."
86
+
87
  paths = []
88
  for f in files:
89
+ try:
90
+ # Gradio provides a temp file path automatically (f.name)
91
+ if hasattr(f, "name") and os.path.exists(f.name):
92
+ temp_path = f.name
93
+ else:
94
+ # fallback in rare case
95
+ with tempfile.NamedTemporaryFile(delete=False, suffix=".pdf") as tmp:
96
+ tmp.write(f.read())
97
+ temp_path = tmp.name
98
+ paths.append(temp_path)
99
+ except Exception as e:
100
+ return f"❌ Error while saving uploaded file: {e}"
101
+
102
+ try:
103
+ global INDEX, CORPUS
104
+ INDEX, CORPUS = build_faiss_index(paths)
105
+ return f"βœ… Successfully indexed {len(paths)} PDF(s). You can now ask questions!"
106
+ except Exception as e:
107
+ return f"❌ Error while building index: {e}"
108
 
109
  # --------------------------
110
+ # πŸ” Retrieve Context
111
  # --------------------------
112
  def retrieve_context(query: str, top_k: int = 4) -> str:
113
  if INDEX is None:
114
+ return "⚠️ Please upload and index PDFs first."
115
  q_emb = embedder.encode([query], convert_to_numpy=True).astype("float32")
116
  distances, indices = INDEX.search(q_emb, top_k)
117
  selected = [CORPUS[i] for i in indices[0] if 0 <= i < len(CORPUS)]
118
  return "\n\n---\n\n".join(selected)
119
 
120
+ # --------------------------
121
+ # 🧠 Query via Groq LLM
122
+ # --------------------------
123
  SYSTEM_PROMPT = (
124
  "You are a helpful Civil Engineering assistant. "
125
  "Use ONLY the provided ASTM or uploaded document context to answer. "
 
129
  def ask_groq(query: str, top_k: int = 4, model: str = "llama-3.3-70b-versatile") -> str:
130
  if INDEX is None:
131
  return "⚠️ Please upload PDFs first."
132
+
133
  context = retrieve_context(query, top_k)
134
+ if not context.strip():
135
+ return "⚠️ No relevant information found in the uploaded PDFs."
136
+
137
  prompt = f"""{SYSTEM_PROMPT}
138
 
139
  Context:
 
142
  Question:
143
  {query}
144
  """
145
+
146
+ try:
147
+ completion = client.chat.completions.create(
148
+ model=model,
149
+ messages=[{"role": "user", "content": prompt}],
150
+ temperature=0.2,
151
+ )
152
+ return completion.choices[0].message.content
153
+ except Exception as e:
154
+ return f"❌ LLM Error: {e}"
155
 
156
  # --------------------------
157
+ # 🎨 Gradio UI
158
  # --------------------------
159
  def ui_ask(query: str, top_k: int):
160
  try:
 
164
 
165
  with gr.Blocks(title="Civil Engineering RAG (ASTM)") as demo:
166
  gr.Markdown("## πŸ—οΈ Civil Engineering RAG\nUpload ASTM or civil-engineering PDFs, build an index, and ask questions.")
167
+
168
  with gr.Row():
169
+ uploader = gr.File(label="πŸ“„ Upload PDFs", file_count="multiple", file_types=[".pdf"])
170
  status = gr.Textbox(label="Status", interactive=False)
171
  uploader.upload(rebuild_index_from_upload, uploader, status)
172
 
173
  gr.Markdown("---")
174
+ inp = gr.Textbox(label="Your Question", placeholder="e.g., What is the curing time for concrete as per ASTM?")
175
  k = gr.Slider(1, 10, value=4, step=1, label="Top-K passages")
176
  out = gr.Textbox(label="Answer")
177
  btn = gr.Button("Ask")