Deploy to Hugging Face Space

app.py

@@ -0,0 +1,204 @@
+import json
+import gradio as gr
+from openai import OpenAI
+from collect import fetch_reviews
+from triage import route_review, triage_review
+from rag import init_store, add_bug, search_bugs, clear_store
+
+init_store()
+
+def collect_and_triage(review, api_key):
+    review_text = review["text"]
+    route_data = route_review(review_text, api_key)
+    route = route_data.get("route", "bug_report")
+
+    if route != "bug_report":
+        return None, route
+
+    similar = search_bugs(review_text, top_k=2)
+    structured = triage_review(review_text, api_key, similar_bugs=similar)
+    add_bug(structured)
+    return structured.get("title", ""), route
+
+def handle_collect(app_name, max_reviews, api_key_input):
+    api_key = (api_key_input or "").strip()
+    if not api_key:
+        yield "OpenAI API key is required for BYOK."
+        return
+
+    yield f"Fetching reviews for {app_name}..."
+    reviews = fetch_reviews(app_name, max_reviews=int(max_reviews))
+    yield f"Got {len(reviews)} reviews. Triaging..."
+
+    titles = []
+    skipped = {"feature_request": 0, "general_complaint": 0}
+    for review in reviews:
+        title, route = collect_and_triage(review, api_key)
+        if route == "bug_report" and title:
+            titles.append(title)
+        elif route in skipped:
+            skipped[route] += 1
+
+    output = "\n".join([f"{i+1}. {t}" for i, t in enumerate(titles)])
+    yield (
+        f"Done — {len(titles)} bugs saved. "
+        f"Skipped: {skipped['feature_request']} feature request(s), "
+        f"{skipped['general_complaint']} general complaint(s).\n\n{output}"
+    )
+
+def build_triage_output(review_text,api_key):
+    route_data = route_review(review_text, api_key)
+    route = route_data.get("route", "bug_report")
+
+    if route != "bug_report":
+        confidence = route_data.get("confidence", 0)
+        output = (
+            f"Route: {route} (confidence: {confidence})\n\n"
+            "This input is not a bug report, so it was not added to bug store."
+        )
+        return output, None
+
+    similar = search_bugs(review_text, top_k=2)
+    structured = triage_review(review_text, api_key, similar_bugs=similar)
+    add_bug(structured)
+
+    output  = f"Severity: {structured.get('severity','')} | Component: {structured.get('component','')}\n\n"
+    output += f"Bug report:\n```json\n{json.dumps(structured, indent=2)}\n```\n\n"
+    output += "Similar bugs:\n"
+    output += "\n".join([f"- {b.get('title','')} [{b.get('severity','')}]" for b in similar])
+    return output, structured
+
+def handle_triage(review_text, api_key_input):
+    api_key = (api_key_input or "").strip()
+    if not api_key:
+        yield "OpenAI API key is required for BYOK."
+        return
+
+    yield "Triaging review..."
+    output, structured = build_triage_output(review_text, api_key)
+    yield output
+
+    if not structured:
+        return
+
+    client = OpenAI(api_key=api_key)
+    stream = client.chat.completions.create(
+        model="gpt-4o",
+        max_tokens=200,
+        stream=True,
+        messages=[{
+            "role": "user",
+            "content": f"Write a 3 sentence QA incident summary:\n{json.dumps(structured, indent=2)}"
+        }]
+    )
+
+    output += "\nAI Summary:\n\n"
+    for chunk in stream:
+        output += chunk.choices[0].delta.content or ""
+        yield output
+def build_search_output(results, query):
+    output = f"{len(results)} results for: {query}\n\n---\n"
+    output += "\n\n---\n".join([
+        f"{r.get('title','')}\n"
+        f"{r.get('severity','')} | {r.get('component','')} | {r.get('platform','')}\n"
+        f"{r.get('description','')}"
+        for r in results
+    ])
+    return output
+
+
+def get_ai_summary(results, query, api_key):
+    client  = OpenAI(api_key=api_key)
+    context = "\n".join([
+        f"- {r.get('title','')}: {r.get('description','')}"
+        for r in results
+    ])
+    resp = client.chat.completions.create(
+        model="gpt-4o",
+        max_tokens=150,
+        messages=[{
+            "role": "user",
+            "content": f"Query: {query}\nBugs:\n{context}\nSummarise in 2 sentences:"
+        }]
+    )
+    return resp.choices[0].message.content
+
+
+def handle_search(query, api_key_input):
+    api_key = (api_key_input or "").strip()
+    if not api_key:
+        return "OpenAI API key is required for BYOK."
+
+    results = search_bugs(query, top_k=5)
+    output  = build_search_output(results, query)
+    output += f"\n\nAI Summary:\n{get_ai_summary(results, query, api_key)}"
+    return output
+
+
+def handle_clear_bugs():
+    removed = clear_store()
+    init_store()
+    return f"Cleared {removed} bug(s)."
+
+with gr.Blocks(title="QA Bug Triage") as demo:
+    gr.Markdown("# QA Bug Triage Pipeline\nA modern RAG workflow for turning messy app reviews into structured, searchable QA bug intelligence..")
+
+    api_key_box = gr.Textbox(
+        label="OpenAI API key (BYOK)",
+        placeholder="sk-...",
+        type="password",
+        value=""
+    )
+
+    with gr.Tabs():
+
+        with gr.TabItem("1. Collect"):
+            app_name_box = gr.Textbox(label="App name", value="notion")
+            max_box      = gr.Slider(5, 20, value=10, step=5, label="Max reviews")
+            collect_btn  = gr.Button("Fetch and triage", variant="primary")
+            collect_out  = gr.Markdown()
+            collect_btn.click(
+                handle_collect,
+                [app_name_box, max_box, api_key_box],
+                collect_out
+            )
+
+        with gr.TabItem("2. Triage"):
+            review_box = gr.Textbox(
+                label="Paste a review",
+                lines=4,
+                placeholder="App crashes every time I try to upload a photo..."
+            )
+            triage_btn = gr.Button("Triage", variant="primary")
+            triage_out = gr.Markdown()
+            triage_btn.click(
+                handle_triage,
+                [review_box, api_key_box],
+                triage_out
+            )
+
+        with gr.TabItem("3. Search"):
+            search_box = gr.Textbox(
+                label="Search query",
+                placeholder="login crash android"
+            )
+            search_btn = gr.Button("Search", variant="primary")
+            search_out = gr.Markdown()
+            search_btn.click(
+                handle_search,
+                [search_box, api_key_box],
+                search_out
+            )
+
+        with gr.TabItem("4. Clear bugs"):
+            clear_btn = gr.Button("Clear stored bugs", variant="stop")
+            clear_out = gr.Markdown()
+            clear_btn.click(
+                handle_clear_bugs,
+                outputs=clear_out
+            )
+
+        
+if __name__ == "__main__":
+    demo.launch()      
+    

collect.py

@@ -0,0 +1,64 @@
+import time
+from datetime import datetime
+
+def make_review(text: str, rating: int, source: str) -> dict:
+    """Create a review dictionary with the given text, rating, and source."""
+    return {
+        'text': text.strip(),
+        'rating': rating,
+        'source': source,
+        'date': datetime.today().strftime('%Y-%m-%d %H:%M:%S')
+    }
+
+APP_PACKAGES = {
+    "google maps": "com.google.android.apps.maps",
+    "facebook":    "com.facebook.katana",
+    "instagram":   "com.instagram.android",
+    "whatsapp":    "com.whatsapp",
+    "snapchat":    "com.snapchat.android",
+    "notion":      "com.notion.android",
+    "spotify":     "com.spotify.music",
+    "netflix":     "com.netflix.mediaclient",
+}
+
+def get_package_id(app_name: str) -> str:
+    """Return the package ID for the given app name, or a default format if not found."""
+    name_lower = app_name.lower().strip()
+    default_package = f"com.{name_lower.replace(' ', '')}"
+    return APP_PACKAGES.get(name_lower, default_package)
+
+def fetch_reviews(app_name: str, source: str = "Google Play", max_reviews: int = 21) -> list:
+    from google_play_scraper import reviews, Sort
+
+    package_id = get_package_id(app_name)
+    print(f"Fetching reviews for '{app_name}' (package: {package_id}) from {source}...")
+
+    try:
+        results, _ = reviews(
+            package_id,
+            lang='en',
+            country='us',
+            sort=Sort.NEWEST,
+            count=max_reviews
+        )
+
+        cleaned = [
+            make_review(r['content'], r.get('score', 1), "Google Play")
+            for r in filter(lambda item: item.get('content', '').strip(), results)
+        ]
+
+        unique = list({review['text'][:80]: review for review in cleaned}.values())
+
+        messages = [
+            f"No reviews found for '{app_name}' on {source}.",
+            f"Fetched {len(unique)} unique reviews for '{app_name}' from {source}.",
+        ]
+        print(messages[bool(unique)])
+
+        time.sleep(5)
+
+        return unique[:max_reviews]
+    
+    except Exception as e:
+        print(f"Error fetching reviews for '{app_name}' from {source}: {e}")
+        return []

eval/eval.py

@@ -0,0 +1,100 @@
+
+import os
+import json
+import sys
+import argparse
+from dotenv import load_dotenv
+
+load_dotenv()
+
+sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
+
+from rag import search_bugs, init_store
+from openai import OpenAI
+from ragas import evaluate, EvaluationDataset, SingleTurnSample
+from ragas.metrics import Faithfulness, AnswerRelevancy, ContextPrecision
+from ragas.llms import LlamaIndexLLMWrapper
+from llama_index.llms.openai import OpenAI as LlamaOpenAI
+
+DATASET = os.path.join(os.path.dirname(os.path.abspath(__file__)), "eval_dataset.json")
+RESULTS = os.path.join(os.path.dirname(os.path.abspath(__file__)), "results.json")
+
+
+def get_answer(query, contexts, api_key):
+    client  = OpenAI(api_key=api_key)
+    context = "\n".join(contexts)
+    resp    = client.chat.completions.create(
+        model="gpt-4o",
+        max_tokens=150,
+        messages=[{
+            "role": "user",
+            "content": f"Query: {query}\n\nContext:\n{context}\n\nAnswer in 2 sentences:"
+        }]
+    )
+    return resp.choices[0].message.content.strip()
+
+
+def build_sample(item, api_key):
+    query    = item["query"]
+    bugs     = search_bugs(query, top_k=5)
+    contexts = [f"{b['title']}: {b['description']}" for b in bugs]
+    answer   = get_answer(query, contexts, api_key)
+    print(f"query : {query}")
+    print(f"answer: {answer}\n")
+    return SingleTurnSample(
+        user_input=query,
+        response=answer,
+        retrieved_contexts=contexts,
+        reference=item["reference_answer"]
+    )
+
+
+def run_eval(api_key):
+    dataset = json.load(open(DATASET))
+    print(f"Loaded {len(dataset)} queries\n")
+
+    init_store()
+
+    llm           = LlamaOpenAI(model="gpt-4o", api_key=api_key)
+    evaluator_llm = LlamaIndexLLMWrapper(llm)
+
+    samples = [build_sample(item, api_key) for item in dataset]
+
+    results = evaluate(
+        EvaluationDataset(samples=samples),
+        metrics=[
+            Faithfulness(llm=evaluator_llm),
+            AnswerRelevancy(llm=evaluator_llm),
+            ContextPrecision(llm=evaluator_llm),
+        ]
+    )
+
+    df = results.to_pandas()
+
+    print("=" * 40)
+    print("RAGAS RESULTS")
+    print("=" * 40)
+    print(f"Faithfulness      : {df['faithfulness'].mean():.3f}")
+    print(f"Answer Relevancy  : {df['answer_relevancy'].mean():.3f}")
+    print(f"Context Precision : {df['context_precision'].mean():.3f}")
+    print("=" * 40)
+
+    json.dump({
+        "faithfulness":      round(float(df["faithfulness"].mean()), 3),
+        "answer_relevancy":  round(float(df["answer_relevancy"].mean()), 3),
+        "context_precision": round(float(df["context_precision"].mean()), 3),
+    }, open(RESULTS, "w"), indent=2)
+
+    print("Saved to eval/results.json")
+
+
+if __name__ == "__main__":
+    parser = argparse.ArgumentParser()
+    parser.add_argument("--api-key", default=os.getenv("OPENAI_API_KEY"))
+    args = parser.parse_args()
+    
+    if not args.api_key:
+        print("Error: OPENAI_API_KEY not found. Set it in .env or pass --api-key")
+        sys.exit(1)
+    
+    run_eval(args.api_key)

eval/eval_dataset.json

@@ -0,0 +1,42 @@
+[
+  {
+    "query": "app crashes on login",
+    "reference_answer": "The app crashes when users try to log in, preventing access."
+  },
+  {
+    "query": "slow performance on older devices",
+    "reference_answer": "The app experiences slow performance and lag on older devices, affecting usability."
+  },
+  {
+    "query": "UI glitches on dark mode",
+    "reference_answer": "Users report UI glitches and misaligned elements when using the app in dark mode."
+  },
+  {
+    "query": "frequent crashes during video playback",
+    "reference_answer": "The app frequently crashes when users attempt to play videos, disrupting the experience."
+  },
+  {
+    "query": "inability to save settings",
+    "reference_answer": "Users are unable to save their settings, leading to frustration and repeated configuration."
+  },
+  {
+    "query": "notifications not working",
+    "reference_answer": "Users report that they are not receiving notifications from the app, missing important updates."
+  },
+  {
+    "query": "app freezes on startup",
+    "reference_answer": "The app freezes and becomes unresponsive when users try to start it, preventing access."
+  },
+  {
+    "query": "battery drain issue",
+    "reference_answer": "Users experience significant battery drain when using the app, reducing device longevity."
+  },
+  {
+    "query": "incompatibility with latest OS version",
+    "reference_answer": "The app is incompatible with the latest OS version, causing crashes and functionality issues."
+  },
+  {
+    "query": "problems with in-app purchases",
+    "reference_answer": "Users encounter issues with in-app purchases, such as failed transactions and missing items."
+  }
+]

eval/results.json

@@ -0,0 +1,5 @@
+{
+  "faithfulness": 0.292,
+  "answer_relevancy": 0.868,
+  "context_precision": 0.020
+}

rag.py

@@ -0,0 +1,85 @@
+import uuid
+import chromadb
+from chromadb.utils.embedding_functions import DefaultEmbeddingFunction
+from rank_bm25 import BM25Okapi
+
+_collection = None
+_all_bugs   = []
+
+def init_store():
+    global _collection, _all_bugs
+
+    client      = chromadb.PersistentClient(path="./chroma_db")
+    _collection = client.get_or_create_collection(
+        name="bug_reports",
+        embedding_function=DefaultEmbeddingFunction()
+    )
+
+    data = _collection.get(include=["metadatas"])
+    _all_bugs = data["metadatas"] or []
+
+    print(f"[rag] Ready - {len(_all_bugs)} bugs loaded")
+
+def add_bug(bug: dict):
+    init_store()
+
+    bug_id = bug.get("bug_id") or f"BUG-{uuid.uuid4().hex[:6].upper()}"
+    text = f"{bug.get('title', '')}. {bug.get('description', '')}"
+
+    metadata = {
+        "bug_id": bug_id,
+        "title": str(bug.get("title", "")),
+        "severity": str(bug.get("severity", "unknown")),
+        "component": str(bug.get("component", "unknown")),
+        "platform": str(bug.get("platform", "unknown")),
+        "frequency": str(bug.get("frequency_estimate", "unknown")),
+        "description": str(bug.get("description", ""))[:400],
+    }
+
+    _collection.upsert(ids=[bug_id], documents=[text], metadatas=[metadata])
+    _all_bugs.append(metadata)
+
+def search_bugs(query: str, top_k: int = 5):
+    init_store()
+
+    results = _collection.query(query_texts=[query], n_results=top_k)
+    sem_bugs = results.get("metadatas", [[]])[0]
+
+    if not _all_bugs:
+        return sem_bugs
+
+    corpus = [f"{bug.get('title', '')}. {bug.get('description', '')}" for bug in _all_bugs]
+    tokenized_corpus = [doc.split() for doc in corpus]
+    bm25 = BM25Okapi(tokenized_corpus)
+    bm25_scores = bm25.get_scores(query.split())
+    bm25_indices = sorted(range(len(_all_bugs)), key=lambda i: bm25_scores[i], reverse=True)[:top_k]
+    bm25_bugs = [_all_bugs[i] for i in bm25_indices]
+
+    sem_rank = {bug["bug_id"]: idx + 1 for idx, bug in enumerate(sem_bugs)}
+    bm25_rank = {bug["bug_id"]: idx + 1 for idx, bug in enumerate(bm25_bugs)}
+    bug_by_id = {bug["bug_id"]: bug for bug in sem_bugs + bm25_bugs}
+    candidate_ids = set(sem_rank) | set(bm25_rank)
+
+    rrf_k = 60
+    default_rank = 10**6
+    ranked_ids = sorted(
+        candidate_ids,
+        key=lambda bug_id: (1 / (rrf_k + sem_rank.get(bug_id, default_rank))) + (1 / (rrf_k + bm25_rank.get(bug_id, default_rank))),
+        reverse=True,
+    )
+
+    return [bug_by_id[bug_id] for bug_id in ranked_ids[:top_k]]
+
+
+def clear_store():
+    global _all_bugs
+
+    init_store()
+    data = _collection.get()
+    ids = data.get("ids", []) or []
+
+    if ids:
+        _collection.delete(ids=ids)
+
+    _all_bugs = []
+    return len(ids)

requirements.txt

@@ -0,0 +1,15 @@
+openai
+gradio
+numpy
+llama-index
+llama-index-vector-stores-chroma
+llama-index-retrievers-bm25
+llama-index-llms-openai
+llama-index-postprocessor-cohere-rerank
+chromadb
+ragas
+beautifulsoup4
+requests
+cohere
+google-play-scraper
+rank-bm25

triage.py

@@ -0,0 +1,135 @@
+import json
+import uuid
+from openai import OpenAI
+
+# ── system prompt ─────────────────────────────────────────────────────────────
+
+SYSTEM_PROMPT = """You are a bug triage assistant who reads customer reviews and extracts structured bug reports.
+
+Your output must always be valid with these exact fields:
+
+{
+"title": "A concise title summarizing the bug",
+"severity": "One of 'critical', 'major', 'minor', or 'trivial'",
+"component": "The app component affected by the bug (e.g., 'login', 'payment', 'UI')",
+"platform": "The platform where the bug occurs (e.g., 'Android', 'iOS')",
+"frequency_estimate": "An estimate of how often the bug occurs (e.g., 'always', 'often', 'sometimes', 'rarely')",
+"symptom": "A detailed description of the symptoms experienced by users",
+"user_impact": "A description of how the bug impacts users (e.g., 'prevents login', 'causes crashes', 'leads to data loss')",
+"recommendation_label": "A recommended action for the development team (e.g., 'investigate immediately', 'schedule for next release', 'monitor user feedback')",
+}
+
+severity guide:
+- critical: The bug causes complete failure of a core feature, data loss, or security vulnerabilities. It severely impacts user experience and requires immediate attention.
+- high: The bug significantly impairs functionality or causes frequent crashes, but does not result in complete failure. It should be addressed as soon as possible.
+- medium: The bug causes noticeable issues or inconveniences but has workarounds available. It should be fixed in a timely manner.
+- low: The bug has minimal impact on functionality or user experience, such as minor UI glitches or typos. It can be scheduled for future releases.
+
+Rules:
+- Return only the JSON object with the specified fields. Do not include any explanations, apologies, or additional text.
+- If review is vague, make your best guess from context.
+- Never leave a field empty — use Unknown or Other as fallback."""
+
+
+def _strip_code_fences(raw: str) -> str:
+    return raw.strip().removeprefix("```json").removeprefix("```").removesuffix("```").strip()
+
+
+def _loads_or_default(raw: str, default: dict, error_prefix: str) -> dict:
+    try:
+        return json.loads(raw)
+    except json.JSONDecodeError:
+        print(f"{error_prefix}: {raw}")
+        return default
+
+
+# ── function 1: route_review ──────────────────────────────────────────────────
+
+def route_review(review_text: str, api_key: str) -> dict:
+    client = OpenAI(api_key=api_key)
+
+    response = client.chat.completions.create(
+        model="gpt-4o",
+        max_tokens=50,
+        messages=[
+                    {
+                        "role": "user",
+                        "content": f"""Classify this review into exactly one category:
+                                - bug_report        : describes a crash, freeze, error, or malfunction
+                                - feature_request   : asks for something new or improved
+                                - general_complaint : vague dissatisfaction, no specific technical issue
+
+                        Reply with JSON only, no explanation:
+                        {{"route": "...", "confidence": 0.0}}
+
+                        Review: {review_text}"""
+                    }
+                ]
+            )
+
+    raw = response.choices[0].message.content.strip()
+    default = {"route": "bug_report", "confidence": 0.8}
+    return _loads_or_default(raw, default, "Failed to parse routing response")
+
+
+# ── function 2: triage_review ─────────────────────────────────────────────────
+
+def triage_review(review_text: str, api_key: str, similar_bugs: list = None) -> dict:
+    client = OpenAI(api_key=api_key)
+
+    examples = [
+        {
+            "title": bug.get("title", ""),
+            "severity": bug.get("severity", ""),
+            "component": bug.get("component", ""),
+            "platform": bug.get("platform", ""),
+            "frequency_estimate": bug.get("frequency", ""),
+        }
+        for bug in (similar_bugs or [])[:2]
+    ]
+    few_shot_text = "".join([f"```json\n{json.dumps(example, indent=2)}\n```\n" for example in examples])
+
+    user_message = f"""Triage this customer review and return the JSON bug report.
+Here are some examples of previously triaged bugs:
+
+{few_shot_text}
+Review:
+\"\"\"{review_text}\"\"\"
+
+JSON output:"""
+
+    fallback = {
+        "title": "Needs manual review",
+        "severity": "medium",
+        "component": "Other",
+        "platform": "Unknown",
+        "frequency_estimate": "unknown",
+        "symptom": review_text[:150],
+        "user_impact": "Unknown — review manually",
+        "recommended_label": "P3 - minor",
+    }
+
+    try:
+        response = client.chat.completions.create(
+            model="gpt-4o",
+            max_tokens=500,
+            messages=[
+                {
+                    "role": "system",
+                    "content": SYSTEM_PROMPT,
+                },
+                {
+                    "role": "user",
+                    "content": user_message,
+                },
+            ],
+        )
+        raw = _strip_code_fences(response.choices[0].message.content)
+        structured = _loads_or_default(raw, fallback, "[triage] parse error")
+    except Exception as e:
+        print(f"[triage] error: {e}")
+        structured = fallback
+
+    structured["bug_id"] = f"BUG-{uuid.uuid4().hex[:6].upper()}"
+    structured["description"] = structured.get("symptom", "")
+    return structured