removed few insights

main.py

@@ -8,6 +8,7 @@ from auth import hash_password, verify_password, create_token
 from deps import get_db, get_current_user
 from config import API_BASE_URL
 from models import User, Log, CtrReport, Restaurant, Review
+from utils import extract_points
 
 
 app = FastAPI()
@@ -279,4 +280,80 @@ def get_few_reviews(
         "total_reviews": len(reviews),
         "selected_count": len(selected_reviews),
         "selected_reviews": selected_reviews
+    }
+
+
+
+@app.get("/few-insights")
+def get_all_insights(
+    eps: float = 0.41,
+    min_samples: int = 2,
+    db: Session = Depends(get_db)
+):
+
+    reports = (
+        db.query(CtrReport)
+        .order_by(CtrReport.id.desc())
+        .all()
+    )
+
+    if not reports:
+        return {
+            "count": 0,
+            "insights": [],
+            "selected_insights": []
+        }
+
+    # flatten all insight points
+    all_points = []
+
+    for report in reports:
+
+        if not report.insights:
+            continue
+
+        extracted = extract_points(report.insights)
+
+        all_points.extend(extracted)
+
+    # remove duplicates while preserving order
+    unique_points = list(dict.fromkeys(all_points))
+
+    selected_insights = []
+
+    if unique_points:
+
+        try:
+            response = requests.post(
+                f"{API_BASE_URL}/get_representatives",
+                json={
+                    "texts": unique_points,
+                    "eps": eps,
+                    "min_samples": min_samples
+                },
+                timeout=60
+            )
+
+        except requests.RequestException as exc:
+            raise HTTPException(
+                502,
+                f"Representative insight service failed: {exc}"
+            )
+
+        if response.status_code != 200:
+            raise HTTPException(
+                502,
+                "Representative insight service error"
+            )
+
+        payload = response.json()
+
+        selected_insights = payload.get(
+            "representatives",
+            []
+        )
+
+    return {
+        "count": len(selected_insights),
+        "selected_insights": selected_insights
     }

utils.py

@@ -0,0 +1,35 @@
+import re
+
+def extract_points(markdown_text: str) -> list[str]:
+    """
+    Extracts numbered/bulleted markdown points as clean strings.
+    
+    Removes:
+    - markdown bold/italic/code formatting
+    - numbering/bullets
+    - extra spaces/newlines
+    """
+
+    lines = markdown_text.splitlines()
+    points = []
+
+    for line in lines:
+        line = line.strip()
+
+        if not line:
+            continue
+
+        # Match numbered or bulleted list items
+        if re.match(r"^(\d+\.\s+|[-*]\s+)", line):
+
+            # Remove numbering/bullets
+            line = re.sub(r"^(\d+\.\s+|[-*]\s+)", "", line)
+
+            # Remove markdown formatting
+            line = re.sub(r"\*\*(.*?)\*\*", r"\1", line)   # bold
+            line = re.sub(r"\*(.*?)\*", r"\1", line)       # italic
+            line = re.sub(r"`(.*?)`", r"\1", line)         # inline code
+
+            points.append(line.strip())
+
+    return points