feat: ResearchPilot v2 — Next.js frontend, 358k embeddings pipeline, LaTeX math rendering, dual-GPU Kaggle scaling

.gitignore

@@ -1,14 +1,46 @@
+# ── Python ──
 venv/
-.env
 __pycache__/
 *.pyc
+*.pyo
+*.egg-info/
+dist/
+build/
+
+# ── Environment & Secrets ──
+.env
+.env.*
+!.env.example
+
+# ── Data (too large for GitHub) ──
 data/raw/
 data/processed/
+data/chunks/
 data/embeddings/
+data/qdrant_db/
+existing_chunks.zip
+
+# ── Logs ──
+logs/
 *.log
+
+# ── OS ──
 .DS_Store
-data/chunks/
-data/processed/
-data/raw/
-data/embeddings/
-data/qdrant_db/
+Thumbs.db
+desktop.ini
+
+# ── IDE ──
+.vscode/
+.idea/
+
+# ── Frontend (handled by frontend-next/.gitignore) ──
+frontend-next/node_modules/
+frontend-next/.next/
+
+# ── Old frontend ──
+frontend/
+
+# ── Misc ──
+*.zip
+output.txt
+Kaggle_Guide.md

config/settings.py

@@ -41,12 +41,12 @@ for directory in [RAW_DIR, PROCESSED_DIR, CHUNKS_DIR, EMBEDDINGS_DIR, LOGS_DIR]:
 # ------------------------------------------
 # DATA INGESTION SETTINGS
 # ------------------------------------------
-ARXIV_CATEGORIES = ["cs.LG", "cs.AI"]    # Machine Learning + AI
-MAX_PAPERS_PER_FETCH = 100               # Papers per API call
-TOTAL_PAPERS_TARGET  = 100               # Total papers to collect
-ARXIV_API_DELAY_SECONDS = 3.0            # ArXiv rate limit: be respectful
-PDF_DOWNLOAD_TIMEOUT = 30                # Seconds before giving up on a PDF
-MAX_DOWNLOAD_RETRIES = 3                 # Retry failed downloads N times
+ARXIV_CATEGORIES = ["cs.LG", "cs.AI", "stat.ML"]    # Machine Learning + AI
+MAX_PAPERS_PER_FETCH = 100                          # Papers per API call
+TOTAL_PAPERS_TARGET  = 700                          # Total papers to collect
+ARXIV_API_DELAY_SECONDS = 3.0                       # ArXiv rate limit: be respectful
+PDF_DOWNLOAD_TIMEOUT = 30                           # Seconds before giving up on a PDF
+MAX_DOWNLOAD_RETRIES = 3                            # Retry failed downloads N times
 
 # ------------------------------------------
 # DOCUMENT PROCESSING SETTINGS
@@ -57,7 +57,7 @@ MAX_TEXT_LENGTH = 500_000   # Skip papers larger than 100k chars (corrupted)
 # ------------------------------------------
 # CHUNKING SETTINGS
 # ------------------------------------------
-CHUNK_SIZE = 512        # Charaters per chunk
+CHUNK_SIZE = 512        # Characters per chunk
 CHUNK_OVERLAP = 50      # Overlap between consecutive chunks
 MIN_CHUNK_SIZE = 100    # Discard chunks smaller than this
 
@@ -82,7 +82,7 @@ TOP_K_RERANK = 5                                    # Keep top 5 after reranking
 GROQ_API_KEY = os.getenv('GROQ_API_KEY')            # Loaded from .env
 LLM_MODEL_NAME = 'llama-3.3-70b-versatile'          # Groq model ID
 LLM_TEMPERATURE = 0.1                               # Low = More factual/consistent 
-LLM_MAX_TOKENS = 1024                               # Max response tokens
+LLM_MAX_TOKENS = 2048                               # Max response tokens
 
 # ------------------------------------------
 # API SETTINGS

frontend-next/.gitignore

@@ -0,0 +1,41 @@
+# See https://help.github.com/articles/ignoring-files/ for more about ignoring files.
+
+# dependencies
+/node_modules
+/.pnp
+.pnp.*
+.yarn/*
+!.yarn/patches
+!.yarn/plugins
+!.yarn/releases
+!.yarn/versions
+
+# testing
+/coverage
+
+# next.js
+/.next/
+/out/
+
+# production
+/build
+
+# misc
+.DS_Store
+*.pem
+
+# debug
+npm-debug.log*
+yarn-debug.log*
+yarn-error.log*
+.pnpm-debug.log*
+
+# env files (can opt-in for committing if needed)
+.env*
+
+# vercel
+.vercel
+
+# typescript
+*.tsbuildinfo
+next-env.d.ts

frontend-next/AGENTS.md

@@ -0,0 +1,5 @@
+<!-- BEGIN:nextjs-agent-rules -->
+# This is NOT the Next.js you know
+
+This version has breaking changes — APIs, conventions, and file structure may all differ from your training data. Read the relevant guide in `node_modules/next/dist/docs/` before writing any code. Heed deprecation notices.
+<!-- END:nextjs-agent-rules -->

frontend-next/CLAUDE.md

@@ -0,0 +1 @@
+@AGENTS.md

frontend-next/README.md

@@ -0,0 +1,36 @@
+This is a [Next.js](https://nextjs.org) project bootstrapped with [`create-next-app`](https://nextjs.org/docs/app/api-reference/cli/create-next-app).
+
+## Getting Started
+
+First, run the development server:
+
+```bash
+npm run dev
+# or
+yarn dev
+# or
+pnpm dev
+# or
+bun dev
+```
+
+Open [http://localhost:3000](http://localhost:3000) with your browser to see the result.
+
+You can start editing the page by modifying `app/page.tsx`. The page auto-updates as you edit the file.
+
+This project uses [`next/font`](https://nextjs.org/docs/app/building-your-application/optimizing/fonts) to automatically optimize and load [Geist](https://vercel.com/font), a new font family for Vercel.
+
+## Learn More
+
+To learn more about Next.js, take a look at the following resources:
+
+- [Next.js Documentation](https://nextjs.org/docs) - learn about Next.js features and API.
+- [Learn Next.js](https://nextjs.org/learn) - an interactive Next.js tutorial.
+
+You can check out [the Next.js GitHub repository](https://github.com/vercel/next.js) - your feedback and contributions are welcome!
+
+## Deploy on Vercel
+
+The easiest way to deploy your Next.js app is to use the [Vercel Platform](https://vercel.com/new?utm_medium=default-template&filter=next.js&utm_source=create-next-app&utm_campaign=create-next-app-readme) from the creators of Next.js.
+
+Check out our [Next.js deployment documentation](https://nextjs.org/docs/app/building-your-application/deploying) for more details.

frontend-next/app/globals.css

@@ -0,0 +1,928 @@
+@import url("https://fonts.googleapis.com/css2?family=Inter:wght@300;400;500;600;700&family=Outfit:wght@300;400;500;600;700;800&family=JetBrains+Mono:wght@400;500;700&family=Dancing+Script:wght@600;700&display=swap");
+
+:root {
+    --bg: #030508;
+    --panel-bg: rgba(10, 14, 23, 0.6);
+    --border: rgba(255, 255, 255, 0.08);
+    --text-main: #f8fafc;
+    --text-muted: #94a3b8;
+    --accent: #00f0ff;
+    --accent-2: #8a2be2;
+    --success: #10b981;
+    --danger: #ef4444;
+}
+
+* {
+    box-sizing: border-box;
+    margin: 0;
+    padding: 0;
+}
+
+body {
+    background-color: var(--bg);
+    color: var(--text-main);
+    font-family: "Outfit", system-ui, sans-serif;
+    overflow-x: hidden;
+    -webkit-font-smoothing: antialiased;
+}
+
+button {
+    all: unset;
+    cursor: pointer;
+    display: inline-flex;
+    box-sizing: border-box;
+}
+input,
+textarea,
+select {
+    all: unset;
+    box-sizing: border-box;
+}
+a {
+    text-decoration: none;
+    color: inherit;
+}
+
+/* ── The Ambient Grid ── */
+.luminous-grid {
+    position: fixed;
+    inset: 0;
+    background:
+        linear-gradient(rgba(0, 240, 255, 0.04) 1px, transparent 1px),
+        linear-gradient(90deg, rgba(0, 240, 255, 0.04) 1px, transparent 1px);
+    background-size: 50px 50px;
+    mask-image: radial-gradient(circle at center, black 10%, transparent 80%);
+    -webkit-mask-image: radial-gradient(
+        circle at center,
+        black 10%,
+        transparent 80%
+    );
+    z-index: -1;
+    pointer-events: none;
+    animation: bg-pan 40s linear infinite;
+}
+
+@keyframes bg-pan {
+    from {
+        background-position: 0 0;
+    }
+    to {
+        background-position: 50px 50px;
+    }
+}
+
+/* ── Glow Orbs ── */
+.orb {
+    position: fixed;
+    border-radius: 50%;
+    filter: blur(150px);
+    z-index: -2;
+    pointer-events: none;
+    opacity: 0.3;
+}
+.orb-cyan {
+    width: 50vw;
+    height: 50vw;
+    background: var(--accent);
+    top: -20%;
+    right: -10%;
+    animation: pulse-orb 12s alternate infinite;
+}
+.orb-purple {
+    width: 40vw;
+    height: 40vw;
+    background: var(--accent-2);
+    bottom: -10%;
+    left: -20%;
+    animation: pulse-orb 15s alternate-reverse infinite;
+}
+@keyframes pulse-orb {
+    0% {
+        transform: scale(0.9);
+        opacity: 0.2;
+    }
+    100% {
+        transform: scale(1.1);
+        opacity: 0.5;
+    }
+}
+
+/* ── Glass Panels ── */
+.cyber-panel {
+    background: var(--panel-bg);
+    backdrop-filter: blur(24px);
+    -webkit-backdrop-filter: blur(24px);
+    border: 1px solid var(--border);
+    border-radius: 24px;
+    box-shadow: 0 10px 40px -10px rgba(0, 0, 0, 0.8);
+    transition: all 0.3s ease;
+}
+
+/* ── Typography ── */
+.hero-title {
+    font-size: min(4.5rem, 10vw);
+    font-weight: 800;
+    line-height: 1.1;
+    letter-spacing: -0.03em;
+    margin-bottom: 24px;
+    text-align: center;
+}
+.hero-subtitle {
+    font-size: 1.1rem;
+    color: var(--text-muted);
+    text-align: center;
+    max-width: 600px;
+    margin: 0 auto;
+    font-weight: 300;
+    line-height: 1.6;
+}
+.text-gradient-2 {
+    background: linear-gradient(
+        135deg,
+        var(--accent) 20%,
+        var(--accent-2) 100%
+    );
+    -webkit-background-clip: text;
+    -webkit-text-fill-color: transparent;
+    animation: hue-shift 5s linear infinite;
+}
+@keyframes hue-shift {
+    0% {
+        filter: hue-rotate(0deg);
+    }
+    100% {
+        filter: hue-rotate(360deg);
+    }
+}
+
+/* ── Layout ── */
+.app-container {
+    display: flex;
+    flex-direction: column;
+    align-items: center;
+    min-height: 100vh;
+    padding: 0 24px;
+    max-width: 1000px;
+    margin: 0 auto;
+}
+
+/* ── Nav ── */
+.top-nav {
+    display: flex;
+    justify-content: space-between;
+    align-items: center;
+    width: 100%;
+    padding: 32px 0;
+    z-index: 50;
+}
+.brand {
+    display: flex;
+    align-items: center;
+    gap: 16px;
+    font-size: 1.5rem;
+    font-weight: 800;
+    letter-spacing: -0.03em;
+}
+.brand-icon {
+    width: 44px;
+    height: 44px;
+    border-radius: 14px;
+    background: rgba(0, 240, 255, 0.1);
+    border: 1px solid rgba(0, 240, 255, 0.4);
+    display: flex;
+    align-items: center;
+    justify-content: center;
+    color: var(--accent);
+    box-shadow: inset 0 0 20px rgba(0, 240, 255, 0.2);
+}
+.nav-status {
+    font-size: 0.75rem;
+    font-weight: 600;
+    text-transform: uppercase;
+    letter-spacing: 0.15em;
+    padding: 10px 20px;
+    border-radius: 99px;
+    border: 1px solid var(--border);
+    display: flex;
+    align-items: center;
+    gap: 10px;
+    background: rgba(255, 255, 255, 0.03);
+}
+.status-dot {
+    width: 8px;
+    height: 8px;
+    border-radius: 50%;
+}
+.status-online {
+    background: var(--success);
+    box-shadow: 0 0 12px var(--success);
+}
+.status-offline {
+    background: var(--danger);
+    box-shadow: 0 0 12px var(--danger);
+}
+
+/* ── New ChatGPT/WhatsApp Style Search Box ── */
+.search-wrapper {
+    width: 100%;
+    position: relative;
+    z-index: 20;
+    display: flex;
+    flex-direction: column;
+    gap: 12px;
+}
+.chat-input-wrapper {
+    background: rgba(30, 35, 45, 0.9);
+    border: 1px solid rgba(255, 255, 255, 0.1);
+    border-radius: 30px;
+    padding: 6px 6px 6px 20px;
+    display: flex;
+    align-items: center;
+    gap: 12px;
+    box-shadow: 0 8px 30px rgba(0, 0, 0, 0.5);
+    transition: 0.3s;
+}
+.chat-input-wrapper:focus-within {
+    border-color: rgba(0, 240, 255, 0.4);
+    box-shadow:
+        0 0 30px rgba(0, 240, 255, 0.15),
+        0 8px 30px rgba(0, 0, 0, 0.5);
+}
+.chat-input {
+    flex: 1;
+    font-size: 1.05rem;
+    color: #fff;
+    line-height: 1.5;
+    resize: none;
+    outline: none;
+    padding: 6px 0;
+    font-family: inherit;
+}
+.chat-input::placeholder {
+    color: rgba(255, 255, 255, 0.4);
+}
+.chat-input::-webkit-scrollbar,
+.info-modal::-webkit-scrollbar {
+    width: 6px;
+}
+.chat-input::-webkit-scrollbar-track,
+.info-modal::-webkit-scrollbar-track {
+    background: transparent;
+}
+.info-modal::-webkit-scrollbar-track {
+    margin-block: 24px;
+}
+.chat-input::-webkit-scrollbar-thumb,
+.info-modal::-webkit-scrollbar-thumb {
+    background: rgba(255, 255, 255, 0.2);
+    border-radius: 4px;
+}
+.chat-input::-webkit-scrollbar-thumb:hover,
+.info-modal::-webkit-scrollbar-thumb:hover {
+    background: rgba(255, 255, 255, 0.4);
+}
+
+.send-btn {
+    width: 38px;
+    height: 38px;
+    border-radius: 50%;
+    background: var(--text-main);
+    color: #000;
+    display: flex;
+    align-items: center;
+    justify-content: center;
+    transition: 0.2s cubic-bezier(0.175, 0.885, 0.32, 1.275);
+    flex-shrink: 0;
+}
+.send-btn:hover:not(:disabled) {
+    transform: scale(1.08);
+    background: var(--accent);
+    box-shadow: 0 0 15px rgba(0, 240, 255, 0.4);
+}
+.send-btn:disabled {
+    opacity: 0.3;
+    cursor: not-allowed;
+    transform: none;
+    background: var(--text-main);
+}
+.spinner-micro {
+    width: 18px;
+    height: 18px;
+    border: 2px solid rgba(0, 0, 0, 0.2);
+    border-top-color: #000;
+    border-radius: 50%;
+    animation: spin 0.8s linear infinite;
+}
+
+/* ── Search Controls (Settings) ── */
+.search-controls {
+    display: flex;
+    align-items: center;
+    padding: 4px 8px;
+    gap: 16px;
+    margin-top: 4px;
+}
+.controls-group {
+    display: flex;
+    gap: 12px;
+    align-items: center;
+    flex-wrap: wrap;
+}
+.cyber-select {
+    background: rgba(255, 255, 255, 0.04);
+    border: 1px solid var(--border);
+    border-radius: 12px;
+    padding: 8px 14px;
+    font-size: 0.85rem;
+    color: var(--text-main);
+    transition: 0.2s;
+    cursor: pointer;
+}
+.cyber-select:hover {
+    background: rgba(255, 255, 255, 0.1);
+    border-color: rgba(255, 255, 255, 0.3);
+}
+
+/* Custom React Dropdown Override */
+.custom-dropdown-menu {
+    position: absolute;
+    left: 0;
+    width: 100%;
+    background: rgba(12, 14, 20, 0.95);
+    backdrop-filter: blur(24px);
+    border: 1px solid rgba(255, 255, 255, 0.1);
+    border-radius: 12px;
+    overflow: hidden;
+    box-shadow: 0 15px 40px rgba(0,0,0,0.8), 0 0 20px rgba(0, 240, 255, 0.15);
+    z-index: 100;
+    display: flex;
+    flex-direction: column;
+}
+
+.custom-dropdown-item {
+    padding: 12px 16px;
+    font-size: 0.85rem;
+    color: var(--text-muted);
+    text-align: left;
+    transition: 0.2s;
+    background: transparent;
+    width: 100%;
+    display: block;
+}
+
+.custom-dropdown-item:hover {
+    background: rgba(255, 255, 255, 0.05);
+    color: #fff;
+}
+
+.custom-dropdown-item.active {
+    background: rgba(0, 240, 255, 0.1);
+    color: var(--accent);
+    font-weight: 500;
+    border-left: 2px solid var(--accent);
+    padding-left: 14px;
+}
+
+.cyber-btn-outline {
+    padding: 8px 16px;
+    border-radius: 12px;
+    font-size: 0.85rem;
+    font-weight: 500;
+    transition: 0.2s;
+    border: 1px solid var(--border);
+    background: rgba(255, 255, 255, 0.04);
+    color: var(--text-muted);
+}
+.cyber-btn-outline.active {
+    background: rgba(0, 240, 255, 0.1);
+    border-color: rgba(0, 240, 255, 0.4);
+    color: var(--accent);
+}
+.cyber-btn-outline:hover {
+    border-color: rgba(255, 255, 255, 0.3);
+    color: #fff;
+}
+
+.cyber-input {
+    background: rgba(0, 0, 0, 0.4);
+    border: 1px solid var(--border);
+    border-radius: 12px;
+    padding: 8px 14px;
+    color: var(--accent);
+    font-family: "JetBrains Mono", monospace;
+    font-size: 0.9rem;
+    outline: none;
+    transition: 0.2s;
+}
+.cyber-input:focus {
+    border-color: var(--accent);
+}
+
+/* ── Suggestions ── */
+.example-chips {
+    display: flex;
+    flex-wrap: wrap;
+    justify-content: center;
+    gap: 10px;
+    margin-top: 24px;
+}
+.chip {
+    background: rgba(255, 255, 255, 0.03);
+    border: 1px solid rgba(255, 255, 255, 0.08);
+    padding: 10px 18px;
+    border-radius: 99px;
+    font-size: 0.85rem;
+    color: var(--text-muted);
+    transition: 0.3s;
+}
+.chip:hover {
+    background: rgba(0, 240, 255, 0.1);
+    border-color: rgba(0, 240, 255, 0.3);
+    color: #fff;
+    transform: translateY(-2px);
+}
+
+/* ── Results Container ── */
+.results-area {
+    width: 100%;
+    padding-bottom: 100px;
+    display: flex;
+    flex-direction: column;
+    gap: 24px;
+    margin-top: 32px;
+}
+
+/* ── Answer Box (AI Output) ── */
+.answer-box {
+    padding: 32px;
+    background: linear-gradient(
+        145deg,
+        rgba(20, 25, 35, 0.8),
+        rgba(5, 7, 10, 0.9)
+    );
+}
+.answer-header {
+    display: flex;
+    justify-content: space-between;
+    align-items: center;
+    margin-bottom: 24px;
+    border-bottom: 1px solid var(--border);
+    padding-bottom: 16px;
+}
+.answer-label {
+    display: flex;
+    align-items: center;
+    gap: 10px;
+    font-size: 1rem;
+    font-weight: 700;
+    color: var(--accent);
+    letter-spacing: 0.1em;
+    text-transform: uppercase;
+}
+.badge-grounded {
+    font-family: "JetBrains Mono", monospace;
+    font-size: 0.7rem;
+    font-weight: 600;
+    background: rgba(16, 185, 129, 0.1);
+    color: var(--success);
+    padding: 6px 12px;
+    border-radius: 8px;
+    border: 1px solid rgba(16, 185, 129, 0.2);
+    display: flex;
+    align-items: center;
+    gap: 8px;
+}
+
+/* ── AI Output Font Update (Inter, refined for math/reading) ── */
+.answer-text {
+    font-family: "Inter", system-ui, sans-serif;
+    font-size: 1.05rem;
+    line-height: 1.85;
+    color: #e2e8f0;
+    font-weight: 400;
+    letter-spacing: 0.01em;
+}
+.answer-text strong {
+    color: #fff;
+    font-weight: 600;
+}
+
+/* ── Stats ── */
+.stats-grid {
+    display: grid;
+    grid-template-columns: repeat(auto-fit, minmax(200px, 1fr));
+    gap: 16px;
+}
+.stat-card {
+    padding: 20px;
+    display: flex;
+    flex-direction: column;
+    gap: 8px;
+    background: rgba(255, 255, 255, 0.02);
+}
+.stat-header {
+    display: flex;
+    justify-content: space-between;
+    align-items: center;
+}
+.stat-label {
+    font-size: 0.7rem;
+    text-transform: uppercase;
+    color: var(--text-muted);
+    font-weight: 700;
+    letter-spacing: 0.1em;
+}
+.stat-value {
+    font-size: 1.5rem;
+    font-family: "JetBrains Mono", monospace;
+    font-weight: 700;
+    color: #fff;
+}
+
+/* ── Citations (Smaller Cards) ── */
+.section-title {
+    display: flex;
+    align-items: center;
+    gap: 10px;
+    font-size: 1.15rem;
+    font-weight: 700;
+    color: #fff;
+    margin: 16px 0;
+}
+.citations-grid {
+    display: grid;
+    grid-template-columns: repeat(
+        auto-fit,
+        minmax(240px, 1fr)
+    ); /* Reduced to fit more */
+    gap: 16px; /* Reduced gap */
+}
+.citation-card {
+    padding: 16px; /* Reduced padding */
+    background: rgba(0, 0, 0, 0.5);
+    border: 1px solid rgba(255, 255, 255, 0.05);
+    border-radius: 16px; /* Reduced radius */
+    transition: 0.3s cubic-bezier(0.175, 0.885, 0.32, 1.275);
+    display: flex;
+    flex-direction: column;
+    gap: 12px; /* Reduced gap */
+    position: relative;
+}
+.citation-card:hover {
+    background: rgba(20, 20, 20, 0.8);
+    border-color: var(--accent-2);
+    transform: translateY(-4px); /* Smoother smaller lift */
+    box-shadow: 0 10px 20px rgba(138, 43, 226, 0.15);
+}
+.citation-open {
+    position: absolute;
+    top: 16px;
+    right: 16px;
+    color: var(--accent-2);
+    opacity: 0;
+    transition: 0.3s;
+    transform: translateX(-5px);
+}
+.citation-card:hover .citation-open {
+    opacity: 1;
+    transform: translateX(0);
+}
+
+.citation-meta {
+    display: flex;
+    gap: 10px;
+    align-items: center;
+}
+.citation-id {
+    font-family: "JetBrains Mono", monospace;
+    font-size: 0.7rem; /* Smaller */
+    background: rgba(138, 43, 226, 0.15);
+    color: #d8b4fe;
+    padding: 4px 8px; /* Smaller */
+    border-radius: 6px;
+    border: 1px solid rgba(138, 43, 226, 0.3);
+}
+.citation-date {
+    font-family: "JetBrains Mono", monospace;
+    font-size: 0.7rem; /* Smaller */
+    color: var(--text-muted);
+}
+.citation-title {
+    font-size: 0.95rem; /* Smaller reading title */
+    font-weight: 600;
+    color: #f8fafc;
+    line-height: 1.4;
+    padding-right: 20px;
+}
+.citation-authors {
+    font-size: 0.8rem; /* Smaller */
+    color: var(--text-muted);
+    font-style: italic;
+    font-weight: 300;
+}
+
+/* ── Loader ── */
+.loader-view {
+    padding: 80px 0;
+    display: flex;
+    flex-direction: column;
+    align-items: center;
+    gap: 24px;
+    text-align: center;
+}
+.ring-spinner {
+    width: 70px;
+    height: 70px;
+    position: relative;
+}
+.ring {
+    position: absolute;
+    inset: 0;
+    border: 2px solid transparent;
+    border-radius: 50%;
+}
+.ring-1 {
+    border-top-color: var(--accent);
+    animation: spin1 1.5s cubic-bezier(0.5, 0, 0.5, 1) infinite;
+}
+.ring-2 {
+    border-right-color: var(--accent-2);
+    animation: spin2 2s cubic-bezier(0.5, 0, 0.5, 1) infinite;
+}
+@keyframes spin1 {
+    to {
+        transform: rotate(360deg);
+    }
+}
+@keyframes spin2 {
+    to {
+        transform: rotate(-360deg);
+    }
+}
+
+/* ── Year Stepper ── */
+.year-stepper {
+    display: flex;
+    align-items: center;
+    background: rgba(0, 0, 0, 0.4);
+    border: 1px solid var(--border);
+    border-radius: 12px;
+    overflow: hidden;
+    transition: 0.2s;
+}
+.year-stepper:hover {
+    border-color: rgba(255, 255, 255, 0.3);
+}
+.year-stepper:focus-within {
+    border-color: var(--accent);
+}
+.stepper-btn {
+    padding: 8px 14px;
+    background: rgba(255, 255, 255, 0.03);
+    color: var(--accent);
+    font-weight: 700;
+    transition: 0.2s;
+    user-select: none;
+}
+.stepper-btn:hover {
+    background: rgba(0, 240, 255, 0.15);
+    color: #fff;
+}
+.stepper-input {
+    width: 50px;
+    text-align: center;
+    color: #fff;
+    font-family: 'JetBrains Mono', monospace;
+    font-size: 0.95rem;
+    pointer-events: none;
+}
+.stepper-input::-webkit-outer-spin-button,
+.stepper-input::-webkit-inner-spin-button {
+    -webkit-appearance: none;
+    margin: 0;
+}
+.stepper-input[type=number] {
+    -moz-appearance: textfield;
+}
+
+/* ── Professional Header Layout ── */
+.top-nav {
+    position: fixed;
+    top: 0;
+    left: 0;
+    width: 100vw;
+    padding: 24px 2%;
+    display: flex;
+    justify-content: space-between;
+    align-items: center;
+    z-index: 5000;
+    background: transparent;
+    backdrop-filter: blur(16px) !important;
+    -webkit-backdrop-filter: blur(16px) !important;
+    border-bottom: 1px solid rgba(255, 255, 255, 0.05);
+}
+.brand {
+    display: flex;
+    align-items: center;
+    gap: 12px;
+    font-size: 1.25rem;
+    font-weight: 700;
+    letter-spacing: -0.02em;
+    color: #fff;
+}
+.brand-icon {
+    display: flex;
+    align-items: center;
+    justify-content: center;
+    width: 36px;
+    height: 36px;
+    background: rgba(0, 240, 255, 0.1);
+    border: 1px solid rgba(0, 240, 255, 0.3);
+    border-radius: 10px;
+    color: var(--accent);
+}
+.nav-right {
+    display: flex;
+    align-items: center;
+    gap: 16px;
+}
+.github-link {
+    color: var(--text-muted);
+    transition: 0.2s;
+    display: flex;
+    align-items: center;
+    justify-content: center;
+}
+.github-link:hover {
+    color: #fff;
+    transform: scale(1.1);
+}
+.nav-status {
+    display: flex;
+    align-items: center;
+    gap: 6px;
+    background: rgba(255, 255, 255, 0.05);
+    border: 1px solid rgba(255, 255, 255, 0.1);
+    padding: 6px 12px;
+    border-radius: 100px;
+    font-size: 0.7rem;
+    font-weight: 600;
+    color: var(--text-muted);
+    cursor: pointer;
+    transition: 0.3s;
+}
+.nav-status:hover {
+    background: rgba(255, 255, 255, 0.1);
+    color: #fff;
+}
+.status-dot {
+    width: 6px;
+    height: 6px;
+    border-radius: 50%;
+    background: #666;
+}
+.status-online {
+    background: var(--success);
+    box-shadow: 0 0 10px var(--success);
+}
+.status-offline {
+    background: var(--danger);
+    box-shadow: 0 0 10px var(--danger);
+}
+
+/* Custom overrides */
+select {
+    -webkit-appearance: none;
+    -moz-appearance: none;
+    background-image: none;
+}
+
+/* ── Info Modal ── */
+.info-modal-backdrop {
+    position: fixed;
+    inset: 0;
+    background: rgba(0, 0, 0, 0.6);
+    backdrop-filter: blur(12px);
+    -webkit-backdrop-filter: blur(12px);
+    display: flex;
+    align-items: center;
+    justify-content: center;
+    z-index: 9999;
+    padding: 24px;
+}
+.info-modal {
+    position: relative;
+    width: 100%;
+    max-width: 650px;
+    padding: 40px;
+    background: linear-gradient(145deg, rgba(12, 16, 24, 0.9), rgba(5, 7, 10, 0.95));
+    border: 1px solid rgba(255, 255, 255, 0.08);
+    box-shadow: 0 30px 60px rgba(0, 0, 0, 0.8), inset 0 1px 0 rgba(255, 255, 255, 0.1), inset 0 0 80px rgba(0, 240, 255, 0.03);
+    border-radius: 24px;
+    overflow-y: auto;
+    max-height: 85vh;
+}
+.info-modal h2 {
+    font-size: 2.2rem;
+    font-weight: 800;
+    margin-bottom: 6px;
+    background: linear-gradient(135deg, #fff 0%, var(--accent) 100%);
+    background-clip: text;
+    -webkit-background-clip: text;
+    -webkit-text-fill-color: transparent;
+    letter-spacing: -0.02em;
+}
+.info-modal h3 {
+    display: flex;
+    align-items: center;
+    gap: 12px;
+    font-size: 1.15rem;
+    font-weight: 700;
+    color: #fff;
+    margin-top: 32px;
+    margin-bottom: 16px;
+    padding-bottom: 8px;
+    border-bottom: 1px solid rgba(255, 255, 255, 0.05);
+}
+.info-modal h3 svg {
+    color: var(--accent);
+}
+.info-modal p, .info-modal ul {
+    color: var(--text-muted);
+    font-size: 0.95rem;
+    line-height: 1.6;
+}
+.info-modal ul {
+    list-style-type: none;
+    padding-left: 0;
+    display: flex;
+    flex-direction: column;
+    gap: 12px;
+}
+.info-modal li {
+    margin: 0;
+    padding: 14px 18px;
+    background: rgba(255, 255, 255, 0.02);
+    border: 1px solid rgba(255, 255, 255, 0.04);
+    border-radius: 12px;
+    transition: 0.3s;
+    font-size: 0.9rem;
+    color: var(--text-muted);
+}
+.info-modal li:hover {
+    background: rgba(0, 240, 255, 0.05);
+    border-color: rgba(0, 240, 255, 0.2);
+    color: #fff;
+    transform: translateX(4px);
+}
+.info-modal li strong {
+    color: var(--accent);
+    display: block;
+    font-size: 0.95rem;
+    margin-bottom: 4px;
+}
+.info-modal hr {
+    border: none;
+    height: 1px;
+    background: linear-gradient(90deg, transparent, rgba(0, 240, 255, 0.4), transparent);
+    margin: 30px 0;
+}
+.modal-close {
+    position: absolute;
+    top: 24px;
+    right: 24px;
+    color: var(--text-muted);
+    background: rgba(255, 255, 255, 0.05);
+    border: 1px solid rgba(255, 255, 255, 0.1);
+    border-radius: 50%;
+    width: 36px;
+    height: 36px;
+    display: flex;
+    align-items: center;
+    justify-content: center;
+    transition: 0.2s cubic-bezier(0.175, 0.885, 0.32, 1.275);
+}
+.modal-close:hover {
+    background: rgba(239, 68, 68, 0.2);
+    color: var(--danger);
+    border-color: rgba(239, 68, 68, 0.3);
+    transform: rotate(90deg);
+}
+.nav-icon-btn {
+    display: flex;
+    align-items: center;
+    justify-content: center;
+    width: 36px;
+    height: 36px;
+    border-radius: 12px;
+    background: rgba(255, 255, 255, 0.05);
+    border: 1px solid rgba(255, 255, 255, 0.1);
+    color: var(--text-muted);
+    transition: 0.2s cubic-bezier(0.175, 0.885, 0.32, 1.275);
+}
+.nav-icon-btn:hover {
+    background: rgba(0, 240, 255, 0.1);
+    border-color: rgba(0, 240, 255, 0.4);
+    color: var(--accent);
+    transform: translateY(-2px);
+}

frontend-next/app/layout.tsx

@@ -0,0 +1,24 @@
+// app/layout.tsx
+import type { Metadata } from "next";
+import { Inter } from "next/font/google";
+import "./globals.css";
+
+const inter = Inter({ subsets: ["latin"] });
+
+export const metadata: Metadata = {
+    title: "ResearchPilot — ML Research Assistant",
+    description:
+        "AI-powered research assistant for ML papers. Hybrid search + grounded answers.",
+};
+
+export default function RootLayout({
+    children,
+}: {
+    children: React.ReactNode;
+}) {
+    return (
+        <html lang="en">
+            <body className={inter.className}>{children}</body>
+        </html>
+    );
+}

frontend-next/app/page.tsx

@@ -0,0 +1,872 @@
+"use client";
+
+import { useState, useRef, useEffect } from "react";
+import { motion, AnimatePresence } from "framer-motion";
+import { InlineMath, BlockMath } from 'react-katex';
+import 'katex/dist/katex.min.css';
+import {
+    BookOpen,
+    Clock,
+    Zap,
+    AlertCircle,
+    CheckCircle,
+    ExternalLink,
+    Sparkles,
+    Brain,
+    ArrowRight,
+    Layers,
+    Fingerprint,
+    Send,
+    Info,
+    X,
+    Server,
+    Activity,
+    Rocket,
+} from "lucide-react";
+
+// ── Types ─────────────────────────────────────────────────
+interface Citation {
+    paper_id: string;
+    title: string;
+    authors: string[];
+    published_date: string;
+    arxiv_url: string;
+}
+
+interface QueryResult {
+    answer: string;
+    citations: Citation[];
+    query: string;
+    chunks_used: number;
+    retrieval_time_ms: number;
+    generation_time_ms: number;
+    total_time_ms: number;
+    has_context: boolean;
+}
+
+// ── Config ────────────────────────────────────────────────
+const API_URL = process.env.NEXT_PUBLIC_API_URL || "http://localhost:8000";
+
+const EXAMPLE_QUERIES = [
+    "How does LoRA reduce trainable parameters?",
+    "What are challenges in multi-agent RL?",
+    "Explain diffusion models for images",
+];
+
+const CATEGORY_OPTIONS = [
+    { value: "All", label: "All Topics" },
+    { value: "cs.LG", label: "cs.LG", indexed: true },
+    { value: "cs.AI", label: "cs.AI", indexed: true },
+    { value: "stat.ML", label: "stat.ML", indexed: true },
+    { value: "cs.CV", label: "cs.CV", indexed: false, disabled: true },
+    { value: "cs.CL", label: "cs.CL", indexed: false, disabled: true },
+    { value: "cs.RO", label: "cs.RO", indexed: false, disabled: true },
+];
+
+// ── Custom Dropdown Component ─────────────────────────────
+function CustomSelect({
+    options,
+    value,
+    onChange,
+    width = '140px',
+}: {
+    options: { value: string | number; label: string; disabled?: boolean; indexed?: boolean }[];
+    value: string | number;
+    onChange: (val: string | number) => void;
+    width?: string;
+}) {
+    const [isOpen, setIsOpen] = useState(false);
+    const [placement, setPlacement] = useState<"top" | "bottom">("bottom");
+    const ref = useRef<HTMLDivElement>(null);
+
+    useEffect(() => {
+        const handleClickOutside = (e: MouseEvent) => {
+            if (ref.current && !ref.current.contains(e.target as Node)) setIsOpen(false);
+        };
+        document.addEventListener('mousedown', handleClickOutside);
+        return () => document.removeEventListener('mousedown', handleClickOutside);
+    }, []);
+
+    const toggleOpen = () => {
+        if (!isOpen && ref.current) {
+            const rect = ref.current.getBoundingClientRect();
+            // Need ~240px for full menu, pop up if space is tight below
+            if (window.innerHeight - rect.bottom < 240) {
+                setPlacement('top');
+            } else {
+                setPlacement('bottom');
+            }
+        }
+        setIsOpen(!isOpen);
+    };
+
+    const activeLabel = options.find((o) => o.value === value)?.label || value;
+
+    return (
+        <div ref={ref} style={{ position: 'relative', width }}>
+            <button
+                onClick={toggleOpen}
+                className="cyber-select"
+                style={{ width: '100%', display: 'flex', justifyContent: 'space-between', alignItems: 'center' }}
+            >
+                <span style={{ overflow: 'hidden', textOverflow: 'ellipsis', whiteSpace: 'nowrap' }}>{activeLabel}</span>
+                <span style={{ fontSize: '0.7em', opacity: 0.5, marginLeft: '8px' }}>{isOpen ? '▲' : '▼'}</span>
+            </button>
+            <AnimatePresence>
+                {isOpen && (
+                    <motion.div
+                        initial={{ opacity: 0, y: placement === 'top' ? 10 : -10, scale: 0.95 }}
+                        animate={{ opacity: 1, y: 0, scale: 1 }}
+                        exit={{ opacity: 0, y: placement === 'top' ? 5 : -5, scale: 0.95 }}
+                        transition={{ duration: 0.15, ease: 'easeOut' }}
+                        className="custom-dropdown-menu"
+                        style={{
+                            top: placement === 'bottom' ? 'calc(100% + 8px)' : 'auto',
+                            bottom: placement === 'top' ? 'calc(100% + 8px)' : 'auto'
+                        }}
+                    >
+                        {options.map((opt) => (
+                            <button
+                                key={opt.value}
+                                onClick={() => {
+                                    if (opt.disabled) return;
+                                    onChange(opt.value);
+                                    setIsOpen(false);
+                                }}
+                                disabled={opt.disabled}
+                                className={`custom-dropdown-item ${value === opt.value ? 'active' : ''}`}
+                                style={opt.disabled ? { opacity: 0.4, cursor: 'not-allowed' } : {}}
+                            >
+                                <div style={{ display: "flex", alignItems: "center", justifyContent: "space-between", width: "100%" }}>
+                                    <span>{opt.label}</span>
+                                    {opt.indexed !== undefined && (
+                                        <div style={{
+                                            fontSize: "0.65em",
+                                            fontWeight: 600,
+                                            padding: "2px 6px",
+                                            borderRadius: "12px",
+                                            backgroundColor: opt.indexed ? "rgba(16, 185, 129, 0.15)" : "rgba(156, 163, 175, 0.1)",
+                                            color: opt.indexed ? "var(--success)" : "var(--text-muted)",
+                                            marginLeft: "8px"
+                                        }}>
+                                            {opt.indexed ? "INDEXED" : "UNAVAILABLE"}
+                                        </div>
+                                    )}
+                                </div>
+                            </button>
+                        ))}
+                    </motion.div>
+                )}
+            </AnimatePresence>
+        </div>
+    );
+}
+
+function renderLaTeX(text: string) {
+    const blockSplit = text.split(/(\$\$[\s\S]+?\$\$)/g);
+    return blockSplit.map((blockPart, i) => {
+        if (blockPart.startsWith("$$") && blockPart.endsWith("$$")) {
+            const math = blockPart.slice(2, -2);
+            return <BlockMath key={i} math={math} />;
+        }
+        const inlineSplit = blockPart.split(/(\$[\s\S]+?\$)/g);
+        return (
+            <span key={i}>
+                {inlineSplit.map((inlinePart, j) => {
+                    if (inlinePart.startsWith("$") && inlinePart.endsWith("$")) {
+                        const math = inlinePart.slice(1, -1);
+                        return <InlineMath key={j} math={math} />;
+                    }
+                    return (
+                        <span key={j} style={{ whiteSpace: "pre-wrap" }}>
+                            {inlinePart}
+                        </span>
+                    );
+                })}
+            </span>
+        );
+    });
+}
+
+// ── Main Page ─────────────────────────────────────────────
+export default function Home() {
+    const [question, setQuestion] = useState("");
+    const [result, setResult] = useState<QueryResult | null>(null);
+    const [loading, setLoading] = useState(false);
+    const [error, setError] = useState("");
+
+    // Settings
+    const [topK, setTopK] = useState(5);
+    const [category, setCategory] = useState("All");
+    const [yearFilter, setYearFilter] = useState(false);
+    const [yearFrom, setYearFrom] = useState(2024);
+    const [showSettings, setShowSettings] = useState(false);
+
+    // System state
+    const [apiStatus, setApiStatus] = useState<
+        "unknown" | "online" | "offline"
+    >("unknown");
+    const [showInfo, setShowInfo] = useState(false);
+    const [showStatusDetails, setShowStatusDetails] = useState(false);
+
+    const textareaRef = useRef<HTMLTextAreaElement>(null);
+
+    useEffect(() => {
+        if (textareaRef.current) {
+            textareaRef.current.style.height = "auto";
+            textareaRef.current.style.height = `${Math.min(textareaRef.current.scrollHeight, 120)}px`;
+        }
+    }, [question]);
+
+    useEffect(() => {
+        checkHealth();
+    }, []);
+
+    const checkHealth = async () => {
+        try {
+            const r = await fetch(`${API_URL}/health`, {
+                signal: AbortSignal.timeout(5000),
+            });
+            setApiStatus(r.ok ? "online" : "offline");
+        } catch {
+            setApiStatus("offline");
+        }
+    };
+
+    const handleSearch = async () => {
+        if (!question.trim()) return;
+        setLoading(true);
+        setError("");
+        setResult(null);
+
+        try {
+            const payload: Record<string, unknown> = {
+                question: question.trim(),
+                top_k: topK,
+            };
+            if (category !== "All") payload.filter_category = category;
+            if (yearFilter) payload.filter_year_gte = yearFrom;
+
+            const response = await fetch(`${API_URL}/query`, {
+                method: "POST",
+                headers: { "Content-Type": "application/json" },
+                body: JSON.stringify(payload),
+            });
+
+            if (!response.ok)
+                throw new Error(`API returned ${response.status}`);
+            const data: QueryResult = await response.json();
+            setResult(data);
+        } catch (err) {
+            setError(
+                err instanceof Error
+                    ? err.message
+                    : "Failed to connect to API.",
+            );
+        } finally {
+            setLoading(false);
+        }
+    };
+
+    const resetApp = () => {
+        setQuestion("");
+        setResult(null);
+        setError("");
+        setLoading(false);
+    };
+
+    const hasSearched = result || loading || error;
+
+    return (
+        <>
+            <div className="luminous-grid" />
+            <div className="orb orb-cyan" />
+            <div className="orb orb-purple" />
+
+            <AnimatePresence>
+                {showInfo && (
+                    <div className="info-modal-backdrop" onClick={() => setShowInfo(false)}>
+                        <motion.div
+                            initial={{ opacity: 0, scale: 0.95, y: 20 }}
+                            animate={{ opacity: 1, scale: 1, y: 0 }}
+                            exit={{ opacity: 0, scale: 0.95, y: 20 }}
+                            onClick={(e) => e.stopPropagation()}
+                            className="cyber-panel info-modal"
+                        >
+                            <button className="modal-close" onClick={() => setShowInfo(false)}>
+                                <X size={18} />
+                            </button>
+                            <h2>ResearchPilot Console</h2>
+                            <div style={{ display: "flex", alignItems: "center", gap: "12px", marginTop: "16px" }}>
+                                <div style={{ background: "rgba(138, 43, 226, 0.15)", border: "1px solid rgba(138, 43, 226, 0.4)", padding: "6px 14px", borderRadius: "99px", fontSize: "0.75rem", color: "var(--accent-2)", fontWeight: 700, letterSpacing: "0.05em", textTransform: "uppercase" }}>
+                                    Lead Architect
+                                </div>
+                                <span style={{ fontFamily: "'Dancing Script', cursive", fontSize: "1.8rem", fontWeight: 700, background: "linear-gradient(135deg, #fff 20%, var(--accent-2) 100%)", WebkitBackgroundClip: "text", WebkitTextFillColor: "transparent", letterSpacing: "0.05em", transform: "translateY(-2px)" }}>Subhadip Hensh</span>
+                            </div>
+                            
+                            <hr />
+                            
+                            <h3><Server size={18} /> System Overview</h3>
+                            <p style={{ fontSize: "1rem", lineHeight: 1.7, marginBottom: "16px" }}>ResearchPilot is a high-performance RAG (Retrieval-Augmented Generation) engine tailored for Machine Learning literature. It features hybrid sparse-dense searching, advanced cross-encoder reranking, and GPU-driven vector indexing via Qdrant.</p>
+                            
+                            <h3><Activity size={18} /> Current Operational Capacity</h3>
+                            <ul>
+                                <li><strong>Current Index</strong> Synthesizing 51,019 dense embeddings isolated from ~700 major AI & ML papers.</li>
+                                <li><strong>Data Categories</strong> Fully indexed on core Machine Learning (cs.LG) and AI (cs.AI).</li>
+                            </ul>
+
+                            <h3><Layers size={18} /> Core Technology Stack</h3>
+                            <ul>
+                                <li><strong>Frontend Application</strong> Next.js 16 (App Router), React, Framer Motion, Vanilla CSS (Glassmorphism).</li>
+                                <li><strong>Backend Environment</strong> Python, FastAPI, Uvicorn, Pydantic.</li>
+                                <li><strong>Vector Database Engine</strong> Qdrant (GPU Accelerated Dense Vectors).</li>
+                                <li><strong>RAG Processing Pipeline</strong> SentenceTransformers (BGE-base), BM25 Sparse Search, Cross-Encoder Reranking, Groq LLM (LLaMA 3.3).</li>
+                                <li><strong>Mathematics Engine</strong> KaTeX & React-KaTeX for fully dynamic native LaTeX equations.</li>
+                            </ul>
+
+                            <h3><Rocket size={18} /> Phase 2: In-Progress Architecture</h3>
+                            <ul>
+                                <li><strong>Massive Data Expansion</strong> Scaling dataset soon to 10,000+ — 20,000+ ML papers spanning NLP, Computer Vision, and Robotics.</li>
+                                <li><strong>Distributed Hardware Execution</strong> Scaling ingestion logic to cloud-based GPU clusters for extreme speed.</li>
+                                <li><strong>Multi-modal Analysis</strong> Soon integrating visual graph and chart processing abilities into the synthesis engine.</li>
+                            </ul>
+                        </motion.div>
+                    </div>
+                )}
+            </AnimatePresence>
+
+            {/* ── Top Nav ── */}
+            <header className="top-nav">
+                    <div className="brand" onClick={resetApp} style={{ cursor: "pointer" }}>
+                        <div className="brand-icon">
+                            <Brain size={22} />
+                        </div>
+                        <span>ResearchPilot</span>
+                    </div>
+                    <div className="nav-right">
+                        <button onClick={() => setShowInfo(true)} className="nav-icon-btn" aria-label="Project Info">
+                            <Info size={16} />
+                        </button>
+                        <div style={{ position: "relative" }}>
+                            <button 
+                                onClick={async () => {
+                                    await checkHealth();
+                                    setShowStatusDetails(!showStatusDetails);
+                                }} 
+                                className="nav-status"
+                            >
+                                <div
+                                    className={`status-dot ${apiStatus === "online" ? "status-online" : apiStatus === "offline" ? "status-offline" : ""}`}
+                                />
+                                {apiStatus === "online"
+                                    ? "Systems Nominal"
+                                    : apiStatus === "offline"
+                                      ? "Offline"
+                                      : "Checking..."}
+                            </button>
+
+                            <AnimatePresence>
+                                {showStatusDetails && (
+                                    <motion.div
+                                        initial={{ opacity: 0, y: 10, scale: 0.95 }}
+                                        animate={{ opacity: 1, y: 0, scale: 1 }}
+                                        exit={{ opacity: 0, y: 10, scale: 0.95 }}
+                                        className="cyber-panel custom-dropdown-menu"
+                                        style={{
+                                            position: "absolute",
+                                            top: "calc(100% + 12px)",
+                                            right: "-40px",
+                                            left: "auto",
+                                            width: "260px",
+                                            padding: "16px",
+                                            zIndex: 100,
+                                            display: "flex",
+                                            flexDirection: "column",
+                                            gap: "8px",
+                                            cursor: "default"
+                                        }}
+                                        onClick={(e) => e.stopPropagation()}
+                                    >
+                                        <h4 style={{ fontSize: "0.85rem", color: "#fff", marginBottom: "4px" }}>System Connection Status</h4>
+                                        <p style={{ fontSize: "0.75rem", color: "var(--text-muted)", lineHeight: 1.5 }}>
+                                            {apiStatus === "online" 
+                                                ? "🟢 Backend API and Qdrant Vector Database are connected and responding correctly. The system is ready for inference."
+                                                : "🔴 Backend API is unreachable. You need to run 'python run_api.py' in your backend directory to enable RAG functionality."}
+                                        </p>
+                                        <button 
+                                            onClick={(e) => {
+                                                e.stopPropagation();
+                                                checkHealth();
+                                            }}
+                                            style={{
+                                                marginTop: "8px",
+                                                fontSize: "0.75rem",
+                                                color: "var(--accent)",
+                                                background: "rgba(0, 240, 255, 0.1)",
+                                                padding: "6px 12px",
+                                                borderRadius: "8px",
+                                                border: "1px solid rgba(0, 240, 255, 0.3)",
+                                                textAlign: "center",
+                                                display: "block",
+                                                width: "100%",
+                                                cursor: "pointer",
+                                                transition: "0.2s"
+                                            }}
+                                            onMouseEnter={(e) => {
+                                                e.currentTarget.style.background = "rgba(0, 240, 255, 0.2)";
+                                            }}
+                                            onMouseLeave={(e) => {
+                                                e.currentTarget.style.background = "rgba(0, 240, 255, 0.1)";
+                                            }}
+                                        >
+                                            Re-verify Connection
+                                        </button>
+                                    </motion.div>
+                                )}
+                            </AnimatePresence>
+                        </div>
+                        <a 
+                            href="https://github.com/07subhadip" 
+                            target="_blank" 
+                            rel="noopener noreferrer" 
+                            className="github-link"
+                            aria-label="GitHub Profile"
+                        >
+                            <svg viewBox="0 0 24 24" width="24" height="24" fill="currentColor">
+                                <path d="M12 2C6.477 2 2 6.484 2 12.017c0 4.425 2.865 8.18 6.839 9.504.5.092.682-.217.682-.483 0-.237-.008-.868-.013-1.703-2.782.605-3.369-1.343-3.369-1.343-.454-1.158-1.11-1.466-1.11-1.466-.908-.62.069-.608.069-.608 1.003.07 1.531 1.032 1.531 1.032.892 1.53 2.341 1.088 2.91.832.092-.647.35-1.088.636-1.338-2.22-.253-4.555-1.113-4.555-4.951 0-1.093.39-1.988 1.029-2.688-.103-.253-.446-1.272.098-2.65 0 0 .84-.27 2.75 1.026A9.564 9.564 0 0112 6.844c.85.004 1.705.115 2.504.337 1.909-1.296 2.747-1.027 2.747-1.027.546 1.379.202 2.398.1 2.651.64.7 1.028 1.595 1.028 2.688 0 3.848-2.339 4.695-4.566 4.943.359.309.678.92.678 1.855 0 1.338-.012 2.419-.012 2.747 0 .268.18.58.688.482A10.019 10.019 0 0022 12.017C22 6.484 17.522 2 12 2z"/>
+                            </svg>
+                        </a>
+                    </div>
+                </header>
+
+            <main className="app-container">
+                {/* ── Central Hero Block ── */}
+                <motion.div
+                    layout
+                    className="search-wrapper"
+                    style={{ marginTop: hasSearched ? "130px" : "15vh" }}
+                    transition={{ type: "spring", bounce: 0.2, duration: 0.8 }}
+                >
+                    <AnimatePresence>
+                        {!hasSearched && (
+                            <motion.div
+                                layout
+                                initial={{ opacity: 0, scale: 0.95 }}
+                                animate={{ opacity: 1, scale: 1 }}
+                                exit={{ opacity: 0, scale: 0.95 }}
+                                transition={{ duration: 0.5 }}
+                                style={{ width: "100%" }}
+                            >
+                                <h1 className="hero-title">
+                                    Decipher the latest
+                                    <br />
+                                    <span className="text-gradient-2">
+                                        ML Research
+                                    </span>
+                                </h1>
+                                <p className="hero-subtitle">
+                                    Neural hybrid search across ArXiv. <br />
+                                    Cross-encoder reranked. LLM synthesized.
+                                </p>
+                                <div style={{ height: "64px" }} />
+                            </motion.div>
+                        )}
+                    </AnimatePresence>
+
+                    {/* ── ChatGPT Style Search Box ── */}
+                    <motion.div
+                        layout
+                        style={{
+                            width: "100%",
+                            margin: "0 auto",
+                            zIndex: 20,
+                        }}
+                    >
+                        <div className="chat-input-wrapper">
+                            <textarea
+                                ref={textareaRef}
+                                value={question}
+                                onChange={(e) => setQuestion(e.target.value)}
+                                placeholder="Message ResearchPilot..."
+                                rows={1}
+                                className="chat-input"
+                                style={{
+                                    minHeight: hasSearched ? "20px" : "28px",
+                                    maxHeight: "120px",
+                                    overflowY: "auto",
+                                }}
+                                onKeyDown={(e) => {
+                                    if (e.key === "Enter" && !e.shiftKey) {
+                                        e.preventDefault();
+                                        handleSearch();
+                                    }
+                                }}
+                            />
+                            <button
+                                onClick={handleSearch}
+                                disabled={loading || !question.trim()}
+                                className="send-btn"
+                            >
+                                {loading ? (
+                                    <div className="spinner-micro" />
+                                ) : (
+                                    <Send
+                                        size={18}
+                                        strokeWidth={2.5}
+                                        style={{ marginLeft: "-2px" }}
+                                    />
+                                )}
+                            </button>
+                        </div>
+
+                        <div className="search-controls">
+                            <button
+                                onClick={() => setShowSettings(!showSettings)}
+                                className="controls-group"
+                                style={{
+                                    color: "var(--text-muted)",
+                                    fontSize: "0.8rem",
+                                    fontWeight: 600,
+                                    padding: "4px",
+                                    cursor: "pointer",
+                                }}
+                            >
+                                <Layers size={14} />
+                                CONFIGURE {showSettings ? "▲" : "▼"}
+                            </button>
+
+                            <div className="controls-group">
+                                <AnimatePresence>
+                                    {showSettings && (
+                                        <motion.div
+                                            initial={{ opacity: 0, height: 0 }}
+                                            animate={{
+                                                opacity: 1,
+                                                height: "auto",
+                                            }}
+                                            exit={{ opacity: 0, height: 0 }}
+                                            style={{
+                                                overflow: "visible",
+                                                display: "flex",
+                                                gap: "12px",
+                                                flexWrap: "wrap",
+                                                alignItems: "center",
+                                            }}
+                                        >
+                                            <CustomSelect
+                                                value={topK}
+                                                onChange={(val) => setTopK(Number(val))}
+                                                options={[
+                                                    { value: 3, label: 'Top 3 Results' },
+                                                    { value: 5, label: 'Top 5 Results' },
+                                                    { value: 10, label: 'Top 10 Results' },
+                                                ]}
+                                            />
+                                            <CustomSelect
+                                                value={category}
+                                                onChange={(val) => setCategory(String(val))}
+                                                options={CATEGORY_OPTIONS}
+                                                width="160px"
+                                            />
+                                            <button
+                                                onClick={() =>
+                                                    setYearFilter(!yearFilter)
+                                                }
+                                                className={`cyber-btn-outline ${yearFilter ? "active" : ""}`}
+                                            >
+                                                YEAR FILTER{" "}
+                                                {yearFilter ? "ON" : "OFF"}
+                                            </button>
+                                            {yearFilter && (
+                                                <div className="year-stepper">
+                                                    <button onClick={() => setYearFrom(y => Math.max(2000, y - 1))} className="stepper-btn">-</button>
+                                                    <input
+                                                        type="number"
+                                                        value={yearFrom}
+                                                        readOnly
+                                                        className="stepper-input"
+                                                    />
+                                                    <button onClick={() => setYearFrom(y => Math.min(2026, y + 1))} className="stepper-btn">+</button>
+                                                </div>
+                                            )}
+                                        </motion.div>
+                                    )}
+                                </AnimatePresence>
+                            </div>
+                        </div>
+                    </motion.div>
+
+                    {/* ── Example Queries ── */}
+                    <AnimatePresence>
+                        {!hasSearched && (
+                            <motion.div
+                                layout
+                                initial={{ opacity: 0 }}
+                                animate={{ opacity: 1 }}
+                                exit={{ opacity: 0, filter: "blur(5px)" }}
+                                className="example-chips"
+                            >
+                                {EXAMPLE_QUERIES.map((q, i) => (
+                                    <motion.button
+                                        key={q}
+                                        initial={{ opacity: 0, y: 10 }}
+                                        animate={{ opacity: 1, y: 0 }}
+                                        transition={{ delay: 0.1 * i + 0.3 }}
+                                        onClick={() => {
+                                            setQuestion(q);
+                                            setTimeout(
+                                                () => handleSearch(),
+                                                50,
+                                            );
+                                        }}
+                                        className="chip"
+                                    >
+                                        {q}
+                                    </motion.button>
+                                ))}
+                            </motion.div>
+                        )}
+                    </AnimatePresence>
+                </motion.div>
+
+                {/* ── Error State ── */}
+                <AnimatePresence>
+                    {error && (
+                        <motion.div
+                            initial={{ opacity: 0, y: 10 }}
+                            animate={{ opacity: 1, y: 0 }}
+                            className="cyber-panel"
+                            style={{
+                                width: "100%",
+                                padding: "24px",
+                                borderColor: "var(--danger)",
+                                marginTop: "24px",
+                            }}
+                        >
+                            <div
+                                style={{
+                                    display: "flex",
+                                    gap: "16px",
+                                    alignItems: "center",
+                                }}
+                            >
+                                <AlertCircle size={32} color="var(--danger)" />
+                                <div>
+                                    <h3
+                                        style={{
+                                            color: "var(--danger)",
+                                            fontSize: "1.2rem",
+                                            marginBottom: "4px",
+                                        }}
+                                    >
+                                        Critical Exception
+                                    </h3>
+                                    <p style={{ color: "var(--text-muted)" }}>
+                                        {error}
+                                    </p>
+                                </div>
+                            </div>
+                        </motion.div>
+                    )}
+                </AnimatePresence>
+
+                {/* ── Loading View ── */}
+                <AnimatePresence>
+                    {loading && (
+                        <motion.div
+                            initial={{ opacity: 0 }}
+                            animate={{ opacity: 1 }}
+                            exit={{ opacity: 0 }}
+                            className="loader-view"
+                            style={{ width: "100%" }}
+                        >
+                            <div className="ring-spinner">
+                                <div className="ring ring-1" />
+                                <div className="ring ring-2" />
+                                <Brain
+                                    size={22}
+                                    style={{
+                                        position: "absolute",
+                                        top: "24px",
+                                        left: "24px",
+                                        color: "var(--text-main)",
+                                    }}
+                                />
+                            </div>
+                            <div>
+                                <h2
+                                    style={{
+                                        fontSize: "1.4rem",
+                                        fontWeight: 600,
+                                        color: "#fff",
+                                        marginBottom: "8px",
+                                    }}
+                                >
+                                    Synthesizing Knowledge
+                                </h2>
+                                <p
+                                    style={{
+                                        color: "var(--text-muted)",
+                                        fontSize: "0.95rem",
+                                    }}
+                                >
+                                    Running Vector Search & LLM Inference
+                                </p>
+                            </div>
+                        </motion.div>
+                    )}
+                </AnimatePresence>
+
+                {/* ── Results Output ── */}
+                {result && !loading && (
+                    <motion.div
+                        initial={{ opacity: 0, y: 20 }}
+                        animate={{ opacity: 1, y: 0 }}
+                        transition={{ staggerChildren: 0.1 }}
+                        className="results-area"
+                    >
+                        <motion.div
+                            className="cyber-panel answer-box"
+                            initial={{ opacity: 0, y: 20 }}
+                            animate={{ opacity: 1, y: 0 }}
+                        >
+                            <div className="answer-header">
+                                <div className="answer-label">
+                                    <Sparkles size={20} /> AI Synthesis
+                                </div>
+                                {result.has_context ? (
+                                    <div className="badge-grounded">
+                                        <CheckCircle size={14} /> Grounded
+                                        Sources
+                                    </div>
+                                ) : (
+                                    <div
+                                        className="badge-grounded"
+                                        style={{
+                                            color: "var(--danger)",
+                                            borderColor:
+                                                "rgba(239, 68, 68, 0.3)",
+                                            background:
+                                                "rgba(239, 68, 68, 0.1)",
+                                        }}
+                                    >
+                                        <AlertCircle size={14} /> Hallucination
+                                        Risk
+                                    </div>
+                                )}
+                            </div>
+                            <div className="answer-text">{renderLaTeX(result.answer)}</div>
+                        </motion.div>
+
+                        <motion.div
+                            className="stats-grid"
+                            initial={{ opacity: 0, y: 20 }}
+                            animate={{ opacity: 1, y: 0 }}
+                        >
+                            {[
+                                {
+                                    l: "Execution Time",
+                                    v: `${(result.total_time_ms / 1000).toFixed(1)}s`,
+                                    i: Clock,
+                                    c: "var(--accent)",
+                                },
+                                {
+                                    l: "Vector Data",
+                                    v: `${(result.retrieval_time_ms / 1000).toFixed(1)}s`,
+                                    i: ArrowRight,
+                                    c: "#fff",
+                                },
+                                {
+                                    l: "LLM Generation",
+                                    v: `${(result.generation_time_ms / 1000).toFixed(1)}s`,
+                                    i: Zap,
+                                    c: "var(--accent-2)",
+                                },
+                                {
+                                    l: "Paper Chunks",
+                                    v: result.chunks_used,
+                                    i: Fingerprint,
+                                    c: "var(--success)",
+                                },
+                            ].map((s, i) => (
+                                <div key={i} className="cyber-panel stat-card">
+                                    <div
+                                        className="stat-header"
+                                        style={{ width: "100%" }}
+                                    >
+                                        <span className="stat-label">
+                                            {s.l}
+                                        </span>
+                                        <s.i
+                                            size={16}
+                                            color={s.c}
+                                            style={{ opacity: 0.8 }}
+                                        />
+                                    </div>
+                                    <div
+                                        className="stat-value"
+                                        style={{
+                                            width: "100%",
+                                            textAlign: "left",
+                                            color: s.c,
+                                        }}
+                                    >
+                                        {s.v}
+                                    </div>
+                                </div>
+                            ))}
+                        </motion.div>
+
+                        {result.citations.length > 0 && (
+                            <motion.div
+                                initial={{ opacity: 0, y: 20 }}
+                                animate={{ opacity: 1, y: 0 }}
+                            >
+                                <div className="section-title">
+                                    <BookOpen
+                                        size={18}
+                                        color="var(--accent-2)"
+                                    />
+                                    Extracted Literature
+                                </div>
+                                <div className="citations-grid">
+                                    {result.citations.map((cite, i) => (
+                                        <a
+                                            href={cite.arxiv_url}
+                                            target="_blank"
+                                            rel="noopener noreferrer"
+                                            key={i}
+                                            className="cyber-panel citation-card"
+                                        >
+                                            <div className="citation-open">
+                                                <ExternalLink size={16} />
+                                            </div>
+                                            <div className="citation-meta">
+                                                <span className="citation-id">
+                                                    {cite.paper_id}
+                                                </span>
+                                                <span className="citation-date">
+                                                    {cite.published_date}
+                                                </span>
+                                            </div>
+                                            <h4 className="citation-title">
+                                                {cite.title}
+                                            </h4>
+                                            <div className="citation-authors">
+                                                {cite.authors
+                                                    .slice(0, 3)
+                                                    .join(", ")}
+                                                {cite.authors.length > 3 &&
+                                                    ` +${cite.authors.length - 3} more`}
+                                            </div>
+                                        </a>
+                                    ))}
+                                </div>
+                            </motion.div>
+                        )}
+                    </motion.div>
+                )}
+            </main>
+        </>
+    );
+}

frontend-next/eslint.config.mjs

@@ -0,0 +1,18 @@
+import { defineConfig, globalIgnores } from "eslint/config";
+import nextVitals from "eslint-config-next/core-web-vitals";
+import nextTs from "eslint-config-next/typescript";
+
+const eslintConfig = defineConfig([
+  ...nextVitals,
+  ...nextTs,
+  // Override default ignores of eslint-config-next.
+  globalIgnores([
+    // Default ignores of eslint-config-next:
+    ".next/**",
+    "out/**",
+    "build/**",
+    "next-env.d.ts",
+  ]),
+]);
+
+export default eslintConfig;

frontend-next/next.config.ts

@@ -0,0 +1,7 @@
+import type { NextConfig } from "next";
+
+const nextConfig: NextConfig = {
+  /* config options here */
+};
+
+export default nextConfig;

frontend-next/package.json

@@ -0,0 +1,34 @@
+{
+  "name": "frontend-next",
+  "version": "0.1.0",
+  "private": true,
+  "scripts": {
+    "dev": "next dev",
+    "build": "next build",
+    "start": "next start",
+    "lint": "eslint"
+  },
+  "dependencies": {
+    "clsx": "^2.1.1",
+    "framer-motion": "^12.38.0",
+    "katex": "^0.16.45",
+    "lucide-react": "^1.7.0",
+    "next": "16.2.2",
+    "react": "19.2.4",
+    "react-dom": "19.2.4",
+    "react-katex": "^3.1.0",
+    "tailwind-merge": "^3.5.0"
+  },
+  "devDependencies": {
+    "@tailwindcss/postcss": "^4",
+    "@types/katex": "^0.16.8",
+    "@types/node": "^20",
+    "@types/react": "^19",
+    "@types/react-dom": "^19",
+    "@types/react-katex": "^3.0.4",
+    "eslint": "^9",
+    "eslint-config-next": "16.2.2",
+    "tailwindcss": "^4",
+    "typescript": "^5"
+  }
+}

frontend-next/postcss.config.mjs

@@ -0,0 +1,7 @@
+const config = {
+  plugins: {
+    "@tailwindcss/postcss": {},
+  },
+};
+
+export default config;

frontend-next/tsconfig.json

@@ -0,0 +1,34 @@
+{
+  "compilerOptions": {
+    "target": "ES2017",
+    "lib": ["dom", "dom.iterable", "esnext"],
+    "allowJs": true,
+    "skipLibCheck": true,
+    "strict": true,
+    "noEmit": true,
+    "esModuleInterop": true,
+    "module": "esnext",
+    "moduleResolution": "bundler",
+    "resolveJsonModule": true,
+    "isolatedModules": true,
+    "jsx": "react-jsx",
+    "incremental": true,
+    "plugins": [
+      {
+        "name": "next"
+      }
+    ],
+    "paths": {
+      "@/*": ["./*"]
+    }
+  },
+  "include": [
+    "next-env.d.ts",
+    "**/*.ts",
+    "**/*.tsx",
+    ".next/types/**/*.ts",
+    ".next/dev/types/**/*.ts",
+    "**/*.mts"
+  ],
+  "exclude": ["node_modules"]
+}

merge_embeddings.py

@@ -0,0 +1,115 @@
+"""
+Merge old (local) and new (Kaggle) embeddings into a single set.
+
+This script:
+  1. Loads existing embeddings.npy + chunk_ids.npy  (your ~51k chunks)
+  2. Loads new_embeddings.npy + new_chunk_ids.npy  (from Kaggle batch)
+  3. Deduplicates by chunk_id
+  4. Saves the merged result back as embeddings.npy + chunk_ids.npy
+  5. Backs up the originals first
+
+Run from project root:
+    python merge_embeddings.py
+"""
+
+import json
+import shutil
+import numpy as np
+from pathlib import Path
+
+EMBEDDINGS_DIR = Path("data/embeddings")
+
+# File paths
+old_emb_file  = EMBEDDINGS_DIR / "embeddings.npy"
+old_ids_file  = EMBEDDINGS_DIR / "chunk_ids.npy"
+new_emb_file  = EMBEDDINGS_DIR / "new_embeddings.npy"
+new_ids_file  = EMBEDDINGS_DIR / "new_chunk_ids.npy"
+
+# Backup dir
+backup_dir = EMBEDDINGS_DIR / "backup_before_merge"
+backup_dir.mkdir(exist_ok=True)
+
+
+def main():
+    print("=" * 60)
+    print("  RESEARCHPILOT — EMBEDDING MERGE TOOL")
+    print("=" * 60)
+
+    # ── Step 1: Validate files exist ──
+    for f in [old_emb_file, old_ids_file, new_emb_file, new_ids_file]:
+        if not f.exists():
+            print(f"❌ Missing file: {f}")
+            return
+    print("✅ All required files found.\n")
+
+    # ── Step 2: Load old embeddings ──
+    print("Loading OLD embeddings...")
+    old_embeddings = np.load(str(old_emb_file))
+    old_ids = list(np.load(str(old_ids_file), allow_pickle=True))
+    print(f"   Old: {old_embeddings.shape[0]:,} chunks, dim={old_embeddings.shape[1]}")
+
+    # ── Step 3: Load new embeddings  ──
+    print("Loading NEW embeddings (from Kaggle)...")
+    new_embeddings = np.load(str(new_emb_file))
+    new_ids = list(np.load(str(new_ids_file), allow_pickle=True))
+    print(f"   New: {new_embeddings.shape[0]:,} chunks, dim={new_embeddings.shape[1]}")
+
+    # ── Step 4: Deduplicate ──
+    print("\nDeduplicating...")
+    old_id_set = set(old_ids)
+    keep_indices = []
+    for i, cid in enumerate(new_ids):
+        if cid not in old_id_set:
+            keep_indices.append(i)
+
+    unique_new_embeddings = new_embeddings[keep_indices]
+    unique_new_ids = [new_ids[i] for i in keep_indices]
+    duplicates_removed = len(new_ids) - len(unique_new_ids)
+    print(f"   Duplicates skipped: {duplicates_removed}")
+    print(f"   Unique new chunks:  {len(unique_new_ids):,}")
+
+    # ── Step 5: Merge ──
+    print("\nMerging...")
+    merged_embeddings = np.vstack([old_embeddings, unique_new_embeddings])
+    merged_ids = old_ids + unique_new_ids
+    print(f"   MERGED TOTAL: {merged_embeddings.shape[0]:,} chunks")
+
+    # ── Step 6: Backup originals ──
+    print("\nBacking up originals...")
+    shutil.copy2(old_emb_file, backup_dir / "embeddings_old.npy")
+    shutil.copy2(old_ids_file, backup_dir / "chunk_ids_old.npy")
+    print(f"   Backed up to: {backup_dir}")
+
+    # ── Step 7: Save merged files ──
+    print("\nSaving merged embeddings...")
+    np.save(str(old_emb_file), merged_embeddings)
+    np.save(str(old_ids_file), np.array(merged_ids, dtype=object))
+
+    # Update the index file
+    index = {
+        "total_embeddings": len(merged_ids),
+        "embedding_dimension": int(merged_embeddings.shape[1]),
+        "model_name": "BAAI/bge-base-en-v1.5",
+        "chunk_id_sample": merged_ids[:5],
+    }
+    with open(EMBEDDINGS_DIR / "embedding_index.json", "w") as f:
+        json.dump(index, f, indent=2)
+
+    print(f"   ✅ embeddings.npy  → {merged_embeddings.shape}")
+    print(f"   ✅ chunk_ids.npy   → {len(merged_ids):,} IDs")
+    print(f"   ✅ embedding_index.json updated")
+
+    # ── Summary ──
+    size_mb = (EMBEDDINGS_DIR / "embeddings.npy").stat().st_size / 1e6
+    print(f"\n{'=' * 60}")
+    print(f"  MERGE COMPLETE!")
+    print(f"  Old:    {len(old_ids):,} chunks")
+    print(f"  + New:  {len(unique_new_ids):,} chunks")
+    print(f"  = Total: {len(merged_ids):,} chunks")
+    print(f"  File size: {size_mb:.0f} MB")
+    print(f"{'=' * 60}")
+    print(f"\n👉 Now run:  python run_indexing.py --recreate")
+
+
+if __name__ == "__main__":
+    main()

reindex_light.py

@@ -0,0 +1,117 @@
+"""
+Re-index Qdrant with a LIMITED number of chunks (for low-RAM machines).
+
+Your full 358k embeddings stay on disk untouched.
+This only controls how many get loaded into the Qdrant search index.
+
+Usage:
+    python reindex_light.py              (default: 100,000 chunks)
+    python reindex_light.py --limit 50000
+"""
+
+import sys
+import json
+import numpy as np
+from pathlib import Path
+
+from src.utils.logger import setup_logger, get_logger
+from src.vectorstore.qdrant_store import QdrantStore
+from src.embeddings.embedding_cache import EmbeddingCache
+from config.settings import CHUNKS_DIR
+
+setup_logger()
+logger = get_logger(__name__)
+
+
+def main():
+    # Parse limit from args
+    limit = 100_000
+    if "--limit" in sys.argv:
+        idx = sys.argv.index("--limit")
+        limit = int(sys.argv[idx + 1])
+
+    print(f"{'=' * 60}")
+    print(f"  LIGHTWEIGHT RE-INDEXER (RAM-safe)")
+    print(f"  Chunk limit: {limit:,}")
+    print(f"{'=' * 60}\n")
+
+    # Step 1: Load chunk files from disk (only up to limit)
+    print("Step 1: Loading chunk files...")
+    chunk_ids = []
+    texts = []
+    metadata = []
+
+    chunk_files = sorted(CHUNKS_DIR.glob("*_semantic.json"))
+    print(f"   Found {len(chunk_files)} chunk files on disk")
+
+    for cf in chunk_files:
+        if len(chunk_ids) >= limit:
+            break
+
+        with open(cf, 'r', encoding='utf-8') as f:
+            raw = json.load(f)
+
+        # Handle both formats
+        if isinstance(raw, dict) and "chunks" in raw:
+            chunk_list = raw["chunks"]
+        elif isinstance(raw, list):
+            chunk_list = raw
+        else:
+            continue
+
+        for chunk in chunk_list:
+            if len(chunk_ids) >= limit:
+                break
+            chunk_ids.append(chunk['chunk_id'])
+            texts.append(chunk['text'])
+            metadata.append({k: v for k, v in chunk.items() if k != 'text'})
+
+    print(f"   Loaded {len(chunk_ids):,} chunks (limit: {limit:,})\n")
+
+    # Step 2: Load embeddings and match to chunks
+    print("Step 2: Loading embedding cache...")
+    cache = EmbeddingCache()
+    cache.load()
+    embeddings_matrix, cached_ids = cache.get_all()
+    print(f"   Cache has {len(cached_ids):,} embeddings")
+
+    # Build lookup
+    id_to_row = {cid: i for i, cid in enumerate(cached_ids)}
+
+    # Match chunks to embeddings
+    valid = [(i, id_to_row[cid]) for i, cid in enumerate(chunk_ids) if cid in id_to_row]
+    print(f"   Matched {len(valid):,} chunks with embeddings\n")
+
+    chunk_indices = [v[0] for v in valid]
+    embed_indices = [v[1] for v in valid]
+
+    final_chunk_ids = [chunk_ids[i] for i in chunk_indices]
+    final_texts = [texts[i] for i in chunk_indices]
+    final_metadata = [metadata[i] for i in chunk_indices]
+    final_embeddings = embeddings_matrix[embed_indices]
+
+    # Step 3: Recreate Qdrant collection
+    print("Step 3: Rebuilding Qdrant collection...")
+    store = QdrantStore()
+    store.create_collection(recreate=True)
+
+    # Step 4: Index
+    print(f"Step 4: Indexing {len(final_chunk_ids):,} chunks into Qdrant...")
+    total = store.index_chunks(
+        embeddings=final_embeddings,
+        chunk_ids=final_chunk_ids,
+        metadata=final_metadata,
+        texts=final_texts,
+    )
+
+    print(f"\n{'=' * 60}")
+    print(f"  ✅ INDEXING COMPLETE")
+    print(f"  Chunks indexed: {total:,}")
+    print(f"  Collection:     {store.get_collection_info()}")
+    print(f"  RAM usage:      ~{total * 768 * 4 / 1e6:.0f} MB (vectors only)")
+    print(f"{'=' * 60}")
+    print(f"\n  👉 Now run: python run_api.py")
+
+
+if __name__ == "__main__":
+    main()

run_api.py

@@ -22,6 +22,6 @@ if __name__ == "__main__":
         "src.api.main:app",
         host    = API_HOST,
         port    = API_PORT,
-        reload  = API_RELOAD,     # Auto-restart on code changes (dev only)
-        workers = 1,              # Single worker for dev (no GPU sharing issues)
+        reload  = False,              # Disable auto-reload (saves ~10s scanning 3000+ data files)
+        workers = 1,
     )

src/ingestion/arxiv_fetcher.py

@@ -43,7 +43,7 @@ logger = get_logger(__name__)
 
 class PaperMetadata(BaseModel):
     """
-    Pydantic model defininf the exact schema for a paper's metadata.
+    Pydantic model defining the exact schema for a paper's metadata.
 
     WHY PYDANTIC:
     Pydantic enforces data types at runtime. If ArXiv returns a date
@@ -212,7 +212,7 @@ class ArXivFetcher:
                 abstract            = result.summary,
                 authors             = [str(a) for a in result.authors],
                 categories          = result.categories,
-                primary_category    = str(result.primary_category) if result.primary_category else result.categories[0] if result.categories else "cs.LG",
+                primary_categories    = str(result.primary_category) if result.primary_category else result.categories[0] if result.categories else "cs.LG",
                 published_date      = result.published.strftime("%Y-%m-%d"),
                 updated_date        = result.updated.strftime("%Y-%m-%d"),
                 arxiv_url           = result.entry_id,
@@ -267,7 +267,7 @@ class ArXivFetcher:
         search = arxiv.Search(
             query       = category_query,
             max_results = max_papers * 2,    # Fetch extra account for skips
-            sort_by     = arxiv.SortCriterion.SubmittedDate,
+            sort_by     = arxiv.SortCriterion.Relevance,
             sort_order  = arxiv.SortOrder.Descending,
         )
 

src/rag/prompt_templates.py

@@ -13,20 +13,29 @@ Key principles we apply:
 """
 
 SYSTEM_PROMPT = """You are ResearchPilot, an expert AI research assistant 
-specialized in machine learning and AI research papers.
+specialized in machine learning, AI, and statistics.
 
-Your job is to answer questions based EXCLUSIVELY on the research paper 
-excerpts provided in the context below. 
+Your job is to answer questions based on the research paper excerpts 
+provided in the context below.
 
 STRICT RULES:
-    1. Only use information from the provided context excerpts
-    2. Always cite the paper title and ID when using information from it
-    3. If the context does not contain enough information to answer, 
-    say "The provided papers do not contain sufficient information 
-    to answer this question" - do NOT make up information
-    4. Be precise and technical - your users are ML researchers and engineers
-    5. When multiple papers discuss the same topic, synthesize their findings
-    6. Keep answers focused and well-structured
+1. Only use information from the provided context excerpts
+2. Always cite the paper title and ID when using information from it
+3. If the context does not contain enough information to answer,
+   clearly state what IS available and what is missing - do NOT fabricate
+4. Be precise and technical — your users are ML researchers and engineers
+5. When multiple papers discuss the same topic, synthesize their findings
+
+FORMATTING RULES:
+6. For ALL mathematical expressions, use LaTeX notation:
+    - Inline math: $expression$ (e.g. $\\hat{y} = \\sigma(Wx + b)$)
+    - Block math: $$expression$$ for standalone equations
+    - Examples:
+        Loss function:  $$\\mathcal{L} = -\\sum_{i} y_i \\log(\\hat{y}_i)$$
+        Attention:      $$\\text{Attention}(Q,K,V) = \\text{softmax}\\left(\\frac{QK^T}{\\sqrt{d_k}}\\right)V$$
+7. Use markdown formatting: **bold** for key terms, numbered lists for steps
+8. For algorithm explanations, structure as: Intuition -> Math -> Steps
+9. Write comprehensive, detailed answers — do not truncate explanations
 """
 
 

src/vectorstore/indexer.py

@@ -80,14 +80,24 @@ class VectorIndexer:
 
         for cf in chunk_files: 
             with open(cf, 'r', encoding = "utf-8") as f:
-                chunks = json.load(f)
-
-            for chunk in chunks:
+                raw = json.load(f)
+
+            # Handle both formats:
+            #   Old local format: [{chunk_id: ..., text: ...}, ...]
+            #   New Kaggle format: {"paper_id": "...", "chunks": [...]}
+            if isinstance(raw, dict) and "chunks" in raw:
+                chunk_list = raw["chunks"]
+            elif isinstance(raw, list):
+                chunk_list = raw
+            else:
+                logger.warning(f"Unexpected format in {cf.name}, skipping")
+                continue
+
+            for chunk in chunk_list:
                 chunk_ids.append(chunk['chunk_id'])
                 texts.append(chunk["text"])
 
-
-                # Everything expect that goes into metadata
+                # Everything except text goes into metadata
                 metadata.append(
                     {
                         k: v for k, v in chunk.items()