a new beginning

.gitattributes

@@ -1,35 +1,2 @@
-*.7z filter=lfs diff=lfs merge=lfs -text
-*.arrow filter=lfs diff=lfs merge=lfs -text
-*.bin filter=lfs diff=lfs merge=lfs -text
-*.bz2 filter=lfs diff=lfs merge=lfs -text
-*.ckpt filter=lfs diff=lfs merge=lfs -text
-*.ftz filter=lfs diff=lfs merge=lfs -text
-*.gz filter=lfs diff=lfs merge=lfs -text
-*.h5 filter=lfs diff=lfs merge=lfs -text
-*.joblib filter=lfs diff=lfs merge=lfs -text
-*.lfs.* filter=lfs diff=lfs merge=lfs -text
-*.mlmodel filter=lfs diff=lfs merge=lfs -text
-*.model filter=lfs diff=lfs merge=lfs -text
-*.msgpack filter=lfs diff=lfs merge=lfs -text
-*.npy filter=lfs diff=lfs merge=lfs -text
-*.npz filter=lfs diff=lfs merge=lfs -text
-*.onnx filter=lfs diff=lfs merge=lfs -text
-*.ot filter=lfs diff=lfs merge=lfs -text
-*.parquet filter=lfs diff=lfs merge=lfs -text
-*.pb filter=lfs diff=lfs merge=lfs -text
-*.pickle filter=lfs diff=lfs merge=lfs -text
-*.pkl filter=lfs diff=lfs merge=lfs -text
-*.pt filter=lfs diff=lfs merge=lfs -text
-*.pth filter=lfs diff=lfs merge=lfs -text
-*.rar filter=lfs diff=lfs merge=lfs -text
-*.safetensors filter=lfs diff=lfs merge=lfs -text
-saved_model/**/* filter=lfs diff=lfs merge=lfs -text
-*.tar.* filter=lfs diff=lfs merge=lfs -text
-*.tar filter=lfs diff=lfs merge=lfs -text
-*.tflite filter=lfs diff=lfs merge=lfs -text
-*.tgz filter=lfs diff=lfs merge=lfs -text
-*.wasm filter=lfs diff=lfs merge=lfs -text
-*.xz filter=lfs diff=lfs merge=lfs -text
-*.zip filter=lfs diff=lfs merge=lfs -text
-*.zst filter=lfs diff=lfs merge=lfs -text
-*tfevents* filter=lfs diff=lfs merge=lfs -text
+faiss_index/index.faiss filter=lfs diff=lfs merge=lfs -text
+faiss_index/index.pkl filter=lfs diff=lfs merge=lfs -text

.gitignore

@@ -0,0 +1,216 @@
+# Byte-compiled / optimized / DLL files
+__pycache__/
+*.py[codz]
+*$py.class
+
+# C extensions
+*.so
+
+# Distribution / packaging
+.Python
+build/
+develop-eggs/
+dist/
+downloads/
+eggs/
+.eggs/
+lib/
+lib64/
+parts/
+sdist/
+var/
+wheels/
+share/python-wheels/
+*.egg-info/
+.installed.cfg
+*.egg
+MANIFEST
+
+# PyInstaller
+#   Usually these files are written by a python script from a template
+#   before PyInstaller builds the exe, so as to inject date/other infos into it.
+*.manifest
+*.spec
+
+# Installer logs
+pip-log.txt
+pip-delete-this-directory.txt
+
+# Unit test / coverage reports
+htmlcov/
+.tox/
+.nox/
+.coverage
+.coverage.*
+.cache
+nosetests.xml
+coverage.xml
+*.cover
+*.py.cover
+.hypothesis/
+.pytest_cache/
+cover/
+
+# Translations
+*.mo
+*.pot
+
+# Django stuff:
+*.log
+local_settings.py
+db.sqlite3
+db.sqlite3-journal
+
+# Flask stuff:
+instance/
+.webassets-cache
+
+# Scrapy stuff:
+.scrapy
+
+# Sphinx documentation
+docs/_build/
+
+# PyBuilder
+.pybuilder/
+target/
+
+# Jupyter Notebook
+.ipynb_checkpoints
+
+# IPython
+profile_default/
+ipython_config.py
+
+# pyenv
+#   For a library or package, you might want to ignore these files since the code is
+#   intended to run in multiple environments; otherwise, check them in:
+# .python-version
+
+# pipenv
+#   According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control.
+#   However, in case of collaboration, if having platform-specific dependencies or dependencies
+#   having no cross-platform support, pipenv may install dependencies that don't work, or not
+#   install all needed dependencies.
+# Pipfile.lock
+
+# UV
+#   Similar to Pipfile.lock, it is generally recommended to include uv.lock in version control.
+#   This is especially recommended for binary packages to ensure reproducibility, and is more
+#   commonly ignored for libraries.
+# uv.lock
+
+# poetry
+#   Similar to Pipfile.lock, it is generally recommended to include poetry.lock in version control.
+#   This is especially recommended for binary packages to ensure reproducibility, and is more
+#   commonly ignored for libraries.
+#   https://python-poetry.org/docs/basic-usage/#commit-your-poetrylock-file-to-version-control
+# poetry.lock
+# poetry.toml
+
+# pdm
+#   Similar to Pipfile.lock, it is generally recommended to include pdm.lock in version control.
+#   pdm recommends including project-wide configuration in pdm.toml, but excluding .pdm-python.
+#   https://pdm-project.org/en/latest/usage/project/#working-with-version-control
+# pdm.lock
+# pdm.toml
+.pdm-python
+.pdm-build/
+
+# pixi
+#   Similar to Pipfile.lock, it is generally recommended to include pixi.lock in version control.
+# pixi.lock
+#   Pixi creates a virtual environment in the .pixi directory, just like venv module creates one
+#   in the .venv directory. It is recommended not to include this directory in version control.
+.pixi
+
+# PEP 582; used by e.g. github.com/David-OConnor/pyflow and github.com/pdm-project/pdm
+__pypackages__/
+
+# Celery stuff
+celerybeat-schedule
+celerybeat.pid
+
+# Redis
+*.rdb
+*.aof
+*.pid
+
+# RabbitMQ
+mnesia/
+rabbitmq/
+rabbitmq-data/
+
+# ActiveMQ
+activemq-data/
+
+# SageMath parsed files
+*.sage.py
+
+# Environments
+.env
+.envrc
+.venv
+env/
+venv/
+ENV/
+env.bak/
+venv.bak/
+
+# Spyder project settings
+.spyderproject
+.spyproject
+
+# Rope project settings
+.ropeproject
+
+# mkdocs documentation
+/site
+
+# mypy
+.mypy_cache/
+.dmypy.json
+dmypy.json
+
+# Pyre type checker
+.pyre/
+
+# pytype static type analyzer
+.pytype/
+
+# Cython debug symbols
+cython_debug/
+
+# PyCharm
+#   JetBrains specific template is maintained in a separate JetBrains.gitignore that can
+#   be found at https://github.com/github/gitignore/blob/main/Global/JetBrains.gitignore
+#   and can be added to the global gitignore or merged into this file.  For a more nuclear
+#   option (not recommended) you can uncomment the following to ignore the entire idea folder.
+# .idea/
+
+# Abstra
+#   Abstra is an AI-powered process automation framework.
+#   Ignore directories containing user credentials, local state, and settings.
+#   Learn more at https://abstra.io/docs
+.abstra/
+
+# Visual Studio Code
+#   Visual Studio Code specific template is maintained in a separate VisualStudioCode.gitignore 
+#   that can be found at https://github.com/github/gitignore/blob/main/Global/VisualStudioCode.gitignore
+#   and can be added to the global gitignore or merged into this file. However, if you prefer, 
+#   you could uncomment the following to ignore the entire vscode folder
+# .vscode/
+
+# Ruff stuff:
+.ruff_cache/
+
+# PyPI configuration file
+.pypirc
+
+# Marimo
+marimo/_static/
+marimo/_lsp/
+__marimo__/
+
+# Streamlit
+.streamlit/secrets.toml

.python-version

@@ -0,0 +1 @@
+3.12

README.md

@@ -1,14 +0,0 @@
----
-title: Nlp Assignment
-emoji: 👁
-colorFrom: red
-colorTo: pink
-sdk: gradio
-sdk_version: 5.49.1
-app_file: app.py
-pinned: false
-license: mit
-short_description: A RAG based academic paper recommendation system
----
-
-Check out the configuration reference at https://huggingface.co/docs/hub/spaces-config-reference

faiss_index/index.pkl

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:6272ece86301fc582e0a66a6d4e035e90b9635656fb8b72a821da8541c244ef7
+size 31873745

main.py

@@ -0,0 +1,135 @@
+import os
+import getpass
+import torch
+from dotenv import load_dotenv
+import gradio as gr
+import faiss
+from typing import List, TypedDict
+from sentence_transformers import CrossEncoder
+from langchain_community.vectorstores import FAISS
+from langchain_community.docstore.in_memory import InMemoryDocstore
+from langchain_core.prompts import ChatPromptTemplate
+from tabulate import tabulate
+from langchain.chat_models import init_chat_model
+from langchain_huggingface import HuggingFaceEmbeddings
+
+load_dotenv()
+
+PROJECT_PATH = os.path.dirname(os.path.abspath(__file__))
+FAISS_INDEX_DIR = os.path.join(PROJECT_PATH, "faiss_index")
+os.makedirs(FAISS_INDEX_DIR, exist_ok=True)
+EMBEDDING_MODEL = 'sentence-transformers/all-MiniLM-L6-v2'
+LLM_MODEL = 'gemini-2.5-flash'
+RERANKER_MODEL = "BAAI/bge-reranker-base"
+device = "cuda" if torch.cuda.is_available() else "cpu"
+reranker_model = CrossEncoder(RERANKER_MODEL, device=device)
+
+if not os.environ.get("GOOGLE_API_KEY"):
+    os.environ["GOOGLE_API_KEY"] = getpass.getpass("Enter API key for Google Gemini: ")
+
+embeddings = HuggingFaceEmbeddings(model_name=EMBEDDING_MODEL)
+llm = init_chat_model(LLM_MODEL, model_provider="google_genai")
+
+print("Loading FAISS index...")
+vector_store = FAISS.load_local(
+    FAISS_INDEX_DIR,
+    embeddings=embeddings,
+    allow_dangerous_deserialization=True,
+)
+
+print("FAISS index loaded successfully")
+
+class State(TypedDict):
+    question: str
+    context: List
+    answer: str
+
+def retrieve(question, faiss_broad_k=30, metadata_min_keep=8, top_k=5, semantic_threshold=0.35):
+    docs_with_scores = vector_store.similarity_search_with_score(question, k=faiss_broad_k)
+    print(f"[retrieve] FAISS returned {len(docs_with_scores)} candidates")
+
+    SCOPE_B_KEYWORDS = [
+        "machine learning", "artificial intelligence", "deep learning", "robotics",
+        "data science", "neural network", "quantum computing", "automation",
+        "computer vision", "nlp", "natural language", "algorithm", "software",
+        "engineering", "big data", "reinforcement learning"
+    ]
+
+    meta_filtered = []
+    for doc, _ in docs_with_scores:
+        text = f"{doc.metadata.get('title','')} {doc.metadata.get('concepts','')} {doc.page_content}".lower()
+        if any(k in text for k in SCOPE_B_KEYWORDS):
+            meta_filtered.append(doc)
+
+    if len(meta_filtered) == 0:
+        qlow = question.lower()
+        if not any(k in qlow for k in SCOPE_B_KEYWORDS):
+            return []
+        meta_filtered = [doc for doc, _ in docs_with_scores[:metadata_min_keep]]
+
+    rerank_inputs = [[question, doc.page_content] for doc in meta_filtered]
+    scores = reranker_model.predict(rerank_inputs)
+    reranked = sorted(zip(meta_filtered, scores), key=lambda x: x[1], reverse=True)
+
+    filtered = [(doc, s) for doc, s in reranked if s >= semantic_threshold]
+    final_docs = [doc for doc, _ in filtered[:top_k]]
+
+    return final_docs
+
+def generate_with_table(question: str, docs: List):
+    if not docs:
+        return f"I cannot find anything based on the search term **{question}**."
+
+    rows = []
+    for d in docs:
+        m = d.metadata
+        rows.append([
+            m.get("title", ""),
+            m.get("pub_year", ""),
+            m.get("authors", ""),
+            m.get("concepts", "")
+        ])
+    headers = ["Title", "Year", "Authors", "Concepts"]
+    papers_table = tabulate(rows, headers=headers, tablefmt="pipe")
+
+    docs_content = "\n\n".join(doc.page_content for doc in docs)
+
+    prompt_template = ChatPromptTemplate.from_messages([
+        (
+            "system",
+            "You are an expert RAG system. Answer the user's question based ONLY on the provided context. "
+            "After your answer, append a Markdown table of the retrieved papers. "
+            "If there are results: say how many you found for [Search term]. "
+            "If none: say 'I cannot find anything based on the search term [Search term]'. "
+            "Output format: [Summary] \n\n [Markdown Table]. Context: {context}"
+        ),
+        ("human", "Question: {question}. Table of papers: \n\n{papers_table}"),
+    ])
+
+    messages = prompt_template.invoke({
+        "question": question,
+        "context": docs_content,
+        "papers_table": papers_table
+    })
+
+    response = llm.invoke(messages)
+    return response.content
+
+def rag_pipeline(question: str):
+    if not question.strip():
+        return "Please enter a question."
+
+    docs = retrieve(question)
+    response = generate_with_table(question, docs)
+    return response
+
+demo = gr.Interface(
+    fn=rag_pipeline,
+    inputs=gr.Textbox(label="Enter your research query:", placeholder="e.g., deep learning in robotics"),
+    outputs=gr.Markdown(label="Response"),
+    title="📚 Research Paper RAG Assistant",
+    description="Retrieves and summarizes papers related to your query using FAISS, CrossEncoder, and Gemini."
+)
+
+if __name__ == "__main__":
+    demo.launch()

pyproject.toml

@@ -0,0 +1,28 @@
+[project]
+name = "nlp-assesment"
+version = "0.1.0"
+description = "Add your description here"
+readme = "README.md"
+requires-python = ">=3.12"
+dependencies = [
+    "accelerate>=1.11.0",
+    "bitsandbytes>=0.48.1",
+    "dotenv>=0.9.9",
+    "faiss-cpu>=1.12.0",
+    "gradio>=5.23.1",
+    "hf-xet>=1.1.10",
+    "langchain>=1.0.0a14",
+    "langchain-chroma>=0.2.6",
+    "langchain-community>=0.3.31",
+    "langchain-experimental>=0.0.42",
+    "langchain-google-genai>=3.0.0",
+    "langchain-huggingface>=1.0.0",
+    "libsql>=0.1.11",
+    "neo4j>=6.0.2",
+    "pandas>=2.3.3",
+    "pydantic>=2.12.2",
+    "sentence-transformers>=5.1.1",
+    "tabulate>=0.9.0",
+    "torch>=2.8.0",
+    "transformers>=4.57.1",
+]