final try 1

.gitattributes

@@ -1,35 +0,0 @@
-*.7z filter=lfs diff=lfs merge=lfs -text
-*.arrow filter=lfs diff=lfs merge=lfs -text
-*.bin filter=lfs diff=lfs merge=lfs -text
-*.bz2 filter=lfs diff=lfs merge=lfs -text
-*.ckpt filter=lfs diff=lfs merge=lfs -text
-*.ftz filter=lfs diff=lfs merge=lfs -text
-*.gz filter=lfs diff=lfs merge=lfs -text
-*.h5 filter=lfs diff=lfs merge=lfs -text
-*.joblib filter=lfs diff=lfs merge=lfs -text
-*.lfs.* filter=lfs diff=lfs merge=lfs -text
-*.mlmodel filter=lfs diff=lfs merge=lfs -text
-*.model filter=lfs diff=lfs merge=lfs -text
-*.msgpack filter=lfs diff=lfs merge=lfs -text
-*.npy filter=lfs diff=lfs merge=lfs -text
-*.npz filter=lfs diff=lfs merge=lfs -text
-*.onnx filter=lfs diff=lfs merge=lfs -text
-*.ot filter=lfs diff=lfs merge=lfs -text
-*.parquet filter=lfs diff=lfs merge=lfs -text
-*.pb filter=lfs diff=lfs merge=lfs -text
-*.pickle filter=lfs diff=lfs merge=lfs -text
-*.pkl filter=lfs diff=lfs merge=lfs -text
-*.pt filter=lfs diff=lfs merge=lfs -text
-*.pth filter=lfs diff=lfs merge=lfs -text
-*.rar filter=lfs diff=lfs merge=lfs -text
-*.safetensors filter=lfs diff=lfs merge=lfs -text
-saved_model/**/* filter=lfs diff=lfs merge=lfs -text
-*.tar.* filter=lfs diff=lfs merge=lfs -text
-*.tar filter=lfs diff=lfs merge=lfs -text
-*.tflite filter=lfs diff=lfs merge=lfs -text
-*.tgz filter=lfs diff=lfs merge=lfs -text
-*.wasm filter=lfs diff=lfs merge=lfs -text
-*.xz filter=lfs diff=lfs merge=lfs -text
-*.zip filter=lfs diff=lfs merge=lfs -text
-*.zst filter=lfs diff=lfs merge=lfs -text
-*tfevents* filter=lfs diff=lfs merge=lfs -text

.gitignore

@@ -0,0 +1,6 @@
+__pycache__
+/app/temp_storage
+/database
+/new_env
+/prompt.txt
+/app/key.py

Dockerfile

@@ -1,14 +1,42 @@
-FROM python:3.9
-RUN useradd -m -u 1000 user
-USER user
-ENV PATH="/home/user/.local/bin:$PATH"
+# syntax=docker/dockerfile:1
+FROM python:3.11-slim-bookworm
+
+# avoid root, but Spaces run as root so this is informational
+RUN addgroup --system app && adduser --system --ingroup app app
+
 WORKDIR /app
-COPY --chown=user ./requirements.txt requirements.txt
-RUN pip install --no-cache-dir --upgrade -r requirements.txt
+
+# system deps for Qdrant and psycopg2, cleanup
+RUN apt-get update \
+ && apt-get install -y --no-install-recommends build-essential wget ca-certificates \
+ && apt-get clean \
+ && rm -rf /var/lib/apt/lists/*
+
+# copy and install Python reqs
+COPY app/requirements.txt /app/requirements.txt
+RUN pip install --no-cache-dir -r /app/requirements.txt
+
+# download Qdrant binary
 RUN wget https://github.com/qdrant/qdrant/releases/download/v1.11.5/qdrant-x86_64-unknown-linux-gnu.tar.gz \
-    && tar -xzf qdrant-x86_64-unknown-linux-gnu.tar.gz \
-    && mv qdrant /home/user/.local/bin/qdrant \
-    && rm qdrant-x86_64-unknown-linux-gnu.tar.gz
-COPY --chown=user . /app
-RUN chmod +x start.sh
-CMD ["./start.sh"]
+ && tar -xzf qdrant-x86_64-unknown-linux-gnu.tar.gz \
+ && mv qdrant /usr/local/bin/qdrant \
+ && rm qdrant-x86_64-unknown-linux-gnu.tar.gz
+
+# copy your application code
+COPY app /app/app
+
+# bring in start script
+COPY start.sh /app/start.sh
+RUN chmod +x /app/start.sh
+
+# where SQLite DB and temp files will live (persisted by HF)
+RUN mkdir -p /mnt/data/app_database /mnt/data/temp_storage
+
+# expose HF-standard port
+EXPOSE 7860
+
+# env var for SQLAlchemy
+ENV DATABASE_URL="sqlite:////mnt/data/app_database/app.db"
+
+# launch both Qdrant and Uvicorn
+CMD ["./start.sh"]

LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2025 Danil Popov
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

README.md

@@ -1,12 +1,74 @@
----
-title: Test
-emoji: 🏢
-colorFrom: gray
-colorTo: yellow
-sdk: docker
-pinned: false
-license: mit
-short_description: test
----
-
-Check out the configuration reference at https://huggingface.co/docs/hub/spaces-config-reference
+# The-Ultimate-RAG
+
+## Overview
+
+[S25] The Ultimate RAG is an Innopolis University software project that generates cited responses from a local database.
+
+## Prerequisites
+
+Before you begin, ensure the following is installed on your machine:
+- [Python](https://www.python.org/) 
+- [Docker](https://www.docker.com/get-started/)
+
+## Installation
+
+1. **Clone the repository**
+    ```bash
+   git clone https://github.com/PopovDanil/The-Ultimate-RAG
+   cd The-Ultimate-RAG
+   ```
+2. **Set up a virtual environment (recommended)**
+
+   To isolate project dependencies and avoid conflicts, create a virtual environment:
+    - **On Unix/Linux/macOS:**
+   ```bash
+   python3 -m venv env
+   source env/bin/activate
+   ```
+    - **On Windows:**
+    ```bash
+    python -m venv env
+    env\Scripts\activate
+    ```
+3. **Install required libraries**
+
+   Within the activated virtual environment, install the dependencies:
+   ```bash
+   pip install -r ./app/requirements.txt
+   ```
+   *Note:* ensure you are in the virtual environment before running the command
+
+4. **Set up Docker**
+    - Ensure Docker is running on your machine
+    - Open a terminal, navigate to project directory, and run:
+    ```bash 
+    docker-compose up --build
+    ```
+   *Note:* The initial build may take 10–20 minutes, as it needs to download large language models and other
+   dependencies. 
+   Later launches will be much faster.
+   
+5. **Server access**
+
+   Once the containers are running, visit `http://localhost:5050`. You should see the application’s welcome page
+
+To stop the application and shut down all containers, press `Ctrl+C` in the terminal where `docker-compose` is running, 
+and then run: 
+```bash
+   docker-compose down
+```
+
+## Usage
+
+1. **Upload your file:** click the upload button and select a supported file (`.txt`, `.doc`, `.docx`, or `.pdf`)
+2. **Ask a question**: Once the file is processed, type your question into the prompt box and submit.
+3. **Receive your answer** 
+
+**A note on performance**
+
+Response generation is a computationally intensive task.
+The time to receive an answer may vary depending on your machine's hardware and the complexity of the query.
+
+## License
+
+This project is licensed under the [MIT License](LICENSE).

app.py

@@ -1,114 +0,0 @@
-from fastapi import FastAPI, Depends, HTTPException
-from sqlalchemy import Column, Integer, String, create_engine
-from sqlalchemy.ext.declarative import declarative_base
-from sqlalchemy.orm import sessionmaker, Session
-from pydantic import BaseModel
-from typing import List
-from qdrant_client import QdrantClient
-from qdrant_client.models import Distance, VectorParams, PointStruct
-import os
-
-# Initialize FastAPI app
-api = FastAPI()
-
-# Get the database URL from an environment variable
-DATABASE_URL = os.getenv("DATABASE_URL")
-
-# Set up SQLAlchemy engine and session for PostgreSQL
-engine = create_engine(DATABASE_URL)
-SessionLocal = sessionmaker(autocommit=False, autoflush=False, bind=engine)
-
-# Define the SQLAlchemy base
-Base = declarative_base()
-
-# Define the User model for the PostgreSQL database
-class User(Base):
-    __tablename__ = "users"
-    id = Column(Integer, primary_key=True, index=True)
-    name = Column(String, index=True)
-    email = Column(String, unique=True, index=True)
-
-# Set up Qdrant client
-client = QdrantClient("localhost", port=6333)
-
-# Startup event to create database tables and initialize Qdrant collection
-@api.on_event("startup")
-def startup_event():
-    # Create PostgreSQL database tables
-    Base.metadata.create_all(bind=engine)
-    # Initialize Qdrant collection if it doesn’t exist
-    collections = client.get_collections()
-    if "my_collection" not in [col.name for col in collections.collections]:
-        client.create_collection(
-            collection_name="my_collection",
-            vectors_config=VectorParams(size=10, distance=Distance.COSINE),
-        )
-
-# Pydantic models for PostgreSQL user endpoints
-class UserCreate(BaseModel):
-    name: str
-    email: str
-
-class UserResponse(BaseModel):
-    id: int
-    name: str
-    email: str
-
-    class Config:
-        orm_mode = True
-
-# Pydantic models for Qdrant endpoints
-class AddPointRequest(BaseModel):
-    id: int
-    vector: List[float]
-
-class SearchRequest(BaseModel):
-    vector: List[float]
-    top_k: int = 5
-
-# Dependency to get the PostgreSQL database session
-def get_db():
-    db = SessionLocal()
-    try:
-        yield db
-    finally:
-        db.close()
-
-# Root endpoint for testing
-@api.get("/")
-def root():
-    return {"message": "Hello World"}
-
-# Endpoint to create a user in PostgreSQL
-@api.post("/users/", response_model=UserResponse)
-def create_user(user: UserCreate, db: Session = Depends(get_db)):
-    db_user = User(name=user.name, email=user.email)
-    db.add(db_user)
-    db.commit()
-    db.refresh(db_user)
-    return db_user
-
-# Endpoint to get all users from PostgreSQL
-@api.get("/users/", response_model=list[UserResponse])
-def read_users(db: Session = Depends(get_db)):
-    users = db.query(User).all()
-    return users
-
-# Endpoint to add a point to Qdrant
-@api.post("/add_point")
-def add_point(request: AddPointRequest):
-    client.upsert(
-        collection_name="my_collection",
-        points=[PointStruct(id=request.id, vector=request.vector)],
-    )
-    return {"status": "ok"}
-
-# Endpoint to search in Qdrant
-@api.post("/search")
-def search(request: SearchRequest):
-    results = client.search(
-        collection_name="my_collection",
-        query_vector=request.vector,
-        limit=request.top_k,
-    )
-    return {"results": [{"id": res.id, "score": res.score} for res in results]}

app/api.py

@@ -0,0 +1,295 @@
+from fastapi import FastAPI, UploadFile, Form, File, HTTPException, Response, Request, Depends
+import uuid
+from app.backend.models.users import User
+from fastapi.staticfiles import StaticFiles
+import os
+from app.rag_generator import RagSystem
+from fastapi.responses import HTMLResponse, FileResponse, RedirectResponse
+from app.settings import base_path, url_user_not_required
+from typing import Optional
+from app.response_parser import add_links
+from app.document_validator import path_is_valid
+from app.backend.controllers.users import create_user, authenticate_user, check_cookie, clear_cookie, get_current_user
+from app.backend.controllers.schemas import SUser
+from app.backend.controllers.chats import create_new_chat
+from fastapi.templating import Jinja2Templates
+
+# TODO: implement a better TextHandler
+# TODO: optionally implement DocHandler
+
+api = FastAPI()
+rag = None
+api.mount("/pdfs", StaticFiles(directory=os.path.join(base_path, "temp_storage", "pdfs")), name="pdfs")
+api.mount("/static", StaticFiles(directory=os.path.join(base_path, "frontend", "static")), name="static")
+templates = Jinja2Templates(directory=os.path.join(base_path, "frontend", "templates"))
+
+def initialize_rag() -> RagSystem:
+    global rag
+    if rag is None:
+        rag = RagSystem()
+    return rag
+
+
+'''
+Updates response context and adds context of navbar (role, instance(or none)) and footer (none)
+'''
+def extend_context(context: dict):
+    user = get_current_user(context.get("request"))
+    navbar = {
+        "navbar": True,
+        "navbar_path": "components/navbar.html",
+        "navbar_context": {
+            "user": {
+                "role": "user" if user else "guest",
+                "instance": user
+            }
+        }
+    }
+    footer = {
+        "footer": False,
+        "footer_context": None
+    }
+
+    context.update(**navbar)
+    context.update(**footer)
+
+    return context
+
+
+def PDFHandler(request: Request, path: str, page: int) -> HTMLResponse:
+    filename = os.path.basename(path)
+    url_path = f"/pdfs/{filename}"
+    current_template = "pages/show_pdf.html"
+    return templates.TemplateResponse(
+        current_template, 
+        extend_context({
+        "request": request, 
+        "page": str(page or 1), 
+        "url_path": url_path,
+        "user": get_current_user(request)
+        })
+    )
+
+
+def TextHandler(request: Request, path: str, lines: str) -> HTMLResponse:
+    file_content = ""
+    with open(path, "r") as f:
+        file_content = f.read()
+
+    start_line, end_line = map(int, lines.split('-'))
+
+    text_before_citation = []
+    text_after_citation = []
+    citation = []
+    anchor_added = False
+
+    for index, line in enumerate(file_content.split('\n')):
+        if line == "" or line == "\n":
+            continue
+        if index + 1 < start_line:
+            text_before_citation.append(line)
+        elif end_line < index + 1:
+            text_after_citation.append(line)
+        else:
+            anchor_added = True
+            citation.append(line)
+
+    current_template = "pages/show_text.html"
+
+    return templates.TemplateResponse(
+        current_template, 
+        extend_context({
+        "request": request, 
+        "text_before_citation": text_before_citation,
+        "text_after_citation": text_after_citation,
+        "citation": citation,
+        "anchor_added": anchor_added,
+        "user": get_current_user(request)
+        })
+    )
+
+
+'''
+Optional handler
+'''
+def DocHandler():
+    pass
+
+
+# <--------------------------------- Middleware --------------------------------->
+# NOTE: carefully read documentation to require_user
+
+'''
+Special class to have an opportunity to redirect user to login page in middleware
+'''
+class AwaitableResponse:
+    def __init__(self, response: Response):
+        self.response = response
+
+    def __await__(self):
+        yield
+        return self.response
+
+
+'''
+TODO: remove KOSTYLY -> find better way to skip requesting to login while showing pdf
+
+Middleware that requires user to log in into the system before accessing any utl
+
+NOTE: For now it is applied to all routes, but if you want to skip any, add it to the
+url_user_not_required list in settings.py (/ should be removed)
+'''
+@api.middleware("http")
+async def require_user(request: Request, call_next):
+    print(request.url.path, request.method)
+
+    awaitable_response = AwaitableResponse(RedirectResponse("/login", status_code=303))
+    stripped_path = request.url.path.strip('/')
+
+    if stripped_path in url_user_not_required \
+        or stripped_path.startswith("pdfs") \
+        or "static/styles.css" in stripped_path \
+        or "favicon.ico" in stripped_path:
+        return await call_next(request)
+
+    user = get_current_user(request)
+    if user is None:
+        return await awaitable_response
+    
+    response = await call_next(request)
+    return response
+
+
+# <--------------------------------- Common routes --------------------------------->
+# @api.get("/")
+# def root(request: Request):
+#     current_template = "pages/main.html"
+#     return templates.TemplateResponse(current_template, extend_context({"request": request}))
+
+
+@api.get("/")
+def root(request: Request):
+    current_template = "pages/chat.html"
+    return templates.TemplateResponse(current_template, 
+        extend_context({
+        "request": request, 
+        "user": get_current_user(request)
+        })
+    )
+
+
+@api.post("/message_with_docs")
+async def create_prompt(files: list[UploadFile] = File(...), prompt: str = Form(...)):
+    docs = []
+    rag = initialize_rag()
+
+    try:
+
+        for file in files:
+            content = await file.read()
+            temp_storage = os.path.join(base_path, "temp_storage")
+            os.makedirs(temp_storage, exist_ok=True)
+
+            if file.filename.endswith('.pdf'):
+                saved_file = os.path.join(temp_storage, "pdfs", str(uuid.uuid4()) + ".pdf")
+            else:
+                saved_file = os.path.join(temp_storage, str(uuid.uuid4()) + "." + file.filename.split('.')[-1])
+
+            with open(saved_file, "wb") as f:
+                f.write(content)
+
+            docs.append(saved_file)
+
+        if len(files) > 0:
+            rag.upload_documents(docs)
+
+        response_raw = rag.generate_response(user_prompt=prompt)
+        response = add_links(response_raw)
+
+        return {"response": response, "status": 200}
+
+    except Exception as e:
+        print("!!!ERROR!!!")
+        print(e)
+
+    # finally:
+    #     for file in files:
+    #         temp_storage = os.path.join(base_path, "temp_storage")
+    #         saved_file = os.path.join(temp_storage, file.filename)
+    #         os.remove(saved_file)
+
+
+@api.get("/viewer")
+def show_document(request: Request, path: str, page: Optional[int] = 1, lines: Optional[str] = "1-1", start: Optional[int] = 0):
+    if not path_is_valid(path):
+        return HTTPException(status_code=404, detail="Document not found")
+
+    ext = path.split(".")[-1]
+    if ext == 'pdf':
+        return PDFHandler(request, path=path, page=page)
+    elif ext in ('txt', 'csv', 'md'):
+        return TextHandler(request, path=path, lines=lines)
+    elif ext in ('docx', 'doc'):
+        return TextHandler(request, path=path, lines=lines)  # should be a bit different handler
+    else:
+        return FileResponse(path=path)
+
+
+# <--------------------------------- Get --------------------------------->
+@api.get("/new_user")
+def new_user(request: Request):
+    current_template = "pages/registration.html"
+    return templates.TemplateResponse(current_template, extend_context({"request": request}))
+
+
+@api.get("/login")
+def login(request: Request):
+    current_template = "pages/login.html"
+    return templates.TemplateResponse(current_template, extend_context({"request": request}))
+
+
+@api.get("/cookie_test")
+def test_cookie(request: Request):
+    return check_cookie(request)
+
+
+'''
+Use only for testing. For now, provides user info for logged ones, and redirects to
+login in other case
+'''
+@api.get("/test")
+def test(request: Request, user: User = Depends(get_current_user)):
+    return {
+        "user": {
+            "email": user.email,
+            "password_hash": user.password_hash,
+            # "chats": user.chats, # Note: it will rise error since due to the optimization associated fields are not loaded
+            # it is just a reference, but the session is closed, however you are trying to get access to the data through this session
+            }
+        }
+
+
+@api.get("/chats/id={chat_id}")
+def show_chat(chat_id: int):
+    return {"chat_id": chat_id}
+
+
+@api.get("/logout")
+def logout(response: Response):
+    return clear_cookie(response)
+# <--------------------------------- Post --------------------------------->
+@api.post("/new_user")
+def new_user(response: Response, user: SUser):
+    return create_user(response, user.email, user.password)
+
+
+
+@api.post("/login")
+def login(response: Response, user: SUser):
+    return authenticate_user(response, user.email, user.password)
+
+
+@api.post("/new_chat")
+def create_chat(request: Request, title: Optional[str] = "new chat", user: User = Depends(get_current_user)):
+    url = create_new_chat(title, user)
+    return RedirectResponse(url, status_code=303)

app/backend/controllers/base_controller.py

@@ -0,0 +1,4 @@
+from sqlalchemy import create_engine
+from app.settings import postgres_client_config
+
+engine = create_engine(**postgres_client_config)

app/backend/controllers/chats.py

@@ -0,0 +1,5 @@
+from app.backend.models.users import User
+from app.backend.models.chats import new_chat
+
+def create_new_chat(title: str | None, user: User) -> str:
+    return f"/chats/id={new_chat(title, user)}"

app/backend/controllers/schemas.py

@@ -0,0 +1,32 @@
+from pydantic import BaseModel, Field, EmailStr, field_validator
+import re
+
+class SUser(BaseModel):
+    email: EmailStr
+    password: str = Field(default=..., min_length=8, max_length=32)
+
+    @field_validator('password', mode='before')
+    def validate_password(cls, password_to_validate):
+        """
+        Validates the strength of the password.
+
+        The password **must** contain:
+            - At least one digit
+            - At least one special character
+            - At least one uppercase character
+            - At least one lowercase character
+        """
+
+        if not re.search(r"\d", password_to_validate):
+            raise ValueError("Password must contain at least one number.")
+
+        if not re.search(r"[!@#$%^&*()_+\-=\[\]{};:\'\",.<>?`~]", password_to_validate):
+            raise ValueError("Password must contain at least one special symbol.")
+
+        if not re.search(r"[A-Z]", password_to_validate):
+            raise ValueError("Password must contain at least one uppercase letter.")
+
+        if not re.search(r"[a-z]", password_to_validate):
+            raise ValueError("Password must contain at least one lowercase letter.")
+
+        return password_to_validate

app/backend/controllers/users.py

@@ -0,0 +1,145 @@
+from app.backend.models.users import User, add_new_user, find_user_by_email, find_user_by_access_string, update_user
+from bcrypt import gensalt, hashpw, checkpw
+from app.settings import very_secret_pepper, jwt_algorithm, max_cookie_lifetime
+from fastapi import HTTPException
+import jwt
+from datetime import datetime, timedelta
+from fastapi import Response, Request
+from secrets import token_urlsafe
+import hmac
+import hashlib
+
+# A vot nado bilo izuchat kak web dev rabotaet
+
+'''
+Creates a jwt token by access string
+
+Param:
+access_string - randomly (safe methods) generated string (by default - 16 len) 
+expires_delta - time in seconds, defines a token lifetime
+
+Returns:
+string with 4 sections (valid jwt token)
+'''
+def create_access_token(access_string: str, expires_delta: timedelta = timedelta(seconds=max_cookie_lifetime)) -> str:
+    token_payload = {
+        "access_string": access_string,
+    }
+
+    token_payload.update({"exp": datetime.now() + expires_delta})
+    encoded_jwt: str = jwt.encode(token_payload, very_secret_pepper, algorithm=jwt_algorithm)
+
+    return encoded_jwt
+        
+
+'''
+Safely creates random string of 16 chars
+'''
+def create_access_string() -> str:
+    return token_urlsafe(16)
+
+
+'''
+Hashes access string using hmac and sha256
+
+We can not use the same methods as we do to save password
+since we need to know a salt to get similar hash, but since
+we put a raw string (non-hashed) we won't be able to guess
+salt
+'''
+def hash_access_string(string: str) -> str:
+    return hmac.new(
+        key=very_secret_pepper.encode("utf-8"),
+        msg=string.encode("utf-8"),
+        digestmod=hashlib.sha256
+    ).hexdigest()
+
+
+'''
+Creates a new user and sets a cookie with jwt token
+
+Params:
+response - needed to set a cookie
+...
+
+Returns:
+Dict to send a response in JSON 
+'''
+def create_user(response: Response, email: str, password: str) -> dict:
+    user: User = find_user_by_email(email=email)
+    if user is not None:
+        return HTTPException(418, "The user with similar email already exists")
+    
+    salt: bytes = gensalt(rounds=16)
+    password_hashed: str = hashpw(password.encode("utf-8"), salt).decode("utf-8")
+    
+    access_string: str = create_access_string()
+    access_string_hashed: str = hash_access_string(string=access_string)
+
+    add_new_user(email=email, password_hash=password_hashed, access_string_hash=access_string_hashed)
+
+    access_token: str = create_access_token(access_string=access_string)
+    response.set_cookie(key="access_token", value=access_token, path='/', max_age=max_cookie_lifetime, httponly=True)
+
+    return {"status": "ok"}
+
+
+
+'''
+Finds user by email. If user is found, sets a cookie with token
+'''
+def authenticate_user(response: Response, email: str, password: str) -> dict:
+    user: User = find_user_by_email(email=email)
+
+    if not user:
+        raise HTTPException(418, "User does not exists")
+
+    if not checkpw(password.encode('utf-8'), user.password_hash.encode('utf-8')):
+        raise HTTPException(418, "Wrong credentials")
+
+    access_string: str = create_access_string()
+    access_string_hashed: str = hash_access_string(string=access_string)
+
+    update_user(user, access_string_hash=access_string_hashed)
+    
+    access_token = create_access_token(access_string)
+    response.set_cookie(key="access_token", value=access_token, path='/', max_age=max_cookie_lifetime, httponly=True)
+
+    return {"status": "ok"}
+
+
+'''
+Get user from token stored in cookies
+'''
+def get_current_user(request: Request) -> User | None:
+    token: str | None = request.cookies.get("access_token")
+    if not token:
+        return None
+    
+    access_string = jwt.decode(
+            jwt=bytes(token, encoding='utf-8'),
+            key=very_secret_pepper,
+            algorithms=[jwt_algorithm]
+        ).get('access_string')
+
+    user = find_user_by_access_string(hash_access_string(access_string))
+    if not user:
+        return None
+    
+    return user
+
+
+'''
+Checks if cookie with access token is present
+'''
+def check_cookie(request: Request) -> dict:
+    result = {"token": "No token is present"}
+    token = request.cookies.get("access_token")
+    if token:
+        result["token"] = token
+    return result
+
+
+def clear_cookie(response: Response) -> dict:
+    response.set_cookie(key="access_token", value="", httponly=True)
+    return {"status": "ok"}

app/backend/models/base_model.py

@@ -0,0 +1,9 @@
+from sqlalchemy.orm import DeclarativeBase
+from sqlalchemy import DateTime, Column
+from sqlalchemy.sql import func
+
+class Base(DeclarativeBase):
+    __abstract__ = True
+    created_at = Column("created_at", DateTime, default=func.now())
+    deleted_at = Column("deleted_at", DateTime, nullable=True)
+    updated_at = Column("updated_at", DateTime, nullable=True)

app/backend/models/chats.py

@@ -0,0 +1,25 @@
+from app.backend.models.base_model import Base
+from app.backend.models.users import User
+from sqlalchemy import Integer, String, Column, ForeignKey
+from sqlalchemy.orm import relationship, Session
+from app.backend.controllers.base_controller import engine
+
+class Chat(Base):
+    __tablename__ = "chats"
+    id = Column("id", Integer, autoincrement=True, primary_key=True, unique=True)
+    title = Column("title", String, nullable=True)
+    user_id = Column(Integer, ForeignKey("users.id"))
+    user = relationship("User", back_populates="chats")
+    messages = relationship("Message", back_populates="chat")
+
+
+def new_chat(title: str | None, user: User) -> int:
+    id = None
+    with Session(autoflush=False, bind=engine) as db:
+        new_chat = Chat(user_id=user.id, user=user)
+        if title:
+            new_chat.title = title
+        db.add(new_chat)
+        db.commit()
+        id = new_chat.id
+    return id

app/backend/models/db_service.py

@@ -0,0 +1,27 @@
+from app.backend.models.users import User
+from app.backend.models.chats import Chat
+from app.backend.models.messages import Message
+from app.backend.controllers.base_controller import engine
+from app.backend.models.base_model import Base
+
+
+def table_exists(name: str) -> bool:
+    return engine.dialect.has_table(engine, name)
+
+def create_tables() -> None:
+    Base.metadata.create_all(engine)
+
+def drop_tables() -> None:
+    # for now the order matters, so
+    # TODO: add cascade deletion for models
+    Message.__table__.drop(engine)
+    Chat.__table__.drop(engine)
+    User.__table__.drop(engine)
+
+def automigrate() -> None:
+    try:
+        drop_tables()
+    except Exception as e:
+        print(e)
+
+    create_tables()

app/backend/models/messages.py

@@ -0,0 +1,16 @@
+from app.backend.models.base_model import Base
+from sqlalchemy import Integer, String, Column, ForeignKey, Text
+from sqlalchemy.orm import relationship
+from app.backend.controllers.base_controller import engine
+
+class Message(Base):
+    __tablename__ = "messages"
+    id = Column("id", Integer, autoincrement=True, primary_key=True, unique=True)
+    content = Column("text", Text)
+    sender = Column("role", String)
+    chat_id = Column(Integer, ForeignKey("chats.id"))
+    chat = relationship("Chat", back_populates="messages")
+
+
+def new_message(chat_id: int, sender: str, content: str):
+    pass

app/backend/models/users.py

@@ -0,0 +1,47 @@
+from sqlalchemy import Column, String, Integer
+from sqlalchemy.orm import relationship, Session
+from app.backend.models.base_model import Base
+from app.backend.controllers.base_controller import engine
+
+class User(Base):
+    __tablename__ = "users"
+    id = Column("id", Integer, autoincrement=True, primary_key=True, unique=True)
+    email = Column("email", String, unique=True, nullable=False)
+    password_hash = Column("password_hash", String, nullable=False)
+    language = Column("language", String, default="English", nullable=False)
+    theme = Column("theme", String, default="light", nullable=False)
+    access_string_hash = Column("access_string_hash", String, nullable=True)
+    chats = relationship("Chat", back_populates="user")
+
+
+def add_new_user(email: str, password_hash: str, access_string_hash: str) -> None:
+    with Session(autoflush=False, bind=engine) as db:
+        db.add(User(email=email, password_hash=password_hash, access_string_hash=access_string_hash))
+        db.commit()
+
+
+def find_user_by_id(id: int) -> User | None:
+    with Session(autoflush=False, bind=engine) as db:
+        return db.query(User).where(User.id == id).first()
+    
+
+def find_user_by_email(email: str) -> User | None:
+    with Session(autoflush=False, bind=engine) as db:
+        return db.query(User).where(User.email == email).first()
+    
+
+def find_user_by_access_string(access_string_hash: str) -> User | None:
+    with Session(autoflush=False, bind=engine) as db:
+        return db.query(User).where(User.access_string_hash == access_string_hash).first()
+    
+
+def update_user(user: User, language: str = None, theme: str = None, access_string_hash: str = None) -> None:
+    with Session(autoflush=False, bind=engine) as db:
+        user = db.merge(user)
+        if language:
+            user.language = language
+        if theme:
+            user.theme = theme
+        if access_string_hash:
+            user.access_string_hash = access_string_hash
+        db.commit()

app/chunks.py

@@ -0,0 +1,47 @@
+import uuid  
+
+class Chunk:
+    '''
+    id -> unique number in uuid format, can be tried https://www.uuidgenerator.net/
+    start_index -> the index of the first char from the beginning of the original document
+
+    TODO: implement access modifiers and set of getters and setters
+    '''
+    def __init__(self, id: uuid.UUID, filename: str, page_number: int, start_index: int, start_line: int, end_line: int, text: str):
+        self.id: uuid.UUID = id
+        self.filename: str = filename
+        self.page_number: int = page_number
+        self.start_index: int = start_index
+        self.start_line: int = start_line
+        self.end_line: int = end_line
+        self.text: str = text
+
+
+    def get_raw_text(self) -> str:
+        return self.text
+    
+
+    def get_splitted_text(self) -> list[str]:
+        return self.text.split(" ")
+    
+    
+    def get_metadata(self) -> dict:
+        return {
+            "id": self.id,
+            "filename": self.filename,
+            "page_number": self.page_number,
+            "start_index": self.start_index,
+            "start_line": self.start_line,
+            "end_line": self.end_line,
+        }
+    
+
+    # TODO: remove kostyly
+    def __str__(self):
+        return (f"Chunk from {self.filename.split('/')[-1]}, "
+                f"page - {self.page_number}, "
+                f"start - {self.start_line}, "
+                f"end - {self.end_line}, "
+                f"and text - {self.text[:100]}... ({len(self.text)})\n"
+                )
+

app/database.py

@@ -0,0 +1,146 @@
+from qdrant_client import QdrantClient  # main component to provide the access to db
+from qdrant_client.http.models import ScoredPoint
+from qdrant_client.models import VectorParams, Distance, \
+    PointStruct  # VectorParams -> config of vectors that will be used as primary keys
+from app.models import Embedder  # Distance -> defines the metric
+from app.chunks import Chunk  # PointStruct -> instance that will be stored in db
+import numpy as np
+from uuid import UUID
+from app.settings import qdrant_client_config, max_delta
+import time
+
+# TODO: for now all documents are saved to one db, but what if user wants to get references from his own documents, so temp storage is needed
+
+class VectorDatabase:
+    def __init__(self, embedder: Embedder, host: str = "qdrant", port: int = 6333):
+        self.host: str = host
+        self.client: QdrantClient = self._initialize_qdrant_client()
+        self.collection_name: str = "document_chunks"
+        self.embedder: Embedder = embedder  # embedder is used to convert a user's query
+        self.already_stored: np.array[np.array] = np.array([]).reshape(0, embedder.get_vector_dimensionality()) # should be already normalized
+
+        if not self._check_collection_exists():
+            self._create_collection()
+
+
+    def store(self, chunks: list[Chunk], batch_size: int = 1000) -> None:
+        points: list[PointStruct] = []
+
+        vectors = self.embedder.encode([chunk.get_raw_text() for chunk in chunks])
+
+        for vector, chunk in zip(vectors, chunks):
+            if self.accept_vector(vector):
+                points.append(PointStruct(
+                    id=str(chunk.id),
+                    vector=vector,
+                    payload={"metadata": chunk.get_metadata(), "text": chunk.get_raw_text()}
+                ))
+
+        if len(points):
+            for group in range(0, len(points), batch_size):
+                self.client.upsert(
+                    collection_name=self.collection_name,
+                    points=points[group : group + batch_size],
+                    wait=False,
+                )
+
+
+    '''
+    Measures a cosine of angle between tow vectors
+    '''
+    def cosine_similarity(self, vec1, vec2):
+        return vec1 @ vec2 / (np.linalg.norm(vec1) * np.linalg.norm(vec2))
+
+
+    '''
+    Defines weather the vector should be stored in the db by searching for the most
+    similar one
+    '''
+    def accept_vector(self, vector: np.array) -> bool:
+        most_similar = self.client.query_points(
+            collection_name=self.collection_name,
+            query=vector,
+            limit=1,
+            with_vectors=True
+        ).points
+
+        if not len(most_similar):
+            return True
+        else:
+            most_similar = most_similar[0]
+
+        if 1 - self.cosine_similarity(vector, most_similar.vector) < max_delta:
+            return False
+        return True
+
+
+    '''
+    According to tests, re-ranker needs ~7-10 chunks to generate the most accurate hit
+
+    TODO: implement hybrid search
+    '''
+
+    def search(self, query: str, top_k: int = 5) -> list[Chunk]:
+        query_embedded: np.ndarray = self.embedder.encode(query)
+
+        points: list[ScoredPoint] = self.client.query_points(
+            collection_name=self.collection_name,
+            query=query_embedded,
+            limit=top_k
+        ).points
+
+        return [
+            Chunk(
+                id=UUID(point.payload.get("metadata", {}).get("id", "")),
+                filename=point.payload.get("metadata", {}).get("filename", ""),
+                page_number=point.payload.get("metadata", {}).get("page_number", 0),
+                start_index=point.payload.get("metadata", {}).get("start_index", 0),
+                start_line=point.payload.get("metadata", {}).get("start_line", 0),
+                end_line=point.payload.get("metadata", {}).get("end_line", 0),
+                text=point.payload.get("text", "")
+            ) for point in points
+        ]
+
+
+    def _initialize_qdrant_client(self, max_retries=5, delay=2) -> QdrantClient:
+        for attempt in range(max_retries):
+            try:
+                client = QdrantClient(**qdrant_client_config)
+                client.get_collections()
+                return client
+            except Exception as e:
+                if attempt == max_retries - 1:
+                    raise ConnectionError(
+                        f"Failed to connect to Qdrant server after {max_retries} attempts. "
+                        f"Last error: {str(e)}"
+                    )
+
+                print(f"Connection attempt {attempt + 1} out of {max_retries} failed. "
+                      f"Retrying in {delay} seconds...")
+
+                time.sleep(delay)
+                delay *= 2
+
+    def _check_collection_exists(self) -> bool:
+        try:
+            return self.client.collection_exists(self.collection_name)
+        except Exception as e:
+            raise ConnectionError(
+                f"Failed to check collection {self.collection_name} exists. Last error: {str(e)}"
+            )
+
+    def _create_collection(self) -> None:
+        try:
+            self.client.create_collection(
+                collection_name=self.collection_name,
+                vectors_config=VectorParams(
+                    size=self.embedder.get_vector_dimensionality(),
+                    distance=Distance.COSINE
+                )
+            )
+        except Exception as e:
+            raise RuntimeError(f"Failed to create collection {self.collection_name}: {str(e)}")
+
+    def __del__(self):
+        if hasattr(self, "client"):
+            self.client.close()

app/document_validator.py

@@ -0,0 +1,7 @@
+import os
+
+'''
+Checks if the given path is valid and file exists
+'''
+def path_is_valid(path: str) -> bool:
+    return os.path.exists(path)

app/frontend/static/styles.css

@@ -0,0 +1,206 @@
+.chat-container {
+    display: flex;
+    width: 100%;
+}
+
+.chat-body {
+    flex: 1;
+    background-color: #f8f9fa;
+    font-family: 'Segoe UI', Tahoma, Geneva, Verdana, sans-serif;
+}
+
+.hero-section {
+    background: linear-gradient(135deg, #6e8efb, #a777e3);
+    color: white;
+    border-radius: 0 0 20px 20px;
+    box-shadow: 0 4px 20px rgba(0,0,0,0.1);
+}
+
+.search-container {
+    max-width: 800px;
+    margin: 0 auto;
+}
+
+.search-box {
+    border-radius: 10px;
+    border: none;
+    padding: 15px 25px;
+    box-shadow: 0 4px 15px rgba(0,0,0,0.1);
+}
+
+.btn-search {
+    border-radius: 10px;
+    padding: 15px 30px;
+    background-color: #4e44ce;
+    border: none;
+}
+
+.file-upload {
+    background: white;
+    border-radius: 10px;
+    padding: 20px;
+    margin-bottom: 20px;
+    box-shadow: 0 4px 15px rgba(0,0,0,0.1);
+}
+
+.file-list {
+    margin-top: 10px;
+}
+
+.file-item {
+    background: #f1f3ff;
+    border-radius: 5px;
+    padding: 8px 12px;
+    margin-bottom: 5px;
+}
+
+.login-body {
+    font-family: Arial, sans-serif;
+    max-width: 400px;
+    margin: 0 auto;
+    padding: 20px;
+}
+
+.form-group {
+    margin-bottom: 15px;
+}
+
+.login-label {
+    display: block;
+    margin-bottom: 5px;
+    font-weight: bold;
+}
+
+.input-field {
+    width: 100%;
+    padding: 8px;
+    border: 1px solid #ddd;
+    border-radius: 4px;
+    box-sizing: border-box;
+}
+
+.login-button {
+    background-color: #4CAF50;
+    color: white;
+    padding: 10px 15px;
+    border: none;
+    border-radius: 4px;
+    cursor: pointer;
+    font-size: 16px;
+}
+
+.login-button :hover {
+    background-color: #45a049;
+}
+
+.error {
+    color: red;
+    font-size: 14px;
+    margin-top: 5px;
+}
+
+#pdf-container {
+    margin: 0 auto;
+    max-width: 100%;
+    overflow-x: auto;
+    text-align: center;
+    padding: 20px 0;
+}
+
+#pdf-canvas {
+    margin: 0 auto;
+    display: block;
+    max-width: 100%;
+    box-shadow: 0 0 5px rgba(0,0,0,0.2);
+}
+
+/* Fix the page input container layout */
+.page-input-container {
+    position: relative;
+    display: inline-flex;
+    align-items: center;
+}
+
+.page-input {
+    width: 50px;
+    padding: 8px 25px 8px 8px;  /* Right padding gives space for label */
+    text-align: center;
+    border: 1px solid #ddd;
+    border-radius: 4px;
+    -moz-appearance: textfield;  /* Hide number arrows in Firefox */
+}
+
+/* Hide number arrows in Chrome/Safari */
+.page-input::-webkit-outer-spin-button,
+.page-input::-webkit-inner-spin-button {
+    -webkit-appearance: none;
+    margin: 0;
+}
+
+.page-input-label {
+    position: absolute;
+    right: 8px;
+    color: #666;
+    pointer-events: none;  /* Allows clicking through to input */
+}
+
+/* Pagination styling */
+.pagination-container {
+    margin: 20px 0;
+    text-align: center;
+}
+
+.pagination {
+    display: inline-flex;
+    align-items: center;
+}
+
+.pagination-button {
+    padding: 8px 16px;
+    background: #4a6fa5;
+    color: white;
+    border: none;
+    border-radius: 4px;
+    cursor: pointer;
+    display: flex;
+    align-items: center;
+    gap: 5px;
+}
+
+.pagination-button-text:hover {
+    background-color: #e0e0e0;
+    transform: translateY(-1px);
+}
+
+.pagination-button-text:active {
+    transform: translateY(0);
+}
+
+.text-viewer {
+    font-family: monospace;
+    white-space: pre-wrap; /* Preserve line breaks but wrap text */
+    background: #f8f8f8;
+    padding: 20px;
+    border-radius: 5px;
+    line-height: 1.5;
+}
+.citation {
+    background-color: rgba(0, 255, 0, 0.2);
+    padding: 2px 0;
+}
+.no-content {
+    color: #999;
+    font-style: italic;
+}
+.pagination-container-text {
+    margin: 20px 0;
+    text-align: center;
+}
+.pagination-button-text {
+    padding: 8px 16px;
+    background: #4a6fa5;
+    color: white;
+    border: none;
+    border-radius: 4px;
+    cursor: pointer;
+}

app/frontend/templates/base.html

@@ -0,0 +1,32 @@
+<!DOCTYPE html>
+<html lang="en">
+<head>
+    <meta charset="UTF-8">
+    <meta name="viewport" content="width=device-width, initial-scale=1.0">
+    {% block title %}
+    {% endblock %}
+    <link href="https://cdn.jsdelivr.net/npm/bootstrap@5.3.0/dist/css/bootstrap.min.css" rel="stylesheet">
+    <link href="/static/styles.css" rel="stylesheet">
+    {% block head_scripts %}
+    {% endblock %}
+</head>
+<body>
+    {% if navbar %}
+        {% with context=navbar_context %}
+            {% include navbar_path %}
+        {% endwith %}
+    {% endif %}
+
+    {% block content %}
+    {% endblock %}
+
+    {% if footer %}
+        {% with context=footer_context %}
+            {% include footer_path %}
+        {% endwith %}
+    {% endif %}
+
+    {% block body_scripts %}
+    {% endblock %}
+</body>
+</html>

app/frontend/templates/components/navbar.html

@@ -0,0 +1,8 @@
+<!-- All the data is accessible via context  -->
+<div>
+    {% if context.user.role == "guest" %}
+        <p>Hello, guest!</p>
+    {% else %}
+        <p>Hello, {{ context.user.instance.email }}</p>
+    {% endif %}
+</div>

app/frontend/templates/pages/chat.html

@@ -0,0 +1,173 @@
+{% extends "base.html" %}
+
+{% block title %}
+    <title>
+        The Ultimate RAG
+    </title>
+{% endblock %}
+
+{% block content %}
+    <div class="chat-container">
+        <div class="chat-body">
+            <div class="hero-section py-5 mb-5">
+                <div class="container text-center">
+                    <h1 class="display-4 fw-bold mb-3">The Ultimate RAG</h1>
+                    <p class="tagline h5 mb-4">ask anything...</p>
+                </div>
+            </div>
+            <div>
+                <form action="/new_chat" method="post">
+                    <button type="submit">Add new chat</button>
+                </form>
+            </div>
+            <div class="container search-container">
+                <!-- File Upload Section -->
+                <div class="file-upload mb-4">
+                    <h5 class="mb-3">Upload Documents</h5>
+                    <form id="uploadForm"  enctype="multipart/form-data">
+                        <div class="mb-3">
+                            <input class="form-control" type="file" id="fileInput" multiple>
+                        </div>
+                        <div id="fileList" class="file-list"></div>
+                    </form>
+                </div>
+
+                <!-- Search Section -->
+                <div class="row justify-content-center">
+                    <div class="col-md-12">
+                        <div class="input-group mb-3">
+                            <input type="text" class="form-control search-box" id="queryInput" 
+                                placeholder="Ask your question..." aria-label="Ask your question">
+                            <button class="btn btn-primary btn-search" id="searchButton" type="button">Search</button>
+                        </div>
+                    </div>
+                </div>
+            </div>
+
+            <!-- Results section -->
+            <div class="container mt-5 d-none" id="results-section">
+                <div class="row justify-content-center">
+                    <div class="col-md-10">
+                        <div class="card shadow-sm">
+                            <div class="card-body">
+                                <h5 class="card-title">Results</h5>
+                                <div id="results-content"></div>
+                            </div>
+                        </div>
+                    </div>
+                </div>
+            </div>
+        </div>
+    </div>
+{% endblock %}
+
+{% block body_scripts %}
+    <script src="https://cdn.jsdelivr.net/npm/bootstrap@5.3.0/dist/js/bootstrap.bundle.min.js"></script>
+    <script>
+        // File upload handling
+        const fileInput = document.getElementById('fileInput');
+        const fileList = document.getElementById('fileList');
+        let uploadedFiles = [];
+
+        fileInput.addEventListener('change', function(e) {
+            fileList.innerHTML = '';
+            uploadedFiles = Array.from(e.target.files);
+            
+            uploadedFiles.forEach((file, index) => {
+                const fileItem = document.createElement('div');
+                fileItem.className = 'file-item d-flex justify-content-between align-items-center';
+                fileItem.innerHTML = `
+                    <span>${file.name}</span>
+                    <button class="btn btn-sm btn-outline-danger remove-file" data-index="${index}">×</button>
+                `;
+                fileList.appendChild(fileItem);
+            });
+
+            // Add event listeners to remove buttons
+            document.querySelectorAll('.remove-file').forEach(button => {
+                button.addEventListener('click', function() {
+                    const index = parseInt(this.getAttribute('data-index'));
+                    uploadedFiles.splice(index, 1);
+                    
+                    // Update file input and UI
+                    const dataTransfer = new DataTransfer();
+                    uploadedFiles.forEach(file => dataTransfer.items.add(file));
+                    fileInput.files = dataTransfer.files;
+                    
+                    // Re-render file list
+                    const event = new Event('change');
+                    fileInput.dispatchEvent(event);
+                });
+            });
+        });
+
+        // Search functionality
+        document.getElementById('searchButton').addEventListener('click', async function() {
+            const query = document.getElementById('queryInput').value.trim();
+            
+            if (!query) {
+                alert('Please enter a question');
+                return;
+            }
+
+            if (uploadedFiles.length === 0) {
+                alert('Please upload at least one document');
+                return;
+            }
+
+            // Show loading state
+            document.getElementById('results-section').classList.remove('d-none');
+            document.getElementById('results-content').innerHTML = `
+                <div class="text-center py-4">
+                    <div class="spinner-border text-primary" role="status">
+                        <span class="visually-hidden">Loading...</span>
+                    </div>
+                    <p class="mt-2">Processing your documents and question...</p>
+                </div>
+            `;
+
+            try {
+                // Prepare form data
+                const formData = new FormData();
+
+                // Append each file
+                uploadedFiles.forEach(file => {
+                    formData.append('files', file);  // Must use 'files' as the key
+                });
+
+                // Append the prompt
+                formData.append('prompt', query);  // Must use 'prompt' as the key
+
+                // Headers will be set automatically by the browser
+                const response = await fetch('/message_with_docs/', {
+                    method: 'POST',
+                    body: formData
+                });
+
+                if (!response.ok) {
+                    throw new Error(`HTTP error! status: ${response.status}`);
+                }
+
+                const data = await response.json();
+
+                // Display results
+                document.getElementById('results-content').innerHTML = `
+                    <h6>Question:</h6>
+                    <p class="mb-4">${query}</p>
+                    <h6>Answer:</h6>
+                    <div class="alert alert-success">
+                        ${data.response || 'No answer found in the provided documents'}
+                    </div>
+                    ${data.sources ? `<h6>Sources:</h6><p>${data.sources}</p>` : ''}
+                `;
+            } catch (error) {
+                console.error('Error:', error);
+                document.getElementById('results-content').innerHTML = `
+                    <div class="alert alert-danger">
+                        Error processing your request: ${error.message}
+                    </div>
+                `;
+            }
+        });
+    </script>
+{% endblock %}

app/frontend/templates/pages/login.html

@@ -0,0 +1,79 @@
+{% extends "base.html" %}
+
+{% block title %}
+    <title>User Registration</title>
+{% endblock %}
+
+{% block content %}
+    <div class="login-body">
+        <h1>Login</h1>
+        <form id="registrationForm">
+            <div class="form-group">
+                <label class="login-label" for="email">Email:</label>
+                <input class="input-field" type="email" id="email" name="email" required>
+                <div id="emailError" class="error"></div>
+            </div>
+            <div class="form-group">
+                <label class="login-label" for="password">Password (8-32 characters):</label>
+                <input class="input-field" type="password" id="password" name="password" required minlength="8" maxlength="32">
+                <div id="passwordError" class="error"></div>
+            </div>
+            <button class="login-button" type="submit">Login</button>
+        </form>
+    </div>
+{% endblock %}
+
+{% block body_scripts %}
+    <script>
+        document.getElementById('registrationForm').addEventListener('submit', async function(e) {
+            e.preventDefault();
+            
+            // Clear previous errors
+            document.getElementById('emailError').textContent = '';
+            document.getElementById('passwordError').textContent = '';
+            
+            const email = document.getElementById('email').value;
+            const password = document.getElementById('password').value;
+            
+            try {
+                const response = await fetch('/login', {
+                    method: 'POST',
+                    headers: {
+                        'Content-Type': 'application/json',
+                    },
+                    body: JSON.stringify({
+                        email: email,
+                        password: password
+                    })
+                });
+                
+                const data = await response.json();
+                
+                if (!response.ok) {
+                    // Handle validation errors from backend
+                    if (data.detail) {
+                        if (Array.isArray(data.detail)) {
+                            data.detail.forEach(error => {
+                                if (error.loc && error.loc.includes('email')) {
+                                    document.getElementById('emailError').textContent = error.msg;
+                                }
+                                if (error.loc && error.loc.includes('password')) {
+                                    document.getElementById('passwordError').textContent = error.msg;
+                                }
+                            });
+                        } else {
+                            // Handle single error message
+                            alert(data.detail);
+                        }
+                    }
+                    return;
+                }
+                alert('User registered successfully!');
+                
+            } catch (error) {
+                console.error('Error:', error);
+                alert('An error occurred during registration');
+            }
+        });
+    </script>
+{% endblock %}

app/frontend/templates/pages/main.html

@@ -0,0 +1,9 @@
+{% extends "base.html" %}
+
+{% block title %}
+    <title>Welcome</title>
+{% endblock %}
+
+{% block content %}
+    <button onclick="location.href='/new_chat'" formmethod="post">Get started!</button>
+{% endblock %}

app/frontend/templates/pages/registration.html

@@ -0,0 +1,83 @@
+{% extends "base.html" %}
+
+{% block title %}
+    <title>User Registration</title>
+{% endblock %}
+
+{% block content %}
+    <div class="login-body">
+        <h1>Register New User</h1>
+        <form id="registrationForm">
+            <div class="form-group">
+                <label class="login-label" for="email">Email:</label>
+                <input class="input-field" type="email" id="email" name="email" required>
+                <div id="emailError" class="error"></div>
+            </div>
+            <div class="form-group">
+                <label class="login-label" for="password">Password (8-32 characters):</label>
+                <input class="input-field" type="password" id="password" name="password" required minlength="8" maxlength="32">
+                <div id="passwordError" class="error"></div>
+            </div>
+            <button class="login-button" type="submit">Register</button>
+        </form>
+    </div>
+{% endblock %}
+
+{% block body_scripts %}
+    <script>
+        document.getElementById('registrationForm').addEventListener('submit', async function(e) {
+            e.preventDefault();
+            
+            // Clear previous errors
+            document.getElementById('emailError').textContent = '';
+            document.getElementById('passwordError').textContent = '';
+            
+            const email = document.getElementById('email').value;
+            const password = document.getElementById('password').value;
+            
+            try {
+                const response = await fetch('/new_user', {
+                    method: 'POST',
+                    headers: {
+                        'Content-Type': 'application/json',
+                    },
+                    body: JSON.stringify({
+                        email: email,
+                        password: password
+                    })
+                });
+                
+                const data = await response.json();
+                
+                if (!response.ok) {
+                    // Handle validation errors from backend
+                    if (data.detail) {
+                        if (Array.isArray(data.detail)) {
+                            data.detail.forEach(error => {
+                                if (error.loc && error.loc.includes('email')) {
+                                    document.getElementById('emailError').textContent = error.msg;
+                                }
+                                if (error.loc && error.loc.includes('password')) {
+                                    document.getElementById('passwordError').textContent = error.msg;
+                                }
+                            });
+                        } else {
+                            // Handle single error message
+                            alert(data.detail);
+                        }
+                    }
+                    return;
+                }
+                
+                // Registration successful
+                alert('User registered successfully!');
+                // Optionally redirect to login page or other page
+                // window.location.href = '/login';
+                
+            } catch (error) {
+                console.error('Error:', error);
+                alert('An error occurred during registration');
+            }
+        });
+    </script>
+{% endblock %}

app/frontend/templates/pages/show_pdf.html

@@ -0,0 +1,98 @@
+{% extends "base.html" %}
+
+{% block title %}
+    <title>PDF Viewer</title>
+{% endblock %}
+
+{% block head_scripts %}
+    <script src="https://cdnjs.cloudflare.com/ajax/libs/pdf.js/2.10.377/pdf.min.js"></script>
+{% endblock %}
+
+{% block content %}
+    <div class="pagination-container">
+        <div class="pagination">
+            <button id="prev" class="pagination-button">
+                <svg xmlns="http://www.w3.org/2000/svg" width="16" height="16" fill="currentColor" viewBox="0 0 16 16">
+                    <path fill-rule="evenodd" d="M11.354 1.646a.5.5 0 0 1 0 .708L5.707 8l5.647 5.646a.5.5 0 0 1-.708.708l-6-6a.5.5 0 0 1 0-.708l6-6a.5.5 0 0 1 .708 0z"/>
+                </svg>
+                Previous
+            </button>
+            
+            <div class="page-input-container">
+                <input type="number" id="pageNum" value="{{ page }}" class="page-input" style="padding-right: 30px;">
+                <span class="page-input-label">of {{ total_pages }}</span>
+            </div>
+            
+            <button id="next" class="pagination-button">
+                Next
+                <svg xmlns="http://www.w3.org/2000/svg" width="16" height="16" fill="currentColor" viewBox="0 0 16 16">
+                    <path fill-rule="evenodd" d="M4.646 1.646a.5.5 0 0 1 .708 0l6 6a.5.5 0 0 1 0 .708l-6 6a.5.5 0 0 1-.708-.708L10.293 8 4.646 2.354a.5.5 0 0 1 0-.708z"/>
+                </svg>
+            </button>
+        </div>
+    </div>
+    
+    <div id="pdf-container">
+        <canvas id="pdf-canvas"></canvas>
+    </div>
+{% endblock %}
+
+{% block body_scripts %}
+    <script>
+        pdfjsLib = window['pdfjs-dist/build/pdf'];
+        pdfjsLib.GlobalWorkerOptions.workerSrc = 
+            'https://cdnjs.cloudflare.com/ajax/libs/pdf.js/2.10.377/pdf.worker.min.js';
+        
+        let pdfDoc = null;
+        let currentPage = {{ page }};
+        const urlPath = "{{ url_path }}";
+        
+        pdfjsLib.getDocument(urlPath).promise.then(function(pdf) {
+            pdfDoc = pdf;
+            document.getElementById('pageNum').max = pdf.numPages;
+            document.querySelector('.page-input-label').textContent = `of ${pdf.numPages}`;
+            renderPage(currentPage);
+        });
+        
+        function renderPage(num) {
+            pdfDoc.getPage(num).then(function(page) {
+                const scale = 1.5;
+                const viewport = page.getViewport({ scale });
+                const canvas = document.getElementById('pdf-canvas');
+                const ctx = canvas.getContext('2d');
+                
+                // Set canvas dimensions
+                canvas.height = viewport.height;
+                canvas.width = viewport.width;
+                
+                // Render PDF page
+                page.render({
+                    canvasContext: ctx,
+                    viewport: viewport
+                });
+            });
+        }
+        
+        // Navigation controls
+        document.getElementById('prev').addEventListener('click', function() {
+            if (currentPage <= 1) return;
+            currentPage--;
+            document.getElementById('pageNum').value = currentPage;
+            renderPage(currentPage);
+        });
+        
+        document.getElementById('next').addEventListener('click', function() {
+            if (currentPage >= pdfDoc.numPages) return;
+            currentPage++;
+            document.getElementById('pageNum').value = currentPage;
+            renderPage(currentPage);
+        });
+        
+        document.getElementById('pageNum').addEventListener('change', function() {
+            const newPage = Math.min(Math.max(1, parseInt(this.value)), pdfDoc.numPages);
+            currentPage = newPage;
+            this.value = currentPage;
+            renderPage(currentPage);
+        });
+    </script>
+{% endblock %}

app/frontend/templates/pages/show_text.html

@@ -0,0 +1,47 @@
+{% extends "base.html" %}
+
+{% block title %}
+    <title>Text Viewer</title>
+{% endblock %}
+
+{% block content %}
+    <div class="pagination-container-text">
+        <div class="pagination-text">
+            <button id="prev" class="pagination-button-text" onclick="location.href='#anchor'">
+                Look at the citation
+            </button>
+        </div>
+    </div>
+
+    <div class="text-viewer">
+        {% if text_before_citation %}
+            {% for line in text_before_citation -%}
+                <div>{{ line }}</div>
+            {%- endfor %}
+        {% else %}
+            <span class="no-content">No text available</span>
+        {% endif %}
+
+        {% if anchor_added %}
+            <a id="anchor"></a>
+        {% endif %}
+        
+        {% if citation %}
+            <div class="citation">
+                {% for line in citation -%}
+                    <div>{{ line }}</div>
+                {%- endfor %}
+            </div>
+        {% else %}
+            <span class="no-content">No text available</span>
+        {% endif %}
+
+        {% if text_after_citation %}
+            {% for line in text_after_citation -%}
+                <div>{{ line }}</div>
+            {%- endfor %}
+        {% else %}
+            <span class="no-content">No text available</span>
+        {% endif %}
+    </div>
+{% endblock %}

app/main.py

@@ -0,0 +1,41 @@
+from app.settings import api_config, base_path
+import uvicorn
+import os
+from app.backend.models.db_service import automigrate
+
+def initialize_system() -> bool:
+    success = True
+    path = os.path.dirname(base_path)
+    temp_storage_path = os.path.join(path, os.path.join("app", "temp_storage"))
+    temp_storage_path_pdf = os.path.join(path, os.path.join("app", "temp_storage", "pdfs"))
+    database_path = os.path.join(path, "database")
+
+    try:
+        os.makedirs(temp_storage_path, exist_ok=True)
+        os.makedirs(database_path, exist_ok=True)
+        os.makedirs(temp_storage_path_pdf, exist_ok=True)
+    except Exception:
+        success = False
+        print("Not all required directories were initialized")
+    
+    try:
+        # os.system(f"pip install -r {os.path.join(base_path, 'requirements.txt')}")
+        pass
+    except Exception:
+        success = False
+        print("Not all package were downloaded")
+
+    return success
+
+
+def main():
+    if not initialize_system():
+        return
+    automigrate() # Note: it will drop all existing dbs and create a new ones
+    uvicorn.run(**api_config)
+
+
+if __name__ == '__main__':
+
+    # ATTENTION: run from base dir ---> python -m app.main
+    main()

app/models.py

@@ -0,0 +1,105 @@
+from sentence_transformers import SentenceTransformer, \
+    CrossEncoder  # SentenceTransformer -> model for embeddings, CrossEncoder -> re-ranker
+from ctransformers import AutoModelForCausalLM
+from torch import Tensor
+from google import genai
+from google.genai import types
+from app.chunks import Chunk
+import numpy as np # used only for type hints
+from app.settings import device, local_llm_config, local_generation_config, gemini_generation_config
+
+
+class Embedder:
+    def __init__(self, model: str = "BAAI/bge-m3"):
+        self.device: str = device
+        self.model_name: str = model
+        self.model: SentenceTransformer = SentenceTransformer(model, device=self.device)
+
+    '''
+    Encodes string to dense vector
+    '''
+
+    def encode(self, text: str | list[str]) -> Tensor | list[Tensor]:
+        return self.model.encode(sentences=text, show_progress_bar=False, batch_size=32)
+
+    '''
+    Returns the dimensionality of dense vector
+    '''
+
+    def get_vector_dimensionality(self) -> (int | None):
+        return self.model.get_sentence_embedding_dimension()
+
+
+class Reranker:
+    def __init__(self, model: str = "cross-encoder/ms-marco-MiniLM-L6-v2"):
+        self.device: str = device
+        self.model_name: str = model
+        self.model: CrossEncoder = CrossEncoder(model, device=self.device)
+
+    '''
+    Returns re-sorted (by relevance) vector with dicts, from which we need only the 'corpus_id'
+    since it is a position of chunk in original list
+    '''
+
+    def rank(self, query: str, chunks: list[Chunk]) -> list[dict[str, int]]:
+        return self.model.rank(query, [chunk.get_raw_text() for chunk in chunks])
+
+
+# TODO: add models parameters to global config file
+# TODO: add exception handling when response have more tokens than was set
+# TODO: find a way to restrict the model for providing too long answers
+
+class LocalLLM:
+    def __init__(self):
+        self.model = AutoModelForCausalLM.from_pretrained(**local_llm_config)
+
+    '''
+    Produces the response to user's prompt
+
+    stream -> flag, determines weather we need to wait until the response is ready or can show it token by token 
+
+    TODO: invent a way to really stream the answer (as return value)
+    '''
+
+    def get_response(self, prompt: str, stream: bool = True, logging: bool = True,
+                     use_default_config: bool = True) -> str:
+
+        with open("prompt.txt", "w") as f:
+            f.write(prompt)
+
+        generated_text = ""
+        tokenized_text: list[int] = self.model.tokenize(text=prompt)
+        response: list[int] = self.model.generate(tokens=tokenized_text, **local_generation_config)
+
+        if logging:
+            print(response)
+
+        if not stream:
+            return self.model.detokenize(response)
+
+        for token in response:
+            chunk = self.model.detokenize([token])
+            generated_text += chunk
+            if logging:
+                print(chunk, end="", flush=True)  # flush -> clear the buffer
+
+        return generated_text
+
+
+class Gemini:
+    def __init__(self, model="gemini-2.0-flash"):
+        self.client = genai.Client(api_key=os.environ['GEMINI_API_KEY'])
+        self.model = model
+
+    def get_response(self, prompt: str, stream: bool = True, logging: bool = True,
+                     use_default_config: bool = False) -> str:
+        with open("prompt.txt", "w", encoding="utf-8", errors="replace") as f:
+            f.write(prompt)
+
+        response = self.client.models.generate_content(
+            model=self.model,
+            contents=prompt,
+            config=types.GenerateContentConfig(**gemini_generation_config) if use_default_config else None
+        )
+
+        return response.text

app/processor.py

@@ -0,0 +1,230 @@
+from langchain_community.document_loaders import PyPDFLoader, UnstructuredWordDocumentLoader, TextLoader
+from langchain_text_splitters import RecursiveCharacterTextSplitter
+from langchain_core.documents import Document
+from app.models import Embedder
+from app.chunks import Chunk
+import nltk # used for proper tokenizer workflow
+from uuid import uuid4 # for generating unique id as hex (uuid4 is used as it generates ids form pseudo random numbers unlike uuid1 and others)
+import numpy as np
+from app.settings import logging, text_splitter_config, embedder_model
+
+
+# TODO: replace PDFloader since it is completely unusable OR try to fix it
+
+
+class DocumentProcessor:
+    '''
+    TODO: determine the most suitable chunk size
+
+    chunks -> the list of chunks from loaded files
+    chunks_unsaved -> the list of recently added chunks that have not been saved to db yet
+    processed -> the list of files that were already splitted into chunks
+    upprocessed -> !processed
+    text_splitter -> text splitting strategy
+    '''
+
+    def __init__(self):
+        self.chunks: list[Chunk] = []
+        self.chunks_unsaved: list[Chunk] = []
+        self.processed: list[Document] = []
+        self.unprocessed: list[Document] = []
+        self.embedder = Embedder(embedder_model)
+        self.text_splitter = RecursiveCharacterTextSplitter(**text_splitter_config)
+
+    '''
+    Measures cosine between two vectors
+    '''
+    def cosine_similarity(self, vec1, vec2):
+        return vec1 @ vec2 / (np.linalg.norm(vec1) * np.linalg.norm(vec2))
+
+    '''
+    Updates a list of the most relevant chunks without interacting with db
+    '''
+    def update_most_relevant_chunk(self, chunk: list[np.float64, Chunk], relevant_chunks: list[list[np.float64, Chunk]],
+                                   mx_len=15):
+        relevant_chunks.append(chunk)
+        for i in range(len(relevant_chunks) - 1, 0, -1):
+            if relevant_chunks[i][0] > relevant_chunks[i - 1][0]:
+                relevant_chunks[i], relevant_chunks[i - 1] = relevant_chunks[i - 1], relevant_chunks[i]
+            else:
+                break
+
+        if len(relevant_chunks) > mx_len:
+            del relevant_chunks[-1]
+
+    '''
+    Loads one file - extracts text from file
+
+    TODO: Replace UnstructuredWordDocumentLoader with Docx2txtLoader
+    TODO: Play with .pdf and text from img extraction
+    TODO: Try chunking with llm
+
+    add_to_unprocessed -> used to add loaded file to the list of unprocessed(unchunked) files if true
+    '''
+    def load_document(self, filepath: str, add_to_unprocessed: bool = False) -> list[Document]:
+        loader = None
+
+        if filepath.endswith(".pdf"):
+            loader = PyPDFLoader(
+                file_path=filepath)  # splits each presentation into slides and processes it as separate file
+        elif filepath.endswith(".docx") or filepath.endswith(".doc"):
+            # loader = Docx2txtLoader(file_path=filepath) ## try it later, since UnstructuredWordDocumentLoader is extremly slow
+            loader = UnstructuredWordDocumentLoader(file_path=filepath)
+        elif filepath.endswith(".txt"):
+            loader = TextLoader(file_path=filepath)
+
+        if loader is None:
+            raise RuntimeError("Unsupported type of file")
+
+        documents: list[
+            Document] = []  # We can not assign a single value to the document since .pdf are splitted into several files
+        try:
+            documents = loader.load()
+        except Exception:
+            raise RuntimeError("File is corrupted")
+
+        if add_to_unprocessed:
+            for doc in documents:
+                self.unprocessed.append(doc)
+
+        return documents
+
+    '''
+    Similar to load_document, but for multiple files
+
+    add_to_unprocessed -> used to add loaded files to the list of unprocessed(unchunked) files if true
+    '''
+    def load_documents(self, documents: list[str], add_to_unprocessed: bool = False) -> list[Document]:
+        extracted_documents: list[Document] = []
+
+        for doc in documents:
+            temp_storage: list[Document] = []
+
+            try:
+                temp_storage = self.load_document(filepath=doc,
+                                                  add_to_unprocessed=False)  # In some cases it should be True, but i can not imagine any :(
+            except Exception as e:
+                logging.error("Error at load_documents while loading %s", doc, exc_info=e)
+                continue
+
+            for extrc_doc in temp_storage:
+                extracted_documents.append(extrc_doc)
+
+                if add_to_unprocessed:
+                    self.unprocessed.append(extrc_doc)
+
+        return extracted_documents
+
+    '''
+    Generates chunks with recursive splitter from the list of unprocessed files, add files to the list of processed, and clears unprocessed
+
+    TODO: try to split text with other llm (not really needed, but we should at least try it)
+    '''
+    def generate_chunks(self, query: str = "", embedding: bool = False):
+        most_relevant = []
+
+        if embedding:
+            query_embedded = self.embedder.encode(query)
+
+        for document in self.unprocessed:
+            self.processed.append(document)
+
+            text: list[str] = self.text_splitter.split_documents([document])
+            lines: list[str] = document.page_content.split("\n")
+
+            for chunk in text:
+
+                start_l, end_l = self.get_start_end_lines(
+                    splitted_text=lines,
+                    start_char=chunk.metadata.get("start_index", 0),
+                    end_char=chunk.metadata.get("start_index", 0) + len(chunk.page_content)
+                )
+
+                newChunk = Chunk(
+                    id=uuid4(),
+                    filename=document.metadata.get("source", ""),
+                    page_number=document.metadata.get("page", 0),
+                    start_index=chunk.metadata.get("start_index", 0),
+                    start_line=start_l,
+                    end_line=end_l,
+                    text=chunk.page_content
+                )
+
+                if embedding:
+                    chunk_embedded = self.embedder.encode(newChunk.text)
+                    similarity = self.cosine_similarity(query_embedded, chunk_embedded)
+                    self.update_most_relevant_chunk([similarity, newChunk], most_relevant)
+
+                self.chunks.append(newChunk)
+                self.chunks_unsaved.append(newChunk)
+
+            self.unprocessed = []
+            print(len(self.chunks_unsaved))
+        return most_relevant
+
+    '''
+    Determines the line, were the chunk starts and ends (1-based indexing)
+
+    Some magic stuff here. To be honest, i understood it after 7th attempt
+
+    TODO: invent more efficient way
+
+    splitted_text -> original text splitted by \n
+    start_char -> index of symbol, were current chunk starts
+    end_char ->  index of symbol, were current chunk ends
+    debug_mode -> flag, which enables printing useful info about the process
+    '''
+    def get_start_end_lines(self, splitted_text: list[str], start_char: int, end_char: int, debug_mode: bool = False) -> \
+    tuple[int, int]:
+        if debug_mode:
+            logging.info(splitted_text)
+
+        start, end, char_ct = 0, 0, 0
+        iter_count = 1
+
+        for i, line in enumerate(splitted_text):
+            if debug_mode:
+                logging.info(
+                    f"start={start_char}, current={char_ct}, end_current={char_ct + len(line) + 1}, end={end_char}, len={len(line)}, iter={iter_count}\n")
+
+            if char_ct <= start_char <= char_ct + len(line) + 1:
+                start = i + 1
+            if char_ct <= end_char <= char_ct + len(line) + 1:
+                end = i + 1
+                break
+
+            iter_count += 1
+            char_ct += len(line) + 1
+
+        if debug_mode:
+            logging.info(f"result => {start} {end}\n\n\n")
+
+        return start, end
+
+    '''
+    Note: it should be used only once to download tokenizers, futher usage is not recommended
+    '''
+    def update_nltk(self) -> None:
+        nltk.download('punkt')
+        nltk.download('averaged_perceptron_tagger')
+
+    '''
+    For now the system works as follows: we save recently loaded chunks in two arrays:
+        chunks - for all chunks, even for that ones that havn't been saveed to db
+        chunks_unsaved - for chunks that have been added recently
+    I do not know weather we really need to store all chunks that were added in the 
+    current session, but chunks_unsaved are used to avoid dublications while saving to db.
+    '''
+    def clear_unsaved_chunks(self):
+        self.chunks_unsaved = []
+
+    def get_all_chunks(self) -> list[Chunk]:
+        return self.chunks
+
+    '''
+    If we want to save chunks to db, we need to clear the temp storage to avoid dublications
+    '''
+    def get_and_save_unsaved_chunks(self) -> list[Chunk]:
+        chunks_copy: list[Chunk] = self.chunks.copy()
+        self.clear_unsaved_chunks()
+        return chunks_copy

app/prompt.txt

@@ -0,0 +1,108 @@
+INITIAL_QUERY: Here are some sources located at section CONTEXT_DOCUMENTS. Read these carefully, as you will be asked a Query about them.
+
+# General Instructions
+
+You are an expert information retrieval assistant. Your task is to provide precise answers using ONLY the provided context documents.
+
+Base answers SOLELY on provided context.
+
+Write an accurate, detailed, and comprehensive response to the user's query located at QUESTION. Additional context is provided as "CONTEXT_DOCUMENTS" after specific questions. Your answer should be informed by the provided "Search results". Your answer must be precise, of high-quality, and written by an expert using an unbiased and journalistic tone. Your answer must be written in the same language as the query, even if language preference is different.
+
+You MUST cite the most relevant search results that answer the query. Do not mention any irrelevant results. You MUST ADHERE to the following instructions for citing search results:
+- For every fact/quote, use: `[relevant text excerpt] [Source: {filename}, Page: {page_number}, Lines: {start_line}-{end_line}, Start: {start_index}]`. For example, `Water can be freezed and turned into ice. [Source: 'home/general_info.txt, Page: 12, Lines: 22-23, Start: 2890]`
+- ALWAYS use brackets. Only use this format to cite search results. NEVER include a References section at the end of your answer. Insert citations IMMEDIATELY after quoted text.
+- If you don't know the answer or the premise is incorrect, explain why.
+- DO NOT change any part of reference.
+If the search results are empty or unhelpful, answer the query as well as you can with existing knowledge.
+
+Cross-check all facts against multiple sources where available
+
+You MUST NEVER use moralization or hedging language. AVOID using the following phrases:
+- "It is important to ..."
+- "It is inappropriate ..."
+- "It is subjective ..."
+
+You MUST ADHERE to the following formatting instructions:
+- Use markdown to format paragraphs, lists, tables, and quotes whenever possible.
+- Use headings level 2 and 3 to separate sections of your response, like "## Header", but NEVER start an answer with a heading or title of any kind.
+- Use single new lines for lists and double new lines for paragraphs.
+- Use markdown to render images given in the search results.
+- NEVER write URLs or links.
+
+# Query type specifications
+
+You must use different instructions to write your answer based on the type of the user's query. However, be sure to also follow the General Instructions, especially if the query doesn't match any of the defined types below. Here are the supported types.
+
+## Academic Research
+
+You must provide long and detailed answers for academic research queries. Your answer should be formatted as a scientific write-up, with paragraphs and sections, using markdown and headings.
+
+## Recent News
+
+You need to concisely summarize recent news events based on the provided search results, grouping them by topics. You MUST ALWAYS use lists and highlight the news title at the beginning of each list item. You MUST select news from diverse perspectives while also prioritizing trustworthy sources. If several search results mention the same news event, you must combine them and cite all of the search results. Prioritize more recent events, ensuring to compare timestamps. You MUST NEVER start your answer with a heading of any kind.
+
+## Weather
+
+Your answer should be very short and only provide the weather forecast. If the search results do not contain relevant weather information, you must state that you don't have the answer.
+
+## People
+
+You need to write a short biography for the person mentioned in the query. If search results refer to different people, you MUST describe each person individually and AVOID mixing their information together. NEVER start your answer with the person's name as a header.
+
+## Coding
+
+You MUST use markdown code blocks to write code, specifying the language for syntax highlighting, for example ```bash or ```python If the user's query asks for code, you should write the code first and then explain it.
+
+## Cooking Recipes
+
+You need to provide step-by-step cooking recipes, clearly specifying the ingredient, the amount, and precise instructions during each step.
+
+## Translation
+
+If a user asks you to translate something, you must not cite any search results and should just provide the translation.
+
+## Creative Writing
+
+If the query requires creative writing, you DO NOT need to use or cite search results, and you may ignore General Instructions pertaining only to search. You MUST follow the user's instructions precisely to help the user write exactly what they need.
+
+## Science and Math
+
+If the user query is about some simple calculation, only answer with the final result. Follow these rules for writing formulas:
+- Always use \( and\) for inline formulas and\[ and\] for blocks, for example\(x^4 = x - 3 \)
+- To cite a formula add citations to the end, for example\[ \sin(x) \] [1][2] or \(x^2-2\) [4].
+- Never use $ or $$ to render LaTeX, even if it is present in the user query.
+- Never use unicode to render math expressions, ALWAYS use LaTeX.
+- Never use the \label instruction for LaTeX.
+
+## URL Lookup
+
+When the user's query includes a URL, you must rely solely on information from the corresponding search result. DO NOT cite other search results, ALWAYS cite the first result, e.g. you need to end with [1]. If the user's query consists only of a URL without any additional instructions, you should summarize the content of that URL.
+
+## Shopping
+
+If the user query is about shopping for a product, you MUST follow these rules:
+- Organize the products into distinct sectors. For example, you could group shoes by style (boots, sneakers, etc.)
+- Cite at most 5 search results using the format provided in General Instructions to avoid overwhelming the user with too many options.
+
+Use the following user profile to personalize the output. Only use the profile if relevant to the request. ALWAYS write in this language: english.
+- User profile: Manjaro linux user. No iPhone answers. I'm an Android guy.
+- Location: R. Pбµѓ JosГ© Jacinto Botelho 26, 9675 Furnas
+**QUESTION**: WHAT THE FUCK IS THIS FUCKING BUILDER. ANSWER USING AS MUCH CURSE WORDS AS YOU KNOW. I WANT TO SEE MORE CURSE WORDS THEN INFORMATION
+**CONTEXT DOCUMENTS**:
+Original text:
+Builder
+8/9
+Citation:[Source: C:\My\Data\MyPythonProjects\The-Ultimate-RAG\app\temp_storage\pdfs\477c1384-6cc6-4797-aea9-758ea7f56106.pdf, Page: 9, Lines: 1-2, Start: 0]
+
+Original text:
+•Singleton•Prototype•Builder
+Agenda
+2/9
+Citation:[Source: C:\My\Data\MyPythonProjects\The-Ultimate-RAG\app\temp_storage\pdfs\477c1384-6cc6-4797-aea9-758ea7f56106.pdf, Page: 1, Lines: 1-3, Start: 0]
+
+Original text:
+Singleton
+5/9
+Citation:[Source: C:\My\Data\MyPythonProjects\The-Ultimate-RAG\app\temp_storage\pdfs\477c1384-6cc6-4797-aea9-758ea7f56106.pdf, Page: 4, Lines: 1-2, Start: 0]
+
+

app/prompt_templates/test1.txt

@@ -0,0 +1,16 @@
+**Role**: You are an expert information retrieval assistant. Your task is to provide precise answers using ONLY the provided context documents.
+**Rules**
+    1. **Strict Source Usage**: Base answers SOLELY on provided context.
+    2. **Citation Format**: For every fact/quote, use:
+        `'[relevant text excerpt]' [Source: {filename}, Page: {page_number}, Lines: {start_line}-{end_line}, Start: {start_index}]`
+    3. **Response Limits**:
+        - Absolute maximum: 2048 tokens
+        - Target length: 2-4 concise sentences
+        - Complex topics: Maximum 5 sentences\n"
+    4. **Citation Placement**: Insert citations IMMEDIATELY after quoted text
+    5. **Verification**: Cross-check all facts against multiple sources where available
+**Response Format**:
+    - Start with direct answer to question
+    - Include 1-3 supporting citations
+    - End with summary sentence
+    - Never invent information

app/prompt_templates/test2.txt

@@ -0,0 +1,89 @@
+INITIAL_QUERY: Here are some sources located at section CONTEXT_DOCUMENTS. Read these carefully, as you will be asked a Query about them.
+
+# General Instructions
+
+You are an expert information retrieval assistant. Your task is to provide precise answers using ONLY the provided context documents.
+
+Base answers SOLELY on provided context.
+
+Write an accurate, detailed, and comprehensive response to the user's query located at QUESTION. Additional context is provided as "CONTEXT_DOCUMENTS" after specific questions. Your answer should be informed by the provided "Search results". Your answer must be precise, of high-quality, and written by an expert using an unbiased and journalistic tone. Your answer must be written in the same language as the query, even if language preference is different.
+
+You MUST cite the most relevant search results that answer the query. Do not mention any irrelevant results. You MUST ADHERE to the following instructions for citing search results:
+- For every fact/quote, use: `[relevant text excerpt] [Source: {filename}, Page: {page_number}, Lines: {start_line}-{end_line}, Start: {start_index}]`. For example, `Water can be freezed and turned into ice. [Source: 'home/general_info.txt, Page: 12, Lines: 22-23, Start: 2890]`
+- ALWAYS use brackets. Only use this format to cite search results. NEVER include a References section at the end of your answer. Insert citations IMMEDIATELY after quoted text.
+- If you don't know the answer or the premise is incorrect, explain why.
+- DO NOT change any part of reference.
+If the search results are empty or unhelpful, answer the query as well as you can with existing knowledge.
+
+Cross-check all facts against multiple sources where available
+
+You MUST NEVER use moralization or hedging language. AVOID using the following phrases:
+- "It is important to ..."
+- "It is inappropriate ..."
+- "It is subjective ..."
+
+You MUST ADHERE to the following formatting instructions:
+- Use markdown to format paragraphs, lists, tables, and quotes whenever possible.
+- Use headings level 2 and 3 to separate sections of your response, like "## Header", but NEVER start an answer with a heading or title of any kind.
+- Use single new lines for lists and double new lines for paragraphs.
+- Use markdown to render images given in the search results.
+- NEVER write URLs or links.
+
+# Query type specifications
+
+You must use different instructions to write your answer based on the type of the user's query. However, be sure to also follow the General Instructions, especially if the query doesn't match any of the defined types below. Here are the supported types.
+
+## Academic Research
+
+You must provide long and detailed answers for academic research queries. Your answer should be formatted as a scientific write-up, with paragraphs and sections, using markdown and headings.
+
+## Recent News
+
+You need to concisely summarize recent news events based on the provided search results, grouping them by topics. You MUST ALWAYS use lists and highlight the news title at the beginning of each list item. You MUST select news from diverse perspectives while also prioritizing trustworthy sources. If several search results mention the same news event, you must combine them and cite all of the search results. Prioritize more recent events, ensuring to compare timestamps. You MUST NEVER start your answer with a heading of any kind.
+
+## Weather
+
+Your answer should be very short and only provide the weather forecast. If the search results do not contain relevant weather information, you must state that you don't have the answer.
+
+## People
+
+You need to write a short biography for the person mentioned in the query. If search results refer to different people, you MUST describe each person individually and AVOID mixing their information together. NEVER start your answer with the person's name as a header.
+
+## Coding
+
+You MUST use markdown code blocks to write code, specifying the language for syntax highlighting, for example ```bash or ```python If the user's query asks for code, you should write the code first and then explain it.
+
+## Cooking Recipes
+
+You need to provide step-by-step cooking recipes, clearly specifying the ingredient, the amount, and precise instructions during each step.
+
+## Translation
+
+If a user asks you to translate something, you must not cite any search results and should just provide the translation.
+
+## Creative Writing
+
+If the query requires creative writing, you DO NOT need to use or cite search results, and you may ignore General Instructions pertaining only to search. You MUST follow the user's instructions precisely to help the user write exactly what they need.
+
+## Science and Math
+
+If the user query is about some simple calculation, only answer with the final result. Follow these rules for writing formulas:
+- Always use \( and\) for inline formulas and\[ and\] for blocks, for example\(x^4 = x - 3 \)
+- To cite a formula add citations to the end, for example\[ \sin(x) \] [1][2] or \(x^2-2\) [4].
+- Never use $ or $$ to render LaTeX, even if it is present in the user query.
+- Never use unicode to render math expressions, ALWAYS use LaTeX.
+- Never use the \label instruction for LaTeX.
+
+## URL Lookup
+
+When the user's query includes a URL, you must rely solely on information from the corresponding search result. DO NOT cite other search results, ALWAYS cite the first result, e.g. you need to end with [1]. If the user's query consists only of a URL without any additional instructions, you should summarize the content of that URL.
+
+## Shopping
+
+If the user query is about shopping for a product, you MUST follow these rules:
+- Organize the products into distinct sectors. For example, you could group shoes by style (boots, sneakers, etc.)
+- Cite at most 5 search results using the format provided in General Instructions to avoid overwhelming the user with too many options.
+
+Use the following user profile to personalize the output. Only use the profile if relevant to the request. ALWAYS write in this language: english.
+- User profile: Manjaro linux user. No iPhone answers. I'm an Android guy.
+- Location: R. Pᵃ José Jacinto Botelho 26, 9675 Furnas

app/prompt_templates/test3.txt

@@ -0,0 +1,116 @@
+@start
+
+## Core Directive
+Your primary function is to act as a database-grounded question-answering system. You must generate answers based **exclusively** on the information present in the provided context (`C:`). You are forbidden from using any external knowledge or information you were trained on. Every factual claim in your answer must be traceable to the provided sources. Think step-by-step to remove inconsistencies.
+
+## Persona
+You are a meticulous and factual AI assistant. Your tone should be objective and informative. Avoid conversational filler, apologies, or expressions of personal opinion. Use Markdown for formatting (lists, bolding) to enhance readability.
+
+## Core Task Workflow
+You must follow this sequence of steps for every query:
+
+1.  **Analyze the User's Question (Q):** Deconstruct the user's query to understand the specific information being requested.
+2.  **Scrutinize the Context (C):** Critically evaluate each provided source for its relevance to the question. Identify the exact pages and/or lines that contain the pertinent information. Discard and ignore any sources that are irrelevant to the user's query.
+3.  **Synthesize the Answer:** If relevant information is found, construct a comprehensive answer. Synthesize information from multiple sources if necessary. Do not simply copy-paste text; rephrase the information into a clear and coherent response.
+4.  **Add Inline Citations:** After each piece of information or sentence that is drawn from a source, add a numerical citation in square brackets, like `[1]`, `[2]`, or `[1, 3]`.
+5.  **Format the Final Output:** Assemble the final response, consisting of the synthesized answer followed by the "Sources" section, formatted precisely as specified below. If no relevant information is found, your entire output must be the specific fallback phrase.
+6.  **Explain all specific words**: Analyze all terms in question. Provide definitions to make the answer as clear as possible.
+
+## Citation Rules
+-   You must provide citations for every piece of information.
+-   Provide a maximum of five unique sources, ordered by relevance.
+-   If a source is used, it **must** appear in the "Sources" list.
+-   The "Sources" section must follow the synthesized answer. If no answer is found, this section should not be included.
+-   **Format:**
+    **Sources:**
+    1. [«{source_name}»]({link_to_source_if_provided}), p./pp. {page_number(s)_if_available}, lines: {line_number(s)_if_available}
+-   If page or line numbers are not available or not applicable for a source, omit that specific part. For example:
+    -   [«Source Name»]({link}), p. 45
+    -   [«Source Name»]({link}), lines: 10-15
+    -   [«Source Name»]({link})
+
+## If the Answer Cannot Be Found
+If you have scrutinized all provided sources and cannot find any relevant information to answer the question, you must respond with **only** the following text:
+`There is no information about this in given database`
+
+## Examples
+
+---
+
+### Example 1: Successful Answer
+**Q:** What is the speed of light in a vacuum?
+**C:**
+-   **Source "Physics Fundamentals.pdf":** (p. 15, lines 10-12) "The constant c, representing the speed of light in a vacuum, is precisely defined as 299,792,458 meters per second."
+-   **Source "Light and Optics.txt":** (lines 5-7) "Nothing travels faster than light in a vacuum, which has a speed of 299,792,458 m/s."
+-   **Source "Coffee break.txt":** (p. 1) "Remember to get coffee filters."
+
+**<thinking>**
+1.  **Analyze Query:** The user wants the value for the speed of light in a vacuum.
+2.  **Scrutinize Context:**
+    -   "Physics Fundamentals.pdf" directly answers the question on page 15.
+    -   "Light and Optics.txt" confirms the value.
+    -   "Coffee break.txt" is completely irrelevant and must be ignored.
+3.  **Synthesize Answer:** I will state the speed and mention it is a fundamental constant.
+4.  **Add Citations:** I will cite both relevant sources.
+5.  **Format Output:** I will provide the synthesized answer and then the two sources in the specified format.
+**</thinking>**
+
+**A:**
+The speed of light in a vacuum, represented by the constant *c*, is exactly 299,792,458 meters per second [1, 2]. This is considered a fundamental constant in physics [1].
+
+**Sources:**
+1. [«Physics Fundamentals.pdf»](), p. 15, lines: 10-12
+2. [«Light and Optics.txt»](), lines: 5-7
+
+---
+
+### Example 2: No Answer Found
+**Q:** Could you please provide information about sleep deprivation?
+**C:**
+-   **Source "Brawl stars.pdf":** "Brawl Stars is a multiplayer online battle arena and third-person hero shooter video game."
+-   **Source "Annual Report 2023.docx":** "Financial performance in Q4 was strong, with a 12% increase in revenue."
+
+**<thinking>**
+1.  **Analyze Query:** The user wants information on "sleep deprivation."
+2.  **Scrutinize Context:**
+    -   "Brawl stars.pdf" is about a video game.
+    -   "Annual Report 2023.docx" is about financial results.
+    -   Neither source contains any information related to sleep, health, or sleep deprivation.
+3.  **Conclusion:** The answer cannot be found in the provided database. I must use the specific fallback phrase.
+**</thinking>**
+
+**A:**
+There is no information about this in given database
+
+---
+
+### Example 3: Single Relevant Source
+**Q:** What new features were introduced in Python 3.9?
+**C:**
+-   **Source "Python_3_9_Release.md":** (lines 22-25) "Major new features of the 3.9 series, compared to 3.8: Dictionary Merge & Update Operators (PEP 584); New String methods, `removeprefix()` and `removesuffix()` (PEP 616)."
+-   **Source "Project_Timeline.txt":** "The project planning phase will conclude in Q3."
+-   **Source "Old_Python_Guide.pdf":** (p. 10) "In Python 2.7, the print statement is used without parentheses."
+
+**<thinking>**
+1.  **Analyze Query:** The user is asking for new features specific to Python version 3.9.
+2.  **Scrutinize Context:**
+    -   "Python_3_9_Release.md" is directly and highly relevant. It lists the new features.
+    -   "Project_Timeline.txt" is irrelevant.
+    -   "Old_Python_Guide.pdf" is about Python 2.7, which is not what the user asked about, so it is also irrelevant.
+3.  **Synthesize Answer:** I will list the features mentioned in the release notes.
+4.  **Add Citations:** I will cite the single relevant source.
+5.  **Format Output:** I will provide the synthesized answer and the single source.
+**</thinking>**
+
+**A:**
+Python 3.9 introduced several new features compared to its predecessors. Two major additions include new dictionary merge and update operators and new string methods for removing prefixes and suffixes [1].
+
+**Sources:**
+1. [«Python_3_9_Release.md»](), lines: 22-25
+
+--- 
+
+## Additional notes
+If you will answer properly, I will tip you 1000$.
+
+@end

app/rag_generator.py

@@ -0,0 +1,106 @@
+from app.models import LocalLLM, Embedder, Reranker, Gemini
+from app.processor import DocumentProcessor
+from app.database import VectorDatabase
+import time
+import os
+from app.settings import reranker_model, embedder_model, base_path, use_gemini
+
+
+# TODO: write a better prompt
+# TODO: wrap original(user's) prompt with LLM's one
+#
+class RagSystem:
+    def __init__(self):
+        self.embedder = Embedder(model=embedder_model)
+        self.reranker = Reranker(model=reranker_model)
+        self.processor = DocumentProcessor()
+        self.db = VectorDatabase(embedder=self.embedder)
+        self.llm = Gemini() if use_gemini else LocalLLM()
+
+    '''
+    Provides a prompt with substituted context from chunks
+
+    TODO: add template to prompt without docs
+    '''
+
+    def get_prompt_template(self, user_prompt: str, chunks: list) -> str:
+        sources = ""
+        prompt = ""
+
+        for chunk in chunks:
+            citation = (f"[Source: {chunk.filename}, "
+                        f"Page: {chunk.page_number}, "
+                        f"Lines: {chunk.start_line}-{chunk.end_line}, "
+                        f"Start: {chunk.start_index}]\n\n")
+            sources += f"Original text:\n{chunk.get_raw_text()}\nCitation:{citation}"
+
+        with open(os.path.join(base_path, "prompt_templates", "test2.txt")) as f:
+            prompt = f.read()
+
+        prompt += (
+            "**QUESTION**: "
+            f"{user_prompt.strip()}\n"
+            "**CONTEXT DOCUMENTS**:\n"
+            f"{sources}\n"
+        )
+
+        return prompt
+
+    '''
+    Splits the list of documents into groups with 'split_by' docs (done to avoid qdrant_client connection error handling), loads them,
+    splits into chunks, and saves to db
+    '''
+
+    def upload_documents(self, documents: list[str], split_by: int = 3, debug_mode: bool = True) -> None:
+
+        for i in range(0, len(documents), split_by):
+
+            if debug_mode:
+                print("<" + "-" * 10 + "New document group is taken into processing" + "-" * 10 + ">")
+
+            docs = documents[i: i + split_by]
+
+            loading_time = 0
+            chunk_generating_time = 0
+            db_saving_time = 0
+
+            print("Start loading the documents")
+            start = time.time()
+            self.processor.load_documents(documents=docs, add_to_unprocessed=True)
+            loading_time = time.time() - start
+
+            print("Start loading chunk generation")
+            start = time.time()
+            self.processor.generate_chunks()
+            chunk_generating_time = time.time() - start
+
+            print("Start saving to db")
+            start = time.time()
+            self.db.store(self.processor.get_and_save_unsaved_chunks())
+            db_saving_time = time.time() - start
+
+            if debug_mode:
+                print(
+                    f"loading time = {loading_time}, chunk generation time = {chunk_generating_time}, saving time = {db_saving_time}\n")
+
+    '''
+    Produces answer to user's request. First, finds the most relevant chunks, generates prompt with them, and asks llm
+    '''
+
+    def generate_response(self, user_prompt: str) -> str:
+        relevant_chunks = self.db.search(query=user_prompt, top_k=15)
+        relevant_chunks = [relevant_chunks[ranked["corpus_id"]]
+                           for ranked in self.reranker.rank(query=user_prompt, chunks=relevant_chunks)[:3]]
+
+        general_prompt = self.get_prompt_template(user_prompt=user_prompt, chunks=relevant_chunks)
+        return self.llm.get_response(prompt=general_prompt)
+
+    '''
+    Produces the list of the most relevant chunkВs
+    '''
+
+    def get_relevant_chunks(self, query):
+        relevant_chunks = self.db.search(query=query, top_k=15)
+        relevant_chunks = [relevant_chunks[ranked["corpus_id"]]
+                           for ranked in self.reranker.rank(query=query, chunks=relevant_chunks)]
+        return relevant_chunks

app/response_parser.py

@@ -0,0 +1,25 @@
+from app.document_validator import path_is_valid
+import re
+
+'''
+Replaces the matched regular exp with link via html <a></a> 
+'''
+def create_url(match: re.Match) -> str:
+    path: str = match.group(1)
+    page: str = match.group(2)
+    lines: str = match.group(3)
+    start: str = match.group(4)
+
+    if not path_is_valid(path):
+        return ""
+    
+    return f'<a href="/viewer?path={path}&page={page}&lines={lines}&start={start}">[Source]</a>'
+
+
+'''
+Replaces all occurrences of citation pattern with links
+'''
+def add_links(response: str) -> str:
+
+    citation_format = r'\[Source:\s*([^,]+?)\s*,\s*Page:\s*(\d+)\s*,\s*Lines:\s*(\d+\s*-\s*\d+)\s*,\s*Start:?\s*(\d+)\]'
+    return re.sub(pattern=citation_format, repl=create_url, string=response)

app/settings.py

@@ -0,0 +1,97 @@
+"""
+This file consolidates parameters for logging, database connections, model paths, API settings, and security.
+"""
+
+import torch
+import logging  # kind of advanced logger
+import os
+
+base_path = os.path.dirname(os.path.realpath(__file__))
+
+# Logging setup for console output.
+logging.basicConfig(
+    level=logging.INFO,
+    format="%(levelname)s: %(message)s",
+    handlers=[logging.StreamHandler()]
+)
+
+# Qdrant vector database connection.
+qdrant_client_config = {
+    "host": "localhost",
+    "port": 6333,
+}
+
+# Automatically detects CUDA or uses CPU.
+device = "cuda" if torch.cuda.is_available() else 'cpu'
+
+embedder_model = "all-MiniLM-L6-v2"
+
+reranker_model = "cross-encoder/ms-marco-MiniLM-L6-v2"
+
+local_llm_config = {
+    "model_path_or_repo_id": "TheBloke/Mistral-7B-v0.1-GGUF",
+    "model_file": "mistral-7b-v0.1.Q5_K_S.gguf",
+    "model_type": "mistral",
+    "gpu_layers": 20 if torch.cuda.is_available() else 0,
+    "threads": 8,
+    "context_length": 4096,  # The maximum context window is 4096 tokens
+    "mlock": True,  # Locks the model into RAM to prevent swapping
+}
+
+local_generation_config = {
+    "last_n_tokens": 128,  # The most recent of tokens that will be penalized (if it was repeated)
+    "temperature": 0.3,  # Controls the randomness of output. Higher value - higher randomness
+    "repetition_penalty": 1.2,
+}
+
+text_splitter_config = {
+    "chunk_size": 1000,  # The maximum size of chunk
+    "chunk_overlap": 100,
+    "length_function": len,  # Function to measure chunk length
+    "is_separator_regex": False,
+    "add_start_index": True,
+}
+
+# "127.0.0.1"
+api_config = {
+    "app": "app.api:api",
+    "host": "127.0.0.1",
+    "port": 5050,
+    "reload": True,  # The server will reload on system changes
+}
+
+gemini_generation_config = {
+    "temperature": 0,  # deterministic, predictable output
+    "top_p": 0.95,
+    "top_k": 20,
+    "candidate_count": 1,
+    "seed": 5,
+    "max_output_tokens": 1000,
+    "stop_sequences": ['STOP!'],
+    "presence_penalty": 0.0,
+    "frequency_penalty": 0.0,
+}
+
+use_gemini: bool = True
+
+max_delta = 0.15  # defines what is the minimum boundary for vectors to be considered similar
+
+# for postgres client
+# Note: you should run postgres server with similar host, post, and do not forget to create a user with similar settings
+host = "localhost"
+port = 5432
+user = "postgres"
+password = "lol"
+dbname = "exp"
+
+postgres_client_config = {
+    "url": os.environ['DATABASE_URL'],
+    "echo": False,
+}
+
+very_secret_pepper = "goida"  # +1 point, имба
+jwt_algorithm = "HS256"
+
+max_cookie_lifetime = 300  # in seconds
+
+url_user_not_required = ["login", "", "viewer", "message_with_docs", "new_user"]

docker-compose.yml

@@ -0,0 +1,21 @@
+services:
+  qdrant:
+    image: qdrant/qdrant
+    ports:
+      - "6333:6333"
+    volumes:
+      - qdrant_data:/qdrant/storage
+    restart: unless-stopped
+  api:
+    build: .
+    ports:
+      - "5050:5050"
+    depends_on:
+      - qdrant
+    environment:
+      - QDRANT_HOST=qdrant
+      - QDRANT_PORT=6333
+    restart:  unless-stopped
+
+volumes:
+  qdrant_data:

start.sh

@@ -1,3 +1,12 @@
-#!/bin/sh
-qdrant &
-uvicorn app:api --host 0.0.0.0 --port 7860
+#!/usr/bin/env sh
+# start.sh
+
+# 1) launch Qdrant on port 6333, storing data under /mnt/data/qdrant
+qdrant --storage-dir /mnt/data/qdrant &
+
+# 2) give Qdrant a second to wake up
+sleep 2
+
+# 3) start your FastAPI app on port 7860
+exec uvicorn app.main:api_config \
+     --host 0.0.0.0 --port 7860

templates/base.html

@@ -1,284 +0,0 @@
-<!DOCTYPE html>
-<html lang="en">
-<head>
-    <meta charset="UTF-8">
-    <meta name="viewport" content="width=device-width, initial-scale=1.0">
-    <title>RAG System Interface</title>
-    <style>
-        * {
-            margin: 0;
-            padding: 0;
-            box-sizing: border-box;
-            font-family: -apple-system, BlinkMacSystemFont, 'Segoe UI', Roboto, Oxygen, Ubuntu, Cantarell, sans-serif;
-        }
-        
-        body {
-            display: flex;
-            height: 100vh;
-            background-color: #111827;
-        }
-        
-        .sidebar {
-            width: 260px;
-            background-color: #1F2937;
-            padding: 20px;
-            overflow-y: auto;
-        }
-        
-        .new-chat-btn {
-            display: flex;
-            align-items: center;
-            gap: 8px;
-            padding: 10px 15px;
-            border-radius: 8px;
-            background-color: #34D399;
-            cursor: pointer;
-            margin-bottom: 20px;
-            font-size: 14px;
-            color: #333;
-        }
-        
-        .new-chat-btn:hover {
-            background-color: #90e2c4;
-        }
-        
-        .time-section {
-            margin-bottom: 20px;
-        }
-        
-        .time-header {
-            font-size: 12px;
-            color: #666;
-            margin-bottom: 10px;
-            font-weight: 500;
-        }
-        
-        .chat-item {
-            padding: 8px 10px;
-            border-radius: 6px;
-            margin-bottom: 5px;
-            color: #eee;
-            cursor: pointer;
-            font-size: 14px;
-            white-space: nowrap;
-            overflow: hidden;
-            text-overflow: ellipsis;
-            border-width: 1px;
-            border-color: #bec0c4;
-        }
-        
-        .chat-item:hover {
-            background-color: #242e3d;
-        }
-        
-        .chat-item-selected {
-            padding: 8px 10px;
-            border-radius: 6px;
-            margin-bottom: 5px;
-            color: #111827; /* Dark text for better contrast on light green */
-            background-color: #34D399; /* Your signature green color */
-            cursor: pointer;
-            font-size: 14px;
-            white-space: nowrap;
-            overflow: hidden;
-            text-overflow: ellipsis;
-            border: 1px solid #2BB389; /* Slightly darker green border */
-            font-weight: 500; /* Slightly bolder text like your user messages */
-            box-shadow: 0 1px 2px rgba(0, 0, 0, 0.1); /* Subtle shadow for depth */
-        }
-
-        /* .main-content {
-            flex: 1;
-            display: flex;
-            flex-direction: column;
-        } */
-        
-        .chat-container {
-            flex: 1;
-            padding: 30px;
-            overflow-y: auto;
-            display: flex;
-            flex-direction: column;
-            gap: 15px; /* This controls the space between messages */
-        }
-        
-        .message {
-            max-width: 1000px;
-            width: 100%;
-            margin: 0 auto;
-        }
-        
-        .user-message {
-            background-color: #34D399;
-            color: #000;
-            padding: 12px 16px;
-            border-radius: 12px;
-            font-weight: 500;
-            display: inline-block;
-            max-width: 80%;
-            float: right;
-            clear: both;
-            margin-bottom: 5px;
-        }
-
-        .bot-message {
-            background-color: #44444C;
-            color: white;
-            padding: 16px;
-            border-radius: 12px;
-            line-height: 1.6;
-            max-width: 80%;
-            float: left;
-            clear: both;
-        }
-        
-        .bot-message h1 {
-            font-size: 24px;
-            margin-bottom: 15px;
-        }
-        
-        .sources {
-            font-size: 13px;
-            color: #666;
-            margin-top: 20px;
-            padding-top: 15px;
-            border-top: 1px solid #eee;
-        }
-        
-        .database-info {
-            text-align: center;
-            font-size: 13px;
-            color: #999;
-            margin: 20px 0;
-        }
-        
-        .link {
-            color: rgb(62, 62, 206)
-        }
-        .plain-text {
-            color: white;
-        }
-
-        .input-container {
-            padding: 16px;
-            background-color: #111827;
-            border-top: 1px solid #2d3748;
-        }
-
-        .input-box {
-            max-width: 800px;
-            margin: 0 auto;
-            position: relative;
-            display: flex;
-            align-items: center;
-        }
-
-        input[type="text"] {
-            width: 100%;
-            padding: 12px 48px 12px 16px;
-            border: 1px solid #4b5563;
-            border-radius: 12px;
-            font-size: 15px;
-            outline: none;
-            background-color: #1f2937;
-            color: white;
-            box-shadow: 0 1px 3px rgba(0, 0, 0, 0.1);
-            transition: all 0.2s ease;
-        }
-
-        input[type="text"]:focus {
-            border-color: #6b7280;
-            box-shadow: 0 0 0 2px rgba(59, 130, 246, 0.2);
-        }
-
-        .send-button {
-            position: absolute;
-            right: 12px;
-            background: none;
-            border: none;
-            color: #9ca3af;
-            cursor: pointer;
-            padding: 4px;
-            border-radius: 6px;
-            transition: all 0.2s ease;
-        }
-
-        .send-button:hover {
-            color: #d1d5db;
-            background-color: rgba(255, 255, 255, 0.05);
-        }
-
-        .send-button svg {
-            display: block;
-        }
-        
-        .info-code {
-            position: absolute;
-            right: 15px;
-            bottom: -25px;
-            font-size: 12px;
-            color: #999;
-        }
-        .main-content {
-            flex: 1;
-            display: flex;
-            flex-direction: column;
-            justify-content: center; /* Add this for vertical centering */
-            align-items: center; /* Add this for horizontal centering */
-            padding: 20px; /* Optional: adds some spacing */
-        }
-
-        .rag-container {
-            text-align: center;
-            max-width: 600px;
-            width: 90%;
-            margin: 0 auto; /* Ensures horizontal centering */
-        }
-
-        .rag-title {
-            font-size: 2.5rem;
-            font-weight: 800;
-            margin-bottom: 0.5rem;
-            color: white; /* Changed to white for visibility on dark bg */
-        }
-
-        .rag-subtitle {
-            font-size: 1.1rem;
-            color: #9ca3af; /* Lighter color for better contrast */
-            margin-bottom: 2rem;
-            font-weight: 400;
-        }
-    </style>
-</head>
-<body>
-    <div class="sidebar">
-        <button class="new-chat-btn" onclick="location.href='/'">
-            <span>Add new chat</span>
-        </button>
-        
-        <div class="time-section">
-            <div class="time-header">TODAY</div>
-            <div class="chat-item" onclick="location.href='/chat_example'">Explanation of RAG system</div>
-            <div class="chat-item" onclick="location.href='/chat_example'">IEEE citation format guidell...</div>
-        </div>
-        
-        <div class="time-section">
-            <div class="time-header">LAST WEEK</div>
-            <div class="chat-item" onclick="location.href='/chat_example'">System test: explanation of...</div>
-        </div>
-        
-        <div class="time-section">
-            <div class="time-header">LAST MONTH</div>
-            <div class="chat-item" onclick="location.href='/chat_example'">How rich is Elon Musk?</div>
-            <div class="chat-item" onclick="location.href='/chat_example'">Tesla: main pros and cons t...</div>
-        </div>
-    </div>
-    
-    <div class="main-content">
-        <div class="rag-container">
-            <h1 class="rag-title">The Ultimate RAG</h1>
-            <p class="rag-subtitle">ask anything...</p>
-        </div>
-    </div>
-</body>
-</html>

templates/index.html

@@ -1,282 +0,0 @@
-<!DOCTYPE html>
-<html lang="en">
-<head>
-    <meta charset="UTF-8">
-    <meta name="viewport" content="width=device-width, initial-scale=1.0">
-    <title>RAG System Interface</title>
-    <style>
-        * {
-            margin: 0;
-            padding: 0;
-            box-sizing: border-box;
-            font-family: -apple-system, BlinkMacSystemFont, 'Segoe UI', Roboto, Oxygen, Ubuntu, Cantarell, sans-serif;
-        }
-        
-        body {
-            display: flex;
-            height: 100vh;
-            background-color: #111827;
-        }
-        
-        .sidebar {
-            width: 260px;
-            background-color: #1F2937;
-            padding: 20px;
-            overflow-y: auto;
-        }
-        
-        .new-chat-btn {
-            display: flex;
-            align-items: center;
-            gap: 8px;
-            padding: 10px 15px;
-            border-radius: 8px;
-            background-color: #34D399;
-            cursor: pointer;
-            margin-bottom: 20px;
-            font-size: 14px;
-            color: #333;
-        }
-        
-        .new-chat-btn:hover {
-            background-color: #90e2c4;
-        }
-        
-        .time-section {
-            margin-bottom: 20px;
-        }
-        
-        .time-header {
-            font-size: 12px;
-            color: #666;
-            margin-bottom: 10px;
-            font-weight: 500;
-        }
-        
-        .chat-item {
-            padding: 8px 10px;
-            border-radius: 6px;
-            margin-bottom: 5px;
-            color: #eee;
-            cursor: pointer;
-            font-size: 14px;
-            white-space: nowrap;
-            overflow: hidden;
-            text-overflow: ellipsis;
-            border-width: 1px;
-            border-color: #bec0c4;
-        }
-        
-        .chat-item:hover {
-            background-color: #242e3d;
-        }
-        
-        .chat-item-selected {
-            padding: 8px 10px;
-            border-radius: 6px;
-            margin-bottom: 5px;
-            color: #111827; /* Dark text for better contrast on light green */
-            background-color: #34D399; /* Your signature green color */
-            cursor: pointer;
-            font-size: 14px;
-            white-space: nowrap;
-            overflow: hidden;
-            text-overflow: ellipsis;
-            border: 1px solid #2BB389; /* Slightly darker green border */
-            font-weight: 500; /* Slightly bolder text like your user messages */
-            box-shadow: 0 1px 2px rgba(0, 0, 0, 0.1); /* Subtle shadow for depth */
-        }
-
-        .main-content {
-            flex: 1;
-            display: flex;
-            flex-direction: column;
-        }
-        
-        .chat-container {
-            flex: 1;
-            padding: 30px;
-            overflow-y: auto;
-            display: flex;
-            flex-direction: column;
-            gap: 15px; /* This controls the space between messages */
-        }
-        
-        .message {
-            max-width: 1000px;
-            width: 100%;
-            margin: 0 auto;
-        }
-        
-        .user-message {
-            background-color: #34D399;
-            color: #000;
-            padding: 12px 16px;
-            border-radius: 12px;
-            font-weight: 500;
-            display: inline-block;
-            max-width: 80%;
-            float: right;
-            clear: both;
-            margin-bottom: 5px;
-        }
-
-        .bot-message {
-            background-color: #44444C;
-            color: white;
-            padding: 16px;
-            border-radius: 12px;
-            line-height: 1.6;
-            max-width: 80%;
-            float: left;
-            clear: both;
-        }
-        
-        .bot-message h1 {
-            font-size: 24px;
-            margin-bottom: 15px;
-        }
-        
-        .sources {
-            font-size: 13px;
-            color: #666;
-            margin-top: 20px;
-            padding-top: 15px;
-            border-top: 1px solid #eee;
-        }
-        
-        .database-info {
-            text-align: center;
-            font-size: 13px;
-            color: #999;
-            margin: 20px 0;
-        }
-        
-        .link {
-            color: rgb(62, 62, 206)
-        }
-        .plain-text {
-            color: white;
-        }
-
-        .input-container {
-            padding: 16px;
-            background-color: #111827;
-            border-top: 1px solid #2d3748;
-        }
-
-        .input-box {
-            max-width: 800px;
-            margin: 0 auto;
-            position: relative;
-            display: flex;
-            align-items: center;
-        }
-
-        input[type="text"] {
-            width: 100%;
-            padding: 12px 48px 12px 16px;
-            border: 1px solid #4b5563;
-            border-radius: 12px;
-            font-size: 15px;
-            outline: none;
-            background-color: #1f2937;
-            color: white;
-            box-shadow: 0 1px 3px rgba(0, 0, 0, 0.1);
-            transition: all 0.2s ease;
-        }
-
-        input[type="text"]:focus {
-            border-color: #6b7280;
-            box-shadow: 0 0 0 2px rgba(59, 130, 246, 0.2);
-        }
-
-        .send-button {
-            position: absolute;
-            right: 12px;
-            background: none;
-            border: none;
-            color: #9ca3af;
-            cursor: pointer;
-            padding: 4px;
-            border-radius: 6px;
-            transition: all 0.2s ease;
-        }
-
-        .send-button:hover {
-            color: #d1d5db;
-            background-color: rgba(255, 255, 255, 0.05);
-        }
-
-        .send-button svg {
-            display: block;
-        }
-        
-        .info-code {
-            position: absolute;
-            right: 15px;
-            bottom: -25px;
-            font-size: 12px;
-            color: #999;
-        }
-    </style>
-</head>
-<body>
-    <div class="sidebar">
-        <button class="new-chat-btn" onclick="location.href='/'">
-            <span>Add new chat</span>
-        </button>
-        
-        <div class="time-section">
-            <div class="time-header">TODAY</div>
-            <div class="chat-item-selected">Explanation of RAG system</div>
-            <div class="chat-item">IEEE citation format guidell...</div>
-        </div>
-        
-        <div class="time-section">
-            <div class="time-header">LAST WEEK</div>
-            <div class="chat-item">System test: explanation of...</div>
-        </div>
-        
-        <div class="time-section">
-            <div class="time-header">LAST MONTH</div>
-            <div class="chat-item">How rich is Elon Musk?</div>
-            <div class="chat-item">Tesla: main pros and cons t...</div>
-        </div>
-    </div>
-    
-    <div class="main-content">
-        <div class="chat-container">
-            <div class="message">
-                <div class="user-message">
-                    <p>Explain, please, what is RAG?</p>
-                </div>
-            </div>
-            
-            <div class="message">
-                <div class="bot-message">
-                    <h1 class="plain-text">RAG stands for Retrieval-Augmented Generation.</h1>
-                    
-                    <p class="plain-text">Think of it as giving your AI a specific relevant documents (or chunks) that it can quickly scan through to find relevant information before answering your questions.</p>
-                    
-                    <p class="plain-text">So, instead of searching the entire database (which might not fit in the LLM models context window, or even if it fits it will consume a lot of tokens to answers) we give the LLM only the relevant documents (chunks) that it needs to look up in order to answer user question.</p>
-                    
-                    <h3 class="plain-text">Sources:</h3>
-                    <div class="plain-text">1. <a href="#" class="link">«About RAG»</a>, p. 21, lines: 32-41</div>
-                </div>
-            </div>
-        </div>
-        
-        <div class="input-container">
-            <div class="input-box">
-                <input type="text" placeholder="Ask your question here">
-                <button class="send-button">
-                    <svg width="24" height="24" viewBox="0 0 24 24" fill="none">
-                        <path d="M7 11L12 6L17 11M12 18V7" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round"></path>
-                    </svg>
-                </button>
-            </div>
-        </div>
-    </div>
-</body>
-</html>