Merge pull request #20 from cyberalertnepal/PujanDev

.gitignore

@@ -59,3 +59,4 @@ model/
 models/.gitattributes  #<-- This line can stay if you only want to ignore that file, not the whole folder
 
 todo.md
+np_text_model

Dockerfile

@@ -0,0 +1,17 @@
+# Read the doc: https://huggingface.co/docs/hub/spaces-sdks-docker
+# you will also find guides on how best to write your Dockerfile
+
+FROM python:3.9
+
+RUN useradd -m -u 1000 user
+USER user
+ENV PATH="/home/user/.local/bin:$PATH"
+
+WORKDIR /app
+
+COPY --chown=user ./requirements.txt requirements.txt
+RUN pip install --no-cache-dir --upgrade -r requirements.txt
+RUN python -m spacy download en_core_web_sm || echo "Failed to download model"
+
+COPY --chown=user . /app
+CMD ["uvicorn", "app:app", "--host", "0.0.0.0", "--port", "7860"]

README.md

@@ -0,0 +1,9 @@
+---
+title: Ai-Checker
+emoji: 🚀
+colorFrom: yellow
+colorTo: blue
+sdk: docker
+pinned: false
+---
+

app.py

@@ -5,6 +5,7 @@ from slowapi.errors import RateLimitExceeded
 from slowapi.util import get_remote_address
 from fastapi.responses import JSONResponse
 from features.text_classifier.routes import router as text_classifier_router
+from features.nepali_text_classifier.routes import router as nepali_text_classifier_router
 from config import ACCESS_RATE
 import requests
 limiter = Limiter(key_func=get_remote_address, default_limits=[ACCESS_RATE])
@@ -25,7 +26,7 @@ app.add_middleware(SlowAPIMiddleware)
 
 # Include your routes
 app.include_router(text_classifier_router, prefix="/text")
-
+app.include_router(nepali_text_classifier_router,prefix="/NP")
 @app.get("/")
 @limiter.limit(ACCESS_RATE)
 async def root(request: Request):

docs/api_endpoints.md

@@ -0,0 +1,75 @@
+# 🧩 API Endpoints
+
+### English (GPT-2) - `/text/`
+
+| Endpoint                         | Method | Description                               |
+| --------------------------------- | ------ | ----------------------------------------- |
+| `/text/analyse`                  | POST   | Classify raw English text                 |
+| `/text/analyse-sentences`        | POST   | Sentence-by-sentence breakdown            |
+| `/text/analyse-sentance-file`    | POST   | Upload file, per-sentence breakdown       |
+| `/text/upload`                   | POST   | Upload file for overall classification    |
+| `/text/health`                   | GET    | Health check                             |
+
+#### Example: Classify English text
+
+```bash
+curl -X POST http://localhost:8000/text/analyse \
+  -H "Authorization: Bearer <SECRET_TOKEN>" \
+  -H "Content-Type: application/json" \
+  -d '{"text": "This is a sample text for analysis."}'
+```
+
+**Response:**
+```json
+{
+  "result": "AI-generated",
+  "perplexity": 55.67,
+  "ai_likelihood": 66.6
+}
+```
+
+#### Example: File upload
+
+```bash
+curl -X POST http://localhost:8000/text/upload \
+  -H "Authorization: Bearer <SECRET_TOKEN>" \
+  -F 'file=@yourfile.txt;type=text/plain'
+```
+
+---
+
+### Nepali (SentencePiece) - `/NP/`
+
+| Endpoint                         | Method | Description                               |
+| --------------------------------- | ------ | ----------------------------------------- |
+| `/NP/analyse`                    | POST   | Classify Nepali text                      |
+| `/NP/analyse-sentences`          | POST   | Sentence-by-sentence breakdown            |
+| `/NP/upload`                     | POST   | Upload Nepali PDF for classification      |
+| `/NP/file-sentences-analyse`     | POST   | PDF upload, per-sentence breakdown        |
+| `/NP/health`                     | GET    | Health check                             |
+
+#### Example: Nepali text classification
+
+```bash
+curl -X POST http://localhost:8000/NP/analyse \
+  -H "Authorization: Bearer <SECRET_TOKEN>" \
+  -H "Content-Type: application/json" \
+  -d '{"text": "यो उदाहरण वाक्य हो।"}'
+```
+
+**Response:**
+```json
+{
+  "label": "Human",
+  "confidence": 98.6
+}
+```
+
+#### Example: Nepali PDF upload
+
+```bash
+curl -X POST http://localhost:8000/NP/upload \
+  -H "Authorization: Bearer <SECRET_TOKEN>" \
+  -F 'file=@NepaliText.pdf;type=application/pdf'
+```
+

docs/deployment.md

@@ -0,0 +1,105 @@
+
+#  Deployment 
+
+This project is containerized and deployed on **Hugging Face Spaces** using a custom `Dockerfile`. This guide explains the structure of the Dockerfile and key considerations for deploying FastAPI apps on Spaces with Docker SDK.
+
+---
+
+## 📦 Base Image
+
+```dockerfile
+FROM python:3.9
+````
+
+We use the official Python 3.9 image for compatibility and stability across most Python libraries and tools.
+
+---
+
+## 👤 Create a Non-Root User
+
+```dockerfile
+RUN useradd -m -u 1000 user
+USER user
+ENV PATH="/home/user/.local/bin:$PATH"
+```
+
+* Hugging Face Spaces **requires** that containers run as a non-root user with UID `1000`.
+* We also prepend the user's local binary path to `PATH` for Python package accessibility.
+
+---
+
+## 🗂️ Set Working Directory
+
+```dockerfile
+WORKDIR /app
+```
+
+All application files will reside under `/app` for consistency and clarity.
+
+---
+
+## 📋 Install Dependencies
+
+```dockerfile
+COPY --chown=user ./requirements.txt requirements.txt
+RUN pip install --no-cache-dir --upgrade -r requirements.txt
+```
+
+* Copies the dependency list with correct file ownership.
+* Uses `--no-cache-dir` to reduce image size.
+* Ensures the latest compatible versions are installed.
+
+---
+
+## 🔡 Download Language Model (Optional)
+
+```dockerfile
+RUN python -m spacy download en_core_web_sm || echo "Failed to download model"
+```
+
+* Downloads the small English NLP model required by SpaCy.
+* Uses `|| echo ...` to prevent build failure if the download fails (optional safeguard).
+
+---
+
+## 📁 Copy Project Files
+
+```dockerfile
+COPY --chown=user . /app
+```
+
+Copies the entire project source into the container, setting correct ownership for Hugging Face's user-based execution.
+
+---
+
+## 🌐 Start the FastAPI Server
+
+```dockerfile
+CMD ["uvicorn", "app:app", "--host", "0.0.0.0", "--port", "7860"]
+```
+
+* Launches the FastAPI app using `uvicorn`.
+* **Port 7860 is mandatory** for Docker-based Hugging Face Spaces deployments.
+* `app:app` refers to the `FastAPI()` instance in `app.py`.
+
+---
+
+## ✅ Deployment Checklist
+
+* [x] Ensure your main file is named `app.py` or adjust `CMD` accordingly.
+* [x] All dependencies should be listed in `requirements.txt`.
+* [x] If using models like SpaCy, verify they are downloaded or bundled.
+* [x] Test your Dockerfile locally with `docker build` before pushing to Hugging Face.
+
+---
+
+## 📚 References
+
+* Hugging Face Docs: [Spaces Docker SDK](https://huggingface.co/docs/hub/spaces-sdks-docker)
+* Uvicorn Docs: [https://www.uvicorn.org/](https://www.uvicorn.org/)
+* SpaCy Models: [https://spacy.io/models](https://spacy.io/models)
+
+---
+
+Happy deploying!  
+**P.S.** Try not to break stuff. 😅

docs/functions.md

@@ -0,0 +1,53 @@
+# Major  Functions used 
+
+## in Text Classifier (`features/text_classifier/` and `features/text_classifier/`)
+
+- **`load_model()`**  
+  Loads the GPT-2 model and tokenizer from the specified directory paths.
+
+- **`lifespan()`**  
+  Manages the application lifecycle. Initializes the model at startup and handles cleanup on shutdown.
+
+- **`classify_text_sync()`**  
+  Synchronously tokenizes input text and predicts using the GPT-2 model. Returns classification and perplexity.
+
+- **`classify_text()`**  
+  Asynchronously runs `classify_text_sync()` in a thread pool for non-blocking text classification.
+
+- **`analyze_text()`**  
+  **POST** endpoint: Accepts text input, classifies it using `classify_text()`, and returns the result with perplexity.
+
+- **`health()`**  
+  **GET** endpoint: Simple health check for API liveness.
+
+- **`parse_docx()`, `parse_pdf()`, `parse_txt()`**  
+  Utilities to extract and convert `.docx`, `.pdf`, and `.txt` file contents to plain text.
+
+- **`warmup()`**  
+  Downloads the model repository and initializes the model/tokenizer using `load_model()`.
+
+- **`download_model_repo()`**  
+  Downloads the model files from the designated `MODEL` folder.
+
+- **`get_model_tokenizer()`**  
+  Checks if the model already exists; if not, downloads it—otherwise, loads the cached model.
+
+- **`handle_file_upload()`**  
+  Handles file uploads from the `/upload` route. Extracts text, classifies, and returns results.
+
+- **`extract_file_contents()`**  
+  Extracts and returns plain text from uploaded files (PDF, DOCX, TXT).
+
+- **`handle_file_sentence()`**  
+  Processes file uploads by analyzing each sentence (under 10,000 chars) before classification.
+
+- **`handle_sentence_level_analysis()`**  
+  Checks/strips each sentence, then computes AI/human likelihood for each.
+
+- **`analyze_sentences()`**  
+  Splits paragraphs into sentences, classifies each, and returns all results.
+
+- **`analyze_sentence_file()`**  
+  Like `handle_file_sentence()`—analyzes sentences in uploaded files.
+
+## for image_classifier

docs/nestjs_integration.md

@@ -0,0 +1,82 @@
+# Nestjs + fastapi
+
+You can easily call this API from a NestJS microservice.
+
+**.env**
+```env
+FASTAPI_BASE_URL=http://localhost:8000
+SECRET_TOKEN=your_secret_token_here
+```
+
+**fastapi.service.ts**
+
+```typescript
+import { Injectable } from "@nestjs/common";
+import { HttpService } from "@nestjs/axios";
+import { ConfigService } from "@nestjs/config";
+import { firstValueFrom } from "rxjs";
+
+@Injectable()
+export class FastAPIService {
+  constructor(
+    private http: HttpService,
+    private config: ConfigService,
+  ) {}
+
+  async analyzeText(text: string) {
+    const url = `${this.config.get("FASTAPI_BASE_URL")}/text/analyse`;
+    const token = this.config.get("SECRET_TOKEN");
+
+    const response = await firstValueFrom(
+      this.http.post(
+        url,
+        { text },
+        {
+          headers: {
+            Authorization: `Bearer ${token}`,
+          },
+        },
+      ),
+    );
+
+    return response.data;
+  }
+}
+```
+
+**app.module.ts**
+```typescript
+import { Module } from "@nestjs/common";
+import { ConfigModule } from "@nestjs/config";
+import { HttpModule } from "@nestjs/axios";
+import { AppController } from "./app.controller";
+import { FastAPIService } from "./fastapi.service";
+
+@Module({
+  imports: [ConfigModule.forRoot(), HttpModule],
+  controllers: [AppController],
+  providers: [FastAPIService],
+})
+export class AppModule {}
+```
+
+**app.controller.ts**
+```typescript
+import { Body, Controller, Post, Get } from '@nestjs/common';
+import { FastAPIService } from './fastapi.service';
+
+@Controller()
+export class AppController {
+  constructor(private readonly fastapiService: FastAPIService) {}
+
+  @Post('analyze-text')
+  async callFastAPI(@Body('text') text: string) {
+    return this.fastapiService.analyzeText(text);
+  }
+
+  @Get()
+  getHello(): string {
+    return 'NestJS is connected to FastAPI';
+  }
+}
+```

docs/security.md

@@ -0,0 +1,9 @@
+# Security: Bearer Token Auth
+
+All endpoints require authentication via Bearer token:
+
+- Set `SECRET_TOKEN` in `.env`
+- Add header: `Authorization: Bearer <SECRET_TOKEN>`
+
+Unauthorized requests receive `403 Forbidden`.
+

docs/setup.md

@@ -0,0 +1,23 @@
+# Setup & Installation
+
+## 1. Clone the Repository
+```bash
+git clone https://github.com/cyberalertnepal/aiapi
+cd aiapi
+```
+
+## 2. Install Dependencies
+```bash
+pip install -r requirements.txt
+```
+
+## 3. Configure Environment
+Create a `.env` file:
+```env
+SECRET_TOKEN=your_secret_token_here
+```
+
+## 4. Run the API
+```bash
+uvicorn app:app --host 0.0.0.0 --port 8000
+```

docs/structure.md

@@ -0,0 +1,54 @@
+## 🏗️ Project Structure
+
+```
+├── app.py                   # Main FastAPI app entrypoint
+├── config.py                # Configuration loader (.env, settings)
+├── features/
+│   ├── text_classifier/     # English (GPT-2) classifier
+│   │   ├── controller.py
+│   │   ├── inferencer.py
+│   │   ├── model_loader.py
+│   │   ├── preprocess.py
+│   │   └── routes.py
+│   └── nepali_text_classifier/ # Nepali (sentencepiece) classifier
+│       ├── controller.py
+│       ├── inferencer.py
+│       ├── model_loader.py
+│       ├── preprocess.py
+│       └── routes.py
+├── np_text_model/           # Nepali model artifacts (auto-downloaded)
+│   ├── classifier/
+│   │   └── sentencepiece.bpe.model
+│   └── model_95_acc.pth
+├── models/                  # English GPT-2 model/tokenizer (auto-downloaded)
+│   ├── merges.txt
+│   ├── tokenizer.json
+│   └── model_weights.pth
+├── Dockerfile               # Container build config
+├── Procfile                 # Deployment entrypoint (for PaaS)
+├── requirements.txt         # Python dependencies
+├── README.md     
+├── Docs                     # documents
+└── .env                     # Secret token(s), environment config
+```
+### 🌟 Key Files and Their Roles
+
+- **`app.py`**: Entry point initializing FastAPI app and routes.
+- **`Procfile`**: Tells Railway (or similar platforms) how to run the program.
+- **`requirements.txt`**: Tracks all Python dependencies for the project.
+- **`__init__.py`**: Package initializer for the root module and submodules.
+- **`features/text_classifier/`**
+  - **`controller.py`**: Handles logic between routes and the model.
+  - **`inferencer.py`**: Runs inference and returns predictions as well as file system 
+  utilities.
+- **`features/NP/`**
+  - **`controller.py`**: Handles logic between routes and the model.
+  - **`inferencer.py`**: Runs inference and returns predictions as well as file system 
+  utilities.
+  - **`model_loader.py`**: Loads the ML model and tokenizer.
+  - **`preprocess.py`**: Prepares input text for the model.
+  - **`routes.py`**: Defines API routes for text classification.
+
+
+
+-[Main](../README.md)

features/nepali_text_classifier/controller.py

@@ -0,0 +1,131 @@
+import asyncio
+from io import BytesIO
+from fastapi import HTTPException, UploadFile, status, Depends
+from fastapi.security import HTTPBearer, HTTPAuthorizationCredentials
+import os
+
+from features.nepali_text_classifier.inferencer import classify_text
+from  features.nepali_text_classifier.preprocess import *
+import re
+
+security = HTTPBearer()
+
+def contains_english(text: str) -> bool:
+    # Remove escape characters
+    cleaned = text.replace("\n", "").replace("\t", "")
+    return bool(re.search(r'[a-zA-Z]', cleaned))
+
+
+async def verify_token(credentials: HTTPAuthorizationCredentials = Depends(security)):
+    token = credentials.credentials
+    expected_token = os.getenv("MY_SECRET_TOKEN")
+    if token != expected_token:
+        raise HTTPException(
+            status_code=status.HTTP_403_FORBIDDEN,
+            detail="Invalid or expired token"
+        )
+    return token
+
+async def nepali_text_analysis(text: str):
+    end_symbol_for_NP_text(text)
+    words = text.split()
+    if len(words) < 10:
+        raise HTTPException(status_code=400, detail="Text must contain at least 10 words")
+    if len(text) > 10000:
+        raise HTTPException(status_code=413, detail="Text must be less than 10,000 characters")
+
+    result = await asyncio.to_thread(classify_text, text)
+
+    return result
+
+
+#Extract text form uploaded files(.docx,.pdf,.txt)
+async def extract_file_contents(file:UploadFile)-> str:
+    content = await file.read()
+    file_stream = BytesIO(content)
+    if file.content_type == "application/vnd.openxmlformats-officedocument.wordprocessingml.document":
+        return parse_docx(file_stream)
+    elif file.content_type =="application/pdf":
+        return parse_pdf(file_stream)
+    elif file.content_type =="text/plain":
+        return parse_txt(file_stream)
+    else:
+        raise HTTPException(status_code=415,detail="Invalid file type. Only .docx,.pdf and .txt are allowed")
+
+async def handle_file_upload(file: UploadFile):
+    try:
+        file_contents = await extract_file_contents(file)
+        end_symbol_for_NP_text(file_contents)
+        if len(file_contents) > 10000:
+            raise HTTPException(status_code=413, detail="Text must be less than 10,000 characters")
+
+        cleaned_text = file_contents.replace("\n", " ").replace("\t", " ").strip()
+        if not cleaned_text:
+            raise HTTPException(status_code=404, detail="The file is empty or only contains whitespace.")
+        
+        result = await asyncio.to_thread(classify_text, cleaned_text)
+        return result
+    except Exception as e:
+        logging.error(f"Error processing file: {e}")
+        raise HTTPException(status_code=500, detail="Error processing the file")
+
+
+
+async def handle_sentence_level_analysis(text: str):
+    text = text.strip()
+    if len(text) > 10000:
+        raise HTTPException(status_code=413, detail="Text must be less than 10,000 characters")
+    
+    end_symbol_for_NP_text(text)
+
+    # Split text into sentences
+    sentences = [s.strip() + "।" for s in text.split("।") if s.strip()]
+
+    results = []
+    for sentence in sentences:
+        end_symbol_for_NP_text(sentence)
+        result = await asyncio.to_thread(classify_text, sentence)
+        results.append({
+            "text": sentence,
+            "result": result["label"],
+            "likelihood": result["confidence"]
+        })
+
+    return {"analysis": results}
+
+
+async def handle_file_sentence(file:UploadFile):
+    try:
+        file_contents = await extract_file_contents(file)
+        if len(file_contents) > 10000:
+            raise HTTPException(status_code=413, detail="Text must be less than 10,000 characters")
+
+        cleaned_text = file_contents.replace("\n", " ").replace("\t", " ").strip()
+        if not cleaned_text:
+            raise HTTPException(status_code=404, detail="The file is empty or only contains whitespace.")
+        # Ensure text ends with danda so last sentence is included
+
+        # Split text into sentences
+        sentences = [s.strip() + "।" for s in cleaned_text.split("।") if s.strip()]
+
+        results = []
+        for sentence in sentences:
+            end_symbol_for_NP_text(sentence)
+
+            result = await asyncio.to_thread(classify_text, sentence)
+            results.append({
+                "text": sentence,
+                "result": result["label"],
+                "likelihood": result["confidence"]
+            })
+
+        return {"analysis": results}
+
+    except Exception as e:
+        logging.error(f"Error processing file: {e}")
+        raise HTTPException(status_code=500, detail="Error processing the file")
+
+
+def classify(text: str):
+    return classify_text(text)
+

features/nepali_text_classifier/inferencer.py

@@ -0,0 +1,23 @@
+import torch
+from .model_loader import get_model_tokenizer
+import torch.nn.functional as F
+
+device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
+
+
+def classify_text(text: str):
+    model, tokenizer = get_model_tokenizer()
+    inputs = tokenizer(text, return_tensors='pt', padding=True, truncation=True, max_length=512)
+    inputs = {k: v.to(device) for k, v in inputs.items()}
+
+    with torch.no_grad():
+        outputs = model(**inputs)
+        logits = outputs if isinstance(outputs, torch.Tensor) else outputs.logits
+        probs = F.softmax(logits, dim=1)
+        pred = torch.argmax(probs, dim=1).item()
+        prob_percent = probs[0][pred].item() * 100
+
+    return {"label": "Human" if pred == 0 else "AI", "confidence": round(prob_percent, 2)}
+
+
+

features/nepali_text_classifier/model_loader.py

@@ -0,0 +1,54 @@
+import os
+import shutil
+import torch
+import torch.nn as nn
+import torch.nn.functional as F
+import logging
+from huggingface_hub import snapshot_download
+from transformers import AutoTokenizer, AutoModel
+
+# Configs
+REPO_ID = "Pujan-Dev/Nepali-AI-VS-HUMAN"
+BASE_DIR = "./np_text_model"
+TOKENIZER_DIR = os.path.join(BASE_DIR, "classifier")  # <- update this to match your uploaded folder
+WEIGHTS_PATH = os.path.join(BASE_DIR, "model_95_acc.pth")  # <- change to match actual uploaded weight
+device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
+
+# Define model class
+class XLMRClassifier(nn.Module):
+    def __init__(self):
+        super(XLMRClassifier, self).__init__()
+        self.bert = AutoModel.from_pretrained("xlm-roberta-base")
+        self.classifier = nn.Linear(self.bert.config.hidden_size, 2)
+
+    def forward(self, input_ids, attention_mask):
+        outputs = self.bert(input_ids=input_ids, attention_mask=attention_mask)
+        cls_output = outputs.last_hidden_state[:, 0, :]
+        return self.classifier(cls_output)
+
+# Globals for caching
+_model = None
+_tokenizer = None
+
+def download_model_repo():
+    if os.path.exists(BASE_DIR) and os.path.isdir(BASE_DIR):
+        logging.info("Model already downloaded.")
+        return
+    snapshot_path = snapshot_download(repo_id=REPO_ID)
+    os.makedirs(BASE_DIR, exist_ok=True)
+    shutil.copytree(snapshot_path, BASE_DIR, dirs_exist_ok=True)
+
+def load_model():
+    download_model_repo()
+    tokenizer = AutoTokenizer.from_pretrained(TOKENIZER_DIR)
+    model = XLMRClassifier().to(device)
+    model.load_state_dict(torch.load(WEIGHTS_PATH, map_location=device))
+    model.eval()
+    return model, tokenizer
+
+def get_model_tokenizer():
+    global _model, _tokenizer
+    if _model is None or _tokenizer is None:
+        _model, _tokenizer = load_model()
+    return _model, _tokenizer
+

features/nepali_text_classifier/preprocess.py

@@ -0,0 +1,38 @@
+import fitz  # PyMuPDF
+import docx
+from io import BytesIO
+import logging
+from fastapi import HTTPException
+
+
+def parse_docx(file: BytesIO):
+    doc = docx.Document(file)
+    text = ""
+    for para in doc.paragraphs:
+        text += para.text + "\n"
+    return text
+
+
+def parse_pdf(file: BytesIO):
+    try:
+        doc = fitz.open(stream=file, filetype="pdf")
+        text = ""
+        for page_num in range(doc.page_count):
+            page = doc.load_page(page_num)
+            text += page.get_text()
+        return text
+    except Exception as e:
+        logging.error(f"Error while processing PDF: {str(e)}")
+        raise HTTPException(
+            status_code=500, detail="Error processing PDF file")
+
+
+def parse_txt(file: BytesIO):
+    return file.read().decode("utf-8")
+
+
+def end_symbol_for_NP_text(text):
+        if not text.endswith("।"):
+            text += "।"
+
+

features/nepali_text_classifier/routes.py

@@ -0,0 +1,45 @@
+from slowapi import Limiter
+from config import ACCESS_RATE
+from .controller import handle_file_sentence, handle_sentence_level_analysis, nepali_text_analysis
+from .inferencer import classify_text
+from fastapi import APIRouter, File, Request, Depends, HTTPException, UploadFile
+from fastapi.security import HTTPBearer
+from slowapi import Limiter
+from slowapi.util import get_remote_address
+from pydantic import BaseModel
+from .controller import handle_file_upload
+router = APIRouter()
+limiter = Limiter(key_func=get_remote_address)
+security = HTTPBearer()
+
+# Input schema
+class TextInput(BaseModel):
+    text: str
+
+@router.post("/analyse")
+@limiter.limit(ACCESS_RATE)
+async def analyse(request: Request, data: TextInput, token: str = Depends(security)):
+    result = classify_text(data.text)
+    return result
+
+@router.post("/upload")
+@limiter.limit(ACCESS_RATE)
+async def upload_file(request:Request,file:UploadFile=File(...),token:str=Depends(security)):
+    return await handle_file_upload(file)
+
+@router.post("/analyse-sentences")
+@limiter.limit(ACCESS_RATE)
+async def upload_file(request:Request,data:TextInput,token:str=Depends(security)):
+    return await  handle_sentence_level_analysis(data.text)
+
+@router.post("/file-sentences-analyse")
+@limiter.limit(ACCESS_RATE)
+async def analyze_sentance_file(request: Request, file: UploadFile = File(...), token: str = Depends(security)):
+    return await handle_file_sentence(file)
+
+
+@router.get("/health")
+@limiter.limit(ACCESS_RATE)
+def health(request: Request):
+    return {"status": "ok"}
+

features/text_classifier/controller.py

@@ -5,12 +5,12 @@ from io import BytesIO
 
 from fastapi import HTTPException, UploadFile, status, Depends
 from fastapi.security import HTTPBearer, HTTPAuthorizationCredentials
-from nltk.tokenize import sent_tokenize
 
 from .inferencer import classify_text
 from .preprocess import parse_docx, parse_pdf, parse_txt
-
+import spacy
 security = HTTPBearer()
+nlp = spacy.load("en_core_web_sm")
 
 # Verify Bearer token from Authorization header
 async def verify_token(credentials: HTTPAuthorizationCredentials = Depends(security)):
@@ -52,7 +52,7 @@ async def extract_file_contents(file: UploadFile) -> str:
     else:
         raise HTTPException(
             status_code=415,
-            detail="Invalid file type. Only .docx, .pdf, and .txt are allowed."
+            detail="Invalid file type. Only .docx, .pdf and .txt are allowed."
         )
 
 # Classify text from uploaded file
@@ -60,7 +60,7 @@ async def handle_file_upload(file: UploadFile):
     try:
         file_contents = await extract_file_contents(file)
         if len(file_contents) > 10000:
-            return {"message": "File contains more than 10,000 characters."}
+            raise HTTPException(status_code=413, detail="Text must be less than 10,000 characters")
 
         cleaned_text = file_contents.replace("\n", " ").replace("\t", " ").strip()
         if not cleaned_text:
@@ -77,18 +77,22 @@ async def handle_file_upload(file: UploadFile):
         logging.error(f"Error processing file: {e}")
         raise HTTPException(status_code=500, detail="Error processing the file")
 
-# Analyze each sentence in plain text input
+
+
 async def handle_sentence_level_analysis(text: str):
     text = text.strip()
-    if text[-1] != ".":
-        text+="."
+    if not text.endswith("."):
+        text += "."
+
     if len(text) > 10000:
         raise HTTPException(status_code=413, detail="Text must be less than 10,000 characters")
-     
-    sentences = sent_tokenize(text, language="english")
+
+    doc = nlp(text)
+    sentences = [sent.text.strip() for sent in doc.sents]
+
     results = []
     for sentence in sentences:
-        if not sentence.strip():
+        if not sentence:
             continue
         label, perplexity, ai_likelihood = await asyncio.to_thread(classify_text, sentence)
         results.append({
@@ -97,14 +101,13 @@ async def handle_sentence_level_analysis(text: str):
             "perplexity": round(perplexity, 2),
             "ai_likelihood": ai_likelihood
         })
-    return {"analysis": results}
 
-# Analyze each sentence from uploaded file
+    return {"analysis": results}# Analyze each sentence from uploaded file
 async def handle_file_sentence(file: UploadFile):
     try:
         file_contents = await extract_file_contents(file)
         if len(file_contents) > 10000:
-            return {"message": "File contains more than 10,000 characters."}
+            raise HTTPException(status_code=413, detail="Text must be less than 10,000 characters")
 
         cleaned_text = file_contents.replace("\n", " ").replace("\t", " ").strip()
         if not cleaned_text:
@@ -119,7 +122,6 @@ async def handle_file_sentence(file: UploadFile):
         logging.error(f"Error processing file: {e}")
         raise HTTPException(status_code=500, detail="Error processing the file")
 
-# Optional synchronous helper function
 def classify(text: str):
     return classify_text(text)
 

features/text_classifier/model_loader.py

@@ -5,7 +5,6 @@ from transformers import GPT2LMHeadModel, GPT2TokenizerFast, GPT2Config
 from huggingface_hub import snapshot_download
 import torch
 from dotenv import load_dotenv
-import nltk
 load_dotenv()
 REPO_ID = "Pujan-Dev/AI-Text-Detector"
 MODEL_DIR = "./models"
@@ -19,10 +18,6 @@ _model, _tokenizer = None, None
 def warmup():
     global _model, _tokenizer
     # Ensure punkt is available
-    nltk.download("punkt")
-
-    nltk.download('punkt_tab')
-
     download_model_repo()
     _model, _tokenizer = load_model()
     logging.info("Its ready")

readme.md

@@ -1,401 +1,35 @@
-### **FastAPI AI**
+# 🚀 FastAPI AI Detector
 
-This FastAPI app loads a GPT-2 model, tokenizes input text, classifies it, and returns whether the text is AI-generated or human-written.
+A production-ready FastAPI app for detecting AI vs. human-written text in English and Nepali. It uses GPT-2 and SentencePiece-based models, with Bearer token security.
 
-### **install Dependencies**
+## 📂 Documentation
 
-```bash
-pip install -r requirements.txt
-
-```
-
-This command installs all the dependencies listed in the `requirements.txt` file. It ensures that your environment has the required packages to run the project smoothly.
-
-**NOTE: IF YOU HAVE DONE ANY CHANGES DON'NT FORGOT TO PUT IT IN THE REQUIREMENTS.TXT USING `bash pip freeze > requirements.txt `**
-
----
-### Files STructure
-
-```
-├── app.py
-├── features
-│   └── text_classifier
-│       ├── controller.py
-│       ├── inferencer.py
-│       ├── __init__.py
-│       ├── model_loader.py
-│       ├── preprocess.py
-│       └── routes.py
-├── __init__.py
-├── Procfile
-├── readme.md
-└── requirements.txt
-```
-**`app.py`**: Entry point initializing FastAPI app and routes
-**`Procfile`**: Tells Railway how to run the program
-**`requirements.txt`**:Have all the packages that we use in our project 
-**`__init__.py`** : Package initializer for the root module
-**FOLDER :features/text_classifier**
-**`controller.py`** :Handles logic between routes and model
-**`inferencer.py`** : Runs inference and returns predictions as well as files system
-**`__init__.py`** :Initializes the module as a package
-**`model_loader.py`** : Loads the ML model and tokenizer
-**`preprocess.py`** :Prepares input text for the model 
-**`routes.py`** :Defines API routes for text classification
-
-### **Functions**
-
-1. **`load_model()`**
-   Loads the GPT-2 model and tokenizer from the specified directory paths.
-
-2. **`lifespan()`**
-   Manages the application lifecycle. It initializes the model at startup and performs cleanup during shutdown.
-
-3. **`classify_text_sync()`**
-   Synchronously tokenizes the input text and performs classification using the GPT-2 model. Returns both the classification result and perplexity score.
-
-4. **`classify_text()`**
-   Asynchronously runs `classify_text_sync()` in a thread pool for non-blocking text classification.
-
-5. **`analyze_text()`**
-   **POST** endpoint: Accepts text input, classifies it using `classify_text()`, and returns the result along with perplexity.
-
-6. **`health()`**
-   **GET** endpoint: Performs a simple health check to confirm the API is operational.
-
-7. **`parse_docx()`, `parse_pdf()`, `parse_txt()`**
-   Utility functions to extract and convert the contents of `.docx`, `.pdf`, and `.txt` files into plain text for classification.
-
-8. **`warmup()`**
-   Downloads the model repository and initializes the model and tokenizer using the `load_model()` function.
-
-9. **`download_model_repo()`**
-   Handles downloading the model files from the designated `MODEL` folder.
-
-10. **`get_model_tokenizer()`**
-    Similar to `warmup()`, but includes a check to see if the model already exists. If not, it downloads the model; otherwise, it uses the previously downloaded one.
-
-11. **`handle_file_upload()`**
-    Manages file uploads from the `/upload` route. Extracts text from the uploaded file, classifies it, and returns the results.
-
-12. **`extract_file_contents()`**
-    Extracts and returns plain text content from uploaded files (e.g., PDF, DOCX, TXT).
-
-13. **`handle_file_sentence()`**
-    Processes uploaded files by analyzing each sentence. Ensures the total file text is under 10,000 characters before classification.
-
-14. **`handle_sentence_level_analysis()`**
-    Strips and checks each sentence’s length, then evaluates the likelihood of AI vs. human generation for each sentence.
-
-15. **`analyze_sentences()`**
-    Divides long paragraphs into individual sentences, classifies each one, and returns a list of their classification results.
-
-16. **`analyze_sentence_file()`**
-    A route function that analyzes sentences in uploaded files, similar to `handle_file_sentence()`.
-
-
----
-
-### **Code Overview**
-
-### **Running and Load Balancing:**
-
-To run the app in production with load balancing:
-
-```bash
-uvicorn app:app --host 0.0.0.0 --port 8000 
-```
-
-This command launches the FastAPI app.
-
-
-### **Endpoints**
-
-#### 1. **`/text/analyze`**
-
-- **Method:** `POST`
-- **Description:** Classifies whether the text is AI-generated or human-written.
-- **Request:**
-  ```json
-  { "text": "sample text" }
-  ```
-- **Response:**
-  ```json
-  { "result": "AI-generated", "perplexity": 55.67,"ai_likelihood":66.6%}
-```
-
-#### 2. **`/health`**
-
-- **Method:** `GET`
-- **Description:** Returns the status of the API.
-- **Response:**
-  ```json
-  { "status": "ok" }
-  ```
-#### 3. **`/text/upload`**
-- **Method:** `POST`
-- **Description:** Takes the files and check the contains inside and returns the results
-- **Request:** Files
-
-- **Response:**
-```json
- { "result": "AI-generated", "perplexity": 55.67,"ai_likelihood":66.6%}
-```
-#### 4. **`/text/analyze_sentence_file`**
-- **Method:** `POST`
-- **Description:** Takes the files and check the contains inside and returns the results
-- **Request:** Files
-
-- **Response:**
-```json
-{
-  "content": "Artificial Intelligence (AI) and Machine Learning (ML) are rapidly transforming the way we \ninteract with technology. AI refers to the broader concept of machines being able to carry out \ntasks in a way that we would consider \"smart,\" while ML is a subset of AI that focuses on the \ndevelopment of algorithms that allow computers to learn from and make decisions based on \ndata. These technologies are behind innovations such as voice assistants, recommendation \nsystems, self-driving cars, and medical diagnosis tools. By analyzing large amounts of data, \nAI and ML can identify patterns, make predictions, and continuously improve their \nperformance over time, making them essential tools in modern industries ranging from \nhealthcare and finance to education and entertainment. \n \n",
-  "analysis": [
-    {
-      "sentence": "Artificial Intelligence (AI) and Machine Learning (ML) are rapidly transforming the way we interact with technology.",
-      "label": "AI-generated",
-      "perplexity": 8.17,
-      "ai_likelihood": 100
-    },
-    {
-      "sentence": "AI refers to the broader concept of machines being able to carry out tasks in a way that we would consider \"smart,\" while ML is a subset of AI that focuses on the development of algorithms that allow computers to learn from and make decisions based on data.",
-      "label": "AI-generated",
-      "perplexity": 19.34,
-      "ai_likelihood": 89.62
-    },
-    {
-      "sentence": "These technologies are behind innovations such as voice assistants, recommendation systems, self-driving cars, and medical diagnosis tools.",
-      "label": "AI-generated",
-      "perplexity": 40.31,
-      "ai_likelihood": 66.32
-    },
-    {
-      "sentence": "By analyzing large amounts of data, AI and ML can identify patterns, make predictions, and continuously improve their performance over time, making them essential tools in modern industries ranging from healthcare and finance to education and entertainment.",
-      "label": "AI-generated",
-      "perplexity": 26.15,
-      "ai_likelihood": 82.05
-    }
-  ]
-}```
-
-#### 5. **`/text/analyze_sentences`**
-- **Method:** `POST`
-- **Description:** Takes the text and check the contains inside and returns the results
-- **Request:**
-```json
-{
-  "text": "This is an test text. This is an another Text "
-}
-```
-
-- **Response:**
-```json
-{
-  "analysis": [
-    {
-      "sentence": "This is an test text.",
-      "label": "Human-written",
-      "perplexity": 510.28,
-      "ai_likelihood": 0
-    },
-    {
-      "sentence": "This is an another Text",
-      "label": "Human-written",
-      "perplexity": 3926.05,
-      "ai_likelihood": 0
-    }
-  ]
-}```
-
-
----
-
-### **Running the API**
-
-Start the server with:
+- [Project Structure](docs/structure.md)
+- [API Endpoints](docs/api_endpoints.md)
+- [Setup & Installation](docs/setup.md)
+- [Deployment](docs/deployment.md)
+- [Security](docs/security.md)
+- [NestJS Integration](docs/nestjs_integration.md)
+- [Core Functions](docs/functions.md)
 
+## ⚡ Quick Start
 ```bash
-uvicorn app:app --host 0.0.0.0 --port 8000 
+uvicorn app:app --host 0.0.0.0 --port 8000
 ```
+## 🚀 Deployment
 
----
-
-### **🧪 Testing the API**
-
-You can test the FastAPI endpoint using `curl` like this:
-
-```bash
-curl -X POST https://can-org-canspace.hf.space/analyze \
-  -H "Authorization: Bearer SECRET_CODE" \
-  -H "Content-Type: application/json" \
-  -d '{"text": "This is a sample sentence for analysis."}'
-```
-
-- The `-H "Authorization: Bearer SECRET_CODE"` part is used to simulate the **handshake**.
-- FastAPI checks this token against the one loaded from the `.env` file.
-- If the token matches, the request is accepted and processed.
-- Otherwise, it responds with a `403 Unauthorized` error.
+- **Local**: Use `uvicorn` as above.
+- **Railway/Heroku**: Use the provided `Procfile`.
+- **Hugging Face Spaces**: Use the `Dockerfile` for container deployment.
 
 ---
 
-### **API Documentation**
-
-- **Swagger UI:** `https://can-org-canspace.hf.space/docs` -> `/docs`
-- **ReDoc:** `https://can-org-canspace.hf.space/redoc` -> `/redoc`
-
-### **🔐 Handshake Mechanism**
-
-In this part, we're implementing a simple handshake to verify that the request is coming from a trusted source (e.g., our NestJS server). Here's how it works:
-
-- We load a secret token from the `.env` file.
-- When a request is made to the FastAPI server, we extract the `Authorization` header and compare it with our expected secret token.
-- If the token does **not** match, we immediately return a **403 Forbidden** response with the message `"Unauthorized"`.
-- If the token **does** match, we allow the request to proceed to the next step.
-
-The verification function looks like this:
-
-```python
-def verify_token(auth: str):
-    if auth != f"Bearer {EXPECTED_TOKEN}":
-        raise HTTPException(status_code=403, detail="Unauthorized")
-```
-
-This provides a basic but effective layer of security to prevent unauthorized access to the API.
-
-### **Implement it with NEST.js**
-
-NOTE: Make an micro service in NEST.JS and implement it there and call it from app.controller.ts
-
-in fastapi.service.ts file what we have done is
+## 💡 Tips
 
-### Project Structure
-
-```files
-nestjs-fastapi-bridge/
-├── src/
-│   ├── app.controller.ts
-│   ├── app.module.ts
-│   └── fastapi.service.ts
-├── .env
-
-```
+- **Model files auto-download at first start** if not found.
+- **Keep `requirements.txt` up-to-date** after adding dependencies.
+- **All endpoints require the correct `Authorization` header**.
+- **For security**: Avoid committing `.env` to public repos.
 
 ---
 
-### Step-by-Step Setup
-
-#### 1. `.env`
-
-Create a `.env` file at the root with the following:
-
-```environment
-  FASTAPI_BASE_URL=https://can-org-canspace.hf.space/
-  SECRET_TOKEN="SECRET_CODE_TOKEN"
-```
-
-#### 2. `fastapi.service.ts`
-
-```javascript
-  // src/fastapi.service.ts
-  import { Injectable } from "@nestjs/common";
-  import { HttpService } from "@nestjs/axios";
-  import { ConfigService } from "@nestjs/config";
-  import { firstValueFrom } from "rxjs";
-
-  @Injectable()
-  export class FastAPIService {
-    constructor(
-      private http: HttpService,
-      private config: ConfigService,
-    ) {}
-
-    async analyzeText(text: string) {
-      const url = `${this.config.get("FASTAPI_BASE_URL")}/analyze`;
-      const token = this.config.get("SECRET_TOKEN");
-
-      const response = await firstValueFrom(
-        this.http.post(
-          url,
-          { text },
-          {
-            headers: {
-              Authorization: `Bearer ${token}`,
-            },
-          },
-        ),
-      );
-
-      return response.data;
-    }
-  }
-```
-
-#### 3. `app.module.ts`
-
-```javascript
-// src/app.module.ts
-import { Module } from "@nestjs/common";
-import { ConfigModule } from "@nestjs/config";
-import { HttpModule } from "@nestjs/axios";
-import { AppController } from "./app.controller";
-import { FastAPIService } from "./fastapi.service";
-
-@Module({
-  imports: [ConfigModule.forRoot(), HttpModule],
-  controllers: [AppController],
-  providers: [FastAPIService],
-})
-export class AppModule {}
-```
-
----
-
-#### 4. `app.controller.ts`
-
-```javascript
-  // src/app.controller.ts
-  import { Body, Controller, Post, Get, Query } from '@nestjs/common';
-  import { FastAPIService } from './fastapi.service';
-
-  @Controller()
-  export class AppController {
-    constructor(private readonly fastapiService: FastAPIService) {}
-
-    @Post('analyze-text')
-    async callFastAPI(@Body('text') text: string) {
-      return this.fastapiService.analyzeText(text);
-    }
-
-    @Get()
-    getHello(): string {
-      return 'NestJS is connected to FastAPI ';
-    }
-  }
-```
-
-### 🚀 How to Run
-
-Run the server of flask and nest.js:
-
-- for nest.js
-  ```bash
-  npm run start
-  ```
-- for Fastapi
-
-  ```bash
-  uvicorn app:app --reload
-  ```
-
-Make sure your FastAPI service is running at `http://localhost:8000`.
-
-### Test with CURL
-http://localhost:3000/-> Server of nest.js
-```bash
-curl -X POST http://localhost:3000/analyze-text \
-  -H 'Content-Type: application/json' \
-  -d '{"text": "This is a test input"}'
-```
-
-
-

requirements.txt

@@ -7,6 +7,7 @@ python-dotenv
 python-docx
 pydantic
 PyMuPDF
-nltk
 python-multipart
-slowapi
+slowapi
+spacy
+nltk