feat: Implemented sentence-level analysis tools and added file support for analysis

.gitignore

@@ -52,6 +52,10 @@ Thumbs.db
 # ---- Project-specific ----
 Ai-Text-Detector/
 HuggingFace/model/
+
 # ---- Node Projects (if applicable) ----
 node_modules/
+model/
+models/.gitattributes  #<-- This line can stay if you only want to ignore that file, not the whole folder
 
+#

app.py

@@ -2,19 +2,29 @@ from fastapi import FastAPI
 from contextlib import asynccontextmanager
 from features.text_classifier.routes import router as text_classifier_router
 from features.text_classifier.model_loader import warmup
-
+import nltk
 
 @asynccontextmanager
 async def lifespan(app: FastAPI):
-    warmup()  # Download and load model at startup
+    # Ensure punkt is available
+    nltk.download("punkt")
+
+    nltk.download('punkt_tab')
+    
+    # Your model warmup
+    warmup()
     yield
-    # Cleanup lo
+    # Optionally add cleanup here
 
 
-app = FastAPI()
-app.include_router(text_classifier_router, prefix="/text", tags=["Text Classification"])
+# Pass lifespan handler to FastAPI constructor
+app = FastAPI(lifespan=lifespan)
 
+app.include_router(text_classifier_router, prefix="/text", tags=["Text Classification"])
 
 @app.get("/")
 def index():
-    return {"Message": "Fast api is running... ", "Try": "/docs"}
+    return {
+        "Message": "FastAPI is running...",
+        "Try": "/text/analyze or /text/analyze-sentences"
+    }

features/text_classifier/__init__.py

@@ -0,0 +1 @@
+

features/text_classifier/controller.py

@@ -2,19 +2,35 @@ from .inferencer import classify_text
 import asyncio
 from fastapi import HTTPException, UploadFile
 from .preprocess import parse_docx, parse_pdf, parse_txt
+from nltk.tokenize import sent_tokenize
 
 from io import BytesIO
 import logging
 
 
+
 async def handle_text_analysis(text: str):
     text = text.strip()
     if not text or len(text.split()) < 2:
         raise HTTPException(
             status_code=400, detail="Text must contain at least two words"
         )
-    label, perplexity = await asyncio.to_thread(classify_text, text)
-    return {"result": label, "perplexity": round(perplexity, 2)}
+    label, perplexity,ai_likelihood = await asyncio.to_thread(classify_text, text)
+    return {"result": label, "perplexity": round(int(perplexity), 2),"ai_likelihood":ai_likelihood}
+
+
+async def handle_file_sentance(file: UploadFile):
+    try:
+        file_contents = await extract_file_contents(file)
+        if len(file_contents) > 10000:
+            return {"message": "File contains more than 10,000 characters."}
+        cleaned_text = file_contents.replace("\n", "").replace("\t", "")
+        result = await handle_sentence_level_analysis(cleaned_text)
+        return {"content": file_contents, **result}
+    except Exception as e:
+        logging.error(f"Error processing file: {str(e)}")
+        raise HTTPException(status_code=500, detail="Error processing the file")
+
 
 
 async def handle_file_upload(file: UploadFile):
@@ -23,8 +39,8 @@ async def handle_file_upload(file: UploadFile):
         if len(file_contents) > 10000:
             return {"message": "File contains more than 10,000 characters."}
         cleaned_text = file_contents.replace("\n", "").replace("\t", "")
-        label, perplexity = await asyncio.to_thread(classify_text, cleaned_text)
-        return {"result": label, "perplexity": round(perplexity, 2)}
+        label, perplexity,ai_likelihood = await asyncio.to_thread(classify_text, cleaned_text)
+        return {"content":file_contents,"result": label, "perplexity": round(int(perplexity), 2),"ai_likelihood":ai_likelihood}
     except Exception as e:
         logging.error(f"Error processing file: {str(e)}")
         raise HTTPException(status_code=500, detail="Error processing the file")
@@ -49,6 +65,26 @@ async def extract_file_contents(file: UploadFile):
             detail="Invalid file type. Only .docx, .pdf, and .txt are allowed.",
         )
 
+async def handle_sentence_level_analysis(text: str):
+    text = text.strip()
+    if not text or len(text.split()) < 2:
+        raise HTTPException(
+            status_code=400, detail="Text must contain at least two words"
+        )
+
+    sentences = sent_tokenize(text,language="english")
+    results = []
+
+    for sentence in sentences:
+        label, perplexity, likelihood = await asyncio.to_thread(classify_text, sentence)
+        results.append({
+            "sentence": sentence,
+            "label": label,
+            "perplexity": round(perplexity, 2),
+            "ai_likelihood": likelihood
+        })
+
+    return {"analysis": results}
 
 def classify(text: str):
     return classify_text(text)

features/text_classifier/inferencer.py

@@ -3,6 +3,19 @@ from .model_loader import get_model_tokenizer
 
 device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
 
+def perplexity_to_ai_likelihood(ppl: float) -> float:
+    # You can tune these parameters
+    min_ppl = 10     # very confident it's AI
+    max_ppl = 100    # very confident it's human
+
+    # Clamp to bounds
+    ppl = max(min_ppl, min(ppl, max_ppl))
+
+    # Invert and scale: lower perplexity -> higher AI-likelihood
+    likelihood = 1 - ((ppl - min_ppl) / (max_ppl - min_ppl))
+
+    return round(likelihood * 100, 2)
+
 
 def classify_text(text: str):
     model, tokenizer = get_model_tokenizer()
@@ -23,5 +36,5 @@ def classify_text(text: str):
         result = "Probably AI-generated"
     else:
         result = "Human-written"
-
-    return result, perplexity
+    likelihood_result=perplexity_to_ai_likelihood(perplexity)
+    return result, perplexity,likelihood_result

features/text_classifier/routes.py

@@ -1,7 +1,7 @@
 from fastapi import APIRouter, Depends, HTTPException, UploadFile, File
 from fastapi.security import HTTPBearer
 from pydantic import BaseModel
-from .controller import handle_text_analysis, handle_file_upload
+from .controller import handle_text_analysis, handle_file_upload, handle_sentence_level_analysis,handle_file_sentance
 
 router = APIRouter()
 bearer_scheme = HTTPBearer()
@@ -26,3 +26,13 @@ async def upload_file(
 @router.get("/health")
 def health():
     return {"status": "ok"}
+
+@router.post("/analyze-sentences")
+async def analyze_sentences(data: TextInput, token: str = Depends(bearer_scheme)):
+    if not data.text:
+        raise HTTPException(status_code=400, detail="Missing 'text' in request body")
+    return await handle_sentence_level_analysis(data.text)
+
+@router.post("/analyze-sentance-file")
+async def AnalyzeSentanceFile(file:UploadFile=File(...),token:str=Depends(bearer_scheme)):
+    return await handle_file_sentance(file)

readme.md

@@ -45,36 +45,54 @@ This command installs all the dependencies listed in the `requirements.txt` file
 
 ### **Functions**
 
-1. **`load_model()`**  
-   Loads the GPT-2 model and tokenizer from specified paths.
+1. **`load_model()`**
+   Loads the GPT-2 model and tokenizer from the specified directory paths.
 
-2. **`lifespan()`**  
-   Manages the app's lifecycle: loads the model at startup and handles cleanup on shutdown.
+2. **`lifespan()`**
+   Manages the application lifecycle. It initializes the model at startup and performs cleanup during shutdown.
 
-3. **`classify_text_sync()`**  
-   Synchronously tokenizes input text and classifies it using the GPT-2 model. Returns the classification and perplexity.
+3. **`classify_text_sync()`**
+   Synchronously tokenizes the input text and performs classification using the GPT-2 model. Returns both the classification result and perplexity score.
 
-4. **`classify_text()`**  
-   Asynchronously executes `classify_text_sync()` in a thread pool to ensure non-blocking processing.
+4. **`classify_text()`**
+   Asynchronously runs `classify_text_sync()` in a thread pool for non-blocking text classification.
 
-5. **`analyze_text()`**  
-   **POST** endpoint: accepts text input, classifies it using `classify_text()`, and returns the result with perplexity.
+5. **`analyze_text()`**
+   **POST** endpoint: Accepts text input, classifies it using `classify_text()`, and returns the result along with perplexity.
+
+6. **`health()`**
+   **GET** endpoint: Performs a simple health check to confirm the API is operational.
+
+7. **`parse_docx()`, `parse_pdf()`, `parse_txt()`**
+   Utility functions to extract and convert the contents of `.docx`, `.pdf`, and `.txt` files into plain text for classification.
 
-6. **`health()`**  
-   **GET** endpoint: simple health check to confirm the API is running.
-7. **`parse_docx() ,parse_pdf(),parse_txt()`**
-  THis are the function that are used to convert the given docs, pdf or text files into the strings format so that we can classify them.
 8. **`warmup()`**
-    This  function is used to downlaod the repo and init the _model and _tokenizer from load_model() function
+   Downloads the model repository and initializes the model and tokenizer using the `load_model()` function.
+
 9. **`download_model_repo()`**
-    This function is use to download the model from the MODEL folder
-10. **`get_model_tokenizer()`** 
-This function is similler to the warmup but it also check if the model is already exist or not if not exist then download it else let it be or use previous downloaded model
+   Handles downloading the model files from the designated `MODEL` folder.
+
+10. **`get_model_tokenizer()`**
+    Similar to `warmup()`, but includes a check to see if the model already exists. If not, it downloads the model; otherwise, it uses the previously downloaded one.
 
 11. **`handle_file_upload()`**
-This function is use to handle the file upload in the upload route and classify and returns the results.
-12. **`Extract_file_contents()`**
-This function is use to extract the contains from the files and return the text from the files.
+    Manages file uploads from the `/upload` route. Extracts text from the uploaded file, classifies it, and returns the results.
+
+12. **`extract_file_contents()`**
+    Extracts and returns plain text content from uploaded files (e.g., PDF, DOCX, TXT).
+
+13. **`handle_file_sentence()`**
+    Processes uploaded files by analyzing each sentence. Ensures the total file text is under 10,000 characters before classification.
+
+14. **`handle_sentence_level_analysis()`**
+    Strips and checks each sentence’s length, then evaluates the likelihood of AI vs. human generation for each sentence.
+
+15. **`analyze_sentences()`**
+    Divides long paragraphs into individual sentences, classifies each one, and returns a list of their classification results.
+
+16. **`analyze_sentence_file()`**
+    A route function that analyzes sentences in uploaded files, similar to `handle_file_sentence()`.
+
 
 ---
 
@@ -103,8 +121,8 @@ This command launches the FastAPI app.
   ```
 - **Response:**
   ```json
-  { "result": "AI-generated", "perplexity": 55.67 }
-  ```
+  { "result": "AI-generated", "perplexity": 55.67,"ai_likelihood":66.6%}
+```
 
 #### 2. **`/health`**
 
@@ -121,8 +139,75 @@ This command launches the FastAPI app.
 
 - **Response:**
 ```json
- { "result": "AI-generated", "perplexity": 55.67 }
+ { "result": "AI-generated", "perplexity": 55.67,"ai_likelihood":66.6%}
 ```
+#### 4. **`/text/analyze_sentence_file`**
+- **Method:** `POST`
+- **Description:** Takes the files and check the contains inside and returns the results
+- **Request:** Files
+
+- **Response:**
+```json
+{
+  "content": "Artificial Intelligence (AI) and Machine Learning (ML) are rapidly transforming the way we \ninteract with technology. AI refers to the broader concept of machines being able to carry out \ntasks in a way that we would consider \"smart,\" while ML is a subset of AI that focuses on the \ndevelopment of algorithms that allow computers to learn from and make decisions based on \ndata. These technologies are behind innovations such as voice assistants, recommendation \nsystems, self-driving cars, and medical diagnosis tools. By analyzing large amounts of data, \nAI and ML can identify patterns, make predictions, and continuously improve their \nperformance over time, making them essential tools in modern industries ranging from \nhealthcare and finance to education and entertainment. \n \n",
+  "analysis": [
+    {
+      "sentence": "Artificial Intelligence (AI) and Machine Learning (ML) are rapidly transforming the way we interact with technology.",
+      "label": "AI-generated",
+      "perplexity": 8.17,
+      "ai_likelihood": 100
+    },
+    {
+      "sentence": "AI refers to the broader concept of machines being able to carry out tasks in a way that we would consider \"smart,\" while ML is a subset of AI that focuses on the development of algorithms that allow computers to learn from and make decisions based on data.",
+      "label": "AI-generated",
+      "perplexity": 19.34,
+      "ai_likelihood": 89.62
+    },
+    {
+      "sentence": "These technologies are behind innovations such as voice assistants, recommendation systems, self-driving cars, and medical diagnosis tools.",
+      "label": "AI-generated",
+      "perplexity": 40.31,
+      "ai_likelihood": 66.32
+    },
+    {
+      "sentence": "By analyzing large amounts of data, AI and ML can identify patterns, make predictions, and continuously improve their performance over time, making them essential tools in modern industries ranging from healthcare and finance to education and entertainment.",
+      "label": "AI-generated",
+      "perplexity": 26.15,
+      "ai_likelihood": 82.05
+    }
+  ]
+}```
+
+#### 5. **`/text/analyze_sentences`**
+- **Method:** `POST`
+- **Description:** Takes the text and check the contains inside and returns the results
+- **Request:**
+```json
+{
+  "text": "This is an test text. This is an another Text "
+}
+```
+
+- **Response:**
+```json
+{
+  "analysis": [
+    {
+      "sentence": "This is an test text.",
+      "label": "Human-written",
+      "perplexity": 510.28,
+      "ai_likelihood": 0
+    },
+    {
+      "sentence": "This is an another Text",
+      "label": "Human-written",
+      "perplexity": 3926.05,
+      "ai_likelihood": 0
+    }
+  ]
+}```
+
+
 ---
 
 ### **Running the API**
@@ -130,7 +215,7 @@ This command launches the FastAPI app.
 Start the server with:
 
 ```bash
-uvicorn app:app --host 0.0.0.0 --port 8000 --workers 4
+uvicorn app:app --host 0.0.0.0 --port 8000 
 ```
 
 ---

requirements.txt

@@ -8,4 +8,5 @@ python-docx
 PyMuPDF
 pydantic
 fitz
+nltk
 python-multipart