Add config endpoint

app/api/v1/endpoints/config.py

@@ -0,0 +1,37 @@
+from fastapi import APIRouter, Depends, HTTPException
+from app.db.models.config import *
+from app.core.config import settings
+
+
+router = APIRouter()
+
+@router.get("/", response_model=ConfigUpdateRequest)
+async def get_config():
+    return settings
+
+@router.put("/", response_model=ConfigUpdateRequest)
+async def update_config(config: ConfigUpdateRequest):
+    if config.MONGO_DETAILS is not None and config.MONGO_DETAILS != "string":
+        settings.MONGO_DETAILS = config.MONGO_DETAILS
+    if config.MongoDB_NAME is not None and config.MongoDB_NAME != "string":
+        settings.MongoDB_NAME = config.MongoDB_NAME
+    if config.COLLECTION_NAMES is not None and config.COLLECTION_NAMES != "string":
+        settings.COLLECTION_NAMES = config.COLLECTION_NAMES
+    if config.SECRET_KEY is not None and config.SECRET_KEY != "string":
+        settings.SECRET_KEY = config.SECRET_KEY
+    if config.ALGORITHM is not None and config.ALGORITHM != "string":   
+        settings.ALGORITHM = config.ALGORITHM
+    if config.ACCESS_TOKEN_EXPIRE_MINUTES is not None and config.ACCESS_TOKEN_EXPIRE_MINUTES != 0:
+        settings.ACCESS_TOKEN_EXPIRE_MINUTES = config.ACCESS_TOKEN_EXPIRE_MINUTES
+
+    # return {
+    #     "MONGO_DETAILS": settings.MONGO_DETAILS,
+    #     "MongoDB_NAME": settings.MongoDB_NAME,
+    #     "COLLECTION_NAMES": settings.COLLECTION_NAMES,
+    #     "SECRET_KEY": settings.SECRET_KEY,
+    #     "ALGORITHM": settings.ALGORITHM,
+    #     "ACCESS_TOKEN_EXPIRE_MINUTES": settings.ACCESS_TOKEN_EXPIRE_MINUTES
+    # }
+    return settings
+
+

app/api/v1/endpoints/ocr.py

@@ -0,0 +1,101 @@
+# app/api/v1/endpoints/ocr.py
+from fastapi import APIRouter, Depends, HTTPException
+from fastapi import FastAPI, File, UploadFile, HTTPException
+from fastapi.responses import JSONResponse
+from fastapi import Form
+from app.models.users import *
+from app.crud.users import *
+from app.crud.ocr import *
+from app.crud.ocrtemplate import *
+from app.models.ocrtemplate import *
+from app.core.security import get_password_hash
+from app.dependencies import get_current_user
+
+from typing import List
+
+import os
+
+
+router = APIRouter()
+
+@router.post("/")
+async def upload_file(file: UploadFile = File(...)):
+    try:
+        # Save the uploaded file to a temporary location
+        with open(f"/tmp/{file.filename}", "wb") as buffer:
+            buffer.write(await file.read())
+        
+        # Perform OCR on the saved image
+        formatted_output = await do_ocr_tesseract(f"/tmp/{file.filename}")
+        
+        # Return the formatted output
+        return JSONResponse(content=formatted_output)
+
+    except Exception as e:
+        return JSONResponse(content={"error": str(e)}, status_code=500)
+    
+@router.post("/process")
+async def upload_files(files: List[UploadFile] = File(...), template_name: str = Form(...), current_user: User = Depends(get_current_user)):
+    results = []
+    try:
+        for file in files:
+            filepath = f"/tmp/{file.filename}"
+            # Save the uploaded file to a temporary location
+            with open(filepath, "wb") as buffer:
+                buffer.write(await file.read())
+            
+            # Perform OCR on the saved image
+            formatted_output = await do_ocr_tesseract(filepath)
+
+            # Parse the output
+            parsed_data = await parse_ocr_output(formatted_output, template_name)
+
+            save_data = await create_data_from_template(template_name, parsed_data, current_user['user_id'])
+
+            results.append({"filename": file.filename, "output": formatted_output})
+
+            os.remove(filepath)
+        
+        # Return the formatted output
+        return JSONResponse(content=save_data.dict())
+
+    except Exception as e:
+        return JSONResponse(content={"error": str(e)}, status_code=500)
+    
+def format_string(input_string: str) -> str:
+    # Split the string by colon
+    parts = input_string.split(':')
+    
+    # Strip extra spaces from each part
+    formatted_parts = [part.strip() for part in parts]
+    
+    # Join the parts with a single colon
+    formatted_string = ':'.join(formatted_parts)
+    
+    return formatted_string
+   
+async def parse_ocr_output(raw_output: str, template_name: str) -> Dict[str, str]:
+
+    if raw_output is None:
+        return {"error": "No output to parse"}
+    if template_name is None:
+        return {"error": "No template provided"}
+    
+    template = await get_template_by_name(template_name)
+
+    lines = raw_output.splitlines()
+    #  # Decode escape sequences in the string
+    # decoded_output = raw_output.encode().decode('unicode_escape')
+    # print(f"Decoded output: {repr(decoded_output)}")
+
+    # lines = decoded_output.splitlines()  # This will handle \n, \r\n, and \r correctly
+    # print(f"Lines: {lines}")
+
+    parsed_data = {}
+    for line in lines:
+        line = format_string(line)
+        print(f"Processing line: {line}")
+        for field_name, field_value in template.fields.items():
+            if line.startswith(field_value + ":"):
+                parsed_data[field_value] = line.split(':', 1)[1].strip()
+    return parsed_data

app/api/v1/endpoints/ocrtemplate.py

@@ -0,0 +1,49 @@
+from fastapi import APIRouter, HTTPException, Depends
+from typing import List
+from app.models.ocrtemplate import OCRTemplate
+from app.models.users import User
+from app.crud.ocrtemplate import *
+from app.dependencies import get_current_user  # Assume this dependency retrieves the current user # Assume you have a database dependency
+
+router = APIRouter()
+
+# In-memory storage for simplicity; replace with your database logic
+templates = {}
+
+@router.post("/", response_model=OCRTemplate)
+async def create_OCR_template(template: OCRTemplate, current_user: User = Depends(get_current_user)):
+    
+    template = await create_template(template, current_user['user_id'])
+    return template
+
+@router.get("/", response_model=List[Union[OCRTemplateInDB, dict]])
+async def get_OCR_templates_for_user(template_name:Optional[bool] = False, current_user: User = Depends(get_current_user)):
+    
+    templates = await get_all_templates_by_user_id(current_user['user_id'], template_name)
+    return templates
+
+@router.get("/{template_name}", response_model=OCRTemplate)
+async def get_template(template_name: str, current_user: dict = Depends(get_current_user)):
+    user_id = current_user['user_id']
+    template = await get_template_by_name_and_user(template_name, user_id)
+    if not template:
+        raise HTTPException(status_code=404, detail="Template not found")
+    return template
+
+@router.put("/templates", response_model=OCRTemplate)
+async def update_template_endpoint(template: OCRTemplate, user: str = Depends(get_current_user)):
+    print("Updating template:", template)
+    print("User ID:", user['user_id'])
+    updated_template = await update_template(template.template_name, template, user['user_id'])
+    if not updated_template:
+        raise HTTPException(status_code=404, detail="Template not found or could not be updated")
+    return updated_template
+
+@router.delete("/{template_name}")
+async def delete_template_endpoint(template_name: str, current_user: str = Depends(get_current_user)):
+    user_id = current_user['user_id']
+    
+    deleted_template = await delete_template(template_name, user_id)
+    if not deleted_template:
+        raise HTTPException(status_code=404, detail="Template not found or could not be deleted")
+    return {"detail": f"{template_name } templatedeleted"}

app/api/v1/endpoints/user.py

@@ -1,6 +1,6 @@
 # app/api/v1/endpoints/user.py
 from fastapi import APIRouter, Depends, HTTPException
-from app.models.users import *
+from app.models import users
 from app.crud.users import *
 from app.core.security import get_password_hash
 from app.dependencies import get_current_user
@@ -8,7 +8,7 @@ from app.dependencies import get_current_user
 router = APIRouter()
 
 @router.post("/")
-async def register_user(user: User):
+async def register_user(user: users.User):
 
     db_user = await get_user_by_username(user.username)
     if db_user:
@@ -20,12 +20,12 @@ async def register_user(user: User):
     
     user_in_db = UserInDB(**user.dict(), hashed_password=get_password_hash(user.password))
 
-    user_id = await create_user(user_in_db)
+    user = await create_user(user_in_db)
 
-    return {"id": user_id}
+    return user
 
-@router.get("/me/", response_model=User)
-async def read_users_me(current_user: User = Depends(get_current_user)):
+@router.get("/me/", response_model=users.User)
+async def read_users_me(current_user: users.User = Depends(get_current_user)):
     print("Current user:", current_user)
 
     

app/core/config.py

@@ -12,7 +12,7 @@ class Settings(BaseSettings):
     COLLECTION_NAMES: list = ["users", "files", "templates", "extracted data", "external credentials"]
     SECRET_KEY: str = os.getenv("SECRET_KEY")
     ALGORITHM: str = "HS256"
-    ACCESS_TOKEN_EXPIRE_MINUTES: int = 30
+    ACCESS_TOKEN_EXPIRE_MINUTES: int = 100
 
 settings = Settings()
 

app/core/creamodel.py

@@ -0,0 +1,12 @@
+from pydantic import BaseModel, create_model
+from typing import Dict
+
+def create_pydantic_model(name: str, variable_dict: Dict[str, str]) -> BaseModel:
+    fields = {key: (str, ...) for key in variable_dict.values()}
+    return create_model(name, **fields, user_id=(str, ...))
+
+def parse_data(model: BaseModel, data_str: str, user_id:str) -> Dict[str, str]:
+    data_lines = data_str.split('\n')
+    data_dict = {line.split(': ')[0]: line.split(': ')[1] for line in data_lines if ': ' in line}
+    data_dict['user_id'] = user_id
+    return model(**data_dict).dict()

app/crud/ocr.py

@@ -0,0 +1,156 @@
+import asyncio
+from PIL import Image
+import pytesseract
+import re, cv2
+import imutils
+from concurrent.futures import ThreadPoolExecutor
+from app.models.ocrtemplate import *
+from app.core.database import get_database
+from app.core.config import settings
+from typing import Any
+from fastapi import HTTPException
+from pytesseract import Output
+
+pytesseract.pytesseract.tesseract_cmd = r'C:\\Program Files\\Tesseract-OCR\\tesseract.exe'
+
+def identify_structure(line):
+    line = line.strip()
+    
+    if line.count(':') > 1:
+        return 'mixed-column'
+    
+    if ':' in line or re.search(r'^[A-Za-z]+\s+[A-Za-z0-9]+', line):
+        return 'key-value'
+    
+    uppercase_words = re.findall(r'\b[A-Z]+\b', line)
+    numbers = re.findall(r'\b\d+\b', line)
+    
+    if len(uppercase_words) > 1:
+        return 'table-header'
+    
+    if len(numbers) > 1 and len(uppercase_words) <= 1:
+        return 'table-row'
+    
+    return 'text'
+
+def format_extracted_text(text):
+    lines = text.split('\n')
+    lines = [line.strip() for line in lines if line.strip()]
+    
+    formatted_text = []
+    in_table = False
+    
+    for line in lines:
+        structure = identify_structure(line)
+        
+        if structure == 'mixed-column':
+            parts = line.split(':')
+            formatted_parts = [f"{parts[i].strip()}: {parts[i+1].strip()}" for i in range(0, len(parts)-1, 2)]
+            formatted_text.extend(formatted_parts)
+            in_table = False
+        elif structure == 'key-value':
+            formatted_text.append(line)
+            in_table = False
+        elif structure == 'table-header':
+            formatted_text.append(line)
+            in_table = True
+        elif structure == 'table-row' and in_table:
+            formatted_text.append(line)
+        else:
+            if in_table:
+                in_table = False
+                formatted_text.append("\n")
+            formatted_text.append(line)
+    
+    return "\n".join(formatted_text)
+
+def refine_text_formatting(text):
+    text = re.sub(r'\s+', ' ', text)
+    text = re.sub(r'\.\s', '.\n', text)
+    text = re.sub(r'\s*:\s*', ': ', text)
+    return text
+
+def do_ocr(image_path):
+    image = Image.open(image_path)
+    extracted_text = pytesseract.image_to_string(image)
+    formatted_text = format_extracted_text(extracted_text)
+    return formatted_text
+
+async def do_ocr_tesseract(image_path):
+    loop = asyncio.get_event_loop()
+    with ThreadPoolExecutor() as pool:
+        formatted_text = await loop.run_in_executor(pool, do_ocr, image_path)
+    return formatted_text
+
+
+async def create_data_from_template(template_name:str, fields:Dict[str, str], user_id:str) -> OCRTemplateInDB:
+    template = OCRTemplateInDB(template_name=template_name, fields=fields, user_id=user_id)
+    db = get_database(settings.MongoDB_NAME)
+    result = await db["extracted data"].insert_one(template.dict())
+    if template:
+        return template
+    return None
+
+
+def preprocess_image(image: Any) -> Any:
+    # Convert to grayscale
+    gray = cv2.cvtColor(image, cv2.COLOR_BGR2GRAY)
+    
+    # Apply Gaussian blur
+    blur = cv2.GaussianBlur(gray, (3, 3), 0)
+    
+    # Apply Otsu's thresholding
+    thresh = cv2.threshold(blur, 0, 255, cv2.THRESH_BINARY_INV + cv2.THRESH_OTSU)[1]
+
+    # Morph open to remove noise and invert image
+    kernel = cv2.getStructuringElement(cv2.MORPH_RECT, (3, 3))
+    opening = cv2.morphologyEx(thresh, cv2.MORPH_OPEN, kernel, iterations=1)
+    invert = 255 - opening
+    
+    return invert
+
+async def detect_rotation(image_path: str) -> Any:
+    # Load the input image
+    image = cv2.imread(image_path)
+    if image is None:
+        raise HTTPException(status_code=400, detail="Image not found or unable to read")
+
+    # Convert from BGR to RGB channel ordering
+    rgb = cv2.cvtColor(image, cv2.COLOR_BGR2RGB)
+    
+    # Use Tesseract to determine the text orientation
+    results = pytesseract.image_to_osd(rgb, output_type=Output.DICT)
+    
+    # Display the orientation information
+    print("[INFO] detected orientation: {}".format(results["orientation"]))
+    print("[INFO] rotate by {} degrees to correct".format(results["rotate"]))
+    print("[INFO] detected script: {}".format(results["script"]))
+    
+    # Rotate the image to correct the orientation
+    rotated = imutils.rotate_bound(image, angle=results["rotate"])
+    
+    return rotated
+
+async def tesseract_ocr(image_path: str) -> str:
+    # Detect rotation and get the image
+    image = await detect_rotation(image_path)
+    
+    # Preprocess the image
+    preprocessed_image = preprocess_image(image)
+
+    # Perform OCR using Tesseract
+    result = pytesseract.image_to_string(preprocessed_image, config='--psm 6')
+
+    formatted_text = format_extracted_text(result)
+    return formatted_text
+# Example usage
+async def main():
+    image_path = 'KTP.jpg'
+    formatted_text = await do_ocr_tesseract(image_path)
+    formatted_text_pre = await tesseract_ocr(image_path)
+    print(formatted_text)
+    print(formatted_text_pre)
+
+# asyncio.run(main())
+
+

app/crud/ocrtemplate.py

@@ -0,0 +1,73 @@
+from fastapi import FastAPI, HTTPException, Depends
+from app.core.database import get_database
+from app.models.ocrtemplate import OCRTemplate, OCRTemplateInDB
+from app.core.security import get_password_hash, verify_password
+from bson import ObjectId
+from app.core.config import settings
+from typing import List, Optional, Union
+from pymongo import ReturnDocument
+
+
+
+async def create_template(template: OCRTemplate, user_id: str):
+    existing_template = await get_template_by_name_and_user(template.template_name, user_id)
+    if existing_template:
+        raise HTTPException(status_code=401, detail="Template already exists")
+
+    template_dict = template.dict()
+    template_dict["user_id"] = user_id
+    db = get_database(settings.MongoDB_NAME)
+    template = await db["templates"].insert_one(template_dict)
+    if template:
+        return OCRTemplate(**template_dict)
+    return None
+
+
+async def get_all_templates_by_user_id(user_id: str, template_name: Optional[bool] = False)->  Union[List[OCRTemplateInDB], List[dict]]:
+    db = get_database(settings.MongoDB_NAME)
+    templates = await db["templates"].find({"user_id": user_id}).to_list(1000)
+
+    if template_name:
+        # Return only the template_name field
+        return [{"template_name": template["template_name"]} for template in templates]
+
+    return [OCRTemplateInDB(**template) for template in templates]
+   
+async def get_template_by_name(template_name: str) -> OCRTemplate:
+    db = get_database(settings.MongoDB_NAME)
+    template = await db["templates"].find_one({"template_name": template_name})
+    if template:
+        return OCRTemplate(**template)
+    return None
+
+async def get_template_by_name_and_user(template_name: str, user_id: Optional[str]) -> OCRTemplate:
+    query = {"template_name": template_name, "user_id": user_id}
+    print("Query:", query)
+    db = get_database(settings.MongoDB_NAME)
+    template = await db["templates"].find_one(query)
+    print(template)
+    if template:
+        return OCRTemplate(**template)
+    return None
+
+async def update_template(template_name: str, template: OCRTemplate, user_id: str):
+    query = {"template_name": template_name, "user_id": user_id}
+    print("Query:", query)
+    db = get_database(settings.MongoDB_NAME)
+    updated_template = await db["templates"].find_one_and_update(
+        query, 
+        {"$set": template.dict()},
+        return_document=ReturnDocument.AFTER
+    )
+    if updated_template:
+        return OCRTemplate(**updated_template)
+    return None
+
+async def delete_template(template_name: str, user_id: str):
+    query = {"template_name": template_name, "user_id": user_id}
+    print("Query:", query)
+    db = get_database(settings.MongoDB_NAME)
+    deleted_template = await db["templates"].find_one_and_delete(query)
+    if deleted_template:
+        return OCRTemplate(**deleted_template)
+    return None

app/crud/users.py

@@ -4,20 +4,23 @@ from app.models.users import UserInDB, User
 from app.core.security import get_password_hash, verify_password
 from bson import ObjectId
 from app.core.config import settings
+import uuid
 
 
 async def create_user(user: UserInDB):
     user_dict = user.dict()
+    user_dict["user_id"] = str(uuid.uuid4())
+    print("user_id:", user_dict["user_id"])
     user_dict["password"] = get_password_hash(user.password)
     db = get_database(settings.MongoDB_NAME)
     result = await db["users"].insert_one(user_dict)
-    user_dict["_id"] = str(result.inserted_id)
+    
+    
     return User(**user_dict)
 
 async def get_user_by_username(username: str):
     db = get_database(settings.MongoDB_NAME)
     user = await db["users"].find_one({"username": username})
-    
     return user
 
 async def get_user_by_email(email: str):

app/db/models/config.py

@@ -0,0 +1,12 @@
+from fastapi import FastAPI, HTTPException, Depends
+from typing import Optional, List
+from pydantic import BaseModel
+import os
+
+class ConfigUpdateRequest(BaseModel):
+    MONGO_DETAILS: Optional[str] = None
+    MongoDB_NAME: Optional[str] = None
+    COLLECTION_NAMES: Optional[List[str]] = None
+    SECRET_KEY: Optional[str] = None
+    ALGORITHM: Optional[str] = None
+    ACCESS_TOKEN_EXPIRE_MINUTES: Optional[int] = None

app/main.py

@@ -13,13 +13,14 @@ from bson import ObjectId
 from contextlib import asynccontextmanager
 from pydantic import BaseModel, Field
 from datetime import timedelta, datetime
-from dotenv import load_dotenv, dotenv_values
-from app.api.v1.endpoints import user, auth
+from dotenv import dotenv_values
+from app.api.v1.endpoints import user, auth, ocr, ocrtemplate, config
 from app.db.base import *
 from app.core.auth import *
 # from app.router.user import *
 from app.core.database import *
 
+
 # Load environment variables from .env file
 dotenv_values(".env")
 
@@ -50,16 +51,23 @@ async def lifespan(app: FastAPI):
         logger.error(e)
         
 app = FastAPI(lifespan=lifespan)
+# Allow CORS for specific origin with credentials
+origins = [
+    "http://localhost:5000",
+]
 app.add_middleware(
     CORSMiddleware,
-    allow_origins=["*"],
+    allow_origins=origins,
     allow_credentials=True,
     allow_methods=["*"],
     allow_headers=["*"],
 )
 
 app.include_router(user.router, prefix='/api/v1/user', tags=["User"])
+app.include_router(ocrtemplate.router, prefix='/api/v1/ocrtemplate', tags=["OCR Template"])
+app.include_router(ocr.router, prefix='/api/v1/ocr', tags=["OCR"])
 app.include_router(auth.router, tags=["Auth"])
+app.include_router(config.router, prefix='/api/v1/config', tags=["Config"])
 
 
 class Destination(BaseModel):

app/models/ocrtemplate.py

@@ -0,0 +1,12 @@
+from pydantic import BaseModel, Field
+from typing import Dict, Optional
+
+class OCRTemplate(BaseModel):
+    template_name: str
+    fields: Dict[str, str]
+    user_id: Optional[str] = Field(None, description="ID of the user who owns the template. None for common templates.")
+
+class OCRTemplateInDB(BaseModel):
+    template_name: str
+    fields: Dict[str, str]
+    user_id: str

app/models/users.py

@@ -2,8 +2,10 @@
 from pydantic import BaseModel, EmailStr, Field
 from typing import Optional
 from datetime import timedelta, datetime
+import uuid
 
 class User(BaseModel):
+    user_id: str = Field(default_factory=uuid.uuid4())
     username: str
     email: EmailStr
     password: str