app.py

import os
import zipfile
import glob
from fastapi import FastAPI, Query
from fastapi.responses import JSONResponse

app = FastAPI(title="850M Records Search Engine", developer="Gopal Parmar")

MOUNT_PATH = "/data/"
FILE_PATTERN = "data_part_*.zip"

def get_all_parts():
    return sorted(glob.glob(os.path.join(MOUNT_PATH, FILE_PATTERN)))

def detect_magic(filepath):
    with open(filepath, 'rb') as f:
        header = f.read(50)
    
    # Check for known magic bytes
    magic = header[:16]
    if magic.startswith(b'PK'):
        return "ZIP", "Valid ZIP archive"
    elif magic.startswith(b'Rar!'):
        return "RAR", "Valid RAR archive"
    elif magic.startswith(b'SQLite format 3'):
        return "SQLITE", "SQLite database"
    elif magic.startswith(b'\x89PNG'):
        return "PNG", "Image file"
    elif magic.startswith(b'%PDF'):
        return "PDF", "Document"
    elif magic.startswith(b'GIF'):
        return "GIF", "Image"
    elif magic.startswith(b'BM'):
        return "BMP", "Image"
    elif magic.startswith(b'{') or magic.startswith(b'['):
        return "JSON", "JSON data"
    elif magic.startswith(b'<?xml') or magic.startswith(b'<'):
        return "XML", "XML data"
    else:
        # Check if it's encrypted
        entropy = sum(header) / len(header)
        if entropy > 120:
            return "ENCRYPTED", "Likely encrypted (high entropy)"
        else:
            return "BINARY", "Unknown binary data"

@app.get("/")
async def root():
    parts = get_all_parts()
    return {
        "message": "🔥 850M Records Search Engine",
        "developer": "Gopal Parmar",
        "total_parts": len(parts),
        "endpoints": {
            "/magic": "Detect file format",
            "/peek?part=1&lines=10": "View raw lines from part",
            "/stats": "Database statistics"
        }
    }

@app.get("/magic")
async def magic(part: int = 1):
    """Detect file format using magic bytes"""
    parts = get_all_parts()
    if not parts:
        return {"error": "No parts found"}
    if part < 1 or part > len(parts):
        return {"error": f"Part must be between 1 and {len(parts)}"}
    
    filepath = parts[part-1]
    file_type, description = detect_magic(filepath)
    
    with open(filepath, 'rb') as f:
        header = f.read(100)
    
    return {
        "developer": "Gopal Parmar",
        "part": part,
        "file": os.path.basename(filepath),
        "file_type": file_type,
        "description": description,
        "magic_bytes_hex": header[:20].hex(),
        "magic_bytes_raw": str(header[:20])
    }

@app.get("/peek")
async def peek(part: int = 1, lines: int = 10):
    """View raw lines from a specific part (decoded as binary)"""
    parts = get_all_parts()
    if not parts:
        return {"error": "No parts found"}
    if part < 1 or part > len(parts):
        return {"error": f"Part must be between 1 and {len(parts)}"}
    
    filepath = parts[part-1]
    file_type, _ = detect_magic(filepath)
    
    result = {
        "developer": "Gopal Parmar",
        "part": part,
        "file": os.path.basename(filepath),
        "file_type": file_type,
        "size_mb": round(os.path.getsize(filepath) / (1024**2), 2),
        "lines": []
    }
    
    if file_type == "ZIP":
        try:
            with zipfile.ZipFile(filepath, 'r') as zf:
                for name in zf.namelist():
                    with zf.open(name) as f:
                        result["inside_file"] = name
                        raw_lines = []
                        for i, line in enumerate(f):
                            if i >= lines:
                                break
                            try:
                                raw_lines.append(line.decode('utf-8', errors='replace').strip())
                            except:
                                raw_lines.append(str(line))
                        result["lines"] = raw_lines
                        break
        except Exception as e:
            result["error"] = str(e)
    elif file_type in ["SQLITE", "JSON", "XML"]:
        with open(filepath, 'rb') as f:
            raw_lines = []
            for i, line in enumerate(f):
                if i >= lines:
                    break
                try:
                    raw_lines.append(line.decode('utf-8', errors='replace').strip())
                except:
                    raw_lines.append(str(line))
            result["lines"] = raw_lines
    else:
        with open(filepath, 'rb') as f:
            raw_lines = []
            for i, line in enumerate(f):
                if i >= lines:
                    break
                raw_lines.append(str(line))
            result["lines"] = raw_lines
    
    return result

@app.get("/stats")
async def stats():
    parts = get_all_parts()
    total_size = sum(os.path.getsize(p) for p in parts)
    return {
        "developer": "Gopal Parmar",
        "total_parts": len(parts),
        "total_size_gb": round(total_size / (1024**3), 2),
        "parts": [os.path.basename(p) for p in parts]
    }