|
|
| from functools import lru_cache |
| from pathlib import Path |
| import logging |
| import time |
|
|
| from fastapi import FastAPI, HTTPException |
| from fastapi.responses import FileResponse |
| from fastapi.staticfiles import StaticFiles |
| from pydantic import BaseModel, Field |
| from transformers import AutoTokenizer |
|
|
| logging.basicConfig(level=logging.INFO) |
| logger = logging.getLogger("turkish-token-lab") |
|
|
| BASE_DIR = Path(__file__).resolve().parent |
|
|
| MODEL_A = "Qwen/Qwen2.5-0.5B" |
| MODEL_B = "google-bert/bert-base-multilingual-cased" |
|
|
| app = FastAPI(title="Turkish Token Lab API") |
|
|
|
|
| class AnalyzeRequest(BaseModel): |
| text: str = Field(min_length=1, max_length=12000) |
|
|
|
|
| @lru_cache(maxsize=2) |
| def get_tokenizer(model_name: str): |
| logger.info("Tokenizer yükleniyor: %s", model_name) |
|
|
| return AutoTokenizer.from_pretrained( |
| model_name, |
| use_fast=True, |
| ) |
|
|
|
|
| def analyze_one(model_name: str, text: str): |
| started = time.perf_counter() |
|
|
| tokenizer = get_tokenizer(model_name) |
|
|
| |
| ids = tokenizer.encode(text, add_special_tokens=False) |
| tokens = tokenizer.convert_ids_to_tokens(ids) |
|
|
| elapsed_ms = (time.perf_counter() - started) * 1000 |
|
|
| return { |
| "model": model_name, |
| "token_count": len(ids), |
| "tokens": tokens, |
| "elapsed_ms": round(elapsed_ms, 2), |
| } |
|
|
|
|
| @app.get("/api/health") |
| def health(): |
| return {"status": "ok"} |
|
|
|
|
| @app.post("/api/analyze") |
| def analyze(request: AnalyzeRequest): |
| text = request.text.strip() |
|
|
| if not text: |
| raise HTTPException( |
| status_code=400, |
| detail="Metin boş olamaz.", |
| ) |
|
|
| try: |
| result_a = analyze_one(MODEL_A, text) |
| result_b = analyze_one(MODEL_B, text) |
|
|
| return { |
| "a": result_a, |
| "b": result_b, |
| "characters": len(text), |
| "method": "pretrained_tokenizer_comparison", |
| "scientific_note": ( |
| "Bu test iki hazır tokenizer'ı karşılaştırır. " |
| "Sözlük boyutları ve eğitim verileri farklıdır. " |
| "Sonuçlar morfoloji farkındalıklı yöntemin " |
| "üstünlüğünü tek başına kanıtlamaz." |
| ), |
| } |
|
|
| except Exception as exc: |
| logger.exception("Tokenizer analizi başarısız") |
|
|
| raise HTTPException( |
| status_code=503, |
| detail=( |
| "Tokenizer yüklenemedi veya analiz başarısız oldu. " |
| "İnternet bağlantısını, model erişimini ve sunucu " |
| "loglarını kontrol et." |
| ), |
| ) from exc |
|
|
|
|
| @app.get("/") |
| def index(): |
| return FileResponse(BASE_DIR / "index.html") |
|
|
|
|
| |
| app.mount( |
| "/static", |
| StaticFiles(directory=BASE_DIR), |
| name="static", |
| ) |
|
|
