Upload 466 files

a72d1cf verified 8 days ago

7.23 kB

"""Türkçeye özgü metin, sayı, tarih ve birim normalizasyonu.
Üç sayı ayrıştırma kipi desteklenir (ablasyon deneyi için):
  - "tr"   : doğru Türkçe yorum  ->  '1.234,56' = 1234.56, '%4,5' = 4.5, '(1.200)' = -1200
  - "en"   : yaygın naif yaklaşım (İngilizce yerel ayar)  -> '.' ondalık, ',' binlik kabul edilir;
             '1.234' = 1.234 (1000 kat hata), '4,5' = 45 (10 kat hata), '1.234,56' -> ayrıştırılamaz
  - "text" : hiç ayrıştırma yok; hücreler metin olarak saklanır (SQLite SUM('1.234,56') = 1.234 !)
"""
from __future__ import annotations

import re
import unicodedata
from dataclasses import dataclass
from datetime import date

# ------------------------------------------------------------------ metin
_TR_ASCII = str.maketrans("çğıöşüÇĞİÖŞÜâîûÂÎÛ", "cgiosuCGIOSUaiuAIU")


def tr_lower(s: str) -> str:
    """Türkçe kurallarına uygun küçük harf: 'İ'->'i', 'I'->'ı'."""
    if s is None:
        return s
    return str(s).replace("I", "ı").replace("İ", "i").lower()


def tr_upper(s: str) -> str:
    return str(s).replace("i", "İ").replace("ı", "I").upper()


def fold(s: str) -> str:
    """Karşılaştırma için: Türkçe küçük harf + aksan/şapka kaldırma + ASCII'ye katlama.
    'İSTANBUL', 'Istanbul', 'istanbul', 'ıstanbul' -> 'istanbul'."""
    if s is None:
        return ""
    s = tr_lower(str(s)).translate(_TR_ASCII)
    s = unicodedata.normalize("NFKD", s)
    s = "".join(ch for ch in s if not unicodedata.combining(ch))
    return re.sub(r"\s+", " ", s).strip()


def slugify(s: str, maxlen: int = 48) -> str:
    s = fold(s)
    s = s.replace("%", " yuzde ").replace("²", "2").replace("³", "3")
    s = re.sub(r"[^a-z0-9]+", "_", s).strip("_")
    return s[:maxlen].strip("_") or "sutun"


def clean_cell(s) -> str:
    if s is None:
        return ""
    s = str(s).replace(" ", " ").replace("−", "-").replace("–", "-")
    return re.sub(r"\s+", " ", s).strip()


# ------------------------------------------------------------------ sayılar
NULL_TOKENS = {"", "-", "--", "—", "...", "…", "n/a", "na", "yok", "x", ".."}

_RE_TR_THOUSANDS = re.compile(r"^\d{1,3}(\.\d{3})+(,\d+)?$")   # 1.234 | 1.234,56
_RE_TR_PLAIN = re.compile(r"^\d+(,\d+)?$")                      # 1234 | 1234,56
_RE_EN_THOUSANDS = re.compile(r"^\d{1,3}(,\d{3})+(\.\d+)?$")    # 1,234.56
_RE_EN_PLAIN = re.compile(r"^\d+\.\d+$")                        # 3.14
_CURRENCY = re.compile(r"(₺|\bTL\b|\bTRY\b|\bUSD\b|\$|€|\bEUR\b)", re.I)


@dataclass
class ParsedNumber:
    value: float | None
    is_percent: bool = False
    ok: bool = True


def parse_number(raw, mode: str = "tr") -> ParsedNumber:
    """Bir hücreyi sayıya çevirmeyi dener. ok=False ise hücre sayı değildir."""
    s = clean_cell(raw)
    if fold(s) in NULL_TOKENS:
        return ParsedNumber(None, ok=True)
    if mode == "text":
        return ParsedNumber(None, ok=False)
    is_pct = "%" in s
    s = s.replace("%", "")
    s = _CURRENCY.sub("", s).strip()
    neg = False
    if s.startswith("(") and s.endswith(")"):
        neg, s = True, s[1:-1].strip()
    if s.startswith("-") or s.startswith("+"):
        neg, s = s.startswith("-"), s[1:].strip()
    s = s.replace(" ", "")
    if not s or not re.fullmatch(r"[\d.,]+", s):
        return ParsedNumber(None, ok=False)

    val = None
    if mode == "tr":
        if _RE_TR_THOUSANDS.match(s) or _RE_TR_PLAIN.match(s):
            val = float(s.replace(".", "").replace(",", "."))
        elif _RE_EN_THOUSANDS.match(s):          # belgede İngilizce biçim varsa yine de tanı
            val = float(s.replace(",", ""))
        elif _RE_EN_PLAIN.match(s):              # '3.5' gibi tek nokta, 3 hane değil -> ondalık
            val = float(s)
    elif mode == "en":                           # naif: Python float() mantığı
        try:
            val = float(s.replace(",", ""))
        except ValueError:
            val = None
    if val is None:
        return ParsedNumber(None, ok=False)
    return ParsedNumber(-val if neg else val, is_percent=is_pct)


def format_tr(x, dec: int | None = None) -> str:
    """Sonuçları kullanıcıya Türkçe biçimde gösterir."""
    if x is None:
        return "—"
    if isinstance(x, bool):
        return str(x)
    if isinstance(x, int) or (isinstance(x, float) and x.is_integer() and dec is None):
        s = f"{int(x):,}"
        return s.replace(",", ".")
    if isinstance(x, float):
        d = 2 if dec is None else dec
        s = f"{x:,.{d}f}"
        return s.replace(",", "§").replace(".", ",").replace("§", ".")
    return str(x)


# ------------------------------------------------------------------ tarihler
AYLAR = {"ocak": 1, "subat": 2, "mart": 3, "nisan": 4, "mayis": 5, "haziran": 6, "temmuz": 7,
         "agustos": 8, "eylul": 9, "ekim": 10, "kasim": 11, "aralik": 12,
         "oca": 1, "sub": 2, "mar": 3, "nis": 4, "may": 5, "haz": 6, "tem": 7, "agu": 8,
         "eyl": 9, "eki": 10, "kas": 11, "ara": 12}
_RE_DMY = re.compile(r"^(\d{1,2})[./-](\d{1,2})[./-](\d{4})$")
_RE_YMD = re.compile(r"^(\d{4})-(\d{2})-(\d{2})$")
_RE_MON_Y = re.compile(r"^([a-z]+)\s*(\d{4})$")
_RE_Y_Q = re.compile(r"^(\d{4})\s*[/-]?\s*(q|ç|c)\s*([1-4])$")


def parse_date(raw) -> str | None:
    """'31.12.2024' -> '2024-12-31', 'Ocak 2024' -> '2024-01', '2024/Ç1' -> '2024-Q1'."""
    s = fold(clean_cell(raw))
    if not s:
        return None
    m = _RE_DMY.match(s)
    if m:
        d, mo, y = map(int, m.groups())
        try:
            return date(y, mo, d).isoformat()
        except ValueError:
            return None
    m = _RE_YMD.match(s)
    if m:
        return s
    m = _RE_MON_Y.match(s)
    if m and m.group(1) in AYLAR:
        return f"{m.group(2)}-{AYLAR[m.group(1)]:02d}"
    m = _RE_Y_Q.match(s)
    if m:
        return f"{m.group(1)}-Q{m.group(3)}"
    return None


_RE_DATE_RANGE = re.compile(r"(\d{1,2}[./]\d{1,2}[./]?(\d{4})?)\s*[-–]\s*(\d{1,2}[./]\d{1,2}[./](\d{4}))")
_RE_YEAR = re.compile(r"\b(19\d{2}|20\d{2})\b")


# ------------------------------------------------------------------ birimler
UNIT_PATTERNS = [
    (r"\bmilyar\s*(tl|₺)\b", "milyar TL", 1e9),
    (r"\bmilyon\s*(tl|₺)\b", "milyon TL", 1e6),
    (r"\bbin\s*(tl|₺)\b", "bin TL", 1e3),
    (r"\b(tl|₺|türk lirası)\b", "TL", 1.0),
    (r"\bbin\s*mwh\b", "bin MWh", 1e3),
    (r"\bmwh\b", "MWh", 1.0),
    (r"\bgwh\b", "GWh", 1e3),
    (r"%|\byuzde\b", "%", 1.0),
    (r"\bkisi\b", "kişi", 1.0),
    (r"\bton\b", "ton", 1.0),
    (r"\bm2\b|m²", "m²", 1.0),
    (r"\bm3\b|m³", "m³", 1.0),
]


def detect_unit(text: str) -> tuple[str | None, float]:
    """Başlık veya tablo adı içindeki parantezli birimi bulur. ('bin TL', 1000.0)"""
    if not text:
        return None, 1.0
    parts = re.findall(r"\(([^()]*)\)", text) or [text]
    for part in parts:
        f = fold(part)
        for pat, name, mult in UNIT_PATTERNS:
            if re.search(pat, f):
                return name, mult
    return None, 1.0


def strip_units(text: str) -> str:
    """'Ücret (TL)' -> 'Ücret'; '(-)' gibi işaretleri de kaldırır."""
    out = re.sub(r"\((?:[^()]*?)\)", " ", text)
    return re.sub(r"\s+", " ", out).strip()

	"""Türkçeye özgü metin, sayı, tarih ve birim normalizasyonu.

	Üç sayı ayrıştırma kipi desteklenir (ablasyon deneyi için):
	- "tr" : doğru Türkçe yorum -> '1.234,56' = 1234.56, '%4,5' = 4.5, '(1.200)' = -1200
	- "en" : yaygın naif yaklaşım (İngilizce yerel ayar) -> '.' ondalık, ',' binlik kabul edilir;
	'1.234' = 1.234 (1000 kat hata), '4,5' = 45 (10 kat hata), '1.234,56' -> ayrıştırılamaz
	- "text" : hiç ayrıştırma yok; hücreler metin olarak saklanır (SQLite SUM('1.234,56') = 1.234 !)
	"""
	from __future__ import annotations

	import re
	import unicodedata
	from dataclasses import dataclass
	from datetime import date

	# ------------------------------------------------------------------ metin
	_TR_ASCII = str.maketrans("çğıöşüÇĞİÖŞÜâîûÂÎÛ", "cgiosuCGIOSUaiuAIU")


	def tr_lower(s: str) -> str:
	"""Türkçe kurallarına uygun küçük harf: 'İ'->'i', 'I'->'ı'."""
	if s is None:
	return s
	return str(s).replace("I", "ı").replace("İ", "i").lower()


	def tr_upper(s: str) -> str:
	return str(s).replace("i", "İ").replace("ı", "I").upper()


	def fold(s: str) -> str:
	"""Karşılaştırma için: Türkçe küçük harf + aksan/şapka kaldırma + ASCII'ye katlama.
	'İSTANBUL', 'Istanbul', 'istanbul', 'ıstanbul' -> 'istanbul'."""
	if s is None:
	return ""
	s = tr_lower(str(s)).translate(_TR_ASCII)
	s = unicodedata.normalize("NFKD", s)
	s = "".join(ch for ch in s if not unicodedata.combining(ch))
	return re.sub(r"\s+", " ", s).strip()


	def slugify(s: str, maxlen: int = 48) -> str:
	s = fold(s)
	s = s.replace("%", " yuzde ").replace("²", "2").replace("³", "3")
	s = re.sub(r"[^a-z0-9]+", "_", s).strip("_")
	return s[:maxlen].strip("_") or "sutun"


	def clean_cell(s) -> str:
	if s is None:
	return ""
	s = str(s).replace(" ", " ").replace("−", "-").replace("–", "-")
	return re.sub(r"\s+", " ", s).strip()


	# ------------------------------------------------------------------ sayılar
	NULL_TOKENS = {"", "-", "--", "—", "...", "…", "n/a", "na", "yok", "x", ".."}

	_RE_TR_THOUSANDS = re.compile(r"^\d{1,3}(\.\d{3})+(,\d+)?$") # 1.234 \| 1.234,56
	_RE_TR_PLAIN = re.compile(r"^\d+(,\d+)?$") # 1234 \| 1234,56
	_RE_EN_THOUSANDS = re.compile(r"^\d{1,3}(,\d{3})+(\.\d+)?$") # 1,234.56
	_RE_EN_PLAIN = re.compile(r"^\d+\.\d+$") # 3.14
	_CURRENCY = re.compile(r"(₺\|\bTL\b\|\bTRY\b\|\bUSD\b\|\$\|€\|\bEUR\b)", re.I)


	@dataclass
	class ParsedNumber:
	value: float \| None
	is_percent: bool = False
	ok: bool = True


	def parse_number(raw, mode: str = "tr") -> ParsedNumber:
	"""Bir hücreyi sayıya çevirmeyi dener. ok=False ise hücre sayı değildir."""
	s = clean_cell(raw)
	if fold(s) in NULL_TOKENS:
	return ParsedNumber(None, ok=True)
	if mode == "text":
	return ParsedNumber(None, ok=False)
	is_pct = "%" in s
	s = s.replace("%", "")
	s = _CURRENCY.sub("", s).strip()
	neg = False
	if s.startswith("(") and s.endswith(")"):
	neg, s = True, s[1:-1].strip()
	if s.startswith("-") or s.startswith("+"):
	neg, s = s.startswith("-"), s[1:].strip()
	s = s.replace(" ", "")
	if not s or not re.fullmatch(r"[\d.,]+", s):
	return ParsedNumber(None, ok=False)

	val = None
	if mode == "tr":
	if _RE_TR_THOUSANDS.match(s) or _RE_TR_PLAIN.match(s):
	val = float(s.replace(".", "").replace(",", "."))
	elif _RE_EN_THOUSANDS.match(s): # belgede İngilizce biçim varsa yine de tanı
	val = float(s.replace(",", ""))
	elif _RE_EN_PLAIN.match(s): # '3.5' gibi tek nokta, 3 hane değil -> ondalık
	val = float(s)
	elif mode == "en": # naif: Python float() mantığı
	try:
	val = float(s.replace(",", ""))
	except ValueError:
	val = None
	if val is None:
	return ParsedNumber(None, ok=False)
	return ParsedNumber(-val if neg else val, is_percent=is_pct)


	def format_tr(x, dec: int \| None = None) -> str:
	"""Sonuçları kullanıcıya Türkçe biçimde gösterir."""
	if x is None:
	return "—"
	if isinstance(x, bool):
	return str(x)
	if isinstance(x, int) or (isinstance(x, float) and x.is_integer() and dec is None):
	s = f"{int(x):,}"
	return s.replace(",", ".")
	if isinstance(x, float):
	d = 2 if dec is None else dec
	s = f"{x:,.{d}f}"
	return s.replace(",", "§").replace(".", ",").replace("§", ".")
	return str(x)


	# ------------------------------------------------------------------ tarihler
	AYLAR = {"ocak": 1, "subat": 2, "mart": 3, "nisan": 4, "mayis": 5, "haziran": 6, "temmuz": 7,
	"agustos": 8, "eylul": 9, "ekim": 10, "kasim": 11, "aralik": 12,
	"oca": 1, "sub": 2, "mar": 3, "nis": 4, "may": 5, "haz": 6, "tem": 7, "agu": 8,
	"eyl": 9, "eki": 10, "kas": 11, "ara": 12}
	_RE_DMY = re.compile(r"^(\d{1,2})[./-](\d{1,2})[./-](\d{4})$")
	_RE_YMD = re.compile(r"^(\d{4})-(\d{2})-(\d{2})$")
	_RE_MON_Y = re.compile(r"^([a-z]+)\s*(\d{4})$")
	_RE_Y_Q = re.compile(r"^(\d{4})\s[/-]?\s(q\|ç\|c)\s*([1-4])$")


	def parse_date(raw) -> str \| None:
	"""'31.12.2024' -> '2024-12-31', 'Ocak 2024' -> '2024-01', '2024/Ç1' -> '2024-Q1'."""
	s = fold(clean_cell(raw))
	if not s:
	return None
	m = _RE_DMY.match(s)
	if m:
	d, mo, y = map(int, m.groups())
	try:
	return date(y, mo, d).isoformat()
	except ValueError:
	return None
	m = _RE_YMD.match(s)
	if m:
	return s
	m = _RE_MON_Y.match(s)
	if m and m.group(1) in AYLAR:
	return f"{m.group(2)}-{AYLAR[m.group(1)]:02d}"
	m = _RE_Y_Q.match(s)
	if m:
	return f"{m.group(1)}-Q{m.group(3)}"
	return None


	_RE_DATE_RANGE = re.compile(r"(\d{1,2}[./]\d{1,2}[./]?(\d{4})?)\s[-–]\s(\d{1,2}[./]\d{1,2}[./](\d{4}))")
	_RE_YEAR = re.compile(r"\b(19\d{2}\|20\d{2})\b")


	# ------------------------------------------------------------------ birimler
	UNIT_PATTERNS = [
	(r"\bmilyar\s*(tl\|₺)\b", "milyar TL", 1e9),
	(r"\bmilyon\s*(tl\|₺)\b", "milyon TL", 1e6),
	(r"\bbin\s*(tl\|₺)\b", "bin TL", 1e3),
	(r"\b(tl\|₺\|türk lirası)\b", "TL", 1.0),
	(r"\bbin\s*mwh\b", "bin MWh", 1e3),
	(r"\bmwh\b", "MWh", 1.0),
	(r"\bgwh\b", "GWh", 1e3),
	(r"%\|\byuzde\b", "%", 1.0),
	(r"\bkisi\b", "kişi", 1.0),
	(r"\bton\b", "ton", 1.0),
	(r"\bm2\b\|m²", "m²", 1.0),
	(r"\bm3\b\|m³", "m³", 1.0),
	]


	def detect_unit(text: str) -> tuple[str \| None, float]:
	"""Başlık veya tablo adı içindeki parantezli birimi bulur. ('bin TL', 1000.0)"""
	if not text:
	return None, 1.0
	parts = re.findall(r"\(([^()]*)\)", text) or [text]
	for part in parts:
	f = fold(part)
	for pat, name, mult in UNIT_PATTERNS:
	if re.search(pat, f):
	return name, mult
	return None, 1.0


	def strip_units(text: str) -> str:
	"""'Ücret (TL)' -> 'Ücret'; '(-)' gibi işaretleri de kaldırır."""
	out = re.sub(r"\((?:[^()]*?)\)", " ", text)
	return re.sub(r"\s+", " ", out).strip()