# -*- coding: utf-8 -*- """ t2r_core.py - Text2Receipt core data-generation module. ARCHITECTURE (locked in project spec) ------------------------------------- * Rule-based code produces PERFECT ground-truth labels. * A model (later, in 03/app) only has to predict `parse`. * Deterministic `complete()` computes everything calculable (VAT, subtotal, totals, serial numbers, allocation-number flag), so the model never predicts anything arithmetic. RECORD SCHEMA ------------- raw_text : messy free-text Hebrew income note (model INPUT) parse : structured extraction of what the note STATES (model TARGET - the only thing predicted) completed : full, valid fiscal document computed from parse + issuer profile (deterministic) meta : pool (A/B), issuer/doc type, vat rate, paraphrase flag, ... ENTITIES ARE IN HEBREW (client names, business names, service descriptions), matching real Israeli income notes. Universal tech acronyms (API/ERP/SEO/UI) are kept as-is, as a Hebrew-speaking freelancer would write them. ISRAELI FISCAL RULES ENCODED ---------------------------- * VAT 18% since 2025-01-01; legacy 17% for documents dated in 2024. * Exempt dealer (osek patur) / institution: 0% VAT, RECEIPT only. * Allocation number (mispar haktzaa) required when ALL hold: doc_type is a tax invoice, issuer is an authorized dealer / company, client is a registered business, subtotal (before VAT) >= threshold for the document's date (timeline: 25,000 in 2024 -> ... -> 5,000 from 2026-06-01). * Israeli ID / company numbers carry a valid check digit. This module is pure-Python and runs on CPU. The optional paraphrasing pass (HF instruct model) lives in the notebook, not here. """ from __future__ import annotations import random import datetime as _dt from dataclasses import dataclass, field, asdict from typing import Optional # ===================================================================== # 1. ISRAELI FISCAL CONSTANTS & CHECK DIGITS # ===================================================================== VAT_NEW = 0.18 # since 2025-01-01 VAT_LEGACY = 0.17 # documents dated in 2024 VAT_NEW_FROM = _dt.date(2025, 1, 1) # Allocation-number threshold timeline (ILS, before VAT). Effective-from -> threshold. ALLOCATION_THRESHOLDS = [ (_dt.date(2024, 5, 5), 25_000), (_dt.date(2025, 1, 1), 20_000), (_dt.date(2026, 1, 1), 10_000), (_dt.date(2026, 6, 1), 5_000), ] def vat_rate_for_date(d: _dt.date) -> float: """18% from 2025-01-01, otherwise 17% (legacy 2024 documents).""" return VAT_NEW if d >= VAT_NEW_FROM else VAT_LEGACY def allocation_threshold_for_date(d: _dt.date) -> int: """Lowest applicable allocation-number threshold for a document date.""" thr = ALLOCATION_THRESHOLDS[0][1] for eff_from, value in ALLOCATION_THRESHOLDS: if d >= eff_from: thr = value return thr def israeli_check_digit(eight_digits: str) -> int: """Compute the 9th check digit for an Israeli ID / company number. Weights alternate 1,2,1,2,...; products > 9 are reduced by summing digits. """ assert len(eight_digits) == 8 and eight_digits.isdigit() total = 0 for i, ch in enumerate(eight_digits): p = int(ch) * (1 if i % 2 == 0 else 2) total += p if p < 10 else p - 9 return (10 - (total % 10)) % 10 def make_israeli_id(rng: random.Random) -> str: """A 9-digit identifier with a valid Israeli check digit.""" base = "".join(str(rng.randint(0, 9)) for _ in range(8)) return base + str(israeli_check_digit(base)) def make_company_number(rng: random.Random) -> str: """Israeli company number (chet-pe): 9 digits, starts with 5, valid check digit.""" base = "5" + "".join(str(rng.randint(0, 9)) for _ in range(7)) return base + str(israeli_check_digit(base)) def is_valid_israeli_id(num: str) -> bool: return len(num) == 9 and num.isdigit() and israeli_check_digit(num[:8]) == int(num[8]) # ===================================================================== # 2. ENTITY POOLS (A and B are DISJOINT -> B powers the OOD test) # ===================================================================== # Each pool supplies: issuer business names, client first/last names, # and service catalogs - all in HEBREW. Nothing overlaps between A and B, # so a model trained on A and evaluated on B is tested on genuinely unseen # entities. Issuer names are neutral freelancer/studio names (the issuer is # never written in raw_text - it is the self-employed person themselves). POOLS = { "A": { "issuer_biz": [ "סטודיו רון", "מאיר כהן", "תכלת סטודיו", "גלים", "שחף", "נטף שירותים", "אלון ושות'", "מרום", "ליאת לוי", "צוף יצירה", ], "client_first": ["משה", "דנה", "יוסי", "נועה", "אבי", "תמר", "איתן", "שירה"], "client_last": ["כהן", "לוי", "מזרחי", "פרץ", "ביטון", "אזולאי"], "client_biz": ["מעגל בע\"מ", "אור-טק", "שקד אחזקות", "ניר לוגיסטיקה"], "services": { "consulting": [("ייעוץ עסקי", 350, 1200), ("סדנת אסטרטגיה", 800, 3500)], "design": [("עיצוב לוגו", 600, 2500), ("חבילת פוסטים לרשתות", 250, 900)], "tutoring": [("שיעור מתמטיקה פרטי", 120, 260), ("שיעור אנגלית", 130, 240)], "trades": [("החלפת ברז", 200, 650), ("פתיחת סתימה", 250, 800)], "catering": [("מגש אירוח לאירוע", 180, 540), ("מגש קינוחים", 150, 420)], "software": [("עמוד נחיתה", 1500, 6000), ("ריטיינר תיקוני באגים", 400, 1600)], "photography": [("צילום אירוע", 400, 1800), ("סט צילומי מוצר", 300, 1200)], "marketing": [("ניהול רשתות חודשי", 900, 3200), ("הקמת קמפיין פרסום", 500, 2200)], }, "highvalue": { "software": [("פרויקט אתר מלא", 9000, 38000), ("פרויקט אינטגרציית ERP", 12000, 45000)], "consulting": [("ריטיינר ייעוץ רבעוני", 6000, 18000), ("תוכנית טרנספורמציה", 9000, 30000)], "design": [("חבילת זהות מותגית", 7000, 22000), ("בניית מערכת עיצוב", 8000, 26000)], "marketing": [("ריטיינר שיווק שנתי", 6500, 20000), ("קמפיין רב-ערוצי", 7000, 24000)], }, }, "B": { # disjoint Hebrew vocabulary - never seen in training "issuer_biz": [ "סטודיו נגב", "אופק", "מעיין יצירה", "אורלי", "להב", "תבור", "נועם פריים", "גשר", "רקיע", "שלי ברק", ], "client_first": ["רוני", "גל", "עומר", "הילה", "נדב", "מאיה", "עידו", "ליאור"], "client_last": ["פרידמן", "סגל", "אוחיון", "דהאן", "כץ", "ברק"], "client_biz": ["כנרת גרופ", "וולט מערכות", "אשל סחר", "רקיע מדיה"], "services": { "consulting": [("שעת ייעוץ פיננסי", 400, 1300), ("אודיט צמיחה", 900, 3800)], "design": [("מדריך מותג", 700, 2700), ("מוקאפ אריזה", 300, 1000)], "tutoring": [("שיעור פיזיקה", 140, 280), ("שיעור גיטרה", 110, 230)], "trades": [("התקנת גוף תאורה", 220, 700), ("תיקון שקע", 180, 520)], "catering": [("הקמת מזנון", 200, 600), ("קופסת מאפים", 160, 460)], "software": [("בניית מסך מובייל", 1600, 6500), ("אינטגרציית API", 500, 2000)], "photography": [("צילומי פורטרט", 380, 1700), ("סט צילומי נדל\"ן", 320, 1300)], "marketing": [("ריטיינר תוכן", 850, 3100), ("הקמת SEO", 550, 2300)], }, "highvalue": { "software": [("בניית פלטפורמה רב-עמודית", 9500, 40000), ("פרויקט צינור נתונים", 13000, 46000)], "consulting": [("ריטיינר ייעוץ שנתי", 6500, 19000), ("שדרוג תפעולי", 9500, 31000)], "design": [("תוכנית מיתוג מחדש", 7500, 24000), ("פרויקט ערכת UI", 8500, 27000)], "marketing": [("ריטיינר צמיחה שנתי", 7000, 21000), ("תוכנית קמפיין השקה", 7500, 25000)], }, }, } PAYMENT_METHODS = ["cash", "bank_transfer", "credit_card", "check", "bit"] PAYMENT_HE = {"cash": "מזומן", "bank_transfer": "העברה בנקאית", "credit_card": "אשראי", "check": "צ'ק", "bit": "ביט"} DOC_TYPES = ["receipt", "tax_invoice", "tax_invoice_receipt"] DOC_TYPE_HE = {"receipt": "קבלה", "tax_invoice": "חשבונית מס", "tax_invoice_receipt": "חשבונית מס/קבלה"} # issuer fiscal status ISSUER_AUTHORIZED = "authorized_dealer" # osek murshe -> VAT, may issue tax invoice ISSUER_EXEMPT = "exempt_dealer" # osek patur -> 0% VAT, receipt only # ===================================================================== # 3. ISSUER PROFILE # ===================================================================== @dataclass class Issuer: name: str status: str # ISSUER_AUTHORIZED / ISSUER_EXEMPT tax_id: str # ID or company number is_company: bool def make_issuer(rng: random.Random, pool: str) -> Issuer: p = POOLS[pool] name = rng.choice(p["issuer_biz"]) # ~25% exempt dealers, rest authorized; ~40% of authorized are companies status = ISSUER_EXEMPT if rng.random() < 0.25 else ISSUER_AUTHORIZED is_company = status == ISSUER_AUTHORIZED and rng.random() < 0.40 tax_id = make_company_number(rng) if is_company else make_israeli_id(rng) return Issuer(name=name, status=status, tax_id=tax_id, is_company=is_company) # ===================================================================== # 4. RAW-TEXT TEMPLATES (messy, human-like Hebrew income notes) # ===================================================================== # Diversity comes from: formality, abbreviations, typos, missing fields, # amount phrasing (gross vs net), and ordering. The note states only what # a human would jot down; complete() infers the rest. def _amount_phrase(rng: random.Random, amount: int) -> str: forms = [f"{amount} ש\"ח", f"{amount} שקל", f"{amount}₪", f"{amount} שקלים", f"סכום {amount}"] return rng.choice(forms) RAW_TEMPLATES = [ "קיבלתי {amount} מ{client} על {service}", "{client} שילם {amount} עבור {service}", "תקבול {amount} - {service} - {client}", "הכנסה: {service} ל{client}, {amount}, {pay}", "{service} {client} {amount} {pay}", "גבית {amount} מ{client} ({service})", "{client} העביר {amount} ב{pay} על {service}", "{amount} בעבור {service}. לקוח: {client}", "היום {client} - {service} - {amount}", "{service} - {amount} - שולם ב{pay}", ] # light typo injection to mimic real notes TYPO_MAP = {"שילם": "שילמ", "קיבלתי": "קבלתי", "עבור": "עבר", "העביר": "העבירה"} def _maybe_typo(rng: random.Random, text: str) -> str: if rng.random() < 0.15: for k, v in TYPO_MAP.items(): if k in text and rng.random() < 0.5: text = text.replace(k, v, 1) break return text # ===================================================================== # 5. SAMPLE GENERATION (parse target + raw text) # ===================================================================== @dataclass class Meta: pool: str issuer_status: str issuer_is_company: bool doc_type: str vat_rate: float category: str = "" client_is_business: bool = False payment_method: str = "" year: int = 0 paraphrased: bool = False def _pick_date(rng: random.Random) -> _dt.date: """~10% in 2024 (legacy 17% VAT), the rest across 2025-2026.""" if rng.random() < 0.10: start = _dt.date(2024, 1, 1); span = 365 else: start = _dt.date(2025, 1, 1); span = 530 return start + _dt.timedelta(days=rng.randint(0, span - 1)) def generate_sample(rng: random.Random, pool: str) -> dict: """Produce one record's ground truth: issuer, parse target, raw_text, meta.""" p = POOLS[pool] issuer = make_issuer(rng, pool) # ---- client (business client more likely to trigger allocation rule) ---- client_is_business = rng.random() < 0.45 if client_is_business: client_name = rng.choice(p["client_biz"]) client_tax_id = make_company_number(rng) if rng.random() < 0.7 else make_israeli_id(rng) else: client_name = f"{rng.choice(p['client_first'])} {rng.choice(p['client_last'])}" client_tax_id = make_israeli_id(rng) if rng.random() < 0.25 else None # ---- items (1-3 line items from one service category) ---- # Large B2B jobs cross the allocation threshold; ~55% of (non-exempt) business # clients get one so the allocation slice is meaningful. Exempt dealers (osek # patur, turnover-capped) only issue small single-item invoices. exempt = issuer.status == ISSUER_EXEMPT large = client_is_business and not exempt and rng.random() < 0.55 if large: category = rng.choice(list(p["highvalue"].keys())) catalog = p["highvalue"][category] n_items = rng.choices([1, 2, 3], weights=[0.55, 0.30, 0.15])[0] qty_pool, qty_w = [1, 2, 3], [0.6, 0.28, 0.12] elif exempt: category = rng.choice(["tutoring", "trades", "catering", "photography", "design", "marketing"]) catalog = p["services"][category] n_items = 1 qty_pool, qty_w = [1, 2], [0.85, 0.15] else: category = rng.choice(list(p["services"].keys())) catalog = p["services"][category] n_items = rng.choices([1, 2, 3], weights=[0.7, 0.22, 0.08])[0] qty_pool, qty_w = [1, 2, 3, 4], [0.75, 0.15, 0.06, 0.04] items = [] for _ in range(n_items): desc, lo, hi = rng.choice(catalog) unit = rng.randint(lo, hi) qty = rng.choices(qty_pool, weights=qty_w)[0] items.append({"description": desc, "unit_price": float(unit), "quantity": qty}) date = _pick_date(rng) payment = rng.choice(PAYMENT_METHODS) # ---- doc type is constrained by issuer status ---- if issuer.status == ISSUER_EXEMPT: doc_type = "receipt" # exempt dealers cannot issue tax invoices else: doc_type = rng.choices(DOC_TYPES, weights=[0.30, 0.30, 0.40])[0] # amount stated in the note: gross (VAT-inclusive) or net amount_basis = rng.choices(["gross", "net"], weights=[0.6, 0.4])[0] parse = { "client_name": client_name, "client_tax_id": client_tax_id, "client_is_business": client_is_business, "items": items, "amount_basis": amount_basis, "doc_type": doc_type, "payment_method": payment, "date": date.isoformat(), "currency": "ILS", } # ---- raw_text: render a messy note from which `parse` is fully recoverable ---- # States each item's quantity and UNIT price (never only a lump sum), so the # model extracts unit_price/quantity directly; complete() still does all the # arithmetic (line totals, VAT, subtotal, total). def _qty_desc(it): d, q = it["description"], it["quantity"] if q == 1: return d return rng.choice([f"{q} {d}", f"{d} x{q}", f"{d} ({q} יח')"]) def _unit_phrase(it): u = int(it["unit_price"]) return rng.choice([f"{u} כל אחד", f"{u} ליחידה", f"{u} ליח'", f"@{u}"]) if n_items == 1: it = items[0] if it["quantity"] == 1: service_str = it["description"] else: service_str = f"{_qty_desc(it)}, {_unit_phrase(it)}" headline_amount = int(it["unit_price"] * it["quantity"]) else: parts = [f"{_qty_desc(it)} {_unit_phrase(it)}" for it in items] service_str = rng.choice([", ", " + ", " ו"]).join(parts) headline_amount = int(sum(it["unit_price"] * it["quantity"] for it in items)) tmpl = rng.choice(RAW_TEMPLATES) raw = tmpl.format( amount=_amount_phrase(rng, headline_amount), client=client_name, service=service_str, pay=PAYMENT_HE[payment], ) raw = _maybe_typo(rng, raw) meta = Meta(pool=pool, issuer_status=issuer.status, issuer_is_company=issuer.is_company, doc_type=doc_type, vat_rate=(0.0 if exempt else vat_rate_for_date(date)), category=category, client_is_business=client_is_business, payment_method=payment, year=date.year) return {"issuer": asdict(issuer), "parse": parse, "raw_text": raw, "meta": asdict(meta)} # ===================================================================== # 6. complete() - DETERMINISTIC DOCUMENT COMPLETION # ===================================================================== # Everything calculable is computed here. A model only ever predicts `parse`. _SERIAL_COUNTER = {"value": 1000} def _next_serial(rng: random.Random) -> int: _SERIAL_COUNTER["value"] += rng.randint(1, 4) return _SERIAL_COUNTER["value"] def complete(issuer: dict, parse: dict, rng: random.Random) -> dict: """Compute a full, valid fiscal document from the issuer profile + parse. Returns the `completed` document. No randomness affects money; rng only advances serial numbers and the allocation token. """ d = _dt.date.fromisoformat(parse["date"]) exempt = issuer["status"] == ISSUER_EXEMPT vat_rate = 0.0 if exempt else vat_rate_for_date(d) # line totals lines, gross_sum = [], 0.0 for it in parse["items"]: line_total = round(it["unit_price"] * it["quantity"], 2) gross_sum += line_total lines.append({**it, "line_total": line_total}) # interpret stated amount basis -> subtotal (pre-VAT) and vat if exempt: subtotal = round(gross_sum, 2) vat_amount = 0.0 elif parse["amount_basis"] == "gross": subtotal = round(gross_sum / (1 + vat_rate), 2) vat_amount = round(gross_sum - subtotal, 2) else: # net subtotal = round(gross_sum, 2) vat_amount = round(subtotal * vat_rate, 2) total = round(subtotal + vat_amount, 2) # allocation number rule needs_alloc = ( (not exempt) and parse["doc_type"] in ("tax_invoice", "tax_invoice_receipt") and bool(parse.get("client_is_business")) and subtotal >= allocation_threshold_for_date(d) ) allocation_number = None if needs_alloc: allocation_number = f"{rng.randint(10**8, 10**9 - 1)}" # 9-digit token completed = { "doc_type": parse["doc_type"], "doc_type_he": DOC_TYPE_HE[parse["doc_type"]], "serial_number": _next_serial(rng), "issue_date": parse["date"], "issuer": { "name": issuer["name"], "tax_id": issuer["tax_id"], "status": issuer["status"], }, "client": { "name": parse["client_name"], "tax_id": parse["client_tax_id"], "is_business": parse.get("client_is_business", False), }, "lines": lines, "currency": "ILS", "subtotal": subtotal, "vat_rate": vat_rate, "vat_amount": vat_amount, "total": total, "payment_method": parse["payment_method"], "allocation_required": needs_alloc, "allocation_number": allocation_number, } return completed # ===================================================================== # 7. INTEGRITY VALIDATION (zero-violation gate before publishing) # ===================================================================== def validate_record(rec: dict) -> list[str]: """Return a list of integrity violations for one record (empty == valid).""" v = [] issuer, parse, comp = rec["issuer"], rec["parse"], rec["completed"] # check digits if not is_valid_israeli_id(issuer["tax_id"]): v.append("issuer_tax_id_checkdigit") if parse.get("client_tax_id") and not is_valid_israeli_id(parse["client_tax_id"]): v.append("client_tax_id_checkdigit") # exempt dealer constraints if issuer["status"] == ISSUER_EXEMPT: if comp["vat_rate"] != 0.0 or comp["vat_amount"] != 0.0: v.append("exempt_dealer_has_vat") if comp["doc_type"] != "receipt": v.append("exempt_dealer_non_receipt") # VAT rate matches the document date d = _dt.date.fromisoformat(comp["issue_date"]) if issuer["status"] != ISSUER_EXEMPT and comp["vat_rate"] != vat_rate_for_date(d): v.append("vat_rate_date_mismatch") # money identity: subtotal + vat == total (to the agora) if round(comp["subtotal"] + comp["vat_amount"], 2) != comp["total"]: v.append("money_identity") if comp["subtotal"] < 0 or comp["total"] < 0: v.append("negative_money") # allocation-number correctness (both directions) need = ( issuer["status"] != ISSUER_EXEMPT and comp["doc_type"] in ("tax_invoice", "tax_invoice_receipt") and comp["client"]["is_business"] and comp["subtotal"] >= allocation_threshold_for_date(d) ) if need and not comp["allocation_number"]: v.append("allocation_missing") if (not need) and comp["allocation_number"]: v.append("allocation_unexpected") return v # ===================================================================== # 8. DATASET BUILD (A/B pools -> disjoint splits) # ===================================================================== def build_dataset(n_pool_a: int, n_pool_b: int, seed: int = 7) -> dict: """Generate completed+validated records, split into train/val/iid/ood. Pool A -> train / validation / iid_test (80/10/10) Pool B -> ood_test (disjoint entities & templates) """ rng = random.Random(seed) _SERIAL_COUNTER["value"] = 1000 def gen(n, pool): out = [] for _ in range(n): rec = generate_sample(rng, pool) rec["completed"] = complete(rec["issuer"], rec["parse"], rng) out.append(rec) return out pool_a = gen(n_pool_a, "A") pool_b = gen(n_pool_b, "B") rng.shuffle(pool_a) n_tr = int(n_pool_a * 0.8) n_va = int(n_pool_a * 0.1) splits = { "train": pool_a[:n_tr], "validation": pool_a[n_tr:n_tr + n_va], "iid_test": pool_a[n_tr + n_va:], "ood_test": pool_b, "human_test": [], # filled by hand later (real notes) } return splits def integrity_report(splits: dict) -> dict: """Aggregate violations across all splits.""" counts, total, bad = {}, 0, 0 for name, recs in splits.items(): for r in recs: total += 1 vs = validate_record(r) if vs: bad += 1 for x in vs: counts[x] = counts.get(x, 0) + 1 return {"total": total, "violations": bad, "by_type": counts} # ===================================================================== # 9. SELF-TEST (dry run -> proves the module end-to-end) # ===================================================================== if __name__ == "__main__": # check-digit sanity sample_id = make_israeli_id(random.Random(1)) assert is_valid_israeli_id(sample_id), "check-digit generator broken" splits = build_dataset(n_pool_a=900, n_pool_b=100, seed=7) rep = integrity_report(splits) sizes = {k: len(v) for k, v in splits.items()} print("split sizes:", sizes) print("integrity:", rep) assert rep["violations"] == 0, f"INTEGRITY FAILURES: {rep}" for ex in splits["train"][:4]: print("\nraw_text :", ex["raw_text"]) print("client :", ex["parse"]["client_name"], "| items:", [it["description"] for it in ex["parse"]["items"]]) print("doc :", ex["completed"]["doc_type_he"], "| subtotal:", ex["completed"]["subtotal"], "| vat:", ex["completed"]["vat_amount"], "| total:", ex["completed"]["total"], "| alloc:", ex["completed"]["allocation_number"]) print("\nOK - zero integrity violations.")