preprocessing/process_pipeline.py

"""
process_pipeline.py  —  Main MCQ generation pipeline

Changes from original:
  1. Lemmatizer integrated — answer and all options are root forms, not inflected
  2. Stronger sentence quality filter (anaphora, blank position, min word length)
  3. Candidate selection prefers longer, more specific nouns over generic ones
  4. build_question blanks the surface (inflected) form in the sentence but
     stores the root form as the answer — so the question looks natural
"""

from preprocessing.tokenizer import tokenize_text
from nlp.pos_tagger import get_pos_tags
from nlp.ner_tagger import get_entities_from_sentence
from lemmatizer import get_root
from question_generation.option_generator import generate_options


# ---------------------------------------------------------------------------
# SENTENCE QUALITY FILTER
# ---------------------------------------------------------------------------

# Words that make a sentence self-referential or anaphoric — skip these
_ANAPHORIC_STARTS = {
    "இது", "அது", "இவை", "அவை", "இதனால்", "அதனால்",
    "இதன்", "அதன்", "இவர்", "அவர்", "இவர்கள்", "அவர்கள்",
    "எனவே", "ஆகவே", "இதனை", "அதனை",
}

# Words so generic they make bad blanks even if tagged NOUN
_GENERIC_NOUNS = {
    "மனித", "இது", "அது", "உடல்", "பொருள்", "விஷயம்",
    "காரணம்", "நிலை", "வகை", "முறை", "பகுதி", "இடம்",
    "நேரம்", "செயல்", "தன்மை", "வழி",
}

# Placeholder stopwords set; can be populated from a resource file later.
STOPWORDS = set()


def is_valid_sentence(sentence: str) -> bool:
    """
    Returns True only if the sentence is suitable for MCQ generation.

    Rules
    -----
    1. At least 6 words (shorter sentences produce trivial blanks).
    2. Must not start with an anaphoric/pronoun word.
    3. Must not start with a conjunctive adverb (sentence is incomplete alone).
    4. Must contain at least one non-generic noun or proper noun.
    """
    words = sentence.split()

    # Rule 1: length
    if len(words) < 3:
        return False

    # Rule 2 & 3: anaphoric / dependent start
    if words[0] in _ANAPHORIC_STARTS:
        return False

    # Rule 4: sentence must have informational content
    # (quick heuristic — full POS check happens later in pipeline)
    tamil_words = [w for w in words if len(w) >= 3 and w not in _GENERIC_NOUNS]
    if len(tamil_words) < 1:
            return False

    return True


# ---------------------------------------------------------------------------
# CANDIDATE SELECTION
# ---------------------------------------------------------------------------

def select_candidate(pos_tags, entities, sentence):
    from question_generation.cluster_store import CLUSTERS

    words_in_sentence = set(sentence.split())

    def valid(word):
        return (
            len(word) >= 3
            and word not in STOPWORDS
            and word in words_in_sentence
        )

    def score(word, tag):
        s = 0
        # POS priority
        if tag == "PROPN": s += 5
        elif tag == "NOUN": s += 3
        # NER bonus
        if word in entities: s += 10
        # Cluster bonus — can we find distractors?
        for cluster in CLUSTERS.values():
            if word in cluster:
                s += 2
                break
        return s

    # Build scored candidates from all nouns/propns
    candidates = [
        (word, tag, score(word, tag))
        for word, tag in pos_tags
        if tag in ("NOUN", "PROPN") and valid(word)
    ]

    # Also add NER entities not caught by POS
    for e in entities:
        if valid(e) and not any(w == e for w, _, _ in candidates):
            candidates.append((e, "PROPN", score(e, "PROPN")))

    if not candidates:
        return None

    # Pick highest scoring
    best = max(candidates, key=lambda x: x[2])
    surface = best[0]
    root = get_root(surface)
    return surface, root


# ---------------------------------------------------------------------------
# QUESTION BUILDER
# ---------------------------------------------------------------------------

def build_question(sentence: str, surface_word: str) -> str | None:
    """
    Replace the first occurrence of surface_word in sentence with _____.

    We blank the *surface* (inflected) form so the sentence stays
    grammatically natural. The answer stored is the root form.
    """
    if not surface_word or surface_word not in sentence:
        return None

    # Guard: blank should not be the first or last token (bad question shape)
    words = sentence.split()
    if words[0] == surface_word or words[-1] == surface_word:
        return None

    return sentence.replace(surface_word, "_____", 1)


# ---------------------------------------------------------------------------
# MAIN PIPELINE
# ---------------------------------------------------------------------------

def generate_mcq_pipeline(sentence: str) -> dict | None:
    """
    Full pipeline: sentence → MCQ dict or None.

    Returns
    -------
    {
        "question": "... _____ ...",
        "options":  ["A", "B", "C", "D"],   # shuffled, root forms
        "answer":   "root_form_of_answer"
    }
    or None if the sentence is unsuitable.
    """
    sentence = sentence.replace("\n", " ").strip()

    # Step 1: quality gate
    if not is_valid_sentence(sentence):
        return None

    # Step 2: NLP
    tokens    = tokenize_text(sentence)
    pos_tags  = get_pos_tags(sentence)
    entities  = get_entities_from_sentence(sentence)

    # Step 3: pick candidate (surface + root)
    candidate = select_candidate(pos_tags, entities, sentence)
    if not candidate:
        return None

    surface, root = candidate

    # Step 4: build question (blank uses surface form for natural Tamil)
    question = build_question(sentence, surface)
    if not question:
        return None

    # Step 5: generate options — all in root form
    options = generate_options(
        answer_word=root,
        sentence_words=tokens,
        pos_tags=pos_tags,
        question=question,
        context=sentence,
    )

    return {
        "question": question,
        "options":  options,
        "answer":   root,
    }