tests/test_fast_engine.py

"""Fast engine (fast.py) end to end on a Word file, with the models replaced by fakes: the detector flags sentences
with AI stock words, the language model offers versions, one of which changes the meaning."""
from docx import Document

import detector as D
import fast as F
import fluency as FL
import groqai as G
import humanizer as H
import meaning as MEANING

AI = ("Furthermore, the proposed controller plays a crucial role in reducing the harmonic distortion of the grid "
      "current from 4.8% to 1.9% [14].")
AI2 = "These findings provide valuable insights for engineers seeking to design robust inverter control schemes."
HUMAN = ("We built the test rig in the basement lab and ran it for six weeks before the first usable data came out, "
         "mostly because the sensors kept drifting.")


def _ai(t):
    """Share of AI-sounding sentences in the text (a window score behaves like this)."""
    sents = [x for x in D.split_sentences(t) if x.strip()] or [t]
    bad = sum(any(w in x for w in ("crucial", "valuable", "Furthermore")) for x in sents)
    return 0.05 + 0.9 * bad / len(sents)


def test_the_fast_engine_lowers_the_score_and_never_uses_an_unfaithful_version(tmp_path, monkeypatch):
    monkeypatch.setattr(D, "score_texts", lambda texts: [_ai(t) for t in texts])
    monkeypatch.setattr(H, "_detector_scores", lambda texts: [_ai(t) for t in texts])
    monkeypatch.setattr(FL, "acceptable", lambda pairs: [True] * len(pairs))
    monkeypatch.setattr(MEANING, "similarities", lambda pairs: [0.95] * len(pairs))
    seen = {}

    import json

    def request(gen, system, user, max_tokens, deadline=None):
        seen["faithful"] = "KEEP EVERY DETAIL" in system
        seen.setdefault("gens", set()).add(gen)
        items = []
        for i, line in enumerate([x for x in user.splitlines() if "SENTENCE:" in x], 1):
            if "harmonic" in line:
                vs = ["The controller lowers the harmonic distortion of the grid current from 4.8% to 1.9% [14].",
                      "The controller is the main reason the distortion vanished [14]."]
            elif "valuable" in line:
                vs = ["Engineers who design inverter control schemes can use these results."]
            else:
                vs = []
            items.append({"id": i, "versions": vs})
        return json.dumps({"items": items})
    monkeypatch.setattr(F, "_request", request)
    monkeypatch.setattr(F, "GENERATORS", ["gemini:model-a", "groq:model-b"])
    monkeypatch.setenv("GEMINI_API_KEY", "test-key")
    monkeypatch.setattr(G, "KEY", "test-key")
    monkeypatch.setattr(G, "verify", lambda pairs, **k: [True if "lowers" in b or "Engineers" in b
                                                        else "claim changed" for _, b in pairs])
    d = Document()
    d.add_paragraph(HUMAN + " " + AI + " " + AI2)
    d.add_paragraph(HUMAN)
    path = tmp_path / "paper.docx"
    d.save(str(path))
    r = F.humanize_docx(str(path), budget=60)
    text = "\n".join(p.text for p in Document(r["path"]).paragraphs)
    assert r["before"]["pct"] > 0 and r["after"]["pct"] == 0
    assert "lowers the harmonic distortion" in text and "[14]" in text and "4.8% to 1.9%" in text
    assert "vanished" not in text                                 # rejected by the faithfulness check
    assert HUMAN in text                                          # unflagged sentences are left alone
    assert seen["faithful"] is True                               # keep-every-detail prompt
    assert seen["gens"] == {"gemini:model-a", "groq:model-b"}     # every model asked


def test_a_rewrite_of_a_sentence_beginning_does_not_end_the_sentence_early():
    assert F._fit("The model was trained on 12,500 images", "The model learned from 12,500 images.") \
        == "The model learned from 12,500 images"
    assert F._fit("A full sentence.", "Another full sentence.") == "Another full sentence."


def test_a_request_still_running_at_the_deadline_is_dropped_and_the_job_ends_on_time(monkeypatch):
    """Whatever the length of the document, the result comes back within the time budget: a slow model's answer is
    not waited for."""
    import time
    import json

    def request(gen, system, user, max_tokens, deadline=None):
        if gen.startswith("slow"):
            time.sleep(3)                              # answers after the deadline
            return json.dumps({"items": [{"id": 1, "versions": ["Too late."]}]})
        return json.dumps({"items": [{"id": 1, "versions": ["The controller lowers the distortion [14]."]}]})
    monkeypatch.setattr(F, "_request", request)
    monkeypatch.setattr(F, "GENERATORS", ["groq:fast-model", "slow:model"])
    monkeypatch.setitem(F._URLS, "slow", ("http://unused", "SLOW_KEY"))
    monkeypatch.setenv("SLOW_KEY", "k")
    monkeypatch.setattr(G, "KEY", "test-key")
    t = time.time()
    out = F._versions(["The controller reduces the distortion [14]."], ["The controller reduces the distortion [14]."],
                      frozenset(), 2, 0, deadline=time.time() + 1.0)
    assert time.time() - t < 2.0                                  # did not wait for the slow model
    assert out == [["The controller lowers the distortion [14]."]]


def test_an_unchecked_rewrite_is_never_used(tmp_path, monkeypatch):
    """When the faithfulness check cannot run (service busy, spending cap), sentences keep their wording."""
    import json
    monkeypatch.setattr(D, "score_texts", lambda texts: [_ai(t) for t in texts])
    monkeypatch.setattr(H, "_detector_scores", lambda texts: [_ai(t) for t in texts])
    monkeypatch.setattr(FL, "acceptable", lambda pairs: [True] * len(pairs))
    monkeypatch.setattr(MEANING, "similarities", lambda pairs: [0.95] * len(pairs))
    monkeypatch.setattr(F, "_request", lambda gen, system, user, mt, deadline=None: json.dumps(
        {"items": [{"id": i, "versions": ["The controller lowers the harmonic distortion from 4.8% to 1.9% [14]."]}
                   for i in range(1, 4)]}))
    monkeypatch.setattr(F, "GENERATORS", ["groq:model-b"])
    monkeypatch.setattr(G, "KEY", "test-key")
    monkeypatch.setattr(G, "verify", lambda pairs, **k: [None] * len(pairs))    # the check could not run
    d = Document()
    d.add_paragraph(HUMAN + " " + AI + " " + AI2)
    path = tmp_path / "paper.docx"
    d.save(str(path))
    r = F.humanize_docx(str(path), budget=60)
    text = "\n".join(p.text for p in Document(r["path"]).paragraphs)
    assert AI in text and AI2 in text                             # nothing unchecked went in


def test_a_report_run_at_zero_stops_when_a_round_clears_too_few_highlighted_sentences(tmp_path, monkeypatch):
    """At 0% with only the report's sentences left, a round that rewords only one of ten clear of their original
    wording ends the run (each such round cost a minute on a thesis part and changed almost nothing)."""
    import json
    monkeypatch.setattr(D, "score_texts", lambda texts: [0.05] * len(texts))       # our detector: human already
    monkeypatch.setattr(H, "_detector_scores", lambda texts: [0.05] * len(texts))
    monkeypatch.setattr(H, "_ctx_scores", lambda pairs, ctx, cur: [0.05] * len(pairs))
    monkeypatch.setattr(FL, "acceptable", lambda pairs: [True] * len(pairs))
    monkeypatch.setattr(MEANING, "similarities", lambda pairs: [0.95] * len(pairs))
    monkeypatch.setattr(G, "verify", lambda pairs, **k: [True] * len(pairs))
    rounds = []

    def request(gen, system, user, max_tokens, deadline=None):
        rounds.append(1)
        lines = [x.split("SENTENCE:", 1)[1].strip() for x in user.splitlines() if "SENTENCE:" in x]
        items = [{"id": i, "versions": [("Quite unlike before, nobody here wrote anything resembling it."
                                          if i == 1 else "Indeed, " + t[0].lower() + t[1:])]}
                 for i, t in enumerate(lines, 1)]
        return json.dumps({"items": items})
    monkeypatch.setattr(F, "_request", request)
    monkeypatch.setattr(F, "GENERATORS", ["groq:model-b"])
    monkeypatch.setattr(G, "KEY", "test-key")
    sents = ["Rainfall in the northern valley rose steadily over the past decade.",
             "Farmers responded by planting crops that tolerate wet soil better.",
             "Local markets now sell more rice than wheat during the monsoon months.",
             "Village councils built drainage channels along most of the main roads.",
             "School attendance dropped whenever the river burst its banks in spring.",
             "Doctors reported more cases of fever after each flood had passed.",
             "Young people increasingly moved to nearby towns looking for steady jobs.",
             "Older residents kept traditional houses raised on wooden stilts above ground.",
             "Fishing became a reliable source of income for many poor households.",
             "Government grants helped repair bridges damaged by the heavy storms."]
    d = Document()
    d.add_paragraph(" ".join(sents))
    path = tmp_path / "paper.docx"
    d.save(str(path))
    r = F.humanize_docx(str(path), budget=60, also=lambda t: t in sents, also_overlap=0.15)
    assert len(rounds) == 1
    assert any("few highlighted sentences" in x for x in r["log"])