scripts/calibrate_lm.py

"""Quick sanity + rough calibration of the LM detectors (D1/D2/D3).

Prints binoculars/ppl/gltr for clearly-AI-style vs clearly-human samples so
the CAL midpoints in plagdetect/lmdetect.py can be set between the clusters.
(Proper calibration on RAID/HC3 is Phase B — this just sets sane defaults.)
"""
import os
import sys
import time

sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
from plagdetect import lmdetect, semantic  # noqa: E402

AI_SAMPLES = [
    # canonical ChatGPT-style prose
    """Artificial intelligence has become an integral part of modern society,
transforming the way we live, work, and interact with one another. In today's
rapidly evolving technological landscape, it is important to note that AI
systems play a crucial role in numerous industries, ranging from healthcare
and finance to transportation and entertainment. Furthermore, these systems
offer significant advantages in terms of efficiency, accuracy, and
scalability. One of the most notable applications of artificial intelligence
is in the field of healthcare. AI-powered diagnostic tools can analyze
medical images with remarkable precision, enabling early detection of
diseases such as cancer and cardiovascular conditions. Moreover, machine
learning algorithms can process vast amounts of patient data to identify
patterns and predict health outcomes, thereby facilitating personalized
treatment plans. Additionally, AI chatbots and virtual assistants provide
round-the-clock support to patients, answering their questions and guiding
them through their healthcare journeys. In the realm of finance, artificial
intelligence has revolutionized the way institutions manage risk and detect
fraud. Sophisticated algorithms can monitor transactions in real time,
flagging suspicious activities and preventing financial crimes. Furthermore,
AI-driven trading systems can analyze market trends and execute trades with
unprecedented speed and accuracy. It is worth noting that these advancements
have also democratized access to financial services, enabling individuals
from diverse backgrounds to participate in the global economy. In conclusion,
artificial intelligence represents a transformative force that continues to
reshape our world in profound ways. As we navigate the complexities of this
technological revolution, it is essential to strike a balance between
innovation and responsibility, ensuring that AI serves the greater good of
humanity.""",
    """Climate change represents one of the most pressing challenges facing
humanity in the twenty-first century. The scientific consensus is clear:
human activities, particularly the burning of fossil fuels, have led to
unprecedented levels of greenhouse gases in the atmosphere. Consequently,
global temperatures have risen significantly, resulting in a wide array of
environmental impacts. Rising sea levels pose a substantial threat to coastal
communities around the world. As polar ice caps continue to melt, millions of
people living in low-lying areas face the prospect of displacement.
Furthermore, extreme weather events such as hurricanes, droughts, and floods
have become more frequent and intense, causing widespread devastation to
communities and ecosystems alike. The transition to renewable energy sources
represents a critical component of any comprehensive climate strategy. Solar
and wind power have become increasingly cost-effective, making them viable
alternatives to traditional fossil fuels. Moreover, advancements in battery
technology have addressed many of the intermittency challenges associated
with renewable energy. Governments around the world must implement policies
that incentivize clean energy adoption and discourage carbon-intensive
practices. In addition to mitigation efforts, adaptation strategies are
essential for building resilience against the impacts of climate change.
Communities must invest in infrastructure improvements, early warning
systems, and sustainable agricultural practices. Ultimately, addressing
climate change requires a coordinated global effort involving governments,
businesses, and individuals working together toward a common goal.""",
]

HUMAN_SAMPLES = [
    # human academic prose (older, pre-LLM style with quirks)
    """We never went to the lab on Tuesdays, mostly because Hendricks had
claimed the centrifuge for his interminable yeast cultures, and partly --
though nobody said this aloud -- because the radiator near bench four made
the whole room smell of scorched dust. My notebook from that winter is a
mess: half the entries are in pencil because my pen froze in my coat pocket
on the walk over, and there's a coffee ring obscuring the one calculation
that, months later, turned out to matter. Page 73, if you're curious. What
we were trying to do, in retrospect, was sloppy. We'd convinced ourselves
that the anomaly in the December readings was instrumental, so we spent
January recalibrating instead of looking at the data. Stupid. The signal was
right there. Marchetti spotted it eventually, not because she was smarter --
though she was -- but because she was the only one who hadn't sat through
Bowen's seminar on detector drift and so wasn't primed to blame the
hardware. There's a lesson in that, I suppose, about the cost of expertise.
The funny thing is the paper we finally published reads like none of this
happened. Methods, results, discussion: clean as an operating theatre. No
frozen pens, no coffee rings, no Hendricks. I understand why we write
papers that way. I'm less sure it's honest.""",
    """The argument of this essay is simple, although its implications are
not: the eighteenth-century coffee-house was a financial instrument before
it was a social one. Historians have long treated Lloyd's emergence from a
Tower Street coffee-house as a colourful accident of geography. It was no
such thing. Underwriters gathered where shipping news arrived first, and
news arrived where men paid a penny for coffee and the right to linger.
That penny, I want to insist, bought time -- and time, in a port city
running on credit and rumour, was the scarcest commodity of all. Consider
what a merchant actually knew in 1720 about a vessel six weeks out of
Jamaica. Almost nothing. A name in a printed list, perhaps; a captain's
reputation; the price sugar fetched last autumn. Every decision to insure,
to sell, to extend credit was a wager on stale information. The
coffee-house compressed that staleness. A man might hear of a wreck hours
before the printed lists carried it, and hours were fortunes. I dwell on
this because we have inherited a sanitised picture of these rooms, all
witty conversation and Addisonian politeness. The reality was closer to a
trading floor: crowded, loud, occasionally violent, and saturated with
money.""",
]


def show(label, samples):
    vals = []
    for i, s in enumerate(samples):
        t0 = time.time()
        r = lmdetect.detect_lm(s)
        m = r["metrics"]
        vals.append(m)
        print(f"{label}{i}: binoc={m['binoculars']:.4f} ppl={m['ppl']:7.2f} "
              f"gltr10={m['gltr_top10']:.3f} tokens={m['tokens_scored']} "
              f"({time.time()-t0:.1f}s) scores={r['scores']}")
    return vals


if __name__ == "__main__":
    print("lm available:", lmdetect.available())
    print("semantic available:", semantic.available())
    t0 = time.time()
    e = semantic.embed(["the cat sat on the mat",
                        "a feline rested upon the rug",
                        "stock markets fell sharply on tuesday"])
    print(f"semantic embed ok {e.shape} in {time.time()-t0:.1f}s; "
          f"paraphrase cos={float(e[0] @ e[1]):.3f}, "
          f"unrelated cos={float(e[0] @ e[2]):.3f}")
    ai = show("AI", AI_SAMPLES)
    hu = show("HU", HUMAN_SAMPLES)