data/load_datasets.py

"""Download and load all datasets, normalize to a common schema."""

from datasets import load_dataset
import pandas as pd


def load_covid_conspiracy():
    """Load primary COVID conspiracy tweets dataset.

    Labels: support -> 1 (conspiracy), deny/neutral -> 0 (non-conspiracy).
    """
    ds = load_dataset("webimmunization/COVID-19-conspiracy-theories-tweets", split="train")
    df = ds.to_pandas()

    label_map = {"support": 1, "deny": 0, "neutral": 0}
    df["label"] = df["label"].map(label_map)
    df = df.dropna(subset=["label", "tweet"])
    df["label"] = df["label"].astype(int)

    return pd.DataFrame({
        "text": df["tweet"].values,
        "label": df["label"].values,
        "source": "covid_conspiracy",
    })


def load_liar():
    """Load LIAR political statements dataset.

    Labels: pants-fire/false -> 1 (deceptive), mostly-true/true -> 0.
    Discards ambiguous labels (barely-true, half-true).
    """
    # Try multiple loading strategies — the dataset script format is deprecated
    ds = None
    for repo in ["ucsbnlp/liar", "liar"]:
        for kwargs in [{"trust_remote_code": True}, {}]:
            try:
                ds = load_dataset(repo, **kwargs)
                break
            except Exception:
                continue
        if ds is not None:
            break

    if ds is None:
        # Fallback: load from TSV files hosted on GitHub
        print("  LIAR HF load failed, trying raw TSV fallback...")
        try:
            base = "https://raw.githubusercontent.com/thiagorainmaker77/liar_dataset/master"
            cols = ["id", "label", "statement", "subject", "speaker",
                    "job", "state", "party", "barely_true", "false_count",
                    "half_true", "mostly_true", "pants_fire", "context"]
            dfs = []
            for split in ["train", "test", "valid"]:
                url = f"{base}/{split}.tsv"
                dfs.append(pd.read_csv(url, sep="\t", header=None, names=cols))
            df = pd.concat(dfs, ignore_index=True)
        except Exception as e:
            print(f"  Warning: Could not load LIAR dataset: {e}")
            print("  Skipping LIAR dataset.")
            return pd.DataFrame(columns=["text", "label", "source"])
    else:
        dfs = []
        for split in ds:
            dfs.append(ds[split].to_pandas())
        df = pd.concat(dfs, ignore_index=True)

    label_map = {
        "pants-fire": 1,
        "false": 1,
        "mostly-true": 0,
        "true": 0,
    }
    df["label"] = df["label"].map(label_map)
    df = df.dropna(subset=["label", "statement"])
    df["label"] = df["label"].astype(int)

    return pd.DataFrame({
        "text": df["statement"].values,
        "label": df["label"].values,
        "source": "liar",
    })


def load_fake_news():
    """Load fake news dataset (titles only to match tweet-length inputs).

    Labels: 0 = fake -> 1 (conspiracy/misinfo), 1 = real -> 0.
    """
    try:
        ds = load_dataset("GonzaloA/fake_news", split="train")
    except Exception as e:
        print(f"  Warning: Could not load Fake News dataset: {e}")
        return pd.DataFrame(columns=["text", "label", "source"])
    df = ds.to_pandas()

    # Invert: original 0=fake, 1=real -> we want fake=1, real=0
    df["label"] = (1 - df["label"]).astype(int)
    df = df.dropna(subset=["label", "title"])
    # Filter out empty titles
    df = df[df["title"].str.strip().str.len() > 0]

    return pd.DataFrame({
        "text": df["title"].values,
        "label": df["label"].values,
        "source": "fake_news",
    })


def load_propaganda():
    """Load SemEval 2020 Task 11 propaganda techniques dataset.

    Converts span-level annotations to sentence-level binary labels.
    Sentences containing propaganda spans -> 1, clean sentences -> 0.
    """
    try:
        ds = load_dataset("SemEvalWorkshop/sem_eval_2020_task_11")
    except Exception as e:
        print(f"Warning: Could not load propaganda dataset: {e}")
        print("Skipping propaganda dataset. Continuing with other datasets.")
        return pd.DataFrame(columns=["text", "label", "source"])

    # This dataset has a complex structure with articles and span annotations.
    # We'll attempt to extract sentence-level labels.
    records = []
    for split in ds:
        df_split = ds[split].to_pandas()
        if "text" in df_split.columns and "label" in df_split.columns:
            for _, row in df_split.iterrows():
                text = str(row.get("text", "")).strip()
                label = row.get("label", None)
                if text and label is not None:
                    records.append({
                        "text": text,
                        "label": 1 if label else 0,
                        "source": "propaganda",
                    })

    if not records:
        print("Warning: Propaganda dataset yielded no usable records.")
        return pd.DataFrame(columns=["text", "label", "source"])

    return pd.DataFrame(records)


def load_everyday_text():
    """Load diverse everyday text as strong non-conspiracy (label=0) examples.

    Uses the 'ag_news' dataset which covers world, sports, business, and sci/tech
    news — all clearly non-conspiratorial normal text.
    """
    print("  Loading AG News (everyday text)...")
    ds = load_dataset("fancyzhx/ag_news", split="train")
    df = ds.to_pandas()

    # Sample a balanced amount — we don't need all 120k
    df = df.sample(n=min(8000, len(df)), random_state=42)

    return pd.DataFrame({
        "text": df["text"].values,
        "label": 0,
        "source": "everyday_text",
    })


def load_tweet_eval_sentiment():
    """Load casual tweets with known non-conspiracy sentiment (positive/neutral).

    This specifically teaches the model that short informal tweet-style text
    is NOT inherently conspiratorial.
    """
    print("  Loading TweetEval sentiment (casual tweets)...")
    try:
        ds = load_dataset("cardiffnlp/tweet_eval", "sentiment", split="train")
        df = ds.to_pandas()

        # Keep only positive (2) and neutral (1) tweets -> label 0
        df = df[df["label"].isin([1, 2])]
        df = df.sample(n=min(5000, len(df)), random_state=42)

        return pd.DataFrame({
            "text": df["text"].values,
            "label": 0,
            "source": "tweet_sentiment",
        })
    except Exception as e:
        print(f"  Warning: Could not load TweetEval: {e}")
        return pd.DataFrame(columns=["text", "label", "source"])


def load_hard_negatives():
    """Hand-crafted non-conspiracy sentences about sensitive topics.

    These cover the exact same topics that conspiracy theories target
    (government, health, science, pharma, media, finance, 5G, climate, etc.)
    but with factual, neutral, or mildly critical framing.

    This teaches the model: topic alone ≠ conspiracy. The conspiratorial
    *framing* (secret plots, cover-ups, "they", "wake up") is what matters.
    """
    print("  Building hard negative examples (topic-matched non-conspiracy)...")

    examples = [
        # --- Government & Policy (normal discussion) ---
        "The government announced new infrastructure spending plans for the next fiscal year.",
        "Congress passed a bipartisan bill to improve veterans' healthcare benefits.",
        "The city council voted to increase funding for public transportation.",
        "New regulations require companies to disclose their carbon emissions annually.",
        "The president signed an executive order addressing supply chain disruptions.",
        "Local government officials held a town hall meeting to discuss zoning changes.",
        "The education department released new guidelines for standardized testing.",
        "Tax policy changes will affect middle-income households starting next year.",
        "The defense budget was approved after months of negotiations in the senate.",
        "Municipal bonds were issued to fund the construction of a new community center.",
        "The governor declared a state of emergency due to severe flooding.",
        "Immigration policy remains a contentious topic in the upcoming election.",
        "The state legislature is considering a bill to reform the criminal justice system.",
        "Public hearings were held regarding the proposed highway expansion project.",
        "The mayor's office released a statement about the new affordable housing initiative.",
        "Government transparency reports show increased spending on cybersecurity measures.",
        "The department of labor released new employment statistics for the quarter.",
        "Federal agencies are implementing new data privacy regulations this year.",

        # --- Health & Pharma (normal discussion) ---
        "The pharmaceutical industry invested heavily in research and development last year.",
        "Drug prices have been a major concern for patients and healthcare providers.",
        "Clinical trials for the new cancer treatment showed promising results.",
        "The FDA approved a new medication for treating chronic migraines.",
        "Healthcare costs continue to rise faster than inflation in many countries.",
        "Public health officials recommend annual flu shots for vulnerable populations.",
        "The hospital expanded its emergency department to serve more patients.",
        "Medical researchers published findings on a potential treatment for Alzheimer's disease.",
        "Insurance companies are required to cover preventive care at no additional cost.",
        "The World Health Organization issued new guidelines on antibiotic resistance.",
        "Pharmaceutical companies reported record profits in the last quarter.",
        "Mental health awareness campaigns have helped reduce stigma in many communities.",
        "The new drug went through three phases of clinical testing before approval.",
        "Doctors recommend regular screenings for early detection of common cancers.",
        "The generic version of the medication will be available at a lower price point.",
        "Telemedicine usage has increased significantly since the start of the decade.",
        "Nutrition research suggests a Mediterranean diet may reduce heart disease risk.",
        "Vaccination programs have been credited with eradicating smallpox globally.",

        # --- Science & Technology (normal discussion) ---
        "5G networks are being deployed in major cities to improve connectivity.",
        "The telecommunications industry is investing billions in network infrastructure.",
        "Scientists discovered a new species of deep-sea fish near hydrothermal vents.",
        "The space agency successfully launched a satellite to study climate patterns.",
        "Renewable energy production exceeded fossil fuel generation for the first time.",
        "Artificial intelligence is being used to improve medical diagnosis accuracy.",
        "Climate scientists report that global temperatures have risen over the past century.",
        "The university research lab received a grant to study quantum computing.",
        "Genetically modified crops have been shown to increase yield in drought conditions.",
        "The technology company unveiled a new processor with improved energy efficiency.",
        "Astronomers detected gravitational waves from a neutron star collision.",
        "Electric vehicle sales surpassed traditional car sales in several European countries.",
        "Researchers developed a new biodegradable plastic made from plant materials.",
        "The scientific community reached consensus on the human impact on climate change.",
        "Nuclear fusion experiments achieved a net energy gain for the first time.",
        "Ocean temperatures have been monitored by a network of autonomous buoys since 2000.",
        "The peer review process ensures research quality before publication in journals.",
        "Machine learning algorithms are being applied to weather prediction models.",

        # --- Media & Information (normal discussion) ---
        "The newspaper won a Pulitzer Prize for its investigative reporting series.",
        "Media literacy education is being introduced in schools across the country.",
        "The broadcasting company announced changes to its evening news format.",
        "Social media platforms updated their content moderation policies.",
        "Journalists face increasing challenges in conflict zones around the world.",
        "The news outlet published a correction after discovering an error in its report.",
        "Podcast listenership has grown significantly among younger demographics.",
        "The editor emphasized the importance of fact-checking in modern journalism.",
        "Local newspapers are struggling financially due to declining advertising revenue.",
        "The media company launched a new streaming service for documentary content.",

        # --- Finance & Banking (normal discussion) ---
        "The Federal Reserve raised interest rates to combat inflation.",
        "Stock markets experienced volatility following the quarterly earnings reports.",
        "The banking sector reported strong performance despite economic headwinds.",
        "New financial regulations aim to prevent another housing market crisis.",
        "Central banks around the world are exploring digital currency options.",
        "The IMF released its annual economic outlook with revised growth projections.",
        "Student loan reform legislation is being debated in congress.",
        "The credit union offers lower interest rates than traditional commercial banks.",
        "Investment in sustainable funds has grown by thirty percent year over year.",
        "The stock exchange implemented new circuit breakers to prevent flash crashes.",

        # --- Education & Community (normal discussion) ---
        "The school district adopted a new mathematics curriculum for elementary students.",
        "The library expanded its digital collection to include more audiobooks.",
        "Community volunteers organized a cleanup event at the local park.",
        "The university announced a new scholarship program for first-generation students.",
        "Parent-teacher conferences will be held next week at all elementary schools.",
        "The community center is offering free computer classes for senior citizens.",
        "School lunch programs are being updated to include healthier menu options.",
        "The local fire department conducted a safety demonstration at the elementary school.",
        "The book club selected a historical fiction novel for next month's discussion.",
        "The museum opened a new exhibit featuring artifacts from ancient civilizations.",

        # --- Critical but non-conspiratorial opinions ---
        "I disagree with the new tax policy but I understand the economic reasoning behind it.",
        "The government could do a better job communicating its public health guidelines.",
        "Big tech companies have too much market power and need stronger regulation.",
        "I'm skeptical about the effectiveness of this particular education reform.",
        "The pharmaceutical industry should make essential medications more affordable.",
        "Politicians on both sides of the aisle need to work together more effectively.",
        "I think the media sometimes oversimplifies complex scientific issues.",
        "Corporate tax loopholes should be closed to ensure a fairer system.",
        "The military budget could be reallocated to fund more domestic programs.",
        "I believe stronger environmental regulations are needed to protect our waterways.",
        "The healthcare system needs fundamental reform to serve all citizens equally.",
        "Social media companies should be more accountable for misinformation on their platforms.",
        "I worry that income inequality is growing faster than policymakers realize.",
        "Public schools deserve more funding to attract and retain quality teachers.",
        "The criminal justice system disproportionately affects minority communities.",
        "I think we need better oversight of how government contracts are awarded.",

        # --- Corporate/financial criticism (strong opinions, NOT conspiracy) ---
        "Corporate lobbying has too much influence on political decisions in this country.",
        "Credit card companies are offering new rewards programs to attract customers.",
        "Banks charge excessive overdraft fees that disproportionately hurt low-income families.",
        "The banking industry needs stricter regulations to prevent reckless lending practices.",
        "Wall Street bonuses are obscene while regular workers struggle to make ends meet.",
        "Insurance companies deny too many legitimate claims to protect their bottom line.",
        "CEOs earn hundreds of times more than their average employees and that gap keeps growing.",
        "The tech industry has a monopoly problem that regulators need to address.",
        "Oil companies knew about climate change decades ago and chose profits over the planet.",
        "Private equity firms are buying up housing and driving up prices for everyone else.",
        "Pharmaceutical companies spend more on marketing than on research and development.",
        "The airline industry consolidation has led to worse service and higher fares.",
        "Payday lenders exploit vulnerable people with predatory interest rates.",
        "The gig economy allows companies to avoid providing benefits to their workers.",
        "Fast food chains should pay their workers a living wage instead of relying on taxpayers.",
        "The student loan industry profits from keeping people in debt for decades.",
        "Utility companies should invest more in renewable energy instead of raising rates.",
        "Big box retailers have destroyed small businesses in many rural communities.",
        "The meat industry contributes significantly to environmental degradation.",
        "Hedge funds manipulate markets through high-frequency trading algorithms.",
        "Social media companies profit from outrage and division among their users.",
        "The housing market crash was caused by irresponsible lending and weak oversight.",
        "Defense contractors overcharge the government on virtually every major project.",
        "Telecom companies promise fast internet speeds but consistently underdeliver.",
        "The for-profit prison industry creates perverse incentives in the justice system.",

        # --- Strong opinions about government/institutions (NOT conspiracy) ---
        "I think the government should be more transparent about how tax money is spent.",
        "The government wastes billions of dollars every year on inefficient programs.",
        "Politicians care more about getting reelected than actually helping people.",
        "Our tax dollars are being mismanaged and we deserve better accountability.",
        "Government spending is out of control and taxpayers are paying the price.",
        "The government needs to do a better job explaining where our money goes.",
        "Too many politicians are beholden to their donors instead of their constituents.",
        "We need term limits to prevent career politicians from staying in power too long.",
        "The revolving door between government and industry undermines public trust.",
        "Government agencies are often slow and bureaucratic when people need help fast.",
        "Our elected officials should be required to disclose all their financial interests.",
        "The government should stop bailing out large corporations with taxpayer money.",
        "Public trust in government institutions has been declining for decades.",
        "I wish our leaders would focus on real problems instead of partisan fighting.",
        "Government contracts often go to the highest bidder rather than the most qualified.",

        # --- Strong opinions about pharma/healthcare (NOT conspiracy) ---
        "The pharmaceutical industry charges too much for essential medications.",
        "Drug companies prioritize profits over patient access to affordable medicine.",
        "Healthcare should be a right, not a privilege based on how much money you have.",
        "Insulin prices are outrageously high and people are dying because they can't afford it.",
        "The pharmaceutical industry spends more on advertising than on finding new cures.",
        "Hospital billing is deliberately confusing and patients end up overpaying.",
        "Generic drugs should be made available sooner to reduce costs for patients.",
        "The cost of an ambulance ride in this country is absolutely ridiculous.",
        "Mental health services are underfunded and millions of people can't get the help they need.",
        "Prescription drug prices in the US are significantly higher than in other countries.",
        "Health insurance companies make it too difficult to get claims approved.",
        "We need more regulation on drug pricing to make medications affordable for everyone.",
        "The opioid crisis was fueled by irresponsible prescribing and aggressive marketing.",
        "Nursing homes are understaffed because companies cut corners to increase profits.",
        "Medical debt is the leading cause of bankruptcy and that's a policy failure.",

        # --- Strong opinions about media/lobbying/finance (NOT conspiracy) ---
        "Corporate lobbying has too much influence on political decisions in this country.",
        "News organizations should do more investigative reporting on local government spending.",
        "The newspaper published an investigative report on local government spending.",
        "Media consolidation means fewer independent voices in journalism.",
        "Billionaires shouldn't be able to buy newspapers and shape public opinion.",
        "Local journalism is dying and communities are worse off because of it.",
        "Too much money in politics undermines democracy for ordinary citizens.",
        "Campaign finance reform is long overdue in this country.",
        "Lobbyists have more access to lawmakers than the people those lawmakers represent.",
        "The influence of money in politics is corroding our democratic institutions.",
        "Financial deregulation in the early 2000s contributed to the housing crisis.",
        "Executive compensation packages are excessive compared to average worker pay.",
        "Stock buybacks benefit shareholders while workers see stagnant wages.",
        "Banks that are too big to fail shouldn't exist in a healthy economy.",
        "The wealth gap between the richest and poorest Americans keeps growing every year.",

        # --- "Documentary" as a neutral word (NOT conspiracy) ---
        # The model wrongly associates "documentary" with conspiracy framing.
        # These teach it that documentaries are just a film genre.
        "I saw a documentary about penguins last night, it was fascinating.",
        "I saw a documentary, crazy to see what happened in 1960 during the space race.",
        "I saw a documentary, amazing to see how they built the Eiffel Tower.",
        "I saw a documentary, they did a bad job, it was boring and unfocused.",
        "I saw a documentary, they did a good job explaining the topic clearly.",
        "I watched a great documentary about the history of jazz music.",
        "I watched a documentary on Netflix about cooking techniques from around the world.",
        "I watched a documentary last weekend about climbing Mount Everest.",
        "The documentary on the Apollo missions was really well-produced.",
        "The new documentary about ocean life features stunning underwater footage.",
        "A new documentary series explores the history of the space program.",
        "There's a great new documentary about the building of the Hoover Dam.",
        "BBC released a documentary about wildlife in Africa that's worth watching.",
        "PBS aired a documentary on the civil rights movement last week.",
        "The documentary won an award at the Sundance Film Festival.",
        "Netflix is producing a documentary about famous chefs and their restaurants.",
        "I learned a lot from the documentary about ancient civilizations.",
        "The documentary filmmaker spent five years researching her subject.",
        "Documentaries about nature are some of my favorite things to watch.",
        "The school showed us a documentary about the solar system in science class.",
        "I love documentaries that take you behind the scenes of historical events.",
        "The documentary about the moon landing in 1969 was incredibly detailed.",
        "Last night I watched a documentary about the construction of the Golden Gate Bridge.",
        "There is a really good documentary about jazz musicians in New Orleans.",
        "The history channel had a documentary on World War II tanks last night.",
        "A documentary crew followed scientists studying coral reefs for two years.",
        "I just finished watching a documentary about ancient Egypt and it was fascinating.",
        "The streaming service added a new documentary about classical music composers.",
        "The documentary about the famous painter showed clips of all his major works.",
        "It is amazing what documentary filmmakers can capture with modern camera technology.",

        # --- Casual complaints / bad reviews (NOT conspiracy) ---
        # The model wrongly associates "they did a bad job" with conspiracy framing.
        # These teach it that personal complaints about services aren't conspiratorial.
        "I ordered food and they did a bad job, the burger was cold.",
        "I ordered food and they did a good job, everything was fresh.",
        "We went to the new restaurant downtown and they did a terrible job with our order.",
        "The waiter was rude and they did a bad job handling our reservation.",
        "I saw a documentary movie and they did a great job explaining the topic.",
        "I saw a documentary movie and they did a bad job, it was boring and unfocused.",
        "The repair shop fixed my laptop but they did a bad job, the screen is loose now.",
        "The plumbers came today and they did a good job fixing the leak.",
        "The plumbers came today and they did a bad job, water is still dripping.",
        "I hired movers last weekend and they did a bad job, they broke my coffee table.",
        "The hairdresser did a bad job, my haircut is uneven.",
        "The mechanic did a bad job on my car, it's making weird noises again.",
        "I bought this product online and they sent me the wrong item, bad service.",
        "The hotel staff was unhelpful and they did a bad job answering my questions.",
        "The Uber driver took the wrong route, they did a bad job navigating.",
        "The contractor renovating our kitchen did a terrible job, the cabinets are crooked.",
        "I called customer support and they were unhelpful, very disappointing experience.",
        "The dentist appointment was rushed, they did a bad job explaining the procedure.",
        "The cleaners came yesterday but they did a bad job, the bathroom is still dirty.",
        "The babysitter we hired did a bad job, the kids said she was on her phone all night.",
        "I returned the shoes because they were defective, the seller did a bad job with quality.",
        "The new app crashed three times today, the developers did a bad job testing it.",
        "The teacher did a bad job explaining the homework assignment.",
        "The barista made my coffee wrong twice, they did a bad job today.",
        "The delivery was late and they did a bad job packaging my order.",
        "The dry cleaner ruined my shirt, they did a really bad job.",
        "I went to the new gym and they did a bad job with the equipment maintenance.",
        "The landscaper did a bad job trimming the hedges, now they look uneven.",
        "The painter did a bad job, you can see streaks on the wall.",
        "The vet was rough with my dog, they did a bad job handling him.",
        "The food at that restaurant was terrible and the service was even worse.",
        "I had a bad experience with the airline, they lost my luggage.",
        "The customer service rep was rude and unhelpful, what a disappointing experience.",
        "I'm not going back to that store, the staff was completely unprofessional.",
        "Worst hotel I have ever stayed at, the room was filthy and the bed was uncomfortable.",
        "The movie was awful, the plot made no sense and the acting was terrible.",
        "I would not recommend this product to anyone, it broke after one week of use.",
        "The concert was disappointing, the sound quality was bad and the venue was crowded.",
        "I waited two hours for my meal and when it came it was cold and overcooked.",
        "The new update made the app worse, half the features don't work properly anymore.",

        # --- Angry/frustrated opinions that are NOT conspiracy (boundary cases) ---
        "I'm sick of politicians making promises they never keep. We deserve better leadership.",
        "The system is broken and ordinary people are the ones who suffer the most.",
        "It's infuriating that billionaires pay a lower tax rate than schoolteachers.",
        "Our leaders have failed us on healthcare, education, and infrastructure.",
        "I don't trust big corporations to do the right thing without regulation.",
        "The rich keep getting richer while working families can barely afford groceries.",
        "Every election cycle they promise change and nothing ever improves for regular people.",
        "Big oil companies should be held accountable for the environmental damage they've caused.",
        "I'm tired of seeing corporations get bailouts while small businesses close.",
        "The criminal justice system needs a complete overhaul from top to bottom.",
        "It disgusts me that people go bankrupt from medical bills in the wealthiest country on earth.",
        "Our infrastructure is crumbling because politicians would rather fund wars than fix bridges.",
        "The food industry puts profit ahead of nutrition and public health.",
        "Social media is rotting society and the companies running it don't care.",
        "I've lost faith in the ability of our institutions to solve real problems.",
        "Nobody in power actually cares about the middle class anymore.",
        "The education system is failing our children and nobody seems to want to fix it.",
        "These tech companies have way too much power over what information people see.",
        "Rent is out of control because we let investment firms buy up all the housing.",
        "The entire healthcare system is designed to extract maximum money from sick people.",

        # --- Everyday life mixed with institutional topics ---
        "I went to the pharmacy to pick up my prescription and the wait was about twenty minutes.",
        "The local school board meeting ran late because of the budget discussion.",
        "My doctor recommended a new treatment plan based on recent medical research.",
        "I signed up for the town's recycling program and received my new bins today.",
        "The hospital in our neighborhood is hiring nurses and administrative staff.",
        "The city installed new bike lanes along the main road through downtown.",
        "I attended a lecture at the university about the history of space exploration.",
        "The weather service issued a severe thunderstorm warning for the evening.",
        "My kid's school is hosting a science fair next Friday and she's very excited.",
        "The public library just got a new section dedicated to local history.",
        "We visited the air and space museum over the weekend and it was fascinating.",
        "The community garden project received funding from a local business grant.",
        "My neighbor works at the water treatment plant and says they just upgraded the filters.",
        "The farmers market downtown has expanded to include more organic produce vendors.",
        "I read an interesting article about how electric cars compare to hybrid vehicles.",
    ]

    return pd.DataFrame({
        "text": examples,
        "label": 0,
        "source": "hard_negatives",
    })


def load_health_science_news():
    """Load health and science-related factual text as non-conspiracy examples.

    Uses the 'health_fact' dataset which contains health claims labeled by
    veracity — we take the 'true' ones as non-conspiracy examples.
    """
    print("  Loading health/science factual text...")
    try:
        ds = load_dataset("health_fact", split="train", trust_remote_code=True)
        df = ds.to_pandas()
        # Keep only 'true' claims
        df = df[df["label"] == 0]  # 0 = true in health_fact
        df = df.dropna(subset=["claim"])
        df = df[df["claim"].str.strip().str.len() > 20]
        df = df.sample(n=min(3000, len(df)), random_state=42)
        print(f"    -> {len(df)} true health claims loaded")
        return pd.DataFrame({
            "text": df["claim"].values,
            "label": 0,
            "source": "health_science",
        })
    except Exception as e:
        print(f"  Warning: Could not load health_fact dataset: {e}")
        return pd.DataFrame(columns=["text", "label", "source"])


def load_all_datasets():
    """Load all datasets and return a dict of DataFrames."""
    print("Loading COVID conspiracy dataset...")
    covid = load_covid_conspiracy()
    print(f"  -> {len(covid)} samples, label dist: {covid['label'].value_counts().to_dict()}")

    print("Loading LIAR dataset...")
    liar = load_liar()
    print(f"  -> {len(liar)} samples, label dist: {liar['label'].value_counts().to_dict()}")

    print("Loading Fake News dataset...")
    fake_news = load_fake_news()
    print(f"  -> {len(fake_news)} samples, label dist: {fake_news['label'].value_counts().to_dict()}")

    print("Loading Propaganda dataset...")
    propaganda = load_propaganda()
    print(f"  -> {len(propaganda)} samples, label dist: {propaganda['label'].value_counts().to_dict() if len(propaganda) > 0 else 'empty'}")

    print("Loading everyday text datasets...")
    everyday = load_everyday_text()
    print(f"  -> {len(everyday)} samples (all label 0)")

    tweet_sentiment = load_tweet_eval_sentiment()
    print(f"  -> {len(tweet_sentiment)} samples (all label 0)")

    print("Loading hard negative examples...")
    hard_negatives = load_hard_negatives()
    print(f"  -> {len(hard_negatives)} samples (all label 0)")

    print("Loading health/science factual text...")
    health_science = load_health_science_news()
    print(f"  -> {len(health_science)} samples (all label 0)")

    return {
        "covid_conspiracy": covid,
        "liar": liar,
        "fake_news": fake_news,
        "propaganda": propaganda,
        "everyday_text": everyday,
        "tweet_sentiment": tweet_sentiment,
        "hard_negatives": hard_negatives,
        "health_science": health_science,
    }


if __name__ == "__main__":
    datasets = load_all_datasets()
    total = sum(len(df) for df in datasets.values())
    print(f"\nTotal samples across all datasets: {total}")