ok

.gitattributes

@@ -1,2 +1,3 @@
 *.csv filter=lfs diff=lfs merge=lfs -text
+*.parquet filter=lfs diff=lfs merge=lfs -text
 *.pickle filter=lfs diff=lfs merge=lfs -text

.mise.toml

@@ -6,7 +6,7 @@ EMBEDDING_MODEL_REV = "d8c86521100d3556476a063fc2342036d45c106f"
 
 DATA_DIR = "{{config_root}}/data"
 MODEL_DIR = "{{config_root}}/model"
-OUT_DIR = "{{config_root}}/out"
+IMGS_DIR = "{{config_root}}/imgs"
 
 [tasks.deps]
 run = [

data/eval.parquet

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:1ce81584baeb7eb8ca4322bc0f50af105ae3795229718cda1dfa1f600e945f3a
+size 195251

data/train.parquet

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:bf199fc047485c2c453c4d9b80714261ed58152ef34c59903a64f9725d0e4956
+size 6608000

debug.py

@@ -0,0 +1,13 @@
+import polars as pl
+
+from utils.paths import DATA
+
+
+def main() -> None:
+    for name in ["train", "eval"]:
+        df = pl.read_parquet(DATA / (name + ".parquet"))
+        print(df)
+
+
+if __name__ == "__main__":
+    main()

model/model.pickle

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:eef2e93850a38b0e98b68ac0e0a32f4a67408e7327a7646ed9e96019c3dc7583
-size 5293480
+oid sha256:73bf71607b6b99d8576a79ec96cdf97e008134e7d348477f93b8cdcf057db19e
+size 3411728

out/preds.csv

@@ -1,45 +0,0 @@
-url,is_news_article,prediction,is_prediction_correct
-https://quantumcomputingreport.com/quandela-launches-belenos-photonic-quantum-computer-with-doubling-of-qubit-count-and-4000x-power-increase/,true,true,true
-https://www.nqcc.ac.uk/,false,false,true
-https://quantumcomputingreport.com/qsensato-raises-e500k-560k-usd-to-advance-integrated-atomic-quantum-sensors-for-precision-sensing/,true,true,true
-https://quantumcomputingreport.com/zurich-instruments-and-rohde-schwarz-join-australias-national-quantum-computing-testbed-facility/,true,true,true
-https://quantumcomputingreport.com/hbku-launches-qatars-first-quantum-computing-laboratory-backed-by-10m-mod-grant/,true,true,true
-https://quantumcomputingreport.com/quantinuum-releases-%ce%bbambeq-gen-ii-for-scalable-interpretable-quantum-nlp/,true,false,false
-https://quantumcomputingreport.com/quobly-secures-e21m-23-7m-usd-to-industrialize-100-qubit-silicon-quantum-processor/,true,true,true
-https://quantumcomputingreport.com/semiqon-and-nanoacademic-partner-to-advance-silicon-spin-qubit-research-and-education/,true,true,true
-https://quantumcomputingreport.com/united-nations-itu-launches-quantum-for-good-to-align-innovation-with-global-impact/,true,false,false
-https://quantumcomputingreport.com/microsoft-adds-post-quantum-cryptography-to-windows-insider-builds-and-linux/,true,true,true
-https://www.nqcc.ac.uk/technology-and-research/our-research/,false,false,true
-https://quantumcomputingreport.com/podcast-with-scott-davis-ceo-and-co-founder-of-vescent/,false,false,true
-https://quantumzeitgeist.com/building-atoms-the-rise-of-nanotechnology-and-molecular-engineering/,false,true,false
-https://quantumzeitgeist.com/networked-services-technologies-applications-and-challenges-for-advanced-communication/,false,false,true
-https://quantumzeitgeist.com/amazon-braket-sdk-and-multi-platform-quantum-development/,false,true,false
-https://quantumzeitgeist.com/pennylane-and-quantum-machine-learning/,false,false,true
-https://quantumzeitgeist.com/quantum-physics-meets-spiritual-philosophy-exploring-the-intersection-of-string-theory-and-consciousness/,false,false,true
-https://quantumzeitgeist.com/quantum-computing-transforms-financial-derivatives-pricing-for-complex-options-and-risk-analysis/,false,true,false
-https://quantumzeitgeist.com/quantifying-quantum-correlations-in-symmetric-gaussian-states-with-universal-invariants/,true,false,false
-https://www.horseandhound.co.uk/news/horse-life-threatening-stomach-tumour-saved-pioneering-surgery-894298,true,true,true
-https://www.maddyness.com/2025/06/02/vivatech-startups-deals-annonces-ce-que-la-mission-french-tech-prevoit-pour-levenement/,false,false,true
-https://www.cbsnews.com/sanfrancisco/news/padel-a-fast-growing-sport-has-become-a-new-obsession-for-silicon-valley/,false,true,false
-https://www.cloudcomputing-news.net/news/microsoft-launches-its-first-cloud-region-in-malaysia/,true,true,true
-https://padelmagazine.fr/best-padel-racket-awards-2025-les-meilleures-raquettes-de-lannee-devoilees/,false,false,true
-https://www.horseandhound.co.uk/news/polly-dickson-obituary-894506,true,true,true
-https://www.homeselect.paris/en/blog/devenir-proprietaire,false,false,true
-https://www.maddyness.com/2020/10/23/salomon-aiach-interview-facebook-startups/,false,false,true
-https://www.solarpowerportal.co.uk/grid-operators-must-work-together-in-aftermath-of-spain-and-portugal-blackout/,false,true,false
-https://www.cloudcomputing-news.net/news/podcast/nginx-f5-api-proxy-podcast-apac-sprint-two-point-one-podcast-s02-e30/,false,false,true
-https://www.farminguk.com/news/vegan-activists-attempt-to-shut-down-royal-highland-parade_66662.html,true,true,true
-https://dairynews.today/news/world_milk_day_2025_health_innovation_and_sustainability_drive_india_s_milk_movement_9339211.html,false,true,false
-"https://lerail.com/news/95810-signature-du-second-appel-%C3%A0-projets-gares-de-demain-entre-la-r%C3%A9gion-%C3%AEle-de-france,-%C3%AEle-de-france-mobilit%C3%A9s-et-sncf-gares-connexions",true,false,false
-https://lerail.com/news/95984-drive-to-zero-2025,false,false,true
-https://www.horseandhound.co.uk/news/farewell-to-twinshock-warrior-894106,true,true,true
-https://www.farminguk.com/news/new-ai-driven-test-targets-silent-killer-in-uk-cattle_66604.html,true,true,true
-https://www.maddyness.com/2019/05/02/growthhacking-chahab-nastar-scaleups/,false,false,true
-https://www.businesstravelnews.com/Lodging/Hyatt-Creates-New-Unscripted-Collection-Brand,true,false,false
-https://meuble-info.fr/falmec-gessi-le-duo-gagnant-du-point-deau/,true,false,false
-https://www.cloudcomputing-news.net/news/podcast/supply-chain-automation-warehousing-distribution-rpa-best-dematic-podcast-s03-e10/,false,false,true
-https://www.maddyness.com/2025/05/06/mon-petit-placement-tombe-dans-le-giron-de-malakoff-humanis/,true,false,false
-https://lerail.com/technical-articles/79770-southco-s%C3%A9curisation-du-v%C3%A9hicule-%C3%A9lectrique-infrastructure-de-recharge-et-de-stockage-sur-batterie-de-r%C3%A9seau,false,false,true
-https://www.watches-news.com/alpine-eagle-41-xp-cs-platinum/,true,true,true
-https://www.imarcgroup.com/football-market,false,true,false
-https://www.constructionnews.co.uk/contractors/balfour-beatty/balfour-beatty-court-battle-over-serious-trucks-cartel-ends-17-01-2025/,true,true,true

prepare.py

@@ -0,0 +1,45 @@
+import logging
+
+import polars as pl
+
+from utils.embed import embed as embed
+from utils.paths import DATA
+
+logger = logging.getLogger(__name__)
+
+logging.basicConfig(level=logging.INFO)
+
+
+def load_dataset(file_name: str):
+    features = ["meta_title", "meta_description", "content"]
+    return (
+        pl.scan_csv(file_name)
+        .with_columns(
+            pl.concat_str([pl.col(c) for c in features], separator="\n\n").alias(
+                "text"
+            ),
+            pl.col("date").str.to_date().alias("date"),
+        )
+        .rename(
+            {
+                "is_news_article": "is_news",
+                "link_count": "links",
+                "paragraph_count": "paragraphs",
+            }
+        )
+        .select("text", "is_news", "url", "date", "paragraphs", "links")
+        .collect()
+    )
+
+
+def main() -> None:
+    for name in ["train", "eval"]:
+        df = load_dataset(DATA / (name + ".csv"))
+        embeds = embed(df.get_column("text").to_list())
+        df = df.with_columns(pl.Series(embeds).alias("embeds")).write_parquet(
+            DATA / (name + ".parquet")
+        )
+
+
+if __name__ == "__main__":
+    main()

train.py

@@ -1,63 +1,17 @@
 import logging
-import os
 import pickle
-from pathlib import Path
 
 import matplotlib.pyplot as plt
-import numpy as np
 import polars as pl
 import seaborn as sns
 from numpy.typing import NDArray
-from polars import DataFrame
 from sklearn.metrics import auc, confusion_matrix, roc_curve
 from sklearn.svm import SVC
 
-from embed import embed as _embed
-
-logger = logging.getLogger(__name__)
+from utils.paths import DATA, IMGS, MODEL
 
 logging.basicConfig(level=logging.INFO)
 
-DATA = Path(os.environ["DATA_DIR"])
-DATA.mkdir(parents=True, exist_ok=True)
-MODEL = Path(os.environ["MODEL_DIR"])
-MODEL.mkdir(parents=True, exist_ok=True)
-OUT = Path(os.environ["OUT_DIR"])
-OUT.mkdir(parents=True, exist_ok=True)
-
-
-def embed(df: DataFrame):
-    logger.info(f"embed start {df.height}")
-    features = ["content", "meta_title", "meta_description"]
-    embeddings = []
-    for col in features:
-        train_texts = df.select(col).to_series().to_list()
-        embeddings.append(_embed(train_texts))
-    res = np.hstack(embeddings)
-    logger.info(f"embed done {res.shape}")
-    return res
-
-
-def train(df: DataFrame, target: str):
-    logger.info(f"train start {df.height}")
-    X = embed(df)
-    y = df.select(target).to_numpy().ravel()
-    clf = SVC(kernel="linear", probability=True)
-    clf.fit(X, y)
-    logger.info("train done")
-    return clf
-
-
-def save_prediction(eval_df: DataFrame, y_eval: NDArray, y_pred: NDArray) -> None:
-    pl.DataFrame(
-        {
-            "url": eval_df.select("url").to_series().to_list(),
-            "is_news_article": y_eval,
-            "prediction": y_pred,
-            "is_prediction_correct": y_eval == y_pred,
-        }
-    ).write_csv(OUT / "preds.csv")
-
 
 def save_roc_curve(clf, X: NDArray, y: NDArray):
     probs = clf.predict_proba(X)[:, 1]  # Probability for the positive class
@@ -76,7 +30,7 @@ def save_roc_curve(clf, X: NDArray, y: NDArray):
     plt.title("Receiver Operating Characteristic (ROC)")
     plt.legend(loc="lower right")
     plt.tight_layout()
-    plt.savefig(OUT / "roc_curve.png")
+    plt.savefig(IMGS / "roc_curve.png")
     plt.close()
 
 
@@ -94,26 +48,26 @@ def save_confusion_matrix(y: NDArray, pred: NDArray):
     plt.ylabel("Actual")
     plt.title("Confusion Matrix")
     plt.tight_layout()
-    plt.savefig(OUT / "confusion_matrix.png")
+    plt.savefig(IMGS / "confusion_matrix.png")
     plt.close()
 
 
 def main() -> None:
-    target = "is_news_article"
-    train_df = pl.read_csv(DATA / "train.csv")
-    clf = train(train_df, target)
+    train_df = pl.read_parquet(DATA / "train.parquet")
+    clf = SVC(kernel="linear", probability=True)
+    clf.fit(
+        train_df.get_column("embeds").to_numpy(),
+        train_df.get_column("is_news").to_numpy(),
+    )
     with open(MODEL / "model.pickle", "wb") as f:
         pickle.dump(clf, f)
 
-    eval_df = pl.read_csv(DATA / "eval.csv")
-    logger.info(f"eval start {eval_df.height}")
-    eval_X = embed(eval_df)
-    eval_y = eval_df.select(target).to_numpy().ravel()
+    eval_df = pl.read_parquet(DATA / "eval.parquet")
+    eval_X = eval_df.get_column("embeds").to_numpy()
+    eval_y = eval_df.get_column("is_news").to_numpy()
     eval_pred = clf.predict(eval_X)
-    save_prediction(eval_df, eval_y, eval_pred)
     save_confusion_matrix(eval_y, eval_pred)
     save_roc_curve(clf, eval_X, eval_y)
-    logger.info("eval done")
 
 
 if __name__ == "__main__":

usage.py

@@ -1,45 +1,37 @@
-import os
 import pickle
 from functools import cache
-from pathlib import Path
 
-import numpy as np
 import polars as pl
 from huggingface_hub import hf_hub_download
 
-from embed import embed
-
-DATA = Path(os.environ["DATA_DIR"])
-
-features = ["content", "meta_title", "meta_description"]
+from utils.embed import embed
+from utils.paths import DATA
 
 
 @cache
 def get_model():
-    file_name = hf_hub_download("opale-ai/news-classifier", "model/model.pickle")
+    file_name = hf_hub_download(
+        "opale-ai/news-classifier", "model/model.pickle", revision="main"
+    )
     with open(file_name, "rb") as f:
         return pickle.load(f)
 
 
-def record_get():
-    df = pl.read_csv(DATA / "eval.csv")
-    return {col: val for col, val in zip(df.columns, df.sample().row(0))}
-
-
-def record_embed(rec):
-    embeddings = []
-    for f in features:
-        embeddings.append(embed([rec[f]]))
-    return np.hstack(embeddings)
+def get_record():
+    df = pl.read_parquet(DATA / "eval.parquet")
+    return {
+        col: val
+        for col, val in zip(df.columns, df.sample().row(0))
+        if col in ["text", "is_news"]
+    }
 
 
 def main():
     model = get_model()
-    record = record_get()
-    embeds = record_embed(record)
+    record = get_record()
+    embeds = embed([record["text"]])
     (pred,) = model.predict(embeds)
-    print(record["content"])
-    print(f"is news (real): {record['is_news_article']}")
+    print(f"is news (real): {record['is_news']}")
     print(f"is news (pred): {pred}")
 
 

utils/data.py

@@ -0,0 +1,17 @@
+COLUMNS = [
+    "url",
+    "website",
+    "og_type",
+    "meta_description",
+    "meta_title",
+    "content",
+    "date",
+    "days_old",
+    "link_count",
+    "paragraph_count",
+    "average_links",
+    "text_to_html_ratio",
+    "css_title",
+    "is_news_article",
+    "reason",
+]

utils/paths.py

@@ -0,0 +1,9 @@
+import os
+from pathlib import Path
+
+DATA = Path(os.environ["DATA_DIR"])
+DATA.mkdir(parents=True, exist_ok=True)
+MODEL = Path(os.environ["MODEL_DIR"])
+MODEL.mkdir(parents=True, exist_ok=True)
+IMGS = Path(os.environ["IMGS_DIR"])
+IMGS.mkdir(parents=True, exist_ok=True)