chore: use datasets

app.py

@@ -14,24 +14,25 @@ from utils import (plot_distances_tsne,
                    cluster_languages_by_subfamilies,
                    filter_languages_by_families)
 from functools import partial
+import datasets
 
 
-with open("../../results/languages_list.pkl", "rb") as f:
-    languages = pickle.load(f)
+dataset = datasets.load_dataset("mshamrai/language-metric-data", split="train", trust_remote_code=True)
 
-DATASETS = ["wikimedia/wikipedia", "uonlp/CulturaX", "HuggingFaceFW/fineweb-2"]
-MODELS = ["mistralai/Mistral-7B-v0.1", "google/gemma-3-4b-pt", "meta-llama/Llama-3.2-1B"]
+languages = dataset["languages_list"][0]
+average_distances_matrix = np.array(dataset["average_distances_matrix"][0])
+
+DATASETS = dataset["distances_matrices"][0]["dataset_name"]
+MODELS = dataset["distances_matrices"][0]["models"][0]["model_name"]
 
 distance_matrices = {
-    dataset: {
-        model: np.load(os.path.join("../../results", dataset, model, "distances_matrix.npy"))
-        for model in MODELS
+    DATASETS[i]: {
+        MODELS[j]: np.array(dataset["distances_matrices"][0]["models"][i]["matrix"][j])
+        for j in range(len(MODELS))
     }
-    for dataset in DATASETS 
+    for i in range(len(DATASETS)) 
 }
 
-average_distances_matrix = np.load("../../results/average_distances_matrix.npy")
-
 
 def filter_languages_nan(model, dataset, use_average):
     if use_average: