import io import os from datetime import datetime, timezone import numpy as np import pandas as pd import requests try: from huggingface_hub import HfApi except Exception: HfApi = None SOURCE_CONFIG = [ { "name": "IA Fluide / state principal", "url": "https://huggingface.co/datasets/Hlt58/iafluide-state/resolve/main/state.json", "type": "state", }, { "name": "IA Fluide / copie test", "url": "https://huggingface.co/datasets/Hlt58/iafluide-state/resolve/main/state.json", "type": "state", }, { "name": "OWID proxy (converti)", "url": "external", "type": "owid", }, ] DATASET_REPO_ID = "Hlt58/iafluide-state" CONFLICT_HISTORY_URL = "https://huggingface.co/datasets/Hlt58/iafluide-state/resolve/main/conflict_history.csv" def load_json(url): try: r = requests.get(url, timeout=8) r.raise_for_status() return r.json() except Exception: return None def compute_age_minutes(utc_str): try: t = datetime.fromisoformat(utc_str.replace("Z", "+00:00")) now = datetime.now(timezone.utc) return (now - t).total_seconds() / 60.0 except Exception: return None def freshness_score(age_minutes, max_age_minutes, stale_penalty=1.0): if age_minutes is None: return 0.0 if age_minutes <= 0: return 1.0 score = np.exp(-stale_penalty * age_minutes / max_age_minutes) return float(np.clip(score, 0.0, 1.0)) def normalize_confidence_label(value): if not isinstance(value, str): return 0.5 v = value.strip().lower() mapping = { "élevée": 0.9, "haute": 0.9, "moyenne": 0.6, "faible": 0.3, "faible à moyenne": 0.45, "moyenne à élevée": 0.75, } return mapping.get(v, 0.5) def convert_owid_to_state(): return { "utc": datetime.now(timezone.utc).isoformat(), "observed": { "Climat & systèmes planétaires": 0.30, "Ressources vitales (eau, sols, énergie réelle)": 0.28, "Santé humaine & biologie cognitive": 0.25, "Éducation & transmission du savoir": 0.23, "Systèmes économiques déconnectés du réel": 0.32, }, "confidence": { "Climat & systèmes planétaires": "élevée", "Ressources vitales (eau, sols, énergie réelle)": "moyenne", "Santé humaine & biologie cognitive": "moyenne", "Éducation & transmission du savoir": "moyenne", "Systèmes économiques déconnectés du réel": "faible", }, } def load_all_sources(max_age_minutes=1440, stale_penalty=1.0): loaded = [] for cfg in SOURCE_CONFIG: if cfg.get("type") == "state": data = load_json(cfg["url"]) elif cfg.get("type") == "owid": data = convert_owid_to_state() else: data = None if not data: continue observed = data.get("observed", {}) confidence = data.get("confidence", {}) utc = data.get("utc", None) if not isinstance(observed, dict): continue values = {} for k, v in observed.items(): try: values[k] = float(v) except Exception: continue if not values: continue conf_values = {} if isinstance(confidence, dict): for k, v in confidence.items(): conf_values[k] = normalize_confidence_label(v) age = compute_age_minutes(utc) if utc else None fresh = freshness_score(age, max_age_minutes, stale_penalty=stale_penalty) is_valid = bool(age is not None and age <= max_age_minutes) if not is_valid: continue loaded.append({ "name": cfg["name"], "url": cfg["url"], "utc": utc, "age_minutes": age, "freshness_score": fresh, "is_valid": is_valid, "observed": values, "confidence": conf_values, "raw": data, }) return loaded def compute_source_status_table(sources): rows = [] for src in sources: rows.append({ "Nom": src["name"], "URL": src["url"], "UTC": src.get("utc", "unknown"), "Âge (min)": src.get("age_minutes"), "Fraîcheur": src.get("freshness_score"), "Valide": src.get("is_valid"), "Nombre de domaines": len(src["observed"]), }) return pd.DataFrame(rows) def collect_domains(sources): domains = set() for src in sources: domains.update(src["observed"].keys()) return sorted(domains) def build_source_comparison_table(sources): domains = collect_domains(sources) rows = [] for domain in domains: row = {"Domaine": domain} for src in sources: row[src["name"]] = src["observed"].get(domain, None) rows.append(row) return pd.DataFrame(rows) def compute_domain_conflict_table(sources): domains = collect_domains(sources) rows = [] for domain in domains: vals = [] src_names = [] for src in sources: if domain in src["observed"]: vals.append(src["observed"][domain]) src_names.append(src["name"]) if len(vals) == 0: continue vals_arr = np.array(vals, dtype=float) dispersion = float(np.std(vals_arr)) if len(vals_arr) > 1 else 0.0 vmin = float(np.min(vals_arr)) vmax = float(np.max(vals_arr)) conflict_score = float(vmax - vmin) rows.append({ "Domaine": domain, "Sources présentes": len(vals), "Minimum": vmin, "Maximum": vmax, "Dispersion inter-sources": dispersion, "Score de conflit": conflict_score, "Sources": ", ".join(src_names), }) return pd.DataFrame(rows).sort_values("Domaine") def consolidate_sources(sources, tau0=0.15): domains = collect_domains(sources) consolidated_values = {} confidence_by_domain = {} dispersion_by_domain = {} conflict_by_domain = {} for domain in domains: vals = [] weights = [] for src in sources: if domain in src["observed"]: val = src["observed"][domain] base_conf = src["confidence"].get(domain, 0.5) fresh = src.get("freshness_score", 1.0) vals.append(val) weights.append(base_conf * fresh) if len(vals) == 0: continue vals_arr = np.array(vals, dtype=float) w_arr = np.array(weights, dtype=float) if np.sum(w_arr) <= 1e-9: w_arr = np.ones_like(vals_arr) consolidated_val = float(np.average(vals_arr, weights=w_arr)) dispersion = float(np.std(vals_arr)) if len(vals_arr) > 1 else 0.0 conflict = float(np.max(vals_arr) - np.min(vals_arr)) if len(vals_arr) > 1 else 0.0 confidence = ( float(np.mean(w_arr)) * float(np.exp(-dispersion / 0.10)) * float(np.exp(-conflict / 0.15)) ) consolidated_values[domain] = consolidated_val confidence_by_domain[domain] = confidence dispersion_by_domain[domain] = dispersion conflict_by_domain[domain] = conflict x = np.array(list(consolidated_values.values()), dtype=float) mu = float(np.mean(x)) delta = float(np.std(x)) C = float(1 / (1 + delta / tau0)) R = float(mu * (1 - C)) global_confidence = float(np.mean(list(confidence_by_domain.values()))) if confidence_by_domain else 0.0 return { "values": consolidated_values, "confidence_by_domain": confidence_by_domain, "dispersion_by_domain": dispersion_by_domain, "conflict_by_domain": conflict_by_domain, "mu": mu, "delta": delta, "C": C, "R": R, "global_confidence": global_confidence, } def load_conflict_history_df(): try: r = requests.get(CONFLICT_HISTORY_URL, timeout=8) if r.status_code == 200 and r.text.strip(): return pd.read_csv(io.StringIO(r.text)) except Exception: pass return pd.DataFrame( columns=[ "timestamp", "domain", "conflict_score", "dispersion", "min_value", "max_value", "sources_present", ] ) def append_conflict_snapshot_to_dataset(df_conflicts): token = os.getenv("HF_TOKEN") if not token: return False, "HF_TOKEN manquant" if HfApi is None: return False, "huggingface_hub indisponible" hist = load_conflict_history_df() timestamp = datetime.now(timezone.utc).isoformat() rows = [] for _, row in df_conflicts.iterrows(): rows.append({ "timestamp": timestamp, "domain": row["Domaine"], "conflict_score": row["Score de conflit"], "dispersion": row["Dispersion inter-sources"], "min_value": row["Minimum"], "max_value": row["Maximum"], "sources_present": row["Sources présentes"], }) new_df = pd.DataFrame(rows) hist = pd.concat([hist, new_df], ignore_index=True) csv_bytes = hist.to_csv(index=False).encode("utf-8") api = HfApi(token=token) api.upload_file( path_or_fileobj=csv_bytes, path_in_repo="conflict_history.csv", repo_id=DATASET_REPO_ID, repo_type="dataset", commit_message="Update conflict_history.csv from Space", ) return True, "conflict_history.csv mis à jour" def compute_slope(series): if len(series) < 2: return 0.0 x = np.arange(len(series)) y = np.array(series, dtype=float) return float(np.polyfit(x, y, 1)[0]) def classify_trend(slope, threshold=0.001): if slope > threshold: return "hausse" elif slope < -threshold: return "baisse" return "stable" def build_conflict_trend_table(hist_df): if len(hist_df) == 0: return pd.DataFrame(columns=["domain", "conflit_actuel", "pente", "tendance", "n_points"]) rows = [] for domain, sub in hist_df.groupby("domain"): sub = sub.sort_values("timestamp") series = sub["conflict_score"].astype(float).tolist() slope = compute_slope(series) trend = classify_trend(slope) current = float(series[-1]) if len(series) else 0.0 rows.append({ "domain": domain, "conflit_actuel": current, "pente": slope, "tendance": trend, "n_points": len(series), }) return pd.DataFrame(rows).sort_values("domain")