| import io |
| import os |
| from datetime import datetime, timezone |
|
|
| import numpy as np |
| import pandas as pd |
| import requests |
|
|
| try: |
| from huggingface_hub import HfApi |
| except Exception: |
| HfApi = None |
|
|
|
|
| SOURCE_CONFIG = [ |
| { |
| "name": "IA Fluide / state principal", |
| "url": "https://huggingface.co/datasets/Hlt58/iafluide-state/resolve/main/state.json", |
| "type": "state", |
| }, |
| { |
| "name": "IA Fluide / copie test", |
| "url": "https://huggingface.co/datasets/Hlt58/iafluide-state/resolve/main/state.json", |
| "type": "state", |
| }, |
| { |
| "name": "OWID proxy (converti)", |
| "url": "external", |
| "type": "owid", |
| }, |
| ] |
|
|
| DATASET_REPO_ID = "Hlt58/iafluide-state" |
| CONFLICT_HISTORY_URL = "https://huggingface.co/datasets/Hlt58/iafluide-state/resolve/main/conflict_history.csv" |
|
|
|
|
| def load_json(url): |
| try: |
| r = requests.get(url, timeout=8) |
| r.raise_for_status() |
| return r.json() |
| except Exception: |
| return None |
|
|
|
|
| def compute_age_minutes(utc_str): |
| try: |
| t = datetime.fromisoformat(utc_str.replace("Z", "+00:00")) |
| now = datetime.now(timezone.utc) |
| return (now - t).total_seconds() / 60.0 |
| except Exception: |
| return None |
|
|
|
|
| def freshness_score(age_minutes, max_age_minutes, stale_penalty=1.0): |
| if age_minutes is None: |
| return 0.0 |
| if age_minutes <= 0: |
| return 1.0 |
|
|
| score = np.exp(-stale_penalty * age_minutes / max_age_minutes) |
| return float(np.clip(score, 0.0, 1.0)) |
|
|
|
|
| def normalize_confidence_label(value): |
| if not isinstance(value, str): |
| return 0.5 |
|
|
| v = value.strip().lower() |
|
|
| mapping = { |
| "élevée": 0.9, |
| "haute": 0.9, |
| "moyenne": 0.6, |
| "faible": 0.3, |
| "faible à moyenne": 0.45, |
| "moyenne à élevée": 0.75, |
| } |
|
|
| return mapping.get(v, 0.5) |
|
|
|
|
| def convert_owid_to_state(): |
| return { |
| "utc": datetime.now(timezone.utc).isoformat(), |
| "observed": { |
| "Climat & systèmes planétaires": 0.30, |
| "Ressources vitales (eau, sols, énergie réelle)": 0.28, |
| "Santé humaine & biologie cognitive": 0.25, |
| "Éducation & transmission du savoir": 0.23, |
| "Systèmes économiques déconnectés du réel": 0.32, |
| }, |
| "confidence": { |
| "Climat & systèmes planétaires": "élevée", |
| "Ressources vitales (eau, sols, énergie réelle)": "moyenne", |
| "Santé humaine & biologie cognitive": "moyenne", |
| "Éducation & transmission du savoir": "moyenne", |
| "Systèmes économiques déconnectés du réel": "faible", |
| }, |
| } |
|
|
|
|
| def load_all_sources(max_age_minutes=1440, stale_penalty=1.0): |
| loaded = [] |
|
|
| for cfg in SOURCE_CONFIG: |
| if cfg.get("type") == "state": |
| data = load_json(cfg["url"]) |
| elif cfg.get("type") == "owid": |
| data = convert_owid_to_state() |
| else: |
| data = None |
|
|
| if not data: |
| continue |
|
|
| observed = data.get("observed", {}) |
| confidence = data.get("confidence", {}) |
| utc = data.get("utc", None) |
|
|
| if not isinstance(observed, dict): |
| continue |
|
|
| values = {} |
| for k, v in observed.items(): |
| try: |
| values[k] = float(v) |
| except Exception: |
| continue |
|
|
| if not values: |
| continue |
|
|
| conf_values = {} |
| if isinstance(confidence, dict): |
| for k, v in confidence.items(): |
| conf_values[k] = normalize_confidence_label(v) |
|
|
| age = compute_age_minutes(utc) if utc else None |
| fresh = freshness_score(age, max_age_minutes, stale_penalty=stale_penalty) |
| is_valid = bool(age is not None and age <= max_age_minutes) |
|
|
| if not is_valid: |
| continue |
|
|
| loaded.append({ |
| "name": cfg["name"], |
| "url": cfg["url"], |
| "utc": utc, |
| "age_minutes": age, |
| "freshness_score": fresh, |
| "is_valid": is_valid, |
| "observed": values, |
| "confidence": conf_values, |
| "raw": data, |
| }) |
|
|
| return loaded |
|
|
|
|
| def compute_source_status_table(sources): |
| rows = [] |
| for src in sources: |
| rows.append({ |
| "Nom": src["name"], |
| "URL": src["url"], |
| "UTC": src.get("utc", "unknown"), |
| "Âge (min)": src.get("age_minutes"), |
| "Fraîcheur": src.get("freshness_score"), |
| "Valide": src.get("is_valid"), |
| "Nombre de domaines": len(src["observed"]), |
| }) |
| return pd.DataFrame(rows) |
|
|
|
|
| def collect_domains(sources): |
| domains = set() |
| for src in sources: |
| domains.update(src["observed"].keys()) |
| return sorted(domains) |
|
|
|
|
| def build_source_comparison_table(sources): |
| domains = collect_domains(sources) |
| rows = [] |
|
|
| for domain in domains: |
| row = {"Domaine": domain} |
| for src in sources: |
| row[src["name"]] = src["observed"].get(domain, None) |
| rows.append(row) |
|
|
| return pd.DataFrame(rows) |
|
|
|
|
| def compute_domain_conflict_table(sources): |
| domains = collect_domains(sources) |
| rows = [] |
|
|
| for domain in domains: |
| vals = [] |
| src_names = [] |
|
|
| for src in sources: |
| if domain in src["observed"]: |
| vals.append(src["observed"][domain]) |
| src_names.append(src["name"]) |
|
|
| if len(vals) == 0: |
| continue |
|
|
| vals_arr = np.array(vals, dtype=float) |
| dispersion = float(np.std(vals_arr)) if len(vals_arr) > 1 else 0.0 |
| vmin = float(np.min(vals_arr)) |
| vmax = float(np.max(vals_arr)) |
| conflict_score = float(vmax - vmin) |
|
|
| rows.append({ |
| "Domaine": domain, |
| "Sources présentes": len(vals), |
| "Minimum": vmin, |
| "Maximum": vmax, |
| "Dispersion inter-sources": dispersion, |
| "Score de conflit": conflict_score, |
| "Sources": ", ".join(src_names), |
| }) |
|
|
| return pd.DataFrame(rows).sort_values("Domaine") |
|
|
|
|
| def consolidate_sources(sources, tau0=0.15): |
| domains = collect_domains(sources) |
|
|
| consolidated_values = {} |
| confidence_by_domain = {} |
| dispersion_by_domain = {} |
| conflict_by_domain = {} |
|
|
| for domain in domains: |
| vals = [] |
| weights = [] |
|
|
| for src in sources: |
| if domain in src["observed"]: |
| val = src["observed"][domain] |
| base_conf = src["confidence"].get(domain, 0.5) |
| fresh = src.get("freshness_score", 1.0) |
|
|
| vals.append(val) |
| weights.append(base_conf * fresh) |
|
|
| if len(vals) == 0: |
| continue |
|
|
| vals_arr = np.array(vals, dtype=float) |
| w_arr = np.array(weights, dtype=float) |
|
|
| if np.sum(w_arr) <= 1e-9: |
| w_arr = np.ones_like(vals_arr) |
|
|
| consolidated_val = float(np.average(vals_arr, weights=w_arr)) |
| dispersion = float(np.std(vals_arr)) if len(vals_arr) > 1 else 0.0 |
| conflict = float(np.max(vals_arr) - np.min(vals_arr)) if len(vals_arr) > 1 else 0.0 |
| confidence = ( |
| float(np.mean(w_arr)) |
| * float(np.exp(-dispersion / 0.10)) |
| * float(np.exp(-conflict / 0.15)) |
| ) |
|
|
| consolidated_values[domain] = consolidated_val |
| confidence_by_domain[domain] = confidence |
| dispersion_by_domain[domain] = dispersion |
| conflict_by_domain[domain] = conflict |
|
|
| x = np.array(list(consolidated_values.values()), dtype=float) |
|
|
| mu = float(np.mean(x)) |
| delta = float(np.std(x)) |
| C = float(1 / (1 + delta / tau0)) |
| R = float(mu * (1 - C)) |
| global_confidence = float(np.mean(list(confidence_by_domain.values()))) if confidence_by_domain else 0.0 |
|
|
| return { |
| "values": consolidated_values, |
| "confidence_by_domain": confidence_by_domain, |
| "dispersion_by_domain": dispersion_by_domain, |
| "conflict_by_domain": conflict_by_domain, |
| "mu": mu, |
| "delta": delta, |
| "C": C, |
| "R": R, |
| "global_confidence": global_confidence, |
| } |
|
|
|
|
| def load_conflict_history_df(): |
| try: |
| r = requests.get(CONFLICT_HISTORY_URL, timeout=8) |
| if r.status_code == 200 and r.text.strip(): |
| return pd.read_csv(io.StringIO(r.text)) |
| except Exception: |
| pass |
|
|
| return pd.DataFrame( |
| columns=[ |
| "timestamp", |
| "domain", |
| "conflict_score", |
| "dispersion", |
| "min_value", |
| "max_value", |
| "sources_present", |
| ] |
| ) |
|
|
|
|
| def append_conflict_snapshot_to_dataset(df_conflicts): |
| token = os.getenv("HF_TOKEN") |
| if not token: |
| return False, "HF_TOKEN manquant" |
|
|
| if HfApi is None: |
| return False, "huggingface_hub indisponible" |
|
|
| hist = load_conflict_history_df() |
|
|
| timestamp = datetime.now(timezone.utc).isoformat() |
|
|
| rows = [] |
| for _, row in df_conflicts.iterrows(): |
| rows.append({ |
| "timestamp": timestamp, |
| "domain": row["Domaine"], |
| "conflict_score": row["Score de conflit"], |
| "dispersion": row["Dispersion inter-sources"], |
| "min_value": row["Minimum"], |
| "max_value": row["Maximum"], |
| "sources_present": row["Sources présentes"], |
| }) |
|
|
| new_df = pd.DataFrame(rows) |
| hist = pd.concat([hist, new_df], ignore_index=True) |
|
|
| csv_bytes = hist.to_csv(index=False).encode("utf-8") |
|
|
| api = HfApi(token=token) |
| api.upload_file( |
| path_or_fileobj=csv_bytes, |
| path_in_repo="conflict_history.csv", |
| repo_id=DATASET_REPO_ID, |
| repo_type="dataset", |
| commit_message="Update conflict_history.csv from Space", |
| ) |
|
|
| return True, "conflict_history.csv mis à jour" |
|
|
|
|
| def compute_slope(series): |
| if len(series) < 2: |
| return 0.0 |
| x = np.arange(len(series)) |
| y = np.array(series, dtype=float) |
| return float(np.polyfit(x, y, 1)[0]) |
|
|
|
|
| def classify_trend(slope, threshold=0.001): |
| if slope > threshold: |
| return "hausse" |
| elif slope < -threshold: |
| return "baisse" |
| return "stable" |
|
|
|
|
| def build_conflict_trend_table(hist_df): |
| if len(hist_df) == 0: |
| return pd.DataFrame(columns=["domain", "conflit_actuel", "pente", "tendance", "n_points"]) |
|
|
| rows = [] |
|
|
| for domain, sub in hist_df.groupby("domain"): |
| sub = sub.sort_values("timestamp") |
| series = sub["conflict_score"].astype(float).tolist() |
|
|
| slope = compute_slope(series) |
| trend = classify_trend(slope) |
| current = float(series[-1]) if len(series) else 0.0 |
|
|
| rows.append({ |
| "domain": domain, |
| "conflit_actuel": current, |
| "pente": slope, |
| "tendance": trend, |
| "n_points": len(series), |
| }) |
|
|
| return pd.DataFrame(rows).sort_values("domain") |
