Permettre plusieurs recherches simultanées

#4
.gitattributes CHANGED
@@ -42,4 +42,3 @@ application_neo4j/static/notice/notice_html/media/image4.png filter=lfs diff=lfs
42
  application_neo4j/static/notice/notice.docx filter=lfs diff=lfs merge=lfs -text
43
  application_neo4j/static/notice/notice.pdf filter=lfs diff=lfs merge=lfs -text
44
  opengds/*.jar filter=lfs diff=lfs merge=lfs -text
45
- application_neo4j/static/notice/notice_en.pdf filter=lfs diff=lfs merge=lfs -text
 
42
  application_neo4j/static/notice/notice.docx filter=lfs diff=lfs merge=lfs -text
43
  application_neo4j/static/notice/notice.pdf filter=lfs diff=lfs merge=lfs -text
44
  opengds/*.jar filter=lfs diff=lfs merge=lfs -text
 
.gitignore CHANGED
@@ -1,3 +1,2 @@
1
  __pycache__/
2
  *.pyc
3
- .venv/
 
1
  __pycache__/
2
  *.pyc
 
application_neo4j/app.py CHANGED
@@ -15,8 +15,6 @@ from uuid import uuid4
15
  import argparse
16
  import atexit
17
  import math
18
- import json
19
- from datetime import datetime
20
 
21
  app = Flask(__name__, static_url_path="/static/") # Application Flask
22
  app.secret_key = os.urandom(24)
@@ -24,32 +22,9 @@ app.secret_key = os.urandom(24)
24
  # --- Connexion à Neo4j et GDS ---
25
  NEO4J_URI = "bolt://localhost:7687"
26
  GDS_GRAPH_NAME = "genealogie_gds"
27
- GDS_RELATIONSHIP_TYPES = ("IS_IN", "POSTED", "USED_IN")
28
  SEARCH_JOB_TTL_SECONDS = 60 * 60
29
  RESULT_BATCH_SIZE = 25
30
  DEFAULT_SEARCH_MAX_WORKERS = 2
31
- DATABASE_METADATA_PATH = os.environ.get(
32
- "DATABASE_METADATA_PATH",
33
- "/backups/database_metadata.json",
34
- )
35
-
36
-
37
- def database_dates():
38
- """Return display dates from metadata shipped with the loaded dump."""
39
- fallback = {"fr": "01/09/2025", "en": "2025-09-01"}
40
- try:
41
- with open(DATABASE_METADATA_PATH, encoding="utf-8") as metadata_file:
42
- built_at = json.load(metadata_file)["built_at"]
43
- built_date = datetime.fromisoformat(built_at.replace("Z", "+00:00")).date()
44
- except (OSError, KeyError, TypeError, ValueError, json.JSONDecodeError):
45
- return fallback
46
- return {
47
- "fr": built_date.strftime("%d/%m/%Y"),
48
- "en": built_date.isoformat(),
49
- }
50
-
51
-
52
- DATABASE_DATES = database_dates()
53
 
54
 
55
  def positive_int_env(name, default):
@@ -149,38 +124,6 @@ def ensure_graph_projected(
149
  if g_reverse_exists:
150
  gds.graph.get(reverse_graph_name).drop()
151
 
152
- relationship_types_result = gds.run_cypher(
153
- """
154
- CALL db.relationshipTypes()
155
- YIELD relationshipType
156
- RETURN relationshipType
157
- """
158
- )
159
- available_relationship_types = set(
160
- relationship_types_result["relationshipType"].tolist()
161
- )
162
- projected_relationship_types = [
163
- relationship_type
164
- for relationship_type in GDS_RELATIONSHIP_TYPES
165
- if relationship_type in available_relationship_types
166
- ]
167
- if not projected_relationship_types:
168
- raise RuntimeError(
169
- "Aucun type de relation compatible avec la recherche GDS "
170
- "n'est présent dans Neo4j."
171
- )
172
- reverse_relationship_projection = {
173
- relationship_type: {
174
- "type": relationship_type,
175
- "orientation": "REVERSE",
176
- }
177
- for relationship_type in projected_relationship_types
178
- }
179
- print(
180
- "Types de relations projetés dans GDS : "
181
- + ", ".join(projected_relationship_types)
182
- )
183
-
184
  # Native projections accept a jobId, allowing gds.listProgress to
185
  # expose real progress while the first search prepares the graph.
186
  natural_projection_job_id = (
@@ -196,7 +139,7 @@ def ensure_graph_projected(
196
  CALL gds.graph.project(
197
  $graph_name,
198
  '*',
199
- $relationship_projection,
200
  {jobId: $job_id}
201
  )
202
  YIELD graphName, nodeCount, relationshipCount
@@ -205,7 +148,6 @@ def ensure_graph_projected(
205
  {
206
  "graph_name": natural_graph_name,
207
  "job_id": natural_projection_job_id,
208
- "relationship_projection": projected_relationship_types,
209
  },
210
  )
211
  print(f"Graphe '{natural_graph_name}' projeté.")
@@ -223,7 +165,11 @@ def ensure_graph_projected(
223
  CALL gds.graph.project(
224
  $graph_name,
225
  '*',
226
- $relationship_projection,
 
 
 
 
227
  {jobId: $job_id}
228
  )
229
  YIELD graphName, nodeCount, relationshipCount
@@ -232,7 +178,6 @@ def ensure_graph_projected(
232
  {
233
  "graph_name": reverse_graph_name,
234
  "job_id": reverse_projection_job_id,
235
- "relationship_projection": reverse_relationship_projection,
236
  },
237
  )
238
  print(f"Graphe '{reverse_graph_name}' projeté.")
@@ -267,10 +212,7 @@ def set_language_route():
267
  def inject_i18n():
268
  """Inject t() and current_lang into all Jinja2 templates."""
269
  def _t(key, **kwargs):
270
- lang = session.get("lang", "fr")
271
- if key in {"site.nav_subtitle_full", "home.download_date"}:
272
- kwargs.setdefault("date", DATABASE_DATES[lang])
273
- return t(key, lang, **kwargs)
274
  # Dictionnaire JS pour les clés utilisées côté client
275
  js_i18n_keys = [
276
  "js.node_info.name", "js.node_info.type", "js.node_info.followers",
@@ -287,11 +229,6 @@ def inject_i18n():
287
  "search.progress_stage_completed", "search.progress_stage_failed",
288
  "search.progress_stage_cancelled",
289
  "search.progress_elapsed", "search.progress_items",
290
- "search.progress_queue_title",
291
- "search.progress_queue_position", "search.progress_queue_ahead_one",
292
- "search.progress_queue_ahead_many", "search.progress_queue_next",
293
- "search.progress_queue_running_one", "search.progress_queue_running_many",
294
- "search.progress_queue_note",
295
  "search.progress_server_percent",
296
  "search.progress_remaining_step", "search.progress_server_note",
297
  "search.progress_connection_error",
@@ -326,16 +263,13 @@ def autocomplete():
326
  if node_filter and node_filter in ["Model", "Dataset"]: # Mesure de sécurité
327
  label_cypher = f":{node_filter}"
328
 
329
- # Récupère les noms commençant par le préfixe fourni. Les suggestions les
330
- # plus téléchargées sont proposées en premier ; le nom garantit un ordre
331
- # stable lorsque plusieurs éléments ont le même nombre de téléchargements.
332
  cypher = f"""
333
  MATCH (n{label_cypher})
334
  WHERE toLower(n.name) STARTS WITH toLower($prefix)
335
  AND n.name IS NOT NULL
336
- RETURN n.name AS name, labels(n)[0] AS label,
337
- coalesce(n.downloads, 0) AS downloads
338
- ORDER BY downloads DESC, toLower(n.name) ASC
339
  LIMIT 10
340
  """
341
  try:
@@ -408,8 +342,6 @@ def make_search_result(name, depth, is_unlimited, filters, expert):
408
  return {
409
  "template": "expert.html" if expert else "search.html",
410
  "message": None,
411
- "message_key": None,
412
- "message_kwargs": {},
413
  "search": {
414
  "name": name,
415
  "depth": depth,
@@ -421,13 +353,6 @@ def make_search_result(name, depth, is_unlimited, filters, expert):
421
  }
422
 
423
 
424
- def set_search_result_message(result, key, lang, **kwargs):
425
- """Store a translatable message while keeping its current rendering."""
426
- result["message_key"] = key
427
- result["message_kwargs"] = kwargs
428
- result["message"] = t(key, lang, **kwargs)
429
-
430
-
431
  def execute_search_job(job_id, name, depth, is_unlimited, filters, expert, lang):
432
  """Run the existing search pipeline while publishing its real server stage."""
433
  result = make_search_result(name, depth, is_unlimited, filters, expert)
@@ -458,7 +383,6 @@ def execute_search_job(job_id, name, depth, is_unlimited, filters, expert, lang)
458
  name,
459
  None if is_unlimited else depth,
460
  expert,
461
- source_labels=filters,
462
  job_id=job_id,
463
  progress_callback=report_gds_stage,
464
  neo4j_driver=driver,
@@ -466,8 +390,7 @@ def execute_search_job(job_id, name, depth, is_unlimited, filters, expert, lang)
466
  )
467
 
468
  if not gds_result:
469
- set_search_result_message(
470
- result,
471
  "error.node_not_found_expert" if expert else "error.model_not_found",
472
  lang,
473
  name=name,
@@ -489,9 +412,7 @@ def execute_search_job(job_id, name, depth, is_unlimited, filters, expert, lang)
489
 
490
  if expert:
491
  if not graph_data["nodes"] and not graph_data["edges"]:
492
- set_search_result_message(
493
- result, "error.no_neighbors", lang, name=name
494
- )
495
  elif gds_result["source_label"] == "Model":
496
  raise_if_search_cancelled(job_id)
497
  set_search_stage(job_id, "building_highlights")
@@ -501,9 +422,7 @@ def execute_search_job(job_id, name, depth, is_unlimited, filters, expert, lang)
501
  elif gds_result["source_label"] == "Dataset":
502
  result["template"] = "search_dataset.html"
503
  elif not graph_data["nodes"] and not graph_data["edges"]:
504
- set_search_result_message(
505
- result, "error.no_neighbors", lang, name=name
506
- )
507
 
508
  raise_if_search_cancelled(job_id)
509
  update_search_job(
@@ -544,15 +463,11 @@ def execute_search_job(job_id, name, depth, is_unlimited, filters, expert, lang)
544
  )
545
  return
546
  if "Failed to find a node" in str(error):
547
- message_key = "error.node_not_found"
548
- message_kwargs = {"name": name}
549
  else:
550
  print(f"Background GDS search error ({job_id}): {error}")
551
- message_key = "error.gds"
552
- message_kwargs = {"error": str(error)}
553
- set_search_result_message(
554
- result, message_key, lang, **message_kwargs
555
- )
556
  update_search_job(
557
  job_id,
558
  status="failed",
@@ -737,25 +652,6 @@ def search_job_status(job_id):
737
  with search_jobs_lock:
738
  stored_job = search_jobs.get(job_id)
739
  job = dict(stored_job) if stored_job else None
740
- queued_jobs = sorted(
741
- (
742
- (queued_job.get("created_at", 0), queued_job_id)
743
- for queued_job_id, queued_job in search_jobs.items()
744
- if queued_job.get("status") == "queued"
745
- )
746
- )
747
- queued_job_ids = [
748
- queued_job_id for _, queued_job_id in queued_jobs
749
- ]
750
- queue_position = (
751
- queued_job_ids.index(job_id) + 1
752
- if job_id in queued_job_ids
753
- else None
754
- )
755
- running_jobs = sum(
756
- queued_job.get("status") == "running"
757
- for queued_job in search_jobs.values()
758
- )
759
  if not job:
760
  return jsonify({"error": "Search job not found"}), 404
761
 
@@ -767,9 +663,6 @@ def search_job_status(job_id):
767
  "elapsed_seconds": max(0, math.floor(now - started_at)) if started_at else 0,
768
  "progress_percent": None,
769
  "remaining_seconds": None,
770
- "queue_position": queue_position,
771
- "queued_jobs": len(queued_job_ids),
772
- "running_jobs": running_jobs,
773
  "completed_items": job.get("completed_items"),
774
  "total_items": job.get("total_items"),
775
  "result_url": (
@@ -808,16 +701,10 @@ def search_job_result(job_id):
808
  if job["status"] not in ("completed", "failed") or not job.get("result"):
809
  return redirect(url_for("findnode", lang=job["lang"]))
810
 
 
811
  result = job["result"]
812
- message = result["message"]
813
- if result.get("message_key"):
814
- message = t(
815
- result["message_key"],
816
- session.get("lang", "fr"),
817
- **result.get("message_kwargs", {}),
818
- )
819
  template_args = {
820
- "message": message,
821
  "search": result["search"],
822
  "graph_data": result["graph_data"],
823
  }
@@ -882,7 +769,6 @@ def findnode():
882
  name,
883
  depth,
884
  False,
885
- source_labels=current_filters,
886
  )
887
  if not gds_result :
888
  message = t("error.model_not_found", session.get("lang", "fr"), name=name)
@@ -964,7 +850,6 @@ def findnode_expert():
964
  name,
965
  depth,
966
  True,
967
- source_labels=current_filters,
968
  )
969
  if not gds_result :
970
  message = t("error.node_not_found_expert", session.get("lang", "fr"), name=name)
 
15
  import argparse
16
  import atexit
17
  import math
 
 
18
 
19
  app = Flask(__name__, static_url_path="/static/") # Application Flask
20
  app.secret_key = os.urandom(24)
 
22
  # --- Connexion à Neo4j et GDS ---
23
  NEO4J_URI = "bolt://localhost:7687"
24
  GDS_GRAPH_NAME = "genealogie_gds"
 
25
  SEARCH_JOB_TTL_SECONDS = 60 * 60
26
  RESULT_BATCH_SIZE = 25
27
  DEFAULT_SEARCH_MAX_WORKERS = 2
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
28
 
29
 
30
  def positive_int_env(name, default):
 
124
  if g_reverse_exists:
125
  gds.graph.get(reverse_graph_name).drop()
126
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
127
  # Native projections accept a jobId, allowing gds.listProgress to
128
  # expose real progress while the first search prepares the graph.
129
  natural_projection_job_id = (
 
139
  CALL gds.graph.project(
140
  $graph_name,
141
  '*',
142
+ ['IS_IN', 'POSTED', 'USED_IN'],
143
  {jobId: $job_id}
144
  )
145
  YIELD graphName, nodeCount, relationshipCount
 
148
  {
149
  "graph_name": natural_graph_name,
150
  "job_id": natural_projection_job_id,
 
151
  },
152
  )
153
  print(f"Graphe '{natural_graph_name}' projeté.")
 
165
  CALL gds.graph.project(
166
  $graph_name,
167
  '*',
168
+ {
169
+ IS_IN: {type: 'IS_IN', orientation: 'REVERSE'},
170
+ POSTED: {type: 'POSTED', orientation: 'REVERSE'},
171
+ USED_IN: {type: 'USED_IN', orientation: 'REVERSE'}
172
+ },
173
  {jobId: $job_id}
174
  )
175
  YIELD graphName, nodeCount, relationshipCount
 
178
  {
179
  "graph_name": reverse_graph_name,
180
  "job_id": reverse_projection_job_id,
 
181
  },
182
  )
183
  print(f"Graphe '{reverse_graph_name}' projeté.")
 
212
  def inject_i18n():
213
  """Inject t() and current_lang into all Jinja2 templates."""
214
  def _t(key, **kwargs):
215
+ return t(key, session.get("lang", "fr"), **kwargs)
 
 
 
216
  # Dictionnaire JS pour les clés utilisées côté client
217
  js_i18n_keys = [
218
  "js.node_info.name", "js.node_info.type", "js.node_info.followers",
 
229
  "search.progress_stage_completed", "search.progress_stage_failed",
230
  "search.progress_stage_cancelled",
231
  "search.progress_elapsed", "search.progress_items",
 
 
 
 
 
232
  "search.progress_server_percent",
233
  "search.progress_remaining_step", "search.progress_server_note",
234
  "search.progress_connection_error",
 
263
  if node_filter and node_filter in ["Model", "Dataset"]: # Mesure de sécurité
264
  label_cypher = f":{node_filter}"
265
 
266
+ # Récupère les noms commençant par le préfixe fourni
 
 
267
  cypher = f"""
268
  MATCH (n{label_cypher})
269
  WHERE toLower(n.name) STARTS WITH toLower($prefix)
270
  AND n.name IS NOT NULL
271
+ RETURN n.name AS name, labels(n)[0] as label
272
+ ORDER BY size(n.name) ASC
 
273
  LIMIT 10
274
  """
275
  try:
 
342
  return {
343
  "template": "expert.html" if expert else "search.html",
344
  "message": None,
 
 
345
  "search": {
346
  "name": name,
347
  "depth": depth,
 
353
  }
354
 
355
 
 
 
 
 
 
 
 
356
  def execute_search_job(job_id, name, depth, is_unlimited, filters, expert, lang):
357
  """Run the existing search pipeline while publishing its real server stage."""
358
  result = make_search_result(name, depth, is_unlimited, filters, expert)
 
383
  name,
384
  None if is_unlimited else depth,
385
  expert,
 
386
  job_id=job_id,
387
  progress_callback=report_gds_stage,
388
  neo4j_driver=driver,
 
390
  )
391
 
392
  if not gds_result:
393
+ result["message"] = t(
 
394
  "error.node_not_found_expert" if expert else "error.model_not_found",
395
  lang,
396
  name=name,
 
412
 
413
  if expert:
414
  if not graph_data["nodes"] and not graph_data["edges"]:
415
+ result["message"] = t("error.no_neighbors", lang, name=name)
 
 
416
  elif gds_result["source_label"] == "Model":
417
  raise_if_search_cancelled(job_id)
418
  set_search_stage(job_id, "building_highlights")
 
422
  elif gds_result["source_label"] == "Dataset":
423
  result["template"] = "search_dataset.html"
424
  elif not graph_data["nodes"] and not graph_data["edges"]:
425
+ result["message"] = t("error.no_neighbors", lang, name=name)
 
 
426
 
427
  raise_if_search_cancelled(job_id)
428
  update_search_job(
 
463
  )
464
  return
465
  if "Failed to find a node" in str(error):
466
+ message = t("error.node_not_found", lang, name=name)
 
467
  else:
468
  print(f"Background GDS search error ({job_id}): {error}")
469
+ message = t("error.gds", lang, error=str(error))
470
+ result["message"] = message
 
 
 
471
  update_search_job(
472
  job_id,
473
  status="failed",
 
652
  with search_jobs_lock:
653
  stored_job = search_jobs.get(job_id)
654
  job = dict(stored_job) if stored_job else None
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
655
  if not job:
656
  return jsonify({"error": "Search job not found"}), 404
657
 
 
663
  "elapsed_seconds": max(0, math.floor(now - started_at)) if started_at else 0,
664
  "progress_percent": None,
665
  "remaining_seconds": None,
 
 
 
666
  "completed_items": job.get("completed_items"),
667
  "total_items": job.get("total_items"),
668
  "result_url": (
 
701
  if job["status"] not in ("completed", "failed") or not job.get("result"):
702
  return redirect(url_for("findnode", lang=job["lang"]))
703
 
704
+ session["lang"] = job["lang"]
705
  result = job["result"]
 
 
 
 
 
 
 
706
  template_args = {
707
+ "message": result["message"],
708
  "search": result["search"],
709
  "graph_data": result["graph_data"],
710
  }
 
769
  name,
770
  depth,
771
  False,
 
772
  )
773
  if not gds_result :
774
  message = t("error.model_not_found", session.get("lang", "fr"), name=name)
 
850
  name,
851
  depth,
852
  True,
 
853
  )
854
  if not gds_result :
855
  message = t("error.node_not_found_expert", session.get("lang", "fr"), name=name)
application_neo4j/app_algorithms.py CHANGED
@@ -12,7 +12,6 @@ def run_gds_bfs(
12
  source_name: str,
13
  max_depth: int = None,
14
  expert=False,
15
- source_labels=None,
16
  job_id: str = None,
17
  progress_callback=None,
18
  neo4j_driver=None,
@@ -29,55 +28,26 @@ def run_gds_bfs(
29
  L'identifiant et le label de la source, ainsi que les résultats GDS
30
  des parcours descendant et ascendant.
31
  """
32
- requested_source_labels = [
33
- label
34
- for label in (source_labels or [])
35
- if label in ("Model", "Dataset", "Author")
36
- ]
37
  try:
38
  source_id_result = gds.run_cypher(
39
  """
40
  MATCH (n {name: $source_name})
41
- WHERE size($source_labels) = 0
42
- OR any(label IN labels(n) WHERE label IN $source_labels)
43
  RETURN id(n) AS id, labels(n) AS label
 
44
  """,
45
- {
46
- "source_name": source_name,
47
- "source_labels": requested_source_labels,
48
- },
49
  )
50
 
51
  if source_id_result.empty:
52
  print(f"Le modèle ou dataset avec le nom '{source_name}' n'a pas été trouvé.")
53
  return None
54
 
55
- label_preferences = requested_source_labels or [
56
- "Model",
57
- "Dataset",
58
- "Author",
59
- ]
60
- selected_source = None
61
- source_label = None
62
- for preferred_label in label_preferences:
63
- matching_sources = source_id_result[
64
- source_id_result["label"].apply(
65
- lambda node_labels: preferred_label in node_labels
66
- )
67
- ]
68
- if not matching_sources.empty:
69
- selected_source = matching_sources.iloc[0]
70
- source_label = preferred_label
71
- break
72
- if selected_source is None:
73
- selected_source = source_id_result.iloc[0]
74
- source_label = selected_source["label"][0]
75
-
76
  if source_label == "Author" and not expert:
77
  print(f"Le modèle ou dataset avec le nom '{source_name}' n'a pas été trouvé.")
78
  return None
79
 
80
- source_node_id = int(selected_source["id"])
81
  except Exception as e:
82
  print(f"Erreur lors de la recherche de l'ID du nœud source pour '{source_name}': {e}")
83
  return None
@@ -186,50 +156,50 @@ def get_genealogy_highlights(gds: "GraphDataScience", model_name: str, num_highl
186
  # Les classes CSS sont directement des classes Bootstrap 5 pour simplifier le rendu dans le template HTML.
187
  badges_info = {
188
  'desc_cited_1': {
189
- 'text_key': 'badge.desc_cited_1.text',
190
  'class': 'bg-success',
191
- 'title_key': 'badge.desc_cited_1.title'
192
  },
193
  'desc_cited_2': {
194
- 'text_key': 'badge.desc_cited_2.text',
195
  'class': 'bg-success bg-opacity-75',
196
- 'title_key': 'badge.desc_cited_2.title'
197
  },
198
  'desc_downloaded_1': {
199
- 'text_key': 'badge.desc_downloaded_1.text',
200
  'class': 'beta',
201
- 'title_key': 'badge.desc_downloaded_1.title'
202
  },
203
  'desc_downloaded_2': {
204
- 'text_key': 'badge.desc_downloaded_2.text',
205
  'class': 'alpha',
206
- 'title_key': 'badge.desc_downloaded_2.title'
207
  },
208
 
209
  'asc_foundation': {
210
- 'text_key': 'badge.asc_foundation.text',
211
  'class': 'bg-warning text-dark',
212
- 'title_key': 'badge.asc_foundation.title'
213
  },
214
  'asc_cited_1': {
215
- 'text_key': 'badge.asc_cited_1.text',
216
  'class': 'bg-success',
217
- 'title_key': 'badge.asc_cited_1.title'
218
  },
219
  'asc_cited_2': {
220
- 'text_key': 'badge.asc_cited_2.text',
221
  'class': 'bg-success bg-opacity-75',
222
- 'title_key': 'badge.asc_cited_2.title'
223
  },
224
  'asc_downloaded_1': {
225
- 'text_key': 'badge.asc_downloaded_1.text',
226
  'class': 'beta',
227
- 'title_key': 'badge.asc_downloaded_1.title'
228
  },
229
  'asc_downloaded_2': {
230
- 'text_key': 'badge.asc_downloaded_2.text',
231
  'class': 'alpha',
232
- 'title_key': 'badge.asc_downloaded_2.title'
233
  },
234
  }
235
 
 
12
  source_name: str,
13
  max_depth: int = None,
14
  expert=False,
 
15
  job_id: str = None,
16
  progress_callback=None,
17
  neo4j_driver=None,
 
28
  L'identifiant et le label de la source, ainsi que les résultats GDS
29
  des parcours descendant et ascendant.
30
  """
 
 
 
 
 
31
  try:
32
  source_id_result = gds.run_cypher(
33
  """
34
  MATCH (n {name: $source_name})
 
 
35
  RETURN id(n) AS id, labels(n) AS label
36
+ LIMIT 1
37
  """,
38
+ {"source_name": source_name},
 
 
 
39
  )
40
 
41
  if source_id_result.empty:
42
  print(f"Le modèle ou dataset avec le nom '{source_name}' n'a pas été trouvé.")
43
  return None
44
 
45
+ source_label = source_id_result["label"].iloc[0][0]
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
46
  if source_label == "Author" and not expert:
47
  print(f"Le modèle ou dataset avec le nom '{source_name}' n'a pas été trouvé.")
48
  return None
49
 
50
+ source_node_id = int(source_id_result["id"].iloc[0])
51
  except Exception as e:
52
  print(f"Erreur lors de la recherche de l'ID du nœud source pour '{source_name}': {e}")
53
  return None
 
156
  # Les classes CSS sont directement des classes Bootstrap 5 pour simplifier le rendu dans le template HTML.
157
  badges_info = {
158
  'desc_cited_1': {
159
+ 'text': t('badge.desc_cited_1.text', lang),
160
  'class': 'bg-success',
161
+ 'title': t('badge.desc_cited_1.title', lang)
162
  },
163
  'desc_cited_2': {
164
+ 'text': t('badge.desc_cited_2.text', lang),
165
  'class': 'bg-success bg-opacity-75',
166
+ 'title': t('badge.desc_cited_2.title', lang)
167
  },
168
  'desc_downloaded_1': {
169
+ 'text': t('badge.desc_downloaded_1.text', lang),
170
  'class': 'beta',
171
+ 'title': t('badge.desc_downloaded_1.title', lang)
172
  },
173
  'desc_downloaded_2': {
174
+ 'text': t('badge.desc_downloaded_2.text', lang),
175
  'class': 'alpha',
176
+ 'title': t('badge.desc_downloaded_2.title', lang)
177
  },
178
 
179
  'asc_foundation': {
180
+ 'text': t('badge.asc_foundation.text', lang),
181
  'class': 'bg-warning text-dark',
182
+ 'title': t('badge.asc_foundation.title', lang)
183
  },
184
  'asc_cited_1': {
185
+ 'text': t('badge.asc_cited_1.text', lang),
186
  'class': 'bg-success',
187
+ 'title': t('badge.asc_cited_1.title', lang)
188
  },
189
  'asc_cited_2': {
190
+ 'text': t('badge.asc_cited_2.text', lang),
191
  'class': 'bg-success bg-opacity-75',
192
+ 'title': t('badge.asc_cited_2.title', lang)
193
  },
194
  'asc_downloaded_1': {
195
+ 'text': t('badge.asc_downloaded_1.text', lang),
196
  'class': 'beta',
197
+ 'title': t('badge.asc_downloaded_1.title', lang)
198
  },
199
  'asc_downloaded_2': {
200
+ 'text': t('badge.asc_downloaded_2.text', lang),
201
  'class': 'alpha',
202
+ 'title': t('badge.asc_downloaded_2.title', lang)
203
  },
204
  }
205
 
application_neo4j/static/css/style.css CHANGED
@@ -2,79 +2,13 @@
2
  * STYLE GLOBAL & GRAPHIQUE SIGMA
3
  * =================================================================== */
4
 
5
- /* Explicit sizing and minimum widths avoid engine-specific flexbox
6
- * expansion differences (notably between Blink and Gecko). */
7
- html {
8
- -webkit-text-size-adjust: 100%;
9
- text-size-adjust: 100%;
10
- }
11
-
12
- body {
13
- min-width: 320px;
14
- }
15
-
16
- .legacy-browser-warning {
17
- display: none;
18
- margin: 0;
19
- padding: 0.75rem 1rem;
20
- color: #664d03;
21
- background: #fff3cd;
22
- border-bottom: 1px solid #ffecb5;
23
- font-weight: 600;
24
- text-align: center;
25
- }
26
-
27
- img,
28
- svg {
29
- max-width: 100%;
30
- height: auto;
31
- }
32
-
33
- .row > *,
34
- .card,
35
- .card-body,
36
- .flex-grow-1 {
37
- min-width: 0;
38
- }
39
-
40
- .navbar .container {
41
- column-gap: 1rem;
42
- row-gap: 0.75rem;
43
- }
44
-
45
- .navbar-brand {
46
- min-width: 0;
47
- max-width: 100%;
48
- white-space: normal;
49
- }
50
-
51
- .navbar-brand small,
52
- #suggestions-list li {
53
- overflow-wrap: anywhere;
54
- word-break: break-word;
55
- }
56
-
57
- .navbar-actions {
58
- min-width: 0;
59
- margin-left: auto;
60
- }
61
-
62
  #sigma-container {
63
  width: 100%;
64
  height: 600px; /* Un peu plus de hauteur pour un meilleur confort */
65
- position: relative;
66
- overflow: hidden;
67
- border: 1px solid #dee2e6;
68
- border: 1px solid var(--bs-border-color, #dee2e6); /* Variable Bootstrap */
69
  margin-top: 1rem;
70
- border-radius: 0.375rem;
71
- border-radius: var(--bs-border-radius, 0.375rem); /* Variable Bootstrap */
72
- background-color: #f8f9fa;
73
- background-color: var(--bs-light-bg-subtle, #f8f9fa); /* Fond légèrement teinté */
74
- }
75
-
76
- #floating-legend {
77
- max-width: calc(100% - 1rem);
78
  }
79
 
80
  /* ===================================================================
@@ -98,62 +32,35 @@ svg {
98
  /* ===================================================================
99
  * STYLE DES TABLES (DATATABLES)
100
  * =================================================================== */
101
- /* A single, deterministic wrapping policy replaces the previous contradictory
102
- * normal/nowrap declarations. */
103
  #descendance-table th, #descendance-table td,
104
- #ascendance-table th, #ascendance-table td,
105
- #train-table th, #train-table td,
106
- #graph-models-table th, #graph-models-table td {
107
- white-space: nowrap;
108
- vertical-align: middle;
109
  }
110
 
111
- #descendance-table th,
112
- #ascendance-table th,
113
- #train-table th {
114
- min-width: 120px;
115
  }
116
 
117
- #descendance-table th:first-child, #descendance-table td:first-child,
118
- #ascendance-table th:first-child, #ascendance-table td:first-child,
119
- #train-table th:first-child, #train-table td:first-child,
120
- #graph-models-table th:first-child, #graph-models-table td:first-child {
121
  min-width: 200px;
122
- white-space: normal;
123
- overflow-wrap: anywhere;
124
- word-break: break-word;
125
  }
126
 
127
- .table-responsive,
128
- div.dataTables_wrapper,
129
- div.dataTables_wrapper div.dataTables_scroll {
130
- width: 100%;
131
- max-width: 100%;
132
- }
133
-
134
- .table-responsive,
135
- div.dataTables_wrapper div.dataTables_scrollBody {
136
- overflow-x: auto !important;
137
- -webkit-overflow-scrolling: touch;
138
- }
139
-
140
- div.dataTables_wrapper div.dataTables_scrollHeadInner {
141
- min-width: 100%;
142
- }
143
-
144
- div.dataTables_wrapper table.dataTable {
145
- margin-left: 0 !important;
146
- margin-right: 0 !important;
147
- table-layout: auto;
148
  }
149
 
150
  /* Personnalisation de l'input de recherche de DataTables pour qu'il ressemble à un champ Bootstrap */
151
  div.dataTables_wrapper div.dataTables_filter input {
152
- max-width: 100%;
153
- border: 1px solid #dee2e6;
154
- border: 1px solid var(--bs-border-color, #dee2e6);
155
- border-radius: 0.375rem;
156
- border-radius: var(--bs-border-radius, 0.375rem);
157
  padding: 0.375rem 0.75rem;
158
  margin-left: 0.5em;
159
  }
@@ -193,12 +100,8 @@ div.dataTables_wrapper .table thead th.sorting_desc::after {
193
  /* Ce style est pour si vous utilisez une <ul> simple.
194
  Si vous avez bien une <ul class="list-group">, vous pouvez supprimer ce bloc. */
195
  box-shadow: 0 0.5rem 1rem rgba(0, 0, 0, 0.15);
196
- max-width: 100%;
197
- overflow-x: hidden;
198
- border-radius: 0.375rem;
199
- border-radius: var(--bs-border-radius, 0.375rem);
200
- background-color: #f8f9fa;
201
- background-color: var(--bs-light, #f8f9fa);
202
  }
203
  #suggestions-list li:hover {
204
  background-color: #94bfeb; ;
@@ -295,22 +198,11 @@ div.dataTables_wrapper .table thead th.sorting_desc::after {
295
  }
296
 
297
  /* Change la couleur de fond des cases à cocher lorsqu'elles sont cochées */
298
- .btn-primary {
299
  background-color: #6a11cb;
300
  border-color:#6a11cb;
301
  }
302
 
303
- .btn-outline-primary {
304
- color: #6a11cb;
305
- border-color: #6a11cb;
306
- }
307
-
308
- .btn-outline-primary:hover {
309
- color: #fff;
310
- background-color: #6a11cb;
311
- border-color: #6a11cb;
312
- }
313
-
314
  .stretched-link{
315
  color: #6a11cb;
316
  }
@@ -376,31 +268,4 @@ main {
376
  .btn-modern:hover {
377
  background: linear-gradient(135deg, #5a0fbf, #1d63e1);
378
  transform: translateY(-3px);
379
- }
380
-
381
- @media (max-width: 575.98px) {
382
- .navbar-brand,
383
- .navbar-actions {
384
- flex: 1 1 100%;
385
- margin-right: 0;
386
- }
387
-
388
- .navbar-actions {
389
- justify-content: flex-start !important;
390
- }
391
-
392
- #sigma-container {
393
- height: 70vh;
394
- min-height: 420px;
395
- }
396
-
397
- div.dataTables_wrapper div.dataTables_filter label {
398
- display: block;
399
- width: 100%;
400
- }
401
-
402
- div.dataTables_wrapper div.dataTables_filter input {
403
- width: 100%;
404
- margin: 0.375rem 0 0;
405
- }
406
- }
 
2
  * STYLE GLOBAL & GRAPHIQUE SIGMA
3
  * =================================================================== */
4
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
5
  #sigma-container {
6
  width: 100%;
7
  height: 600px; /* Un peu plus de hauteur pour un meilleur confort */
8
+ border: 1px solid var(--bs-border-color); /* Variable Bootstrap */
 
 
 
9
  margin-top: 1rem;
10
+ border-radius: var(--bs-border-radius); /* Variable Bootstrap */
11
+ background-color: var(--bs-light-bg-subtle); /* Fond légèrement teinté */
 
 
 
 
 
 
12
  }
13
 
14
  /* ===================================================================
 
32
  /* ===================================================================
33
  * STYLE DES TABLES (DATATABLES)
34
  * =================================================================== */
35
+ /* Permet au texte de revenir à la ligne naturellement */
 
36
  #descendance-table th, #descendance-table td,
37
+ #ascendance-table th, #ascendance-table td {
38
+ white-space: normal; /* <-- La correction clé ! */
39
+ word-wrap: break-word; /* Force le retour à la ligne des mots longs */
40
+ vertical-align: middle; /* Garde le centrage vertical */
 
41
  }
42
 
43
+ /* On peut donner une largeur minimale aux colonnes pour garder une bonne structure */
44
+ #descendance-table th, #ascendance-table th {
45
+ min-width: 120px; /* Ajustez cette valeur selon vos besoins */
 
46
  }
47
 
48
+ /* Cas spécifique pour les colonnes potentiellement longues comme le nom du modèle */
49
+ #descendance-table th:first-child, #ascendance-table th:first-child {
 
 
50
  min-width: 200px;
 
 
 
51
  }
52
 
53
+ /* Empêche le texte de se couper dans les cellules, force le défilement horizontal */
54
+ #descendance-table th, #descendance-table td,
55
+ #ascendance-table th, #ascendance-table td {
56
+ white-space: nowrap;
57
+ vertical-align: middle; /* Centrage vertical pour un look plus propre */
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
58
  }
59
 
60
  /* Personnalisation de l'input de recherche de DataTables pour qu'il ressemble à un champ Bootstrap */
61
  div.dataTables_wrapper div.dataTables_filter input {
62
+ border: 1px solid var(--bs-border-color);
63
+ border-radius: var(--bs-border-radius);
 
 
 
64
  padding: 0.375rem 0.75rem;
65
  margin-left: 0.5em;
66
  }
 
100
  /* Ce style est pour si vous utilisez une <ul> simple.
101
  Si vous avez bien une <ul class="list-group">, vous pouvez supprimer ce bloc. */
102
  box-shadow: 0 0.5rem 1rem rgba(0, 0, 0, 0.15);
103
+ border-radius: var(--bs-border-radius);
104
+ background-color: var(--bs-light);
 
 
 
 
105
  }
106
  #suggestions-list li:hover {
107
  background-color: #94bfeb; ;
 
198
  }
199
 
200
  /* Change la couleur de fond des cases à cocher lorsqu'elles sont cochées */
201
+ .btn {
202
  background-color: #6a11cb;
203
  border-color:#6a11cb;
204
  }
205
 
 
 
 
 
 
 
 
 
 
 
 
206
  .stretched-link{
207
  color: #6a11cb;
208
  }
 
268
  .btn-modern:hover {
269
  background: linear-gradient(135deg, #5a0fbf, #1d63e1);
270
  transform: translateY(-3px);
271
+ }
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
application_neo4j/static/js/browser_compatibility.js DELETED
@@ -1,14 +0,0 @@
1
- (function () {
2
- "use strict";
3
-
4
- // `document.documentMode` is exposed by Internet Explorer only. The rest of
5
- // the application deliberately targets maintained browsers supported by
6
- // Bootstrap 5.
7
- if (document.documentMode) {
8
- var warning = document.getElementById("legacy-browser-warning");
9
- if (warning) {
10
- warning.hidden = false;
11
- warning.style.display = "block";
12
- }
13
- }
14
- }());
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
application_neo4j/static/js/script.js CHANGED
@@ -193,15 +193,12 @@ document.addEventListener("DOMContentLoaded", () => {
193
  const common_dt_options = {
194
  "language": { "url": datatablesLangUrl },
195
  "pageLength": 10,
196
- "responsive": true,
197
- "scrollX": true,
198
- // Afficher en premier les modèles les plus téléchargés.
199
- "order": [[2, "desc"]],
200
  "columnDefs": [
201
  {
202
  // Appliquer notre plugin 'numeric-string' aux colonnes cibles
203
  "type": "numeric-string",
204
- "targets": [2, 4, 8, 9, 10, 11] // Downloads, Likes, Distance, Ascendants, Descendants, Citations
205
  }
206
  ]
207
  // "searching" est activé par défaut, on peut le customiser
 
193
  const common_dt_options = {
194
  "language": { "url": datatablesLangUrl },
195
  "pageLength": 10,
196
+ "responsive": true,"scrollX": true , "scrollY":true,
 
 
 
197
  "columnDefs": [
198
  {
199
  // Appliquer notre plugin 'numeric-string' aux colonnes cibles
200
  "type": "numeric-string",
201
+ "targets": [2, 4, 6, 7, 8, 9] // Indices des colonnes: Downloads, Likes, Distance, etc.
202
  }
203
  ]
204
  // "searching" est activé par défaut, on peut le customiser
application_neo4j/static/js/script_dataset.js CHANGED
@@ -209,8 +209,7 @@ document.addEventListener("DOMContentLoaded", () => {
209
  const common_dt_options = {
210
  "language": { "url": datatablesLangUrl },
211
  "pageLength": 10,
212
- "responsive": true,
213
- "scrollX": true,
214
  "columnDefs": [
215
  {
216
  // Appliquer notre plugin 'numeric-string' aux colonnes cibles
 
209
  const common_dt_options = {
210
  "language": { "url": datatablesLangUrl },
211
  "pageLength": 10,
212
+ "responsive": true,"scrollX": true , "scrollY":true,
 
213
  "columnDefs": [
214
  {
215
  // Appliquer notre plugin 'numeric-string' aux colonnes cibles
application_neo4j/static/js/search_progress.js CHANGED
@@ -95,7 +95,6 @@
95
  }
96
 
97
  function renderStatus(progress, job) {
98
- const title = progress.querySelector(".search-progress-title");
99
  const stage = progress.querySelector(".search-progress-stage");
100
  const status = progress.querySelector(".search-progress-status");
101
  const note = progress.querySelector(".search-progress-note");
@@ -103,84 +102,6 @@
103
 
104
  stage.textContent = stageLabel(job.stage);
105
 
106
- if (job.status === "queued" || job.stage === "queued") {
107
- title.textContent = translate(
108
- "search.progress_queue_title",
109
- "Recherche en file d’attente"
110
- );
111
- progress.classList.remove("alert-info", "alert-danger", "alert-secondary");
112
- progress.classList.add("alert-warning");
113
-
114
- const queueDetails = [];
115
- if (job.queue_position && job.queued_jobs) {
116
- queueDetails.push(
117
- format(
118
- translate(
119
- "search.progress_queue_position",
120
- "Position dans la file : {position}/{total}"
121
- ),
122
- {
123
- position: job.queue_position,
124
- total: job.queued_jobs,
125
- }
126
- )
127
- );
128
- const jobsAhead = job.queue_position - 1;
129
- if (jobsAhead === 0) {
130
- queueDetails.push(
131
- translate(
132
- "search.progress_queue_next",
133
- "Votre recherche est la prochaine."
134
- )
135
- );
136
- } else {
137
- queueDetails.push(
138
- format(
139
- translate(
140
- jobsAhead === 1
141
- ? "search.progress_queue_ahead_one"
142
- : "search.progress_queue_ahead_many",
143
- jobsAhead === 1
144
- ? "1 recherche devant la vôtre"
145
- : "{count} recherches devant la vôtre"
146
- ),
147
- { count: jobsAhead }
148
- )
149
- );
150
- }
151
- }
152
- if (job.running_jobs) {
153
- queueDetails.push(
154
- format(
155
- translate(
156
- job.running_jobs === 1
157
- ? "search.progress_queue_running_one"
158
- : "search.progress_queue_running_many",
159
- job.running_jobs === 1
160
- ? "1 recherche en cours"
161
- : "{count} recherches en cours"
162
- ),
163
- { count: job.running_jobs }
164
- )
165
- );
166
- }
167
- status.textContent = queueDetails.join(" · ");
168
- note.textContent = translate(
169
- "search.progress_queue_note",
170
- "La recherche démarrera automatiquement dès qu’une capacité sera disponible."
171
- );
172
- bar.style.width = "0%";
173
- bar.removeAttribute("aria-valuenow");
174
- return;
175
- }
176
-
177
- title.textContent = translate(
178
- "search.progress_title",
179
- "Recherche en cours…"
180
- );
181
- progress.classList.remove("alert-warning", "alert-danger", "alert-secondary");
182
- progress.classList.add("alert-info");
183
-
184
  const details = [
185
  format(
186
  translate("search.progress_elapsed", "Temps écoulé : {seconds} s"),
@@ -287,9 +208,6 @@
287
  elapsed_seconds: 0,
288
  progress_percent: null,
289
  remaining_seconds: null,
290
- queue_position: null,
291
- queued_jobs: null,
292
- running_jobs: null,
293
  completed_items: null,
294
  total_items: null,
295
  });
 
95
  }
96
 
97
  function renderStatus(progress, job) {
 
98
  const stage = progress.querySelector(".search-progress-stage");
99
  const status = progress.querySelector(".search-progress-status");
100
  const note = progress.querySelector(".search-progress-note");
 
102
 
103
  stage.textContent = stageLabel(job.stage);
104
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
105
  const details = [
106
  format(
107
  translate("search.progress_elapsed", "Temps écoulé : {seconds} s"),
 
208
  elapsed_seconds: 0,
209
  progress_percent: null,
210
  remaining_seconds: null,
 
 
 
211
  completed_items: null,
212
  total_items: null,
213
  });
application_neo4j/static/js/utils.js CHANGED
@@ -231,9 +231,6 @@ function formatDateFr(dateString) {
231
 
232
  // Fonction pour parser la valeur : enlève les espaces et convertit en nombre si possible
233
  function parseNumericValue(value) {
234
- if (typeof value === 'number' && Number.isFinite(value)) {
235
- return { isNumber: true, value };
236
- }
237
  if (typeof value === 'string') {
238
  // Enlève les espaces (pour les nombres comme "12 345") et les virgules
239
  const cleanedValue = value.replace(/[\s,]/g, '');
@@ -247,7 +244,7 @@ function parseNumericValue(value) {
247
  }
248
 
249
  // Définition du tri ascendant
250
- jQuery.fn.dataTable.ext.type.order['numeric-string-asc'] = function (a, b) {
251
  const valA = parseNumericValue(a);
252
  const valB = parseNumericValue(b);
253
 
@@ -263,19 +260,9 @@ jQuery.fn.dataTable.ext.type.order['numeric-string-asc'] = function (a, b) {
263
  }
264
  };
265
 
266
- // Tri descendant, en conservant les valeurs inconnues après les nombres.
267
- jQuery.fn.dataTable.ext.type.order['numeric-string-desc'] = function (a, b) {
268
- const valA = parseNumericValue(a);
269
- const valB = parseNumericValue(b);
270
-
271
- if (valA.isNumber && valB.isNumber) {
272
- return valB.value - valA.value;
273
- } else if (valA.isNumber && !valB.isNumber) {
274
- return -1;
275
- } else if (!valA.isNumber && valB.isNumber) {
276
- return 1;
277
- }
278
- return String(valB.value).localeCompare(String(valA.value));
279
  };
280
 
281
  /**
 
231
 
232
  // Fonction pour parser la valeur : enlève les espaces et convertit en nombre si possible
233
  function parseNumericValue(value) {
 
 
 
234
  if (typeof value === 'string') {
235
  // Enlève les espaces (pour les nombres comme "12 345") et les virgules
236
  const cleanedValue = value.replace(/[\s,]/g, '');
 
244
  }
245
 
246
  // Définition du tri ascendant
247
+ jQuery.fn.dataTable.ext.order['numeric-string-asc'] = function (a, b) {
248
  const valA = parseNumericValue(a);
249
  const valB = parseNumericValue(b);
250
 
 
260
  }
261
  };
262
 
263
+ // Le tri descendant est simplement l'inverse de l'ascendant
264
+ jQuery.fn.dataTable.ext.order['numeric-string-desc'] = function (a, b) {
265
+ return jQuery.fn.dataTable.ext.order['numeric-string-asc'](a, b) * -1;
 
 
 
 
 
 
 
 
 
 
266
  };
267
 
268
  /**
application_neo4j/static/notice/generate_notice_en.py DELETED
@@ -1,503 +0,0 @@
1
- """Generate the English version of the CNIL GenMod information notice.
2
-
3
- This is a maintainer utility, not a runtime dependency of the Space. Run it
4
- from the repository root after installing ``pypdf`` and ``reportlab``:
5
-
6
- python application_neo4j/static/notice/generate_notice_en.py \
7
- --source application_neo4j/static/notice/notice.pdf
8
- """
9
-
10
- from __future__ import annotations
11
-
12
- import argparse
13
- import math
14
- import tempfile
15
- from pathlib import Path
16
-
17
- from PIL import Image as PillowImage
18
- from pypdf import PdfReader
19
- from reportlab.lib import colors
20
- from reportlab.lib.enums import TA_CENTER
21
- from reportlab.lib.pagesizes import A4
22
- from reportlab.lib.styles import ParagraphStyle, getSampleStyleSheet
23
- from reportlab.lib.units import mm
24
- from reportlab.pdfbase import pdfmetrics
25
- from reportlab.pdfbase.ttfonts import TTFont
26
- from reportlab.platypus import (
27
- Flowable,
28
- Image,
29
- PageBreak,
30
- Paragraph,
31
- SimpleDocTemplate,
32
- Spacer,
33
- Table,
34
- TableStyle,
35
- )
36
-
37
-
38
- CNIL_BLUE = colors.HexColor("#0046A8")
39
- LIGHT_BLUE = colors.HexColor("#EAF2FB")
40
- RED = colors.HexColor("#E52A22")
41
- GOLD = colors.HexColor("#F7B500")
42
-
43
-
44
- class NeuralNetworkDiagram(Flowable):
45
- """Small English recreation of the neural-network diagram."""
46
-
47
- def __init__(self, width=170 * mm, height=54 * mm):
48
- super().__init__()
49
- self.width = width
50
- self.height = height
51
-
52
- def draw(self):
53
- canvas = self.canv
54
- canvas.saveState()
55
- scale_x = self.width / 480
56
- scale_y = self.height / 155
57
- canvas.scale(scale_x, scale_y)
58
-
59
- inputs = [(45, 115), (45, 78), (45, 41)]
60
- hidden_1 = [(190, 130), (190, 97), (190, 64), (190, 31)]
61
- hidden_2 = [(330, 130), (330, 97), (330, 64), (330, 31)]
62
- outputs = [(445, 113), (445, 78), (445, 43)]
63
-
64
- canvas.setStrokeColor(colors.HexColor("#222222"))
65
- canvas.setLineWidth(0.6)
66
- for x1, y1 in inputs:
67
- for x2, y2 in hidden_1:
68
- canvas.line(x1 + 32, y1, x2 - 10, y2)
69
- for x1, y1 in hidden_2:
70
- for x2, y2 in outputs:
71
- canvas.line(x1 + 10, y1, x2 - 32, y2)
72
-
73
- canvas.setFont("DejaVuSans", 7.5)
74
- for index, (x, y) in enumerate(inputs, 1):
75
- canvas.setFillColor(colors.HexColor("#DDEFD6"))
76
- canvas.roundRect(x - 32, y - 10, 64, 20, 4, fill=1, stroke=1)
77
- canvas.setFillColor(colors.black)
78
- canvas.drawCentredString(x, y - 3, f"Input {index}")
79
- for layer in (hidden_1, hidden_2):
80
- for x, y in layer:
81
- canvas.setFillColor(colors.HexColor("#FFF1C9"))
82
- canvas.circle(x, y, 10, fill=1, stroke=1)
83
- for index, (x, y) in enumerate(outputs, 1):
84
- canvas.setFillColor(colors.HexColor("#DCE9FF"))
85
- canvas.roundRect(x - 32, y - 10, 64, 20, 4, fill=1, stroke=1)
86
- canvas.setFillColor(colors.black)
87
- canvas.drawCentredString(x, y - 3, f"Output {index}")
88
-
89
- canvas.setFillColor(colors.white)
90
- path = canvas.beginPath()
91
- path.moveTo(255, 80)
92
- path.lineTo(275, 98)
93
- path.lineTo(295, 80)
94
- path.lineTo(275, 62)
95
- path.close()
96
- canvas.drawPath(path, fill=1, stroke=1)
97
- canvas.setFillColor(colors.black)
98
- canvas.setFont("DejaVuSans", 10)
99
- canvas.drawCentredString(275, 77, "f")
100
- canvas.setFont("DejaVuSans", 7.5)
101
- canvas.drawCentredString(190, 8, "Layer 1")
102
- canvas.drawCentredString(330, 8, "Layer 2")
103
- canvas.drawCentredString(275, 118, "Activation function")
104
- canvas.drawCentredString(405, 145, "Edge parameters")
105
- canvas.restoreState()
106
-
107
-
108
- class HuggingFaceDiagram(Flowable):
109
- """English recreation of the platform example diagram."""
110
-
111
- def __init__(self, width=170 * mm, height=55 * mm):
112
- super().__init__()
113
- self.width = width
114
- self.height = height
115
-
116
- def draw_arrow(self, canvas, x1, y1, x2, y2, color):
117
- canvas.setStrokeColor(color)
118
- canvas.setFillColor(color)
119
- canvas.setLineWidth(1.8)
120
- canvas.line(x1, y1, x2, y2)
121
- angle = math.atan2(y2 - y1, x2 - x1)
122
- for offset in (-0.5, 0.5):
123
- canvas.line(
124
- x2,
125
- y2,
126
- x2 - 7 * math.cos(angle + offset),
127
- y2 - 7 * math.sin(angle + offset),
128
- )
129
-
130
- def box(self, canvas, x, y, width, height, text, color):
131
- canvas.setStrokeColor(color)
132
- canvas.setFillColor(colors.white)
133
- canvas.rect(x, y, width, height, fill=1, stroke=1)
134
- canvas.setFillColor(color)
135
- canvas.setFont("DejaVuSans-Bold", 8)
136
- canvas.drawCentredString(x + width / 2, y + height / 2 - 3, text)
137
-
138
- def draw(self):
139
- canvas = self.canv
140
- canvas.saveState()
141
- scale_x = self.width / 500
142
- scale_y = self.height / 170
143
- canvas.scale(scale_x, scale_y)
144
- self.box(canvas, 5, 118, 70, 24, "USER A", CNIL_BLUE)
145
- self.box(canvas, 105, 105, 105, 48, "", RED)
146
- canvas.setFillColor(RED)
147
- canvas.setFont("DejaVuSans-Bold", 8)
148
- canvas.drawCentredString(157.5, 130, "MACHINE-LEARNING")
149
- canvas.drawCentredString(157.5, 116, "MODEL")
150
- self.box(canvas, 290, 105, 85, 48, "DATASET", RED)
151
- self.box(canvas, 420, 118, 75, 24, "USER B", CNIL_BLUE)
152
- self.box(canvas, 205, 15, 90, 26, "USER C", CNIL_BLUE)
153
- self.box(canvas, 207, 66, 86, 25, "NEW MODEL", RED)
154
-
155
- canvas.setFillColor(GOLD)
156
- canvas.circle(250, 130, 21, fill=1, stroke=0)
157
- canvas.setFillColor(colors.HexColor("#795500"))
158
- canvas.setFont("DejaVuSans-Bold", 8)
159
- canvas.drawCentredString(250, 127, "HF")
160
- canvas.setFont("DejaVuSans-Bold", 7)
161
- canvas.drawCentredString(250, 157, "Hugging Face")
162
-
163
- self.draw_arrow(canvas, 75, 130, 101, 130, colors.black)
164
- self.draw_arrow(canvas, 420, 130, 379, 130, colors.black)
165
- self.draw_arrow(canvas, 210, 130, 225, 130, RED)
166
- self.draw_arrow(canvas, 290, 130, 275, 130, RED)
167
- self.draw_arrow(canvas, 250, 65, 250, 43, colors.black)
168
- self.draw_arrow(canvas, 250, 93, 250, 106, RED)
169
- self.draw_arrow(canvas, 157, 103, 218, 42, colors.HexColor("#2385CC"))
170
- self.draw_arrow(canvas, 332, 103, 282, 42, colors.HexColor("#2385CC"))
171
-
172
- canvas.setFillColor(colors.black)
173
- canvas.setFont("DejaVuSans", 6.8)
174
- canvas.drawString(5, 106, "individual or organisation")
175
- canvas.drawString(402, 106, "individual or organisation")
176
- canvas.drawCentredString(250, 3, "individual or organisation")
177
- canvas.setFillColor(CNIL_BLUE)
178
- canvas.setFont("DejaVuSans-Bold", 13)
179
- canvas.drawString(12, 12, "CNIL")
180
- canvas.restoreState()
181
-
182
-
183
- def extract_figures(source: Path, work_dir: Path) -> tuple[Path, Path]:
184
- reader = PdfReader(str(source))
185
- fox_image = reader.pages[2].images[0].image
186
- fox_crop = fox_image.crop((0, 0, int(fox_image.width * 0.35), fox_image.height))
187
- fox_path = work_dir / "fox.jpg"
188
- fox_crop.save(fox_path, quality=92)
189
-
190
- memorisation_path = work_dir / "memorisation.jpg"
191
- reader.pages[5].images[0].image.save(memorisation_path, quality=92)
192
- return fox_path, memorisation_path
193
-
194
-
195
- def build_styles():
196
- pdfmetrics.registerFont(
197
- TTFont("DejaVuSans", "/usr/share/fonts/truetype/dejavu/DejaVuSans.ttf")
198
- )
199
- pdfmetrics.registerFont(
200
- TTFont(
201
- "DejaVuSans-Bold",
202
- "/usr/share/fonts/truetype/dejavu/DejaVuSans-Bold.ttf",
203
- )
204
- )
205
- styles = getSampleStyleSheet()
206
- styles.add(
207
- ParagraphStyle(
208
- "NoticeTitle",
209
- fontName="DejaVuSans-Bold",
210
- fontSize=20,
211
- leading=24,
212
- textColor=CNIL_BLUE,
213
- spaceAfter=8 * mm,
214
- )
215
- )
216
- styles.add(
217
- ParagraphStyle(
218
- "Section",
219
- fontName="DejaVuSans-Bold",
220
- fontSize=14,
221
- leading=17,
222
- textColor=CNIL_BLUE,
223
- spaceBefore=3 * mm,
224
- spaceAfter=2.5 * mm,
225
- )
226
- )
227
- styles.add(
228
- ParagraphStyle(
229
- "Subsection",
230
- fontName="DejaVuSans-Bold",
231
- fontSize=11,
232
- leading=14,
233
- textColor=colors.HexColor("#222222"),
234
- spaceBefore=2 * mm,
235
- spaceAfter=1.5 * mm,
236
- )
237
- )
238
- styles.add(
239
- ParagraphStyle(
240
- "BodyNotice",
241
- fontName="DejaVuSans",
242
- fontSize=8.7,
243
- leading=11.3,
244
- alignment=4,
245
- spaceAfter=2.1 * mm,
246
- )
247
- )
248
- styles.add(
249
- ParagraphStyle(
250
- "Caption",
251
- fontName="DejaVuSans",
252
- fontSize=7.2,
253
- leading=9,
254
- alignment=TA_CENTER,
255
- textColor=colors.HexColor("#555555"),
256
- spaceAfter=2 * mm,
257
- )
258
- )
259
- return styles
260
-
261
-
262
- def generate(source: Path, output: Path) -> None:
263
- styles = build_styles()
264
- body = styles["BodyNotice"]
265
- title = styles["NoticeTitle"]
266
- section = styles["Section"]
267
- subsection = styles["Subsection"]
268
- caption = styles["Caption"]
269
-
270
- def p(text: str):
271
- return Paragraph(text, body)
272
-
273
- def bullet(text: str):
274
- style = ParagraphStyle(
275
- "BulletNotice",
276
- parent=body,
277
- leftIndent=5 * mm,
278
- firstLineIndent=-3.5 * mm,
279
- bulletIndent=1.5 * mm,
280
- spaceAfter=1.4 * mm,
281
- )
282
- return Paragraph(text, style, bulletText="•")
283
-
284
- with tempfile.TemporaryDirectory(prefix="genmod-notice-") as temp_dir:
285
- fox_path, memorisation_path = extract_figures(source, Path(temp_dir))
286
-
287
- doc = SimpleDocTemplate(
288
- str(output),
289
- pagesize=A4,
290
- rightMargin=18 * mm,
291
- leftMargin=18 * mm,
292
- topMargin=18 * mm,
293
- bottomMargin=16 * mm,
294
- title="A tool for exploring the genealogy of open-source AI models",
295
- author="CNIL",
296
- subject="English translation of the GenMod information notice",
297
- )
298
-
299
- def decorate_page(canvas, document):
300
- canvas.saveState()
301
- canvas.setStrokeColor(CNIL_BLUE)
302
- canvas.setLineWidth(1.2)
303
- canvas.line(18 * mm, 12 * mm, A4[0] - 18 * mm, 12 * mm)
304
- canvas.setFont("DejaVuSans-Bold", 8)
305
- canvas.setFillColor(CNIL_BLUE)
306
- canvas.drawString(18 * mm, 7.5 * mm, "CNIL · GenMod")
307
- canvas.setFont("DejaVuSans", 8)
308
- canvas.drawRightString(
309
- A4[0] - 18 * mm, 7.5 * mm, f"Page {document.page}"
310
- )
311
- canvas.restoreState()
312
-
313
- story = [
314
- Paragraph(
315
- "A tool for exploring the genealogy of open-source AI models",
316
- title,
317
- ),
318
- Paragraph("What is an AI model?", section),
319
- Paragraph("Training", subsection),
320
- p(
321
- "The fields in which AI can be used are vast and difficult to delimit, as they extend to many aspects of everyday life: online searches and purchases, targeted advertising, machine translation, personal digital assistants and connected cities, as well as transport, healthcare and many other areas."
322
- ),
323
- p(
324
- "Under Article 3 of the European Union Artificial Intelligence Act, an AI system is ‘a machine-based system that is designed to operate with varying levels of autonomy and that may exhibit adaptiveness after deployment, and that, for explicit or implicit objectives, infers, from the input it receives, how to generate outputs such as predictions, content, recommendations, or decisions that can influence physical or virtual environments.’"
325
- ),
326
- p(
327
- "These systems incorporate one or more AI models. Such models can be described as algorithms whose operation is determined by a set of attributes and which are designed to perform tasks such as prediction, classification, inference or generation. Deep neural-network models, for example, consist of nodes (neurons) arranged in layers and connected by edges, each of which has a parameter or ‘weight’. During training, these parameters are adjusted to learn the statistical distribution of the training data."
328
- ),
329
- p("For a simple neural network, the model’s attributes might include:"),
330
- bullet("the type and size of each layer (linear, convolutional, attention, etc.);"),
331
- bullet("the weights assigned to each edge (also called parameters);"),
332
- bullet("the activation functions between layers; and"),
333
- bullet("possibly other operations located within or between layers."),
334
- Spacer(1, 1.5 * mm),
335
- NeuralNetworkDiagram(),
336
- Paragraph("Figure 1 — Diagram of a neural network (authors)", caption),
337
- PageBreak(),
338
- Paragraph("Training from examples", subsection),
339
- p(
340
- "When a neural network is trained to recognise images, it is given examples in which the image pixels are associated with an annotation, or label. The model then adjusts its parameters—the weights—to learn to assign the correct label as often as possible."
341
- ),
342
- p(
343
- "The main difference between a deep-learning model and a conventional computer program is that the model learns inference rules autonomously from data. In a conventional program, a task is solved using explicit rules defined in advance by the developer. To sort a list of numbers, for example, the order in which elements are compared is precisely programmed. This works very well for clearly delimited tasks for which explicit rules can be established."
344
- ),
345
- p(
346
- "By contrast, the rules of a deep-learning model are not specified directly. Instead, the model is given a large volume of examples—its training data—so that, during the learning phase, it can identify the statistical regularities or strategies that solve the task. This makes it possible to automate much more complex tasks for which defining every rule by hand would be extremely difficult or impossible."
347
- ),
348
- Paragraph("Using a trained model", subsection),
349
- p(
350
- "Once trained, a model can be used as it is, without further modification, to perform specific tasks automatically. This is called the inference phase. The model receives an input—an image, text or an audio signal, for example—and produces an output based on what it learned during training. It then acts as a ‘black box’: it applies the regularities it has learned without changing its internal structure or learning anything new."
351
- ),
352
- p(
353
- "Consider machine translation. A neural-network model trained on millions of pairs of Spanish and English sentences can translate a new text from English into Spanish at inference time. Linguistic rules are not explicitly implemented in the model; it has learned to match sequences of words by drawing on statistical regularities in the training data."
354
- ),
355
- p(
356
- "Another example is image-to-text models, which generate image captions. A model can be trained to associate images with textual descriptions. Once trained, it can receive a new image—such as a photograph of a dog running in a park—and automatically produce a sentence such as ‘A dog is running on the grass in a park.’"
357
- ),
358
- Table(
359
- [[
360
- Image(str(fox_path), width=58 * mm, height=41 * mm),
361
- Paragraph(
362
- "The image is a close-up portrait of a red fox standing in the snow. The fox is in the centre, its vibrant orange fur lit by golden sunrise or sunset light. It stands alert, ears upright and looking off-camera. The pristine white snow has a bluish tint reflecting the cool colours of the sky. The softly blurred blue-grey background adds depth and highlights the fox as the subject.",
363
- body,
364
- ),
365
- ]],
366
- colWidths=[62 * mm, 105 * mm],
367
- style=TableStyle([
368
- ("VALIGN", (0, 0), (-1, -1), "MIDDLE"),
369
- ("LEFTPADDING", (0, 0), (-1, -1), 2),
370
- ("RIGHTPADDING", (0, 0), (-1, -1), 4),
371
- ("BOX", (0, 0), (-1, -1), 0.4, colors.HexColor("#BBBBBB")),
372
- ("BACKGROUND", (0, 0), (-1, -1), colors.HexColor("#F7F7F7")),
373
- ]),
374
- ),
375
- Paragraph(
376
- "Figure 2 — Example of an image’s textual description (source: imagedescriber.online)",
377
- caption,
378
- ),
379
- PageBreak(),
380
- p(
381
- "These use cases illustrate the power of advances in deep learning to automate complex, often subjective or ambiguous tasks for which writing explicit rules by hand would be difficult or impossible."
382
- ),
383
- Paragraph("Derivatives: fine-tuning, merging, quantisation, etc.", subsection),
384
- p(
385
- "To adapt a neural network more closely to a specific task, optimise its performance or reduce its running costs, several transformations can be applied to an initial pre-trained model. Such modifications are very common in the open-source ecosystem. They make it possible to create new models from one or more initial models, sometimes with additional data. Four common transformations between a target model and a source model are:"
386
- ),
387
- bullet(
388
- "<b>Fine-tuning:</b> a general-purpose source model continues training on a specific dataset to improve its performance on a more precise task. A large language model initially trained on freely accessible internet sources may, for example, be fine-tuned on a company’s business data so that it better understands the company’s specialist vocabulary and expressions."
389
- ),
390
- bullet(
391
- "<b>Quantisation:</b> the precision of the source model’s weights is reduced to lower its memory footprint. Weights initially encoded using 32 bits may, for example, be rounded to the nearest value that can be encoded using 16 bits."
392
- ),
393
- bullet(
394
- "<b>Adaptation:</b> the source model is adjusted so that it can be used with limited computing resources—for example, on a mobile phone—most often using a Low-Rank Adaptation (LoRA) technique."
395
- ),
396
- bullet(
397
- "<b>Merging:</b> layers from different models are combined to improve performance. For example, two LLMs, A and B, may both have been trained on general text corpora. Averaging the weights in their twelfth layers and replacing A’s twelfth layer with that average may produce a model C that performs better than either A or B."
398
- ),
399
- PageBreak(),
400
- Paragraph("A platform for open-source AI: Hugging Face", section),
401
- p(
402
- "To enable AI models to be shared and made available by and for as many people as possible, the Franco-American company Hugging Face, founded in 2016, developed a platform that centralises models and datasets. It also provides software tools for deploying AI models. Today it hosts more open-source models than any other platform—over two million were available in September 2025—and acts as a catalyst for the open-source AI ecosystem."
403
- ),
404
- p("The following example illustrates how the platform works:"),
405
- bullet("User C wants to create a model that automatically detects fraudulent emails."),
406
- bullet(
407
- "User A has published a natural-language-processing model on Hugging Face—for example, Google’s <i>google/gemma-3-27b-it</i>—and User B has published a dataset containing millions of emails labelled as fraudulent or non-fraudulent."
408
- ),
409
- bullet(
410
- "User C downloads the model and dataset, then trains the model on the dataset to specialise it for classification. Once the resulting classifier performs well, User C can publish it on Hugging Face so that anyone can use it as is or train it again on other datasets to improve its performance."
411
- ),
412
- HuggingFaceDiagram(),
413
- Paragraph("Figure 3 — Example of how Hugging Face is used", caption),
414
- p(
415
- "In short, Hugging Face provides tools for building, training and deploying deep-learning models based on open-source technologies and code. It also offers a space where researchers, engineers and enthusiasts can exchange ideas, obtain support and contribute to open-source projects."
416
- ),
417
- Paragraph("Benefits of open-source AI", section),
418
- p(
419
- "The rise of open-source AI shows that powerful and partly transparent models can compete with proprietary solutions while stimulating collective innovation. BLOOM (176 billion parameters, 2022), developed by the BigScience consortium, illustrates this dynamic: trained in 46 languages, it enabled multilingual conversational assistants to be developed in Africa, Latin America and the Arab world, where commercial models remained poorly adapted to local languages."
420
- ),
421
- PageBreak(),
422
- p(
423
- "Similarly, GPT-J and GPT-NeoX (EleutherAI) and Vicuna (LMSYS) provided the basis for open-source projects that enabled universities and start-ups to create specialised chatbots without relying on closed services. These models also made research into bias detection and the robustness of large language models possible."
424
- ),
425
- p(
426
- "In computer vision, Stable Diffusion (Stability AI) transformed visual creation. Its weights were made freely available, opening the way to applications in video games, advertising and audiovisual production, including concept images, storyboards and rapid design. Its open-source code enabled tools such as Automatic1111 and ComfyUI, used by hundreds of thousands of artists and researchers."
427
- ),
428
- p(
429
- "The impact is industrial as well. LLaMA (Meta), initially released to the research community, gave rise to a generation of derivatives—including Zephyr, Nous-Hermes and OpenChat—that are now used for customer support, summarising legal or medical documents and prototyping code."
430
- ),
431
- p(
432
- "In science, projects such as BioGPT (Microsoft Research) and OpenFold (inspired by AlphaFold) demonstrate how opening code and weights accelerates biomedical research by allowing independent laboratories to reproduce and improve results in protein-structure prediction and molecule discovery."
433
- ),
434
- p(
435
- "These achievements show that open source is not limited to reusing models: it enables technological ownership, local adaptation and open innovation in fields as varied as artistic creation, healthcare, education, data science and the cultural industries. Nevertheless, some opacity may remain as to how models are built: with what data and which training algorithm? A CNIL paper and a PEReN paper provide further discussion of this issue."
436
- ),
437
- Paragraph("Privacy issues", section),
438
- Paragraph("Memorisation by AI models", subsection),
439
- p(
440
- "The scientific community has long established that information about the data used to train an AI model can often be extracted from even partial access to the model. In generative AI, a model may reproduce text or images that are very close to examples in its training dataset. In the figure below, when Stable Diffusion is asked to generate an image matching the caption ‘Emma Watson to play Belle in Disney’s Beauty and the Beast’, its output closely resembles an image from the training database."
441
- ),
442
- p(
443
- "This is regurgitation—only one form of memorisation. Statistical methods can sometimes reveal other information, such as whether a particular record belonged to the training dataset, through membership-inference attacks."
444
- ),
445
- Image(str(memorisation_path), width=153 * mm, height=91 * mm),
446
- Paragraph(
447
- "Figure 4 — Source: Louis Hunt (LinkedIn). Original photograph: UN Women.",
448
- caption,
449
- ),
450
- PageBreak(),
451
- p(
452
- "For text models such as chatbots, prominent cases of regurgitation are already widely documented. One version of ChatGPT, for example, was reported to reproduce New York Times articles almost verbatim and to provide personal information such as a person’s name, address and telephone number."
453
- ),
454
- Paragraph("The GDPR and AI models", subsection),
455
- p(
456
- "If information about an AI model’s training database can generally be extracted from the model, what legal regime should apply when that database contains personal data? The European Data Protection Board clarified this question in Opinion 28/2024 on AI models, on which the CNIL’s latest recommendations are based. In particular, the Opinion concludes that the GDPR applies in many cases to AI models trained on personal data because of their capacity for memorisation."
457
- ),
458
- Paragraph("Exercising rights in relation to AI models", subsection),
459
- p(
460
- "For AI models subject to the GDPR, people affected by memorisation have rights in relation to their data, including the rights to object, access and erasure. These rights are not absolute: a controller may depart from them in several situations, for example where a request is manifestly unfounded or excessive (Article 12), or where the controller is unable to identify the person concerned. The CNIL’s guidance on exercising rights provides further details."
461
- ),
462
- p(
463
- "At a time when European bodies are confirming that data-protection law also applies to AI models, the CNIL wishes to study the conditions under which these rights could be exercised within the highly dynamic open-source AI ecosystem."
464
- ),
465
- Spacer(1, 8 * mm),
466
- Table(
467
- [[Paragraph(
468
- "This document is an English translation of the French information notice made available by the CNIL for the GenMod application. In the event of any discrepancy, the French version is the reference version.",
469
- ParagraphStyle(
470
- "TranslationNote",
471
- parent=body,
472
- textColor=CNIL_BLUE,
473
- backColor=LIGHT_BLUE,
474
- borderPadding=8,
475
- ),
476
- )]],
477
- colWidths=[170 * mm],
478
- ),
479
- ]
480
-
481
- doc.build(story, onFirstPage=decorate_page, onLaterPages=decorate_page)
482
-
483
-
484
- def main() -> None:
485
- parser = argparse.ArgumentParser()
486
- parser.add_argument(
487
- "--source",
488
- type=Path,
489
- default=Path("application_neo4j/static/notice/notice.pdf"),
490
- )
491
- parser.add_argument(
492
- "--output",
493
- type=Path,
494
- default=Path("application_neo4j/static/notice/notice_en.pdf"),
495
- )
496
- args = parser.parse_args()
497
- args.output.parent.mkdir(parents=True, exist_ok=True)
498
- generate(args.source, args.output)
499
- print(f"Generated {args.output}")
500
-
501
-
502
- if __name__ == "__main__":
503
- main()
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
application_neo4j/static/notice/notice.pdf CHANGED
@@ -1,3 +1,3 @@
1
  version https://git-lfs.github.com/spec/v1
2
- oid sha256:98c0455e1d8419a184a2471fc69960027976f9cb7d9ee07752516c3b216b4a79
3
- size 1166198
 
1
  version https://git-lfs.github.com/spec/v1
2
+ oid sha256:dced59e9dc43bd1823fd18cee4005415e375e9f77c7a58bc6ffcf3a48e732a3c
3
+ size 409000
application_neo4j/static/notice/notice_en.pdf DELETED
@@ -1,3 +0,0 @@
1
- version https://git-lfs.github.com/spec/v1
2
- oid sha256:abec965848c0fd334609151abd08615455a57872fbd0709961e6c8a8ddb2e93b
3
- size 383279
 
 
 
 
application_neo4j/templates/expert.html CHANGED
@@ -18,18 +18,13 @@
18
  </head>
19
 
20
  <body class="d-flex flex-column min-vh-100">
21
- <div id="legacy-browser-warning" class="legacy-browser-warning" role="alert" hidden>{{ t('site.legacy_browser_warning') }}</div>
22
- <script src="{{ url_for('static', filename='js/browser_compatibility.js') }}"></script>
23
  <header class="navbar navbar-expand-lg navbar-light bg-light border-bottom">
24
  <div class="container">
25
  <a class="navbar-brand" href="{{ url_for('home', lang=current_lang) }}">
26
  <span class="fw-bold">{{ t('site.nav_brand_model_expert') }}</span>
27
  <small class="d-block text-muted">{{ t('site.nav_subtitle') }}</small>
28
  </a>
29
- <div class="navbar-actions d-flex align-items-center justify-content-end flex-wrap gap-2">
30
- <a href="{{ url_for('home', lang=current_lang) }}" class="btn btn-sm btn-dark me-3">
31
- <i class="bi bi-house-door-fill me-1" aria-hidden="true"></i>{{ t('site.home_button') }}
32
- </a>
33
  <a href="{{ url_for('set_language_route', lang='fr') }}" class="btn btn-sm {{ 'btn-primary' if current_lang == 'fr' else 'btn-outline-primary' }}">{{ t('lang.fr') }}</a>
34
  <a href="{{ url_for('set_language_route', lang='en') }}" class="btn btn-sm {{ 'btn-primary' if current_lang == 'en' else 'btn-outline-primary' }}">{{ t('lang.en') }}</a>
35
  </div>
 
18
  </head>
19
 
20
  <body class="d-flex flex-column min-vh-100">
 
 
21
  <header class="navbar navbar-expand-lg navbar-light bg-light border-bottom">
22
  <div class="container">
23
  <a class="navbar-brand" href="{{ url_for('home', lang=current_lang) }}">
24
  <span class="fw-bold">{{ t('site.nav_brand_model_expert') }}</span>
25
  <small class="d-block text-muted">{{ t('site.nav_subtitle') }}</small>
26
  </a>
27
+ <div class="d-flex gap-2">
 
 
 
28
  <a href="{{ url_for('set_language_route', lang='fr') }}" class="btn btn-sm {{ 'btn-primary' if current_lang == 'fr' else 'btn-outline-primary' }}">{{ t('lang.fr') }}</a>
29
  <a href="{{ url_for('set_language_route', lang='en') }}" class="btn btn-sm {{ 'btn-primary' if current_lang == 'en' else 'btn-outline-primary' }}">{{ t('lang.en') }}</a>
30
  </div>
application_neo4j/templates/index.html CHANGED
@@ -5,7 +5,7 @@
5
  <meta name="viewport" content="width=device-width, initial-scale=1.0">
6
  <title>{{ t('site.title_home') }}</title>
7
  <link rel="stylesheet" href="{{ url_for('static', filename='css/style.css') }}">
8
- <link href="https://cdn.jsdelivr.net/npm/bootstrap@5.3.3/dist/css/bootstrap.min.css" rel="stylesheet" integrity="sha384-QWTKZyjpPEjISv5WaRU9OFeRpok6YctnYmDr5pNlyT2bRjXh0JMhjY6hW+ALEwIH" crossorigin="anonymous">
9
 
10
  <style>/* Style simple pour la page d'accueil */
11
  body {
@@ -116,30 +116,11 @@
116
  background: #3498db;
117
  color: #fff;
118
  }
119
-
120
- @media (max-width: 575.98px) {
121
- .lang-switch {
122
- position: static;
123
- justify-content: flex-end;
124
- padding: 12px 15px 0;
125
- }
126
-
127
- .welcome-container {
128
- padding-top: 24px;
129
- }
130
-
131
- .content-card {
132
- padding: 22px;
133
- text-align: left;
134
- }
135
- }
136
 
137
  </style>
138
  </head>
139
 
140
  <body>
141
- <div id="legacy-browser-warning" class="legacy-browser-warning" role="alert" hidden>{{ t('site.legacy_browser_warning') }}</div>
142
- <script src="{{ url_for('static', filename='js/browser_compatibility.js') }}"></script>
143
  <!-- Language selector -->
144
  <div class="lang-switch">
145
  <a href="{{ url_for('set_language_route', lang='fr') }}" class="{{ 'active' if current_lang == 'fr' else '' }}">{{ t('lang.fr') }}</a>
@@ -148,7 +129,7 @@
148
 
149
  <div class="welcome-container">
150
  <h1>{{ t('home.welcome_title') }}</h1>
151
- <a href="{{ url_for('static', filename='notice/notice_en.pdf' if current_lang == 'en' else 'notice/notice.pdf') }}" class="btn-modern" target="_blank">{{ t('home.more_info') }}</a>
152
  </div>
153
  <div class="container content-section">
154
  <div class="row g-4">
 
5
  <meta name="viewport" content="width=device-width, initial-scale=1.0">
6
  <title>{{ t('site.title_home') }}</title>
7
  <link rel="stylesheet" href="{{ url_for('static', filename='css/style.css') }}">
8
+ <link href="https://cdn.jsdelivr.net/npm/bootstrap@5.3.2/dist/css/bootstrap.min.css" rel="stylesheet">
9
 
10
  <style>/* Style simple pour la page d'accueil */
11
  body {
 
116
  background: #3498db;
117
  color: #fff;
118
  }
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
119
 
120
  </style>
121
  </head>
122
 
123
  <body>
 
 
124
  <!-- Language selector -->
125
  <div class="lang-switch">
126
  <a href="{{ url_for('set_language_route', lang='fr') }}" class="{{ 'active' if current_lang == 'fr' else '' }}">{{ t('lang.fr') }}</a>
 
129
 
130
  <div class="welcome-container">
131
  <h1>{{ t('home.welcome_title') }}</h1>
132
+ <a href="{{ url_for('static', filename='notice/notice.pdf') }}" class="btn-modern" target="_blank">{{ t('home.more_info') }}</a>
133
  </div>
134
  <div class="container content-section">
135
  <div class="row g-4">
application_neo4j/templates/infos.html CHANGED
@@ -1,43 +1,18 @@
1
  <!DOCTYPE html>
2
- <html lang="{{ current_lang }}">
3
  <head>
4
  <meta charset="UTF-8">
5
- <meta name="viewport" content="width=device-width, initial-scale=1.0">
6
- <title>{{ t('home.more_info') }}</title>
7
  <style>
8
  body, html { margin: 0; padding: 0; height: 100%; overflow: hidden; }
9
- body { display: flex; flex-direction: column; font-family: sans-serif; }
10
- .legacy-browser-warning { display: none; padding: 0.75rem 1rem; color: #664d03; background: #fff3cd; border-bottom: 1px solid #ffecb5; font-weight: 600; text-align: center; }
11
- .toolbar {
12
- display: flex;
13
- align-items: center;
14
- padding: 0.75rem 1rem;
15
- background: #f8f9fa;
16
- border-bottom: 1px solid #dee2e6;
17
- }
18
- .home-button {
19
- color: #fff;
20
- background: #212529;
21
- border: 1px solid #212529;
22
- border-radius: 0.25rem;
23
- padding: 0.375rem 0.75rem;
24
- text-decoration: none;
25
- font-weight: 600;
26
- }
27
- .home-button:hover { color: #fff; background: #424649; }
28
- .pdf-container { width: 100%; height: calc(100% - 52px); flex: 1; min-height: 0; }
29
- .pdf-container iframe { display: block; border: none; width: 100%; height: 100%; }
30
  </style>
31
  </head>
32
  <body>
33
- <div id="legacy-browser-warning" class="legacy-browser-warning" role="alert" hidden>{{ t('site.legacy_browser_warning') }}</div>
34
- <script src="{{ url_for('static', filename='js/browser_compatibility.js') }}"></script>
35
- <nav class="toolbar" aria-label="{{ t('site.home_button') }}">
36
- <a class="home-button" href="{{ url_for('home', lang=current_lang) }}">⌂ {{ t('site.home_button') }}</a>
37
- </nav>
38
  <div class="pdf-container">
39
  <!-- Cette balise iframe va afficher votre PDF -->
40
  <iframe src="{{ url_for('static', filename='pdf/plus_infos_interface.pdf') }}"></iframe>
41
  </div>
42
  </body>
43
- </html>
 
1
  <!DOCTYPE html>
2
+ <html lang="fr">
3
  <head>
4
  <meta charset="UTF-8">
5
+ <title>Plus d'informations</title>
 
6
  <style>
7
  body, html { margin: 0; padding: 0; height: 100%; overflow: hidden; }
8
+ .pdf-container { width: 100%; height: 100%; }
9
+ .pdf-container iframe { border: none; width: 100%; height: 100%; }
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
10
  </style>
11
  </head>
12
  <body>
 
 
 
 
 
13
  <div class="pdf-container">
14
  <!-- Cette balise iframe va afficher votre PDF -->
15
  <iframe src="{{ url_for('static', filename='pdf/plus_infos_interface.pdf') }}"></iframe>
16
  </div>
17
  </body>
18
+ </html>
application_neo4j/templates/search.html CHANGED
@@ -33,18 +33,13 @@
33
  </head>
34
 
35
  <body class="d-flex flex-column min-vh-100">
36
- <div id="legacy-browser-warning" class="legacy-browser-warning" role="alert" hidden>{{ t('site.legacy_browser_warning') }}</div>
37
- <script src="{{ url_for('static', filename='js/browser_compatibility.js') }}"></script>
38
  <header class="navbar navbar-expand-lg navbar-light bg-light border-bottom">
39
  <div class="container">
40
  <a class="navbar-brand" href="{{ url_for('home', lang=current_lang) }}">
41
  <span class="fw-bold">{{ t('site.nav_brand') }}</span>
42
  <small class="d-block text-muted">{{ t('site.nav_subtitle_full') }}</small>
43
  </a>
44
- <div class="navbar-actions d-flex align-items-center justify-content-end flex-wrap gap-2">
45
- <a href="{{ url_for('home', lang=current_lang) }}" class="btn btn-sm btn-dark me-3">
46
- <i class="bi bi-house-door-fill me-1" aria-hidden="true"></i>{{ t('site.home_button') }}
47
- </a>
48
  <a href="{{ url_for('set_language_route', lang='fr') }}" class="btn btn-sm {{ 'btn-primary' if current_lang == 'fr' else 'btn-outline-primary' }}">{{ t('lang.fr') }}</a>
49
  <a href="{{ url_for('set_language_route', lang='en') }}" class="btn btn-sm {{ 'btn-primary' if current_lang == 'en' else 'btn-outline-primary' }}">{{ t('lang.en') }}</a>
50
  </div>
@@ -179,8 +174,8 @@
179
  <span class="badge {{ badge.class }} on-top"
180
  data-bs-toggle="tooltip"
181
  data-bs-placement="top"
182
- title="{{ t(badge.title_key) }}">
183
- {{ t(badge.text_key) }}
184
  </span>
185
  {% endfor %}
186
  </div>
@@ -305,8 +300,8 @@
305
  <span class="badge {{ badge.class }} on-top"
306
  data-bs-toggle="tooltip"
307
  data-bs-placement="top"
308
- title="{{ t(badge.title_key) }}">
309
- {{ t(badge.text_key) }}
310
  </span>
311
  {% endfor %}
312
  </div>
 
33
  </head>
34
 
35
  <body class="d-flex flex-column min-vh-100">
 
 
36
  <header class="navbar navbar-expand-lg navbar-light bg-light border-bottom">
37
  <div class="container">
38
  <a class="navbar-brand" href="{{ url_for('home', lang=current_lang) }}">
39
  <span class="fw-bold">{{ t('site.nav_brand') }}</span>
40
  <small class="d-block text-muted">{{ t('site.nav_subtitle_full') }}</small>
41
  </a>
42
+ <div class="d-flex gap-2">
 
 
 
43
  <a href="{{ url_for('set_language_route', lang='fr') }}" class="btn btn-sm {{ 'btn-primary' if current_lang == 'fr' else 'btn-outline-primary' }}">{{ t('lang.fr') }}</a>
44
  <a href="{{ url_for('set_language_route', lang='en') }}" class="btn btn-sm {{ 'btn-primary' if current_lang == 'en' else 'btn-outline-primary' }}">{{ t('lang.en') }}</a>
45
  </div>
 
174
  <span class="badge {{ badge.class }} on-top"
175
  data-bs-toggle="tooltip"
176
  data-bs-placement="top"
177
+ title="{{ badge.title }}">
178
+ {{ badge.text }}
179
  </span>
180
  {% endfor %}
181
  </div>
 
300
  <span class="badge {{ badge.class }} on-top"
301
  data-bs-toggle="tooltip"
302
  data-bs-placement="top"
303
+ title="{{ badge.title }}">
304
+ {{ badge.text }}
305
  </span>
306
  {% endfor %}
307
  </div>
application_neo4j/templates/search_dataset.html CHANGED
@@ -33,18 +33,13 @@
33
  </head>
34
 
35
  <body class="d-flex flex-column min-vh-100">
36
- <div id="legacy-browser-warning" class="legacy-browser-warning" role="alert" hidden>{{ t('site.legacy_browser_warning') }}</div>
37
- <script src="{{ url_for('static', filename='js/browser_compatibility.js') }}"></script>
38
  <header class="navbar navbar-expand-lg navbar-light bg-light border-bottom">
39
  <div class="container">
40
  <a class="navbar-brand" href="{{ url_for('home', lang=current_lang) }}">
41
  <span class="fw-bold">{{ t('site.nav_brand') }}</span>
42
  <small class="d-block text-muted">{{ t('site.nav_subtitle') }}</small>
43
  </a>
44
- <div class="navbar-actions d-flex align-items-center justify-content-end flex-wrap gap-2">
45
- <a href="{{ url_for('home', lang=current_lang) }}" class="btn btn-sm btn-dark me-3">
46
- <i class="bi bi-house-door-fill me-1" aria-hidden="true"></i>{{ t('site.home_button') }}
47
- </a>
48
  <a href="{{ url_for('set_language_route', lang='fr') }}" class="btn btn-sm {{ 'btn-primary' if current_lang == 'fr' else 'btn-outline-primary' }}">{{ t('lang.fr') }}</a>
49
  <a href="{{ url_for('set_language_route', lang='en') }}" class="btn btn-sm {{ 'btn-primary' if current_lang == 'en' else 'btn-outline-primary' }}">{{ t('lang.en') }}</a>
50
  </div>
 
33
  </head>
34
 
35
  <body class="d-flex flex-column min-vh-100">
 
 
36
  <header class="navbar navbar-expand-lg navbar-light bg-light border-bottom">
37
  <div class="container">
38
  <a class="navbar-brand" href="{{ url_for('home', lang=current_lang) }}">
39
  <span class="fw-bold">{{ t('site.nav_brand') }}</span>
40
  <small class="d-block text-muted">{{ t('site.nav_subtitle') }}</small>
41
  </a>
42
+ <div class="d-flex gap-2">
 
 
 
43
  <a href="{{ url_for('set_language_route', lang='fr') }}" class="btn btn-sm {{ 'btn-primary' if current_lang == 'fr' else 'btn-outline-primary' }}">{{ t('lang.fr') }}</a>
44
  <a href="{{ url_for('set_language_route', lang='en') }}" class="btn btn-sm {{ 'btn-primary' if current_lang == 'en' else 'btn-outline-primary' }}">{{ t('lang.en') }}</a>
45
  </div>
application_neo4j/translations.py CHANGED
@@ -44,9 +44,7 @@ TRANSLATIONS = {
44
  "site.nav_brand": "Généalogie des Modèles",
45
  "site.nav_brand_model_expert": "Généalogie des Modèles",
46
  "site.nav_subtitle": "Exploration des relations entre modèles et datasets",
47
- "site.nav_subtitle_full": "Exploration des relations entre modèles et datasets (base de données actualisée le {date})",
48
- "site.home_button": "Accueil",
49
- "site.legacy_browser_warning": "Ce navigateur n’est plus pris en charge. Pour un affichage stable, utilisez une version récente de Firefox, Chrome, Edge ou Safari.",
50
  "site.footer": "Application de recherche et visualisation des relations entre modèles et jeux de données publiés sur la plateforme HuggingFace. © 2025",
51
  "site.footer_expert": "Application de recherche et visualisation... © 2025",
52
 
@@ -83,7 +81,7 @@ TRANSLATIONS = {
83
  "home.btn_unsure": "Je ne sais pas",
84
  "home.expert_label": "Vous êtes un chercheur",
85
  "home.btn_expert": "Mode expert",
86
- "home.download_date": "Date de téléchargement de la base de données : {date}",
87
  "home.notice_title": "Mentions d'information sur les traitements de données à caractère personnel",
88
  "home.notice_p1": "Afin d'étudier le développement de la communauté de l'IA open source, et de préparer la possibilité d'exercices de droits des citoyens, le projet vise à étudier la base de données des jeux de données et modèles présents sur la plateforme HuggingFace. Cette base de données permet d'établir un arbre généalogique des modèles.",
89
  "home.notice_p2": "Les données traitées sont le pseudonyme de l'auteur (quand il apparaît dans les métadonnées), le nom du modèle/jeu de données et plusieurs informations inhérentes à ce modèle/jeu de données telles que la date de publication, la licence utilisée ou encore le nombre de téléchargements.",
@@ -100,7 +98,7 @@ TRANSLATIONS = {
100
  "search.page_lead_dataset": "Explorez la généalogie des modèles et datasets",
101
  "search.label_name": "Nom à rechercher",
102
  "search.placeholder_model": "Taper le nom du modèle ou l'id de son repo Hugging Face.",
103
- "search.placeholder_dataset": "Taper le nom du dataset à investiguer.",
104
  "search.label_filters": "Filtres",
105
  "search.filter_model": "Modèle",
106
  "search.filter_dataset": "Dataset",
@@ -110,7 +108,7 @@ TRANSLATIONS = {
110
  "search.btn_search": "Rechercher",
111
  "search.btn_search_simple": "Rechercher",
112
  "search.progress_title": "Recherche en cours…",
113
- "search.progress_stage_queued": "En attente d’une capacité de recherche…",
114
  "search.progress_stage_preparing": "Préparation du graphe…",
115
  "search.progress_stage_descendants": "Recherche des descendants…",
116
  "search.progress_stage_ancestors": "Recherche des ascendants…",
@@ -124,14 +122,6 @@ TRANSLATIONS = {
124
  "search.progress_stage_cancelled": "Recherche annulée.",
125
  "search.progress_elapsed": "Temps écoulé : {seconds} s",
126
  "search.progress_items": "Éléments traités par le serveur : {completed}/{total}",
127
- "search.progress_queue_title": "Recherche en file d’attente",
128
- "search.progress_queue_position": "Position dans la file : {position}/{total}",
129
- "search.progress_queue_ahead_one": "1 recherche devant la vôtre",
130
- "search.progress_queue_ahead_many": "{count} recherches devant la vôtre",
131
- "search.progress_queue_next": "Votre recherche est la prochaine.",
132
- "search.progress_queue_running_one": "1 recherche en cours",
133
- "search.progress_queue_running_many": "{count} recherches en cours",
134
- "search.progress_queue_note": "La recherche démarrera automatiquement dès qu’une capacité sera disponible.",
135
  "search.progress_server_percent": "Progression indiquée par le serveur : {percent} %",
136
  "search.progress_remaining_step": "Temps restant estimé pour cette étape : {seconds} s",
137
  "search.progress_server_note": "La progression repose sur les opérations réellement terminées par le serveur.",
@@ -197,7 +187,7 @@ TRANSLATIONS = {
197
  # ── expert.html ──
198
  "expert.page_title": "Recherche Experte",
199
  "expert.page_lead": "Explorez et filtrez la généalogie des modèles",
200
- "expert.placeholder": "Taper le nom du modèle à investiguer.",
201
  "expert.filter_author": "Auteur",
202
  "expert.connected_component": "Composante connexe du noeud recherché",
203
  "expert.legend_title": "Légendes",
@@ -283,9 +273,7 @@ TRANSLATIONS = {
283
  "site.nav_brand": "Model Genealogy",
284
  "site.nav_brand_model_expert": "Model Genealogy",
285
  "site.nav_subtitle": "Exploration of relations between models and datasets",
286
- "site.nav_subtitle_full": "Exploration of relations between models and datasets (database updated on {date})",
287
- "site.home_button": "Home",
288
- "site.legacy_browser_warning": "This browser is no longer supported. For a stable display, use a recent version of Firefox, Chrome, Edge or Safari.",
289
  "site.footer": "Application for searching and visualizing relations between models and datasets published on the HuggingFace platform. © 2025",
290
  "site.footer_expert": "Application for searching and visualizing... © 2025",
291
 
@@ -322,7 +310,7 @@ TRANSLATIONS = {
322
  "home.btn_unsure": "I don't know",
323
  "home.expert_label": "You are a researcher",
324
  "home.btn_expert": "Expert mode",
325
- "home.download_date": "Database download date: {date}",
326
  "home.notice_title": "Information on the processing of personal data",
327
  "home.notice_p1": "In order to study the development of the open-source AI community and to prepare for the possibility of citizens exercising their rights, the project aims to study the database of datasets and models present on the HuggingFace platform. This database makes it possible to establish a genealogy tree of the models.",
328
  "home.notice_p2": "The data processed includes the author's pseudonym (when it appears in the metadata), the name of the model/dataset and several pieces of information inherent to this model/dataset such as the publication date, the license used or the number of downloads.",
@@ -349,7 +337,7 @@ TRANSLATIONS = {
349
  "search.btn_search": "Search",
350
  "search.btn_search_simple": "Search",
351
  "search.progress_title": "Search in progress…",
352
- "search.progress_stage_queued": "Waiting for search capacity…",
353
  "search.progress_stage_preparing": "Preparing the graph…",
354
  "search.progress_stage_descendants": "Searching descendants…",
355
  "search.progress_stage_ancestors": "Searching ancestors…",
@@ -363,14 +351,6 @@ TRANSLATIONS = {
363
  "search.progress_stage_cancelled": "Search cancelled.",
364
  "search.progress_elapsed": "Elapsed time: {seconds} s",
365
  "search.progress_items": "Items processed by the server: {completed}/{total}",
366
- "search.progress_queue_title": "Search queued",
367
- "search.progress_queue_position": "Queue position: {position}/{total}",
368
- "search.progress_queue_ahead_one": "1 search ahead of yours",
369
- "search.progress_queue_ahead_many": "{count} searches ahead of yours",
370
- "search.progress_queue_next": "Your search is next.",
371
- "search.progress_queue_running_one": "1 search currently running",
372
- "search.progress_queue_running_many": "{count} searches currently running",
373
- "search.progress_queue_note": "Your search will start automatically as soon as capacity becomes available.",
374
  "search.progress_server_percent": "Progress reported by the server: {percent}%",
375
  "search.progress_remaining_step": "Estimated time remaining for this step: {seconds} s",
376
  "search.progress_server_note": "Progress is based on operations actually completed by the server.",
 
44
  "site.nav_brand": "Généalogie des Modèles",
45
  "site.nav_brand_model_expert": "Généalogie des Modèles",
46
  "site.nav_subtitle": "Exploration des relations entre modèles et datasets",
47
+ "site.nav_subtitle_full": "Exploration des relations entre modèles et datasets (base de données actualisée le 01/09/2025)",
 
 
48
  "site.footer": "Application de recherche et visualisation des relations entre modèles et jeux de données publiés sur la plateforme HuggingFace. © 2025",
49
  "site.footer_expert": "Application de recherche et visualisation... © 2025",
50
 
 
81
  "home.btn_unsure": "Je ne sais pas",
82
  "home.expert_label": "Vous êtes un chercheur",
83
  "home.btn_expert": "Mode expert",
84
+ "home.download_date": "Date de téléchargement de la base de donnée : 01/09/2025",
85
  "home.notice_title": "Mentions d'information sur les traitements de données à caractère personnel",
86
  "home.notice_p1": "Afin d'étudier le développement de la communauté de l'IA open source, et de préparer la possibilité d'exercices de droits des citoyens, le projet vise à étudier la base de données des jeux de données et modèles présents sur la plateforme HuggingFace. Cette base de données permet d'établir un arbre généalogique des modèles.",
87
  "home.notice_p2": "Les données traitées sont le pseudonyme de l'auteur (quand il apparaît dans les métadonnées), le nom du modèle/jeu de données et plusieurs informations inhérentes à ce modèle/jeu de données telles que la date de publication, la licence utilisée ou encore le nombre de téléchargements.",
 
98
  "search.page_lead_dataset": "Explorez la généalogie des modèles et datasets",
99
  "search.label_name": "Nom à rechercher",
100
  "search.placeholder_model": "Taper le nom du modèle ou l'id de son repo Hugging Face.",
101
+ "search.placeholder_dataset": "Taper le nom du dataset suspecté.",
102
  "search.label_filters": "Filtres",
103
  "search.filter_model": "Modèle",
104
  "search.filter_dataset": "Dataset",
 
108
  "search.btn_search": "Rechercher",
109
  "search.btn_search_simple": "Rechercher",
110
  "search.progress_title": "Recherche en cours…",
111
+ "search.progress_stage_queued": "En attente du serveur…",
112
  "search.progress_stage_preparing": "Préparation du graphe…",
113
  "search.progress_stage_descendants": "Recherche des descendants…",
114
  "search.progress_stage_ancestors": "Recherche des ascendants…",
 
122
  "search.progress_stage_cancelled": "Recherche annulée.",
123
  "search.progress_elapsed": "Temps écoulé : {seconds} s",
124
  "search.progress_items": "Éléments traités par le serveur : {completed}/{total}",
 
 
 
 
 
 
 
 
125
  "search.progress_server_percent": "Progression indiquée par le serveur : {percent} %",
126
  "search.progress_remaining_step": "Temps restant estimé pour cette étape : {seconds} s",
127
  "search.progress_server_note": "La progression repose sur les opérations réellement terminées par le serveur.",
 
187
  # ── expert.html ──
188
  "expert.page_title": "Recherche Experte",
189
  "expert.page_lead": "Explorez et filtrez la généalogie des modèles",
190
+ "expert.placeholder": "Taper le nom du modèle suspecté.",
191
  "expert.filter_author": "Auteur",
192
  "expert.connected_component": "Composante connexe du noeud recherché",
193
  "expert.legend_title": "Légendes",
 
273
  "site.nav_brand": "Model Genealogy",
274
  "site.nav_brand_model_expert": "Model Genealogy",
275
  "site.nav_subtitle": "Exploration of relations between models and datasets",
276
+ "site.nav_subtitle_full": "Exploration of relations between models and datasets (database updated on 01/09/2025)",
 
 
277
  "site.footer": "Application for searching and visualizing relations between models and datasets published on the HuggingFace platform. © 2025",
278
  "site.footer_expert": "Application for searching and visualizing... © 2025",
279
 
 
310
  "home.btn_unsure": "I don't know",
311
  "home.expert_label": "You are a researcher",
312
  "home.btn_expert": "Expert mode",
313
+ "home.download_date": "Database download date: 01/09/2025",
314
  "home.notice_title": "Information on the processing of personal data",
315
  "home.notice_p1": "In order to study the development of the open-source AI community and to prepare for the possibility of citizens exercising their rights, the project aims to study the database of datasets and models present on the HuggingFace platform. This database makes it possible to establish a genealogy tree of the models.",
316
  "home.notice_p2": "The data processed includes the author's pseudonym (when it appears in the metadata), the name of the model/dataset and several pieces of information inherent to this model/dataset such as the publication date, the license used or the number of downloads.",
 
337
  "search.btn_search": "Search",
338
  "search.btn_search_simple": "Search",
339
  "search.progress_title": "Search in progress…",
340
+ "search.progress_stage_queued": "Waiting for the server…",
341
  "search.progress_stage_preparing": "Preparing the graph…",
342
  "search.progress_stage_descendants": "Searching descendants…",
343
  "search.progress_stage_ancestors": "Searching ancestors…",
 
351
  "search.progress_stage_cancelled": "Search cancelled.",
352
  "search.progress_elapsed": "Elapsed time: {seconds} s",
353
  "search.progress_items": "Items processed by the server: {completed}/{total}",
 
 
 
 
 
 
 
 
354
  "search.progress_server_percent": "Progress reported by the server: {percent}%",
355
  "search.progress_remaining_step": "Estimated time remaining for this step: {seconds} s",
356
  "search.progress_server_note": "Progress is based on operations actually completed by the server.",
database_refresh/README.md DELETED
@@ -1,70 +0,0 @@
1
- # Actualisation hebdomadaire de la base
2
-
3
- La source est le dataset public
4
- [`cfahlgren1/hub-stats`](https://huggingface.co/datasets/cfahlgren1/hub-stats).
5
- Le pipeline télécharge ses configurations `models` et `datasets`, reconstruit
6
- hors ligne une base Neo4j, crée `neo4j.dump`, puis publie ce dump et son fichier
7
- `database_metadata.json` dans le dataset privé configuré.
8
-
9
- Le Space continue à utiliser le dump précédent pendant toute la reconstruction.
10
- Il ne voit la nouvelle base qu'après un redémarrage. La date affichée dans
11
- l'interface vient de `database_metadata.json`; elle n'est donc mise à jour que
12
- si la reconstruction et la publication ont réussi.
13
-
14
- ## Test sans modifier la production
15
-
16
- Le dépôt de dumps accepte une branche de test :
17
-
18
- ```bash
19
- export HF_TOKEN=hf_...
20
- bash database_refresh/run_weekly_refresh.sh \
21
- --dump-revision weekly-refresh
22
- ```
23
-
24
- Pour un test rapide du prétraitement :
25
-
26
- ```bash
27
- bash database_refresh/run_weekly_refresh.sh \
28
- --max-models 10000 \
29
- --max-datasets 10000 \
30
- --prepare-only
31
- ```
32
-
33
- ## Job Hugging Face hebdomadaire
34
-
35
- Le Job peut monter en lecture la branche du Space qui contient ce script et
36
- utiliser l'image Neo4j correspondant exactement à celle de l'application :
37
-
38
- Le compte qui crée le Job doit avoir au minimum le rôle `write` dans
39
- l'organisation CNIL. Le token utilisé par la CLI doit explicitement autoriser
40
- `Start and manage Jobs`. Puisque le même token est transmis au Job avec
41
- `--secrets HF_TOKEN`, il doit également pouvoir écrire dans
42
- `cnil/genmod-dump-neo4j` et redémarrer le Space ciblé. L'option
43
- `--namespace cnil` est indispensable pour créer et facturer le Job sous
44
- l'organisation plutôt que sous le compte personnel.
45
-
46
- ```bash
47
- hf jobs scheduled run "@weekly" \
48
- --namespace cnil \
49
- --name genmod-weekly-database-refresh \
50
- --flavor cpu-basic \
51
- --timeout 12h \
52
- --no-concurrency \
53
- --secrets HF_TOKEN \
54
- --env NEO4J_DUMP_REPO=cnil/genmod-dump-neo4j \
55
- --env NEO4J_DUMP_REVISION=weekly-refresh \
56
- --env SPACES_TO_RESTART=cnil/genmod-faster \
57
- --volume hf://spaces/cnil/genmod@refresh-hub-database:/workspace:ro \
58
- neo4j:2025.09.0-community \
59
- bash /workspace/database_refresh/run_weekly_refresh.sh
60
- ```
61
-
62
- La révision `weekly-refresh` permet de valider le nouveau dump avec
63
- `genmod-faster` sans remplacer le dump de production. Après validation, il
64
- suffit de programmer le même Job avec `NEO4J_DUMP_REVISION=main`.
65
- Pour la production, `SPACES_TO_RESTART` doit alors être remplacé par
66
- `cnil/genmod`.
67
-
68
- Les Hugging Face Jobs nécessitent un solde positif et sont facturés uniquement
69
- pendant leur exécution. Un Space CPU Basic ne constitue pas à lui seul un cron
70
- fiable : il peut être mis en veille et son disque est éphémère.
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
database_refresh/refresh_database.py DELETED
@@ -1,484 +0,0 @@
1
- #!/usr/bin/env python3
2
- """Build and publish a Neo4j dump from the public cfahlgren1/hub-stats dataset."""
3
-
4
- from __future__ import annotations
5
-
6
- import argparse
7
- import json
8
- import os
9
- import shutil
10
- import subprocess
11
- import sys
12
- from datetime import datetime, timezone
13
- from pathlib import Path
14
-
15
- import duckdb
16
- from huggingface_hub import CommitOperationAdd, HfApi, hf_hub_download
17
-
18
-
19
- SOURCE_REPO = "cfahlgren1/hub-stats"
20
- DEFAULT_DUMP_REPO = "cnil/genmod-dump-neo4j"
21
- PARQUET_REVISION = "refs/convert/parquet"
22
-
23
-
24
- def log(message: str) -> None:
25
- print(f"[genmod-refresh] {message}", flush=True)
26
-
27
-
28
- def download_sources(work_dir: Path) -> tuple[Path, Path]:
29
- cache_dir = work_dir / "hf-cache"
30
- log(f"Downloading the model snapshot from {SOURCE_REPO}")
31
- models = Path(
32
- hf_hub_download(
33
- repo_id=SOURCE_REPO,
34
- repo_type="dataset",
35
- revision=PARQUET_REVISION,
36
- filename="models/train/0000.parquet",
37
- cache_dir=cache_dir,
38
- )
39
- )
40
- log(f"Downloading the dataset snapshot from {SOURCE_REPO}")
41
- datasets = Path(
42
- hf_hub_download(
43
- repo_id=SOURCE_REPO,
44
- repo_type="dataset",
45
- revision=PARQUET_REVISION,
46
- filename="datasets/train/0000.parquet",
47
- cache_dir=cache_dir,
48
- )
49
- )
50
- return models, datasets
51
-
52
-
53
- def sql_path(path: Path) -> str:
54
- return str(path).replace("'", "''")
55
-
56
-
57
- def create_views(
58
- connection: duckdb.DuckDBPyConnection,
59
- models_path: Path,
60
- datasets_path: Path,
61
- max_models: int | None,
62
- max_datasets: int | None,
63
- ) -> None:
64
- model_limit = f" LIMIT {max_models}" if max_models else ""
65
- dataset_limit = f" LIMIT {max_datasets}" if max_datasets else ""
66
- connection.execute(
67
- f"""
68
- CREATE VIEW source_models AS
69
- SELECT * EXCLUDE (_dedupe_rank)
70
- FROM (
71
- SELECT
72
- *,
73
- row_number() OVER (
74
- PARTITION BY id
75
- ORDER BY lastModified DESC NULLS LAST, _id DESC NULLS LAST
76
- ) AS _dedupe_rank
77
- FROM read_parquet('{sql_path(models_path)}')
78
- WHERE id IS NOT NULL AND trim(id) <> ''
79
- )
80
- WHERE _dedupe_rank = 1
81
- {model_limit}
82
- """
83
- )
84
- connection.execute(
85
- f"""
86
- CREATE VIEW source_datasets AS
87
- SELECT * EXCLUDE (_dedupe_rank)
88
- FROM (
89
- SELECT
90
- *,
91
- row_number() OVER (
92
- PARTITION BY id
93
- ORDER BY lastModified DESC NULLS LAST, _id DESC NULLS LAST
94
- ) AS _dedupe_rank
95
- FROM read_parquet('{sql_path(datasets_path)}')
96
- WHERE id IS NOT NULL AND trim(id) <> ''
97
- )
98
- WHERE _dedupe_rank = 1
99
- {dataset_limit}
100
- """
101
- )
102
- connection.execute(
103
- """
104
- CREATE VIEW base_model_edges AS
105
- SELECT DISTINCT
106
- base.id AS parent_id,
107
- child.id AS child_id,
108
- COALESCE(child.baseModels.relation, 'derived') AS relation_name
109
- FROM source_models AS child,
110
- UNNEST(child.baseModels.models) AS nested(base)
111
- WHERE child.baseModels IS NOT NULL
112
- AND base.id IS NOT NULL
113
- AND trim(base.id) <> ''
114
- AND child.id IS NOT NULL
115
- """
116
- )
117
- connection.execute(
118
- """
119
- CREATE VIEW model_dataset_edges AS
120
- SELECT DISTINCT
121
- substr(tag, 9) AS dataset_id,
122
- model.id AS model_id
123
- FROM source_models AS model,
124
- UNNEST(model.tags) AS nested(tag)
125
- WHERE starts_with(tag, 'dataset:')
126
- AND length(trim(substr(tag, 9))) > 0
127
- AND model.id IS NOT NULL
128
- """
129
- )
130
-
131
-
132
- def export_csv(
133
- connection: duckdb.DuckDBPyConnection,
134
- output_dir: Path,
135
- filename: str,
136
- header: str,
137
- query: str,
138
- ) -> Path:
139
- path = output_dir / filename
140
- header_path = output_dir / filename.replace(".csv", "-header.csv")
141
- header_path.write_text(header + "\n", encoding="utf-8")
142
- connection.execute(
143
- f"""
144
- COPY ({query})
145
- TO '{sql_path(path)}'
146
- (FORMAT CSV, HEADER false, DELIMITER ',', QUOTE '"', ESCAPE '"')
147
- """
148
- )
149
- log(f"Created {filename}")
150
- return path
151
-
152
-
153
- def prepare_csv_files(
154
- models_path: Path,
155
- datasets_path: Path,
156
- output_dir: Path,
157
- max_models: int | None = None,
158
- max_datasets: int | None = None,
159
- ) -> dict[str, Path]:
160
- output_dir.mkdir(parents=True, exist_ok=True)
161
- database_path = output_dir / "refresh.duckdb"
162
- connection = duckdb.connect(str(database_path))
163
- connection.execute("SET preserve_insertion_order = false")
164
- connection.execute("SET threads = 2")
165
- create_views(connection, models_path, datasets_path, max_models, max_datasets)
166
-
167
- files: dict[str, Path] = {}
168
- files["models"] = export_csv(
169
- connection,
170
- output_dir,
171
- "models.csv",
172
- "modelId:ID(Model),name,downloads:long,task,createdAt,parameters,likes:long,license",
173
- """
174
- WITH actual_models AS (
175
- SELECT
176
- id,
177
- id AS name,
178
- downloadsAllTime AS downloads,
179
- pipeline_tag AS task,
180
- CAST(createdAt AS VARCHAR) AS created_at,
181
- CASE
182
- WHEN safetensors.total >= 1000000000
183
- THEN printf('%.1fB', safetensors.total / 1000000000.0)
184
- WHEN safetensors.total >= 1000000
185
- THEN printf('%.1fM', safetensors.total / 1000000.0)
186
- WHEN safetensors.total >= 1000
187
- THEN printf('%.1fK', safetensors.total / 1000.0)
188
- WHEN safetensors.total IS NOT NULL
189
- THEN CAST(safetensors.total AS VARCHAR)
190
- END AS parameters,
191
- likes,
192
- json_extract_string(cardData, '$.license') AS license
193
- FROM source_models
194
- WHERE id IS NOT NULL AND trim(id) <> ''
195
- ),
196
- missing_parents AS (
197
- SELECT DISTINCT parent_id AS id
198
- FROM base_model_edges
199
- WHERE parent_id NOT IN (SELECT id FROM actual_models)
200
- )
201
- SELECT id, name, downloads, task, created_at, parameters, likes, license
202
- FROM actual_models
203
- UNION ALL
204
- SELECT id, id, NULL, NULL, NULL, NULL, NULL, NULL
205
- FROM missing_parents
206
- """,
207
- )
208
- files["datasets"] = export_csv(
209
- connection,
210
- output_dir,
211
- "datasets.csv",
212
- "datasetId:ID(Dataset),name,downloads:long,createdAt_dataset",
213
- """
214
- WITH actual_datasets AS (
215
- SELECT
216
- id,
217
- id AS name,
218
- downloadsAllTime AS downloads,
219
- CAST(createdAt AS VARCHAR) AS created_at
220
- FROM source_datasets
221
- WHERE id IS NOT NULL AND trim(id) <> ''
222
- ),
223
- missing_datasets AS (
224
- SELECT DISTINCT dataset_id AS id
225
- FROM model_dataset_edges
226
- WHERE dataset_id NOT IN (SELECT id FROM actual_datasets)
227
- )
228
- SELECT id, name, downloads, created_at
229
- FROM actual_datasets
230
- UNION ALL
231
- SELECT id, id, NULL, NULL
232
- FROM missing_datasets
233
- """,
234
- )
235
- files["authors"] = export_csv(
236
- connection,
237
- output_dir,
238
- "authors.csv",
239
- "authorId:ID(Author),name,type,followers:long",
240
- """
241
- SELECT author, author, 'unknown', NULL
242
- FROM (
243
- SELECT author FROM source_models
244
- UNION
245
- SELECT author FROM source_datasets
246
- )
247
- WHERE author IS NOT NULL AND trim(author) <> ''
248
- """,
249
- )
250
- files["base_model_edges"] = export_csv(
251
- connection,
252
- output_dir,
253
- "base-model-edges.csv",
254
- ":START_ID(Model),:END_ID(Model),name",
255
- "SELECT parent_id, child_id, relation_name FROM base_model_edges",
256
- )
257
- files["model_dataset_edges"] = export_csv(
258
- connection,
259
- output_dir,
260
- "model-dataset-edges.csv",
261
- ":START_ID(Dataset),:END_ID(Model),name",
262
- """
263
- SELECT dataset_id, model_id, 'A été utilisé dans ce modèle'
264
- FROM model_dataset_edges
265
- """,
266
- )
267
- files["author_model_edges"] = export_csv(
268
- connection,
269
- output_dir,
270
- "author-model-edges.csv",
271
- ":START_ID(Author),:END_ID(Model),name",
272
- """
273
- SELECT DISTINCT author, id, 'A publié'
274
- FROM source_models
275
- WHERE author IS NOT NULL AND trim(author) <> '' AND id IS NOT NULL
276
- """,
277
- )
278
- files["author_dataset_edges"] = export_csv(
279
- connection,
280
- output_dir,
281
- "author-dataset-edges.csv",
282
- ":START_ID(Author),:END_ID(Dataset),name",
283
- """
284
- SELECT DISTINCT author, id, 'A publié'
285
- FROM source_datasets
286
- WHERE author IS NOT NULL AND trim(author) <> '' AND id IS NOT NULL
287
- """,
288
- )
289
- connection.close()
290
- database_path.unlink(missing_ok=True)
291
- return files
292
-
293
-
294
- def header_for(path: Path) -> Path:
295
- return path.with_name(path.name.replace(".csv", "-header.csv"))
296
-
297
-
298
- def build_dump(files: dict[str, Path], output_dir: Path, neo4j_admin: str) -> Path:
299
- def group(name: str) -> str:
300
- return f"{header_for(files[name])},{files[name]}"
301
-
302
- command = [
303
- neo4j_admin,
304
- "database",
305
- "import",
306
- "full",
307
- "neo4j",
308
- "--overwrite-destination=true",
309
- "--id-type=string",
310
- "--threads=2",
311
- "--verbose",
312
- f"--nodes=Model={group('models')}",
313
- f"--nodes=Dataset={group('datasets')}",
314
- f"--nodes=Author={group('authors')}",
315
- f"--relationships=USED_IN={group('base_model_edges')}",
316
- f"--relationships=USED_IN={group('model_dataset_edges')}",
317
- f"--relationships=POSTED={group('author_model_edges')}",
318
- f"--relationships=POSTED={group('author_dataset_edges')}",
319
- ]
320
- log("Building the offline Neo4j database")
321
- subprocess.run(command, check=True)
322
-
323
- dump_dir = output_dir / "dump"
324
- dump_dir.mkdir(exist_ok=True)
325
- log("Creating neo4j.dump")
326
- subprocess.run(
327
- [
328
- neo4j_admin,
329
- "database",
330
- "dump",
331
- "neo4j",
332
- f"--to-path={dump_dir}",
333
- "--overwrite-destination=true",
334
- ],
335
- check=True,
336
- )
337
- return dump_dir / "neo4j.dump"
338
-
339
-
340
- def write_metadata(
341
- output_dir: Path,
342
- source_revision: str,
343
- model_count: int,
344
- dataset_count: int,
345
- ) -> Path:
346
- metadata = {
347
- "built_at": datetime.now(timezone.utc).isoformat(),
348
- "source_repo": SOURCE_REPO,
349
- "source_revision": source_revision,
350
- "model_count": model_count,
351
- "dataset_count": dataset_count,
352
- }
353
- path = output_dir / "database_metadata.json"
354
- path.write_text(json.dumps(metadata, indent=2) + "\n", encoding="utf-8")
355
- return path
356
-
357
-
358
- def parquet_unique_id_count(path: Path) -> int:
359
- connection = duckdb.connect()
360
- count = connection.execute(
361
- f"""
362
- SELECT count(DISTINCT id)
363
- FROM read_parquet('{sql_path(path)}')
364
- WHERE id IS NOT NULL AND trim(id) <> ''
365
- """
366
- ).fetchone()[0]
367
- connection.close()
368
- return int(count)
369
-
370
-
371
- def publish_dump(
372
- api: HfApi,
373
- dump_path: Path,
374
- metadata_path: Path,
375
- repo_id: str,
376
- revision: str,
377
- ) -> None:
378
- if revision != "main":
379
- api.create_branch(
380
- repo_id=repo_id,
381
- repo_type="dataset",
382
- branch=revision,
383
- exist_ok=True,
384
- )
385
- log(f"Publishing the dump to {repo_id}@{revision}")
386
- api.create_commit(
387
- repo_id=repo_id,
388
- repo_type="dataset",
389
- revision=revision,
390
- operations=[
391
- CommitOperationAdd(
392
- path_in_repo="neo4j.dump",
393
- path_or_fileobj=str(dump_path),
394
- ),
395
- CommitOperationAdd(
396
- path_in_repo="database_metadata.json",
397
- path_or_fileobj=str(metadata_path),
398
- ),
399
- ],
400
- commit_message="Refresh Neo4j graph from cfahlgren1/hub-stats",
401
- )
402
-
403
-
404
- def restart_spaces(api: HfApi, space_ids: list[str]) -> None:
405
- for space_id in space_ids:
406
- log(f"Restarting Space {space_id}")
407
- api.restart_space(repo_id=space_id)
408
-
409
-
410
- def parse_args() -> argparse.Namespace:
411
- parser = argparse.ArgumentParser()
412
- parser.add_argument("--work-dir", type=Path, default=Path("/tmp/genmod-refresh"))
413
- parser.add_argument("--dump-repo", default=os.getenv("NEO4J_DUMP_REPO", DEFAULT_DUMP_REPO))
414
- parser.add_argument("--dump-revision", default=os.getenv("NEO4J_DUMP_REVISION", "main"))
415
- parser.add_argument("--neo4j-admin", default=os.getenv("NEO4J_ADMIN", "neo4j-admin"))
416
- parser.add_argument("--models-parquet", type=Path)
417
- parser.add_argument("--datasets-parquet", type=Path)
418
- parser.add_argument("--max-models", type=int)
419
- parser.add_argument("--max-datasets", type=int)
420
- parser.add_argument("--prepare-only", action="store_true")
421
- parser.add_argument("--no-upload", action="store_true")
422
- parser.add_argument("--keep-work-dir", action="store_true")
423
- parser.add_argument("--restart-space", action="append", default=[])
424
- return parser.parse_args()
425
-
426
-
427
- def main() -> int:
428
- args = parse_args()
429
- if args.work_dir.exists() and not args.keep_work_dir:
430
- shutil.rmtree(args.work_dir)
431
- args.work_dir.mkdir(parents=True, exist_ok=True)
432
-
433
- api = HfApi()
434
- source_revision = api.dataset_info(SOURCE_REPO).sha
435
- if bool(args.models_parquet) != bool(args.datasets_parquet):
436
- raise SystemExit("Provide both --models-parquet and --datasets-parquet.")
437
- if args.models_parquet:
438
- models_path, datasets_path = args.models_parquet, args.datasets_parquet
439
- else:
440
- models_path, datasets_path = download_sources(args.work_dir)
441
-
442
- csv_dir = args.work_dir / "csv"
443
- files = prepare_csv_files(
444
- models_path,
445
- datasets_path,
446
- csv_dir,
447
- max_models=args.max_models,
448
- max_datasets=args.max_datasets,
449
- )
450
- if args.prepare_only:
451
- log(f"CSV preparation completed in {csv_dir}")
452
- return 0
453
-
454
- dump_path = build_dump(files, args.work_dir, args.neo4j_admin)
455
- model_count = args.max_models or parquet_unique_id_count(models_path)
456
- dataset_count = args.max_datasets or parquet_unique_id_count(datasets_path)
457
- metadata_path = write_metadata(
458
- args.work_dir,
459
- source_revision,
460
- model_count,
461
- dataset_count,
462
- )
463
- if not args.no_upload:
464
- if not os.getenv("HF_TOKEN"):
465
- raise SystemExit("HF_TOKEN is required to upload the refreshed dump.")
466
- publish_dump(
467
- api,
468
- dump_path,
469
- metadata_path,
470
- args.dump_repo,
471
- args.dump_revision,
472
- )
473
- configured_spaces = [
474
- value.strip()
475
- for value in os.getenv("SPACES_TO_RESTART", "").split(",")
476
- if value.strip()
477
- ]
478
- restart_spaces(api, list(dict.fromkeys(configured_spaces + args.restart_space)))
479
- log("Refresh completed successfully")
480
- return 0
481
-
482
-
483
- if __name__ == "__main__":
484
- sys.exit(main())
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
database_refresh/run_weekly_refresh.sh DELETED
@@ -1,7 +0,0 @@
1
- #!/bin/bash
2
- set -e
3
- apt-get update
4
- apt-get install -y python3-pip wget
5
- pip3 install duckdb==1.3.2 huggingface-hub==0.31.4
6
- wget -qO /tmp/r.py https://huggingface.co/spaces/cnil/genmod/resolve/refresh-hub-database/database_refresh/refresh_database.py
7
- exec python3 /tmp/r.py
 
 
 
 
 
 
 
 
start.sh CHANGED
@@ -1,37 +1,11 @@
1
- set -euo pipefail
2
-
3
- DUMP_REPO="${GENMOD_DUMP_REPO:-cnil/genmod-dump-neo4j}"
4
- DUMP_REVISION="${GENMOD_DUMP_REVISION:-main}"
5
- DUMP_BASE_URL="https://huggingface.co/datasets/${DUMP_REPO}/resolve/${DUMP_REVISION}"
6
- AUTH_HEADER="Authorization: Bearer ${HF_TOKEN}"
7
-
8
- # Load the database backup. Metadata is optional for compatibility with the
9
- # original dump; the application then falls back to its historical date.
10
- wget --header="${AUTH_HEADER}" "${DUMP_BASE_URL}/system.dump" -O /backups/system.dump
11
- wget --header="${AUTH_HEADER}" "${DUMP_BASE_URL}/neo4j.dump" -O /backups/neo4j.dump
12
- if ! wget --header="${AUTH_HEADER}" "${DUMP_BASE_URL}/database_metadata.json" \
13
- -O /backups/database_metadata.json; then
14
- rm -f /backups/database_metadata.json
15
- fi
16
  neo4j-admin database load --expand-commands system --from-path=/backups --overwrite-destination=true
17
  neo4j-admin database load --expand-commands neo4j --from-path=/backups --overwrite-destination=true
18
 
19
  # start database
20
  /startup/docker-entrypoint.sh neo4j &
21
-
22
- # The Community importer cannot create indexes through --schema. Ensure that
23
- # dumps produced by the refresh Job receive the required indexes before the
24
- # application starts serving searches.
25
- until cypher-shell -u neo4j -p genealogiemodeles \
26
- "RETURN 1;" >/dev/null 2>&1; do
27
- sleep 2
28
- done
29
- cypher-shell -u neo4j -p genealogiemodeles \
30
- "CREATE INDEX IF NOT EXISTS FOR (m:Model) ON (m.name);
31
- CREATE INDEX IF NOT EXISTS FOR (a:Author) ON (a.name);
32
- CREATE INDEX IF NOT EXISTS FOR (d:Dataset) ON (d.name);
33
- CALL db.awaitIndexes(600);"
34
-
35
  # start tool
36
  python3 /application_neo4j/app.py neo4j genealogiemodeles &
37
  # wait for any process to exit
 
1
+ # load database backup
2
+ wget --no-clobber --header="Authorization: Bearer $HF_TOKEN" https://huggingface.co/datasets/cnil/genmod-dump-neo4j/resolve/main/system.dump -P /backups/
3
+ wget --no-clobber --header="Authorization: Bearer $HF_TOKEN" https://huggingface.co/datasets/cnil/genmod-dump-neo4j/resolve/main/neo4j.dump -P /backups/
 
 
 
 
 
 
 
 
 
 
 
 
4
  neo4j-admin database load --expand-commands system --from-path=/backups --overwrite-destination=true
5
  neo4j-admin database load --expand-commands neo4j --from-path=/backups --overwrite-destination=true
6
 
7
  # start database
8
  /startup/docker-entrypoint.sh neo4j &
 
 
 
 
 
 
 
 
 
 
 
 
 
 
9
  # start tool
10
  python3 /application_neo4j/app.py neo4j genealogiemodeles &
11
  # wait for any process to exit