Auto-sync from demo at Fri Nov 7 11:01:47 UTC 2025

graphgen/operators/read/read_files.py

@@ -1,3 +1,6 @@
+from pathlib import Path
+from typing import Any, Dict, List
+
 from graphgen.models import (
     CSVReader,
     JSONLReader,
@@ -8,6 +11,7 @@ from graphgen.models import (
     RDFReader,
     TXTReader,
 )
+from graphgen.utils import logger
 
 _MAPPING = {
     "jsonl": JSONLReader,
@@ -23,17 +27,36 @@ _MAPPING = {
 }
 
 
+def _build_reader(suffix: str, cache_dir: str | None):
+    suffix = suffix.lower()
+    if suffix == "pdf" and cache_dir is not None:
+        return _MAPPING[suffix](output_dir=cache_dir)
+    return _MAPPING[suffix]()
+
+
 def read_files(file_path: str, cache_dir: str | None = None) -> list[dict]:
-    suffix = file_path.split(".")[-1].lower()
-    if suffix == "pdf":
-        if cache_dir is not None:
-            reader = _MAPPING[suffix](output_dir=cache_dir)
-        else:
-            reader = _MAPPING[suffix]()
-    elif suffix in _MAPPING:
-        reader = _MAPPING[suffix]()
-    else:
-        raise ValueError(
-            f"Unsupported file format: {suffix}. Supported formats are: {list(_MAPPING.keys())}"
-        )
-    return reader.read(file_path)
+    path = Path(file_path).expanduser()
+    if not path.exists():
+        raise FileNotFoundError(f"input_path not found: {file_path}")
+
+    if path.is_file():
+        suffix = path.suffix.lstrip(".")
+        reader = _build_reader(suffix, cache_dir)
+        return reader.read(str(path))
+
+    support_suffix = set(_MAPPING.keys())
+    files_to_read = [
+        p for p in path.rglob("*") if p.suffix.lstrip(".").lower() in support_suffix
+    ]
+    logger.info("Found %d file(s) under folder %s", len(files_to_read), file_path)
+
+    all_docs: List[Dict[str, Any]] = []
+    for p in files_to_read:
+        try:
+            suffix = p.suffix.lstrip(".")
+            reader = _build_reader(suffix, cache_dir)
+            all_docs.extend(reader.read(str(p)))
+        except Exception as e:  # pylint: disable=broad-except
+            logger.exception("Error reading %s: %s", p, e)
+
+    return all_docs