Change path setting

api/data_pipeline.py

@@ -221,7 +221,8 @@ def read_all_documents(path: str, is_ollama_embedder: bool = None, excluded_dirs
         Returns:
             bool: True if the file should be processed, False otherwise
         """
-        file_path_parts = os.path.normpath(file_path).split(os.sep)
+        # Normalize the file path for cross-platform compatibility
+        normalized_path = os.path.normpath(file_path).replace(os.sep, '/')
         file_name = os.path.basename(file_path)
 
         if use_inclusion:
@@ -232,7 +233,9 @@ def read_all_documents(path: str, is_ollama_embedder: bool = None, excluded_dirs
             if included_dirs:
                 for included in included_dirs:
                     clean_included = included.strip("./").rstrip("/")
-                    if clean_included in file_path_parts:
+                    # Check if the directory appears in the path - handle both .git and git patterns
+                    path_parts = normalized_path.split('/')
+                    if clean_included in path_parts or f".{clean_included}" in path_parts:
                         is_included = True
                         break
 
@@ -261,14 +264,23 @@ def read_all_documents(path: str, is_ollama_embedder: bool = None, excluded_dirs
             # Check if file is in an excluded directory
             for excluded in excluded_dirs:
                 clean_excluded = excluded.strip("./").rstrip("/")
-                if clean_excluded in file_path_parts:
+                # Check if the directory appears in the path - handle both .git and git patterns
+                path_parts = normalized_path.split('/')
+                if clean_excluded in path_parts or f".{clean_excluded}" in path_parts:
                     is_excluded = True
                     break
 
-            # Check if file matches excluded file patterns
+            # Check if file matches excluded file patterns  
             if not is_excluded:
                 for excluded_file in excluded_files:
-                    if file_name == excluded_file:
+                    # Handle pattern matching more robustly
+                    if excluded_file.startswith("*."):
+                        # Handle wildcard patterns like "*.pyc"
+                        extension = excluded_file[1:]  # Remove the *
+                        if file_name.endswith(extension):
+                            is_excluded = True
+                            break
+                    elif file_name == excluded_file:
                         is_excluded = True
                         break