Update app.py

app.py

@@ -99,11 +99,11 @@ def download_gitlab_repo():
         print("Upload complete")
 
 
-def get_all_files_in_folder(temp_dir, partial_path):
+def get_all_files_in_folder(temp_dir, folder_path):
     
     all_files = [] 
     print("inner method of get all files in folder")
-    target_dir = os.path.join(temp_dir, partial_path)
+    target_dir = os.path.join(temp_dir, folder_path)
     print(target_dir)
 
     for root, dirs, files in os.walk(target_dir):
@@ -119,9 +119,8 @@ def get_file(temp_dir, file_path):
     return full_path
 
 
-
-def process_directory(directory):
-    code_partial_paths = ['kadi_apy/lib/resources/']
+#getFilesFromRepo
+def process_directory(directory, folder_paths, file_paths):
 
 
     zip_filename = next((file for file in os.listdir(directory) if file.endswith('.zip')), None) # zip_filename: kadi-apy-master-2a244f1af1483b48f8f9c0d99ce2744a0950c834.zip
@@ -142,6 +141,37 @@ def process_directory(directory):
 
             tmpsubdirpath= os.path.join(tmpdirname, unzipped_root[0])  # /tmp/tmpux1v52wy/kadi-apy-master-2a244f1af1483b48f8f9c0d99ce2744a0950c834
             print("tempsubdirpath: ", tmpsubdirpath)
+
+            if folder_paths:
+                for folder_path in folder_paths:
+                    files += get_all_files_in_folder(tmpsubdirpath, folder_paths) 
+            if file_paths:
+                files += [get_file(tmpsubdirpath, file_path) for file_path in file_paths] 
+
+
+            print(f"Total number of files: {len(files)}")
+            for file_path in files:
+                #print(f"Paths of files: {iles}")
+                file_ext = os.path.splitext(file_path)[1]
+
+                if os.path.getsize(file_path) == 0:
+                    print(f"Skipping an empty file: {file_path}")
+                    continue
+
+                with open(file_path, 'rb') as f:
+                    if file_ext in ['.rst', '.md', '.txt', '.html', '.json', '.yaml', '.py']:
+                        text = f.read().decode('utf-8')
+                    elif file_ext in ['.svg']:
+                        text = f"SVG file content from {file_path}"
+                    elif file_ext in ['.png', '.ico']:
+                        text = f"Image metadata from {file_path}"
+                    else
+                        continue
+
+                    all_texts.append(text)
+                    file_references.append(file_path)
+
+    return all_texts, file_references
                 
  #   with tempfile.TemporaryDirectory() as tmpdirname:
     # Unzip the file into the temporary directory
@@ -150,7 +180,8 @@ def process_directory(directory):
 
 
   #      unzipped_root = os.listdir(tmpdirname)
-    
+
+            
 def process_directory5(directory, partial_paths=None, file_paths=None):
     all_texts = []
     file_references = []
@@ -211,7 +242,7 @@ def process_directory5(directory, partial_paths=None, file_paths=None):
                         text = f"SVG file content from {file_path}"
                     elif file_ext in ['.png', '.ico']:
                         text = f"Image metadata from {file_path}"
-                    else:
+                    else
                         continue
 
                     all_texts.append(text)
@@ -413,15 +444,16 @@ def rag_workflow(query):
 def initialize():
     global docstore, codestore, chunks, llm
     #download_gitlab_repo()
-    code_partial_paths = ['kadi_apy/lib/resources/']
+    #code_partial_paths = ['kadi_apy/lib/resources/']
     code_file_path = []
     #doc_partial_paths = []
     #doc_partial_paths = ['docs/source/setup/']
     #doc_file_paths = ['docs/source/usage/lib.rst']
 
 
-
-    process_directory(DATA_DIR)
+    code_partial_paths = ['kadi_apy/lib/resources/']
+    
+    process_directory(DATA_DIR,code_partial_paths, code_file_path)
     
     #code_files, code_file_references = process_directory5(DATA_DIR, code_partial_paths, code_file_path)