reduce upload time

api/external_services.py

@@ -51,21 +51,23 @@ class InitiazlizeGithubService:
 
         return False
 
-    def load_repo_data(self, owner, repo):
+    def load_repo_data(self, owner, repo, file_type):
         if self.validate_owner_repo(owner, repo):
             loader = GithubRepositoryReader(
                 self.github_client,
                 owner=self.owner,
                 repo=self.repo,
                 filter_file_extensions=(
-                    [".py", ".js", ".ts", ".md", "ipynb"],
+                    [file_type],
                     GithubRepositoryReader.FilterType.INCLUDE,
                 ),
                 verbose=False,
                 concurrent_requests=25,
             )
 
-            print(f"Loading {self.repo} repository by {self.owner}")
+            print(
+                f"Loading {self.repo} repository by {self.owner}, file type: {file_type}"
+            )
 
             docs = loader.load_data(branch="main")
             print("Documents uploaded:")

api/main.py

@@ -31,8 +31,10 @@ async def scrape_and_upload_to_activeloop(repo_request: GitHubRepoRequest):
     print(f"repo from user: {repo_request.githubRepoUrl}")
 
     owner, repo = github_service.parse_github_url(repo_request.githubRepoUrl)
-    docs = github_service.load_repo_data(owner, repo)
-    activeloop_service.upload_to_activeloop(docs)
+    file_types = [".py", ".js", ".ts", ".md", "ipynb"]
+    for file_type in file_types:
+        docs = github_service.load_repo_data(owner, repo, file_type)
+        activeloop_service.upload_to_activeloop(docs)
 
     return {"status": "success", "message": "Repo processed successfully"}