conf files

Dockerfile

@@ -0,0 +1,14 @@
+# read the doc: https://huggingface.co/docs/hub/spaces-sdks-docker
+# you will also find guides on how best to write your Dockerfile
+
+FROM python:3.9
+
+WORKDIR /code
+
+COPY ./requirements.txt /code/requirements.txt
+
+RUN pip install --no-cache-dir --upgrade -r /code/requirements.txt
+
+COPY . .
+
+CMD ["uvicorn", "api.main:app", "--host", "0.0.0.0", "--port", "7860"]

api/external_services.py

@@ -0,0 +1,111 @@
+import re
+import os
+from fastapi import HTTPException
+from dotenv import load_dotenv
+from llama_index import download_loader
+from llama_hub.github_repo import GithubRepositoryReader, GithubClient
+from llama_index import VectorStoreIndex
+from llama_index.vector_stores import DeepLakeVectorStore
+from llama_index.storage.storage_context import StorageContext
+import yaml
+
+load_dotenv()
+
+# Fetch and set API keys
+openai_api_key = os.getenv("OPENAI_API_KEY")
+
+
+# Check for OpenAI API key
+if not openai_api_key:
+    raise EnvironmentError("OpenAI API key not found in environment variables")
+
+
+def get_validate_token(token_name):
+    token = os.getenv(token_name)
+    if not token:
+        raise EnvironmentError(f"{token_name} not found in environment variables")
+    return token
+
+
+class InitiazlizeGithubService:
+    def __init__(self):
+        self.owner = None
+        self.repo = None
+        self.github_token = get_validate_token("GITHUB_TOKEN")  # Check for GitHub Token
+        self.github_client = self.initialize_github_client(self.github_token)
+        download_loader("GithubRepositoryReader")
+
+    def initialize_github_client(self, github_token):
+        return GithubClient(github_token)
+
+    def parse_github_url(self, url):
+        pattern = r"https://github\.com/([^/]+)/([^/]+)"
+        match = re.match(pattern, url)
+        return match.groups() if match else (None, None)
+
+    def validate_owner_repo(self, owner, repo):
+        if bool(owner) and bool(repo):
+            self.owner = owner
+            self.repo = repo
+            return True
+
+        return False
+
+    def load_repo_data(self, owner, repo):
+        if self.validate_owner_repo(owner, repo):
+            loader = GithubRepositoryReader(
+                self.github_client,
+                owner=self.owner,
+                repo=self.repo,
+                filter_file_extensions=(
+                    [".py", ".js", ".ts", ".md"],
+                    GithubRepositoryReader.FilterType.INCLUDE,
+                ),
+                verbose=False,
+                concurrent_requests=5,
+            )
+
+            print(f"Loading {self.repo} repository by {self.owner}")
+
+            docs = loader.load_data(branch="main")
+            print("Documents uploaded:")
+            for doc in docs:
+                print(doc.metadata)
+
+            return docs
+
+        else:
+            raise HTTPException(
+                status_code=400,
+                detail="Invalid GitHub URL. Please enter a valid GitHub URL",
+            )
+
+
+class InitiazlizeActiveloopService:
+    def __init__(self):
+        self.active_loop_token = get_validate_token(
+            "ACTIVELOOP_TOKEN"
+        )  # Check for Activeloop Token
+        self.dataset_path = self.get_user_info("dataset_path")
+        self.vector_store = DeepLakeVectorStore(
+            dataset_path=f"hub://{self.dataset_path}",
+            overwrite=True,
+            runtime={"tensor_db": True},
+        )
+
+        self.storage_context = StorageContext.from_defaults(
+            vector_store=self.vector_store
+        )
+
+    def upload_to_activeloop(self, docs):
+        self.index = VectorStoreIndex.from_documents(
+            docs, storage_context=self.storage_context
+        )
+        self.query_engine = self.index.as_query_engine()
+
+    def get_user_info(self, user_info):
+        with open("resources.yaml", "r") as file:
+            yaml_data = yaml.safe_load(file)
+
+        retrieved_info = yaml_data["info"][user_info]
+        return retrieved_info

api/main.py

@@ -0,0 +1,59 @@
+import textwrap
+from fastapi import FastAPI
+from pydantic import BaseModel
+from dotenv import load_dotenv
+
+from external_services import InitiazlizeGithubService, InitiazlizeActiveloopService
+
+# Load environment variables
+load_dotenv()
+
+
+app = FastAPI()
+
+
+class GitHubRepoRequest(BaseModel):
+    githubRepoUrl: str
+
+
+class UserCodeRequest(BaseModel):
+    userCode: str
+
+
+@app.post("/upload")
+async def scrape_and_upload_to_activeloop(repo_request: GitHubRepoRequest):
+    # Add logic to scrape and upload to ActiveLoop
+    # Example: Scrape GitHub repo and upload to ActiveLoop
+    # Implement your scraping and upload logic here
+    github_service = InitiazlizeGithubService()
+    activeloop_service = InitiazlizeActiveloopService()
+
+    print(f"repo from user: {repo_request.githubRepoUrl}")
+
+    owner, repo = github_service.parse_github_url(repo_request.githubRepoUrl)
+    docs = github_service.load_repo_data(owner, repo)
+    activeloop_service.upload_to_activeloop(docs)
+
+    return {"status": "success", "message": "Repo processed successfully"}
+
+
+@app.post("/retrieve")
+async def find_similar_code_and_explain(code_request: UserCodeRequest):
+    # Add logic to find similar code and provide explanations or improvements
+    # Example: Search in ActiveLoop DB
+    # Implement your search and analysis logic here
+    activeloop_service = InitiazlizeActiveloopService()
+
+    print(f"code from user: {code_request.userCode}")
+
+    # intro_question = "What is the repository about?"
+    intro_question = code_request.userCode
+    print(f"Test question: {intro_question}")
+    print("=" * 50)
+
+    answer = activeloop_service.query_engine.query(intro_question)
+    print(f"Answer: {textwrap.fill(str(answer), 100)} \n")
+
+    return {
+        "answer": answer,
+    }

requirements.txt

@@ -0,0 +1,5 @@
+llama-index==0.9.38
+deeplake==3.8.17
+openai==1.10.0
+python-dotenv==1.0.1
+llama-hub==0.0.76

resources.yaml

@@ -0,0 +1,2 @@
+info:
+  dataset_path: manufe_test/code_retriever