Ruff

app.py

@@ -31,20 +31,39 @@ def get_query_issue_information(issue_no, token):
     return request.json()
 
 
-def run_find_similar_issues(token, n_issues, issue_no, query):
+def run_find_similar_issues(token, n_issues, issue_no, query, issue_types):
     if issue_no == "":
         issue_no = None
     if query == "":
         query = None
 
-    similar_issues = get_similar_issues(issue_no=issue_no, query=query, token=token, top_k=n_issues)
+    if len(issue_types) == 0:
+        raise ValueError("At least one issue type must be selected")
+
+    similar_issues = []
+    similar_pulls = []
+    if "Issue" in issue_types:
+        similar_issues = get_similar_issues(issue_no=issue_no, query=query, token=token, top_k=n_issues, issue_type="issue")
+    if "Pull Request" in issue_types:
+        similar_pulls = get_similar_issues(issue_no=issue_no, query=query, token=token, top_k=n_issues, issue_type="pull")
 
     issues_html = [f"<a href='{issue['html_url']}' target='_blank'>#{issue['number']} - {issue['title']}</a>" for issue in similar_issues]
     issues_html = "<br>".join(issues_html)
-    return issues_html
 
+    pulls_html = [f"<a href='{issue['html_url']}' target='_blank'>#{issue['number']} - {issue['title']}</a>" for issue in similar_pulls]
+    pulls_html = "<br>".join(pulls_html)
+
+    final = ""
+    if len(issues_html) > 0:
+        final += f"<h2>Issues</h2>{issues_html}"
+    if len(pulls_html) > 0:
+        final += f"<h2>Pull Requests</h2>{pulls_html}"
+
+    # return issues_html
+    return final
 
-def update_issues():
+
+def update():
     # Archive the stored issues
     if os.path.exists("issues.json"):
         date_time = datetime.datetime.now().strftime("%Y-%m-%d_%H-%M-%S")
@@ -63,6 +82,12 @@ def update_issues():
         model_id="all-mpnet-base-v2",
         update=True
     )
+    build_embeddings.embed_issues(
+        input_filename="issues_dict.json",
+        issue_type="pull",
+        model_id="all-mpnet-base-v2",
+        update=True
+    )
 
 
 with gr.Blocks(title="Github Bot") as demo:
@@ -76,8 +101,9 @@ with gr.Blocks(title="Github Bot") as demo:
             with gr.Column():
                 token = gr.Textbox(label="Github Token", placeholder="Your github token for authentication. This is not stored anywhere.")
                 n_issues = gr.Slider(1, 50, value=5, step=1, label="Number of similar issues", info="Choose between 1 and 50")
-                update_button = gr.Button(value="Update issues")
-                update_button.click(update_issues)
+                issue_types = gr.CheckboxGroup(["Issue", "Pull Request"], label="Issue types")
+                update_button = gr.Button(value="Update issues", trigger_mode="once")
+                update_button.click(update, outputs=[], inputs=[])
 
         with gr.Row():
                 submit_button = gr.Button(value="Submit")
@@ -85,7 +111,7 @@ with gr.Blocks(title="Github Bot") as demo:
         with gr.Row():
             with gr.Row():
                 issues_html = gr.HTML(label="Issue text", elem_id="issue_html")
-                submit_button.click(run_find_similar_issues, outputs=[issues_html], inputs=[token, n_issues, issue_no, query])
+                submit_button.click(run_find_similar_issues, outputs=[issues_html], inputs=[token, n_issues, issue_no, query, issue_types])
 
     with gr.Tab("Find maintainers to ping"):
         with gr.Row():

defaults.py

@@ -3,3 +3,4 @@ import os
 OWNER = "huggingface"
 REPO = "transformers"
 TOKEN = os.environ.get("GITHUB_TOKEN")
+GITHUB_API_VERSION = "2022-11-28"

fetch.py

@@ -2,16 +2,16 @@
 Script to fetch issues from the transformers repo and save them to a json file
 """
 
-import json
-
 import argparse
+import datetime
+import logging
+import json
+import os
 
 import requests
-import os
 import numpy as np
-import json
-import datetime
-import logging
+
+from defaults import OWNER, REPO, GITHUB_API_VERSION, TOKEN
 
 logging.basicConfig(level=logging.INFO)
 
@@ -19,11 +19,7 @@ logger = logging.getLogger(__name__)
 
 today = datetime.datetime.now().strftime("%Y_%m_%d_%H_%M_%S")
 
-OWNER = "huggingface"
-REPO = "transformers"
-GITHUB_API_VERSION = "2022-11-28"
-TOKEN = os.environ.get("GITHUB_TOKEN")
-JSON_FILE = f"issues.json"
+JSON_FILE = "issues.json"
 UPDATE_FILE = False
 OVERWRITE_FILE = True
 
@@ -65,7 +61,7 @@ def get_issues(
     url = f"https://api.github.com/repos/{owner}/{repo}/issues"
     headers = {
         "Accept": "application/vnd.github+json",
-        f"Authorization": f"{token}",
+        "Authorization": f"{token}",
         "X-GitHub-Api-Version": f"{github_api_version}",
         "User-Agent": "amyeroberts",
     }

find_similar_issues.py

@@ -1,4 +1,3 @@
-import pprint
 import json
 import argparse
 import requests
@@ -17,11 +16,12 @@ def load_embeddings():
     embeddings = np.load("issue_embeddings.npy")
     return embeddings
 
-def load_issue_information():
+
+def load_issue_information(issue_type="issue"):
     """
     Function to load issue information from file
     """
-    with open("embedding_index_to_issue.json", "r") as f:
+    with open(f"embedding_index_to_{issue_type}.json", "r") as f:
         embedding_index_to_issue = json.load(f)
 
     with open("issues_dict.json", "r") as f:
@@ -45,23 +45,20 @@ def get_issue(issue_no, token=TOKEN, owner=OWNER, repo=REPO):
     """
     Function to get issue from GitHub
     """
-    url = f"https://api.github.com/repos/{owner}/{repo}/issues"
+    url = f"https://api.github.com/repos/{owner}/{repo}/issues/{issue_no}"
     headers = {
         "Accept": "application/vnd.github+json",
-        f"Authorization": "{token}",
+        "Authorization": f"{token}",
         "X-GitHub-Api-Version": "2022-11-28",
         "User-Agent": "amyeroberts",
     }
-    request = requests.get(
-        f"https://api.github.com/repos/{OWNER}/{REPO}/issues/{issue_no}",
-        headers=headers,
-    )
+    request = requests.get(url, headers=headers)
     if request.status_code != 200:
         raise ValueError(f"Request failed with status code {request.status_code}")
     return request.json()
 
 
-def get_similar_issues(issue_no, query, top_k=5, token=TOKEN, owner=OWNER, repo=REPO):
+def get_similar_issues(issue_no, query, top_k=5, token=TOKEN, owner=OWNER, repo=REPO, issue_type="issue"):
     """
     Function to find similar issues
     """
@@ -86,7 +83,7 @@ def get_similar_issues(issue_no, query, top_k=5, token=TOKEN, owner=OWNER, repo=
     most_similar_indices = np.argsort(cosine_similarities)
     most_similar_indices = most_similar_indices[0][::-1]
 
-    embedding_index_to_issue, issues = load_issue_information()
+    embedding_index_to_issue, issues = load_issue_information(issue_type=issue_type)
 
     similar_issues = []
     for i in most_similar_indices[:top_k]:

get_topic.py

@@ -1,57 +0,0 @@
-
-import json
-
-with open("issues_dict.json", "r") as f:
-    issues = json.load(f)
-
-topic_maintainers_map ={
-    "text models": ["@ArthurZucker", "@younesbelkada"],
-    "vision models": "@amyeroberts",
-    "speech models": "@sanchit-gandhi",
-    "graph models": "@clefourrier",
-    "flax": "@sanchit-gandhi",
-    "generate": "@gante",
-    "pipelines": "@Narsil",
-    "tensorflow": ["@gante", "@Rocketknight1"],
-    "tokenizers": "@ArthurZucker",
-    "trainer": ["@muellerzr", "@pacman100"],
-    "deepspeed": "@pacman100",
-    "ray/raytune": ["@richardliaw", "@amogkam"],
-    "Big Model Inference": "@SunMarc",
-    "quantization (bitsandbytes, autogpt)": ["@SunMarc", "@younesbelkada"],
-    "Documentation": ["@stevhliu", "@MKhalusova"],
-    "accelerate": "different repo",
-    "datasets": "different repo",
-    "diffusers": "different repo",
-    "rust tokenizers": "different repo",
-    "Flax examples": "@sanchit-gandhi",
-    "PyTorch vision examples": "@amyeroberts",
-    "PyTorch text examples": "@ArthurZucker",
-    "PyTorch speech examples": "@sanchit-gandhi",
-    "PyTorch generate examples": "@gante",
-    "TensorFlow": "@Rocketknight1",
-    "Research projects and examples": "not maintained",
-}
-
-
-issue_no = 2781
-issue = issues[str(issue_no)]
-
-
-from transformers import AutoTokenizer, LlamaForCausalLM
-
-model = LlamaForCausalLM.from_pretrained("meta-llama/Llama-2-7b-hf")
-tokenizer = AutoTokenizer.from_pretrained("meta-llama/Llama-2-7b-hf")
-
-# prompt = f"Which of the following topics {list(topic_maintainers_map.keys())} is this issue about:\n{issue['body']}"
-prompt = f"QUESTION: What is the provided issue about? Pick up to 3 topics from the following list: {list(topic_maintainers_map.keys())} \nISSUE START:\n{issue['body']} \n ISSUE END. \n ANSWER:"
-inputs = tokenizer(prompt, return_tensors="pt")
-
-prefix_len = inputs.input_ids.shape[1]
-
-# Generate
-generate_ids = model.generate(inputs.input_ids, max_length=30 + prefix_len)
-outputs = tokenizer.batch_decode(generate_ids, skip_special_tokens=True, clean_up_tokenization_spaces=False)[0]
-print(outputs[prefix_len:])
-
-print("TITLE", issue["number"] + " " + issue["title"])

retrieval.py

@@ -1,6 +1,5 @@
 import argparse
 import json
-import pprint
 
 import numpy as np
 from sentence_transformers import SentenceTransformer

update_stored_issues.py

@@ -1,38 +1,20 @@
 """
 Module which updates any of the issues to reflect changes in the issue state
 """
-import json
-import datetime
-from defaults import TOKEN, OWNER, REPO
-
-GITHUB_API_VERSION = "2022-11-28"
-
-
-
-# Get the issues that have been updated since the last update
-
-import json
-
 import argparse
-
-import requests
-import os
-import numpy as np
 import json
-import datetime
 import logging
+import os
 
-logging.basicConfig(level=logging.INFO)
+import numpy as np
+import requests
 
-logger = logging.getLogger(__name__)
+from defaults import TOKEN, OWNER, REPO, GITHUB_API_VERSION
 
-today = datetime.datetime.now().strftime("%Y_%m_%d_%H_%M_%S")
+logging.basicConfig(level=logging.INFO)
+logger = logging.getLogger(__name__)
 
-OWNER = "huggingface"
-REPO = "transformers"
-GITHUB_API_VERSION = "2022-11-28"
-TOKEN = os.environ.get("GITHUB_TOKEN")
-JSON_FILE = f"issues.json"
+JSON_FILE = "issues.json"
 
 
 def update_issues(
@@ -62,7 +44,7 @@ def update_issues(
     url = f"https://api.github.com/repos/{owner}/{repo}/issues"
     headers = {
         "Accept": "application/vnd.github+json",
-        f"Authorization": f"{token}",
+        "Authorization": f"{token}",
         "X-GitHub-Api-Version": f"{github_api_version}",
         "User-Agent": "amyeroberts",
     }