added date filter and comment filter and show date, label refs #5

app.py

@@ -1,11 +1,13 @@
+from datetime import datetime, date, timedelta
 from typing import Iterable
 import streamlit as st
 import torch
 from langchain.embeddings import HuggingFaceEmbeddings
 from langchain.vectorstores import Qdrant
 from qdrant_client import QdrantClient
-from qdrant_client.http.models import Filter, FieldCondition, MatchValue
+from qdrant_client.http.models import Filter, FieldCondition, MatchValue, Range
 from config import DB_CONFIG
+from model import Issue
 
 
 @st.cache_resource
@@ -25,11 +27,28 @@ EMBEDDINGS = load_embeddings()
 
 
 def make_filter_obj(options: list[dict[str]]):
+    # print(options)
     must = []
     for option in options:
-        must.append(
-            FieldCondition(key=option["key"], match=MatchValue(value=option["value"]))
-        )
+        if "value" in option:
+            must.append(
+                FieldCondition(
+                    key=option["key"], match=MatchValue(value=option["value"])
+                )
+            )
+        elif "range" in option:
+            range_ = option["range"]
+            must.append(
+                FieldCondition(
+                    key=option["key"],
+                    range=Range(
+                        gt=range_.get("gt"),
+                        gte=range_.get("gte"),
+                        lt=range_.get("lt"),
+                        lte=range_.get("lte"),
+                    ),
+                )
+            )
     filter = Filter(must=must)
     return filter
 
@@ -52,22 +71,47 @@ def main(
     query: str,
     repo_name: str,
     query_options: str,
-) -> Iterable[tuple[str, tuple[str, str]]]:
+    start_date: date,
+    end_date: date,
+    include_comments: bool,
+) -> Iterable[tuple[Issue, float, str]]:
     options = [{"key": "metadata.repo_name", "value": repo_name}]
+    if start_date is not None and end_date is not None:
+        options.append(
+            {
+                "key": "metadata.created_at",
+                "range": {
+                    "gte": int(datetime.fromisoformat(str(start_date)).timestamp()),
+                    "lte": int(
+                        datetime.fromisoformat(
+                            str(end_date + timedelta(days=1))
+                        ).timestamp()
+                    ),
+                },
+            }
+        )
+    if not include_comments:
+        options.append({"key": "metadata.type_", "value": "issue"})
+    filter = make_filter_obj(options=options)
     if query_options == "Empty":
         query_options = ""
     query_str = f"{query_options}{query}"
-    filter = make_filter_obj(options=options)
     docs = get_similay(query_str, filter)
     for doc, score in docs:
         text = doc.page_content
         metadata = doc.metadata
         # print(metadata)
-        title = metadata.get("title")
-        url = metadata.get("url")
-        id_ = metadata.get("id")
-        is_comment = metadata.get("type_") == "comment"
-        yield title, url, id_, text, score, is_comment
+        issue = Issue(
+            repo_name=repo_name,
+            id=metadata.get("id"),
+            title=metadata.get("title"),
+            created_at=metadata.get("created_at"),
+            user=metadata.get("user"),
+            url=metadata.get("url"),
+            labels=metadata.get("labels"),
+            type_=metadata.get("type_"),
+        )
+        yield issue, score, text
 
 
 with st.form("my_form"):
@@ -94,6 +138,20 @@ with st.form("my_form"):
         ],
         label="Query options",
     )
+    date_min = date(2022, 1, 1)
+    date_max = date.today()
+    date_col1, date_col2 = st.columns(2)
+    start_date = date_col1.date_input(
+        label="Select a start date",
+        value=date_min,
+        format="YYYY-MM-DD",
+    )
+    end_date = date_col2.date_input(
+        label="Select a end date",
+        value=date_max,
+        format="YYYY-MM-DD",
+    )
+    include_comments = st.checkbox(label="Include Issue comments", value=True)
 
     submitted = st.form_submit_button("Submit")
     if submitted:
@@ -101,8 +159,18 @@ with st.form("my_form"):
         st.header("Search Results")
         st.divider()
         with st.spinner("Searching..."):
-            results = main(query, repo_name, query_options)
-            for title, url, id_, text, score, is_comment in results:
+            results = main(
+                query, repo_name, query_options, start_date, end_date, include_comments
+            )
+            for issue, score, text in results:
+                title = issue.title
+                url = issue.url
+                id_ = issue.id
+                score = round(score, 3)
+                created_at = datetime.fromtimestamp(issue.created_at)
+                user = issue.user
+                labels = issue.labels
+                is_comment = issue.type_ == "comment"
                 with st.container():
                     if not is_comment:
                         st.subheader(f"#{id_} - {title}")
@@ -110,6 +178,7 @@ with st.form("my_form"):
                         st.subheader(f"comment with {title}")
                     st.write(url)
                     st.write(text)
-                    st.write(score)
+                    st.write("score:", score, "Date:", created_at.date(), "User:", user)
+                    st.write(f"{labels=}")
                     # st.markdown(html, unsafe_allow_html=True)
                     st.divider()

gh_issue_loader.py

@@ -1,21 +1,10 @@
-from dataclasses import dataclass, asdict
+from dataclasses import asdict
 import json
 from typing import Iterator
 from dateutil.parser import parse
 from langchain.docstore.document import Document
 from langchain.document_loaders.base import BaseLoader
-
-
-@dataclass
-class Issue:
-    repo_name: str
-    id: int
-    title: str
-    created_at: int
-    user: str
-    url: str
-    labels: list[str]
-    type_: str
+from gh_issue_loader import Issue
 
 
 def date_to_int(dt_str: str) -> int:

model.py

@@ -0,0 +1,13 @@
+from dataclasses import dataclass
+
+
+@dataclass(frozen=True)
+class Issue:
+    repo_name: str
+    id: int
+    title: str
+    created_at: int
+    user: str
+    url: str
+    labels: list[str]
+    type_: str