Upload folder using huggingface_hub

.gitattributes

@@ -1,35 +1,5 @@
-*.7z filter=lfs diff=lfs merge=lfs -text
-*.arrow filter=lfs diff=lfs merge=lfs -text
-*.bin filter=lfs diff=lfs merge=lfs -text
-*.bz2 filter=lfs diff=lfs merge=lfs -text
-*.ckpt filter=lfs diff=lfs merge=lfs -text
-*.ftz filter=lfs diff=lfs merge=lfs -text
-*.gz filter=lfs diff=lfs merge=lfs -text
-*.h5 filter=lfs diff=lfs merge=lfs -text
-*.joblib filter=lfs diff=lfs merge=lfs -text
-*.lfs.* filter=lfs diff=lfs merge=lfs -text
-*.mlmodel filter=lfs diff=lfs merge=lfs -text
-*.model filter=lfs diff=lfs merge=lfs -text
-*.msgpack filter=lfs diff=lfs merge=lfs -text
-*.npy filter=lfs diff=lfs merge=lfs -text
-*.npz filter=lfs diff=lfs merge=lfs -text
-*.onnx filter=lfs diff=lfs merge=lfs -text
-*.ot filter=lfs diff=lfs merge=lfs -text
-*.parquet filter=lfs diff=lfs merge=lfs -text
-*.pb filter=lfs diff=lfs merge=lfs -text
-*.pickle filter=lfs diff=lfs merge=lfs -text
-*.pkl filter=lfs diff=lfs merge=lfs -text
-*.pt filter=lfs diff=lfs merge=lfs -text
-*.pth filter=lfs diff=lfs merge=lfs -text
-*.rar filter=lfs diff=lfs merge=lfs -text
-*.safetensors filter=lfs diff=lfs merge=lfs -text
-saved_model/**/* filter=lfs diff=lfs merge=lfs -text
-*.tar.* filter=lfs diff=lfs merge=lfs -text
-*.tar filter=lfs diff=lfs merge=lfs -text
-*.tflite filter=lfs diff=lfs merge=lfs -text
-*.tgz filter=lfs diff=lfs merge=lfs -text
-*.wasm filter=lfs diff=lfs merge=lfs -text
-*.xz filter=lfs diff=lfs merge=lfs -text
-*.zip filter=lfs diff=lfs merge=lfs -text
-*.zst filter=lfs diff=lfs merge=lfs -text
-*tfevents* filter=lfs diff=lfs merge=lfs -text
+index/**/* filter=lfs diff=lfs merge=lfs -text
+sparse_index/_0.fdt filter=lfs diff=lfs merge=lfs -text
+sparse_index/_0_Lucene90_0.doc filter=lfs diff=lfs merge=lfs -text
+sparse_index/_0_Lucene90_0.pos filter=lfs diff=lfs merge=lfs -text
+sparse_index/_0_Lucene90_0.tim filter=lfs diff=lfs merge=lfs -text

README.md

@@ -1,10 +1,13 @@
 ---
-title: Xsumcli
-emoji: 🐢
-colorFrom: gray
-colorTo: purple
-sdk: static
+title: None
+emoji: 🚀
+colorFrom: blue
+colorTo: blue
+sdk: gradio
+sdk_version: 3.12.0
+app_file: app.py
 pinned: false
+license: apache-2.0
 ---
 
-Check out the configuration reference at https://huggingface.co/docs/hub/spaces-config-reference
+Check out the configuration reference at https://huggingface.co/docs/hub/spaces-config-reference

app.py

@@ -0,0 +1,110 @@
+from typing import List, NewType, Optional, Union
+
+import gradio as gr
+
+from spacerini_utils.index import fetch_index_stats
+from spacerini_utils.search import _search, init_searcher, SearchResult
+
+HTML = NewType('HTML', str)
+
+searcher = init_searcher(sparse_index_path="sparse_index")
+
+def get_docid_html(docid: Union[int, str]) -> HTML:
+    docid_html = (
+            f"<a "
+            f'class="underline-on-hover"'
+            'title="This dataset is licensed apache-2.0"'
+            f'style="color:#2D31FA;"'
+            'href="https://huggingface.co/datasets/🚀"'
+            'target="_blank"><b>🔒xsum</b></a><span style="color: #7978FF;">/'+f'{docid}</span>'
+        )
+
+    return docid_html
+
+
+def process_results(results: List[SearchResult], language: str, highlight_terms: Optional[List[str]] = None) -> HTML:
+    if len(results) == 0:
+        return """<br><p style='font-family: Arial; color:Silver; text-align: center;'>
+                No results retrieved.</p><br><hr>"""
+
+    results_html = ""
+    for result in results:
+        tokens = result["text"].split()
+
+        tokens_html = []
+        if highlight_terms:
+            for token in tokens:
+                if token in highlight_terms:
+                    tokens_html.append("<b>{}</b>".format(token))
+                else:
+                    tokens_html.append(token)
+
+        tokens_html = " ".join(tokens_html)
+        meta_html = (
+            """
+                <p class='underline-on-hover' style='font-size:12px; font-family: Arial; color:#585858; text-align: left;'>
+            """
+        )
+        docid_html = get_docid_html(result["docid"])
+        results_html += """{}
+            <p style='font-size:20px; font-family: Arial; color:#7978FF; text-align: left;'>Document ID: {}</p>
+            <p style='font-size:14px; font-family: Arial; color:#7978FF; text-align: left;'>Score: {}</p>
+            <p style='font-size:12px; font-family: Arial; color:MediumAquaMarine'>Language: {}</p>
+            <p style='font-family: Arial;font-size:15px;'>{}</p>
+            <br>
+        """.format(
+            meta_html, docid_html, result["score"], language, tokens_html
+        )
+    return results_html + "<hr>"
+
+
+def search(query: str, language: str, num_results: int = 10) -> HTML:
+    results_dict = _search(searcher, query, num_results=num_results)
+    return process_results(results_dict, language)
+
+
+stats = fetch_index_stats('sparse_index/')
+
+description = f"""# <h2 style="text-align: center;"> 🚀 🔎 None 🔍 🚀 </h2>
+<p style="text-align: center;font-size:15px;">None</p>
+<p style="text-align: center;font-size:20px;">Dataset Statistics: Total Number of Documents = <b>{stats["documents"]}</b>, Number of Terms = <b>{stats["total_terms"]}</b> </p>"""
+
+demo = gr.Blocks(
+    css=".underline-on-hover:hover { text-decoration: underline; } .flagging { font-size:12px; color:Silver; }"
+)
+
+with demo:
+    with gr.Row():
+        gr.Markdown(value=description)
+    with gr.Row():
+        query = gr.Textbox(lines=1, max_lines=1, placeholder="Type your query here...", label="Query")
+    with gr.Row():
+        lang = gr.Dropdown(
+            choices=[
+                "en",
+                "detect_language",
+                "all",
+            ],
+            value="en",
+            label="Language",
+        )
+    with gr.Row():
+            k = gr.Slider(1, 100, value=10, step=1, label="Max Results")
+    with gr.Row():
+        submit_btn = gr.Button("Submit")
+    with gr.Row():
+        results = gr.HTML(label="Results")
+
+
+    def submit(query: str, lang: str, k: int):
+        query = query.strip()
+        if query is None or query == "":
+            return "", ""
+        return {
+            results: search(query, lang, k),
+        }
+
+    query.submit(fn=submit, inputs=[query, lang, k], outputs=[results])
+    submit_btn.click(submit, inputs=[query, lang, k], outputs=[results])
+
+demo.launch(enable_queue=True, debug=True)

packages.txt

@@ -0,0 +1 @@
+default-jdk

requirements.txt

@@ -0,0 +1,4 @@
+pyserini
+datasets
+faiss-cpu
+torch

spacerini_utils/index.py

@@ -0,0 +1,20 @@
+import os
+from typing import Any
+from typing import Dict
+
+from pyserini.index.lucene import IndexReader
+
+
+def fetch_index_stats(index_path: str) -> Dict[str, Any]:
+    """
+    Fetch index statistics
+    index_path : str
+        Path to index directory
+    Returns
+    -------
+    Dictionary of index statistics
+    Dictionary Keys ==> total_terms, documents, unique_terms
+    """
+    assert os.path.exists(index_path), f"Index path {index_path} does not exist"
+    index_reader = IndexReader(index_path)
+    return index_reader.stats()

spacerini_utils/search.py

@@ -0,0 +1,135 @@
+import json
+from typing import List, Literal, Protocol, Tuple, TypedDict, Union
+
+from pyserini.analysis import get_lucene_analyzer
+from pyserini.index import IndexReader
+from pyserini.search import DenseSearchResult, JLuceneSearcherResult
+from pyserini.search.faiss.__main__ import init_query_encoder
+from pyserini.search.faiss import FaissSearcher
+from pyserini.search.hybrid import HybridSearcher
+from pyserini.search.lucene import LuceneSearcher
+
+EncoderClass = Literal["dkrr", "dpr", "tct_colbert", "ance", "sentence", "contriever", "auto"]
+
+
+class AnalyzerArgs(TypedDict):
+    language: str
+    stemming: bool
+    stemmer: str
+    stopwords: bool
+    huggingFaceTokenizer: str
+
+
+class SearchResult(TypedDict):
+    docid: str
+    text: str
+    score: float
+    language: str
+
+
+class Searcher(Protocol):
+    def search(self, query: str, **kwargs) -> List[Union[DenseSearchResult, JLuceneSearcherResult]]:
+        ...
+
+
+def init_searcher_and_reader(
+    sparse_index_path: str = None,
+    bm25_k1: float = None,
+    bm25_b: float = None,
+    analyzer_args: AnalyzerArgs = None,
+    dense_index_path: str = None,
+    encoder_name_or_path: str = None,
+    encoder_class: EncoderClass = None, 
+    tokenizer_name: str = None,
+    device: str = None,
+    prefix: str = None
+) -> Tuple[Union[FaissSearcher, HybridSearcher, LuceneSearcher], IndexReader]:
+    """
+    Initialize and return an approapriate searcher
+    
+    Parameters
+    ----------
+    sparse_index_path: str
+        Path to sparse index
+    dense_index_path: str
+        Path to dense index
+    encoder_name_or_path: str
+        Path to query encoder checkpoint or encoder name
+    encoder_class: str
+        Query encoder class to use. If None, infer from `encoder`
+    tokenizer_name: str
+        Tokenizer name or path
+    device: str
+        Device to load Query encoder on. 
+    prefix: str
+        Query prefix if exists
+    
+    Returns
+    -------
+    Searcher: FaissSearcher | HybridSearcher | LuceneSearcher
+        A sparse, dense or hybrid searcher
+    """
+    reader = None
+    if sparse_index_path:
+        ssearcher = LuceneSearcher(sparse_index_path)
+        if analyzer_args:
+            analyzer = get_lucene_analyzer(**analyzer_args)
+            ssearcher.set_analyzer(analyzer)
+            if bm25_k1 and bm25_b:
+                ssearcher.set_bm25(bm25_k1, bm25_b)
+
+    if dense_index_path:
+        encoder = init_query_encoder(
+            encoder=encoder_name_or_path,
+            encoder_class=encoder_class,
+            tokenizer_name=tokenizer_name,
+            topics_name=None,
+            encoded_queries=None,
+            device=device,
+            prefix=prefix
+        )
+
+        reader = IndexReader(sparse_index_path)
+        dsearcher = FaissSearcher(dense_index_path, encoder)
+
+        if sparse_index_path:
+            hsearcher = HybridSearcher(dense_searcher=dsearcher, sparse_searcher=ssearcher)
+            return hsearcher, reader
+        else:
+            return dsearcher, reader
+    
+    return ssearcher, reader
+
+
+def _search(searcher: Searcher, reader: IndexReader, query: str, num_results: int = 10) -> List[SearchResult]:
+    """
+    Parameters:
+    -----------
+    searcher: FaissSearcher | HybridSearcher | LuceneSearcher
+        A sparse, dense or hybrid searcher
+    query: str
+        Query for which to retrieve results
+    num_results: int
+        Maximum number of results to retrieve
+    
+    Returns:
+    --------
+    Dict:
+    """
+    def _get_dict(r: Union[DenseSearchResult, JLuceneSearcherResult]):
+        if isinstance(r, JLuceneSearcherResult):
+            return json.loads(r.raw)
+        elif isinstance(r, DenseSearchResult):
+            # Get document from sparse_index using index reader
+            return json.loads(reader.doc(r.docid).raw())
+    
+    search_results = searcher.search(query, k=num_results)
+    all_results = [
+        SearchResult(
+            docid=result["id"],
+            text=result["contents"],
+            score=search_results[idx].score   
+        ) for idx, result in enumerate(map(lambda r: _get_dict(r), search_results))
+    ]
+
+    return all_results

sparse_index/_0.fdt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:a9653b671af279f143c0dd3a61f4b162688f24048df7f79577cc7a620ef06bd2
+size 25713651

sparse_index/_0_Lucene90_0.doc

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:3565da3ce3e8c08edf7c6a1af79f74f2c5256aab8649d7aeb57a6ebb78441970
+size 2539833

sparse_index/_0_Lucene90_0.pos

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e7af63495f7ed503de8060d9d034f20867f6efd55c03e3446804772b997c9a84
+size 3890273

sparse_index/_0_Lucene90_0.tim

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:311a4a8d73261eac691ce45614c02f1510ff1e672e35bae7ff03676f66e8bbf8
+size 660334