download service added

README.md

@@ -8,4 +8,132 @@ pinned: false
 short_description: Semantic search API for NDL Core datasets
 ---
 
-Check out the configuration reference at https://huggingface.co/docs/hub/spaces-config-reference
+# NDL Core Data API
+
+A FastAPI-based service that provides semantic search and data download capabilities for NDL Core datasets. The API uses LanceDB for vector search with sentence transformers for embedding.
+
+## Base URL
+
+```
+https://hkir-dev-ndl-core-data-api.hf.space
+```
+
+## Endpoints
+
+### Search
+
+**GET** `/search`
+
+Perform semantic search across NDL Core datasets using natural language queries.
+
+**Parameters:**
+| Parameter | Type | Required | Default | Description |
+|-----------|------|----------|---------|-------------|
+| `query` | string | Yes | - | Natural language search query |
+| `limit` | integer | No | 5 | Maximum number of results to return |
+
+**Example:**
+```bash
+curl "https://hkir-dev-ndl-core-data-api.hf.space/search?query="Police%20use%20of%20force"&limit=10"
+```
+
+**Response:**
+```json
+[
+  {
+    "identifier": "UUID1",
+    "title": "Police use of force dataset1",
+    "description": "...",
+    "format": "parquet",
+    ...
+  },
+  {
+    "identifier": "UUID2",
+    "title": "Police use of force dataset2",
+    "description": "...",
+    "format": "text",
+    ...
+  },
+]
+```
+
+see [NDL Corpus](https://huggingface.co/datasets/hkir-dev/ndl-core-corpus) the definition of all fields.
+
+---
+
+### Download
+
+**GET** `/download`
+
+Get download information for one or more datasets by their identifiers.
+
+**Parameters:**
+| Parameter | Type | Required | Description |
+|-----------|------|----------|-------------|
+| `identifiers` | string | Yes | Comma-separated list of dataset identifiers |
+
+**Example:**
+```bash
+curl "https://hkir-dev-ndl-core-data-api.hf.space/download?identifiers=UUID1,UUID2"
+```
+
+**Response:**
+```json
+[
+  {
+    "identifier": "UUID1",
+    "format": "parquet",
+    "data": ["https://huggingface.co/datasets/hkir-dev/ndl-core-structured-data/resolve/main/some.parquet"]
+  },
+  {
+    "identifier": "UUID2",
+    "format": "text",
+    "data": ["https://hkir-dev-ndl-core-data-api.hf.space/download/text/UUID2"]
+  }
+]
+```
+
+**Response Fields:**
+- `identifier` - The requested dataset identifier
+- `format` - Either `text` or `parquet`
+- `data` - Array of download URLs
+- `error` - Error message (only present if the request failed)
+
+---
+
+### Download Text File
+
+**GET** `/download/text/{identifier}`
+
+Stream text content as a downloadable `.txt` file.
+
+**Parameters:**
+| Parameter | Type | Required | Description |
+|-----------|------|----------|-------------|
+| `identifier` | path | Yes | The dataset identifier |
+
+**Example:**
+```bash
+curl -O "https://hkir-dev-ndl-core-data-api.hf.space/download/text/UUID2"
+```
+
+**Response:**
+- Returns a `text/plain` file download with `Content-Disposition: attachment`
+
+**Errors:**
+- `404` - No record found with the given identifier
+- `400` - Record exists but is not in text format
+
+---
+
+## Data Sources
+
+- **Vector Index:** [hkir-dev/ndl-core-rag-index](https://huggingface.co/datasets/hkir-dev/ndl-core-rag-index)
+- **Structured Data:** [hkir-dev/ndl-core-structured-data](https://huggingface.co/datasets/hkir-dev/ndl-core-structured-data)
+
+## Technology Stack
+
+- **Framework:** FastAPI
+- **Vector Database:** LanceDB
+- **Embeddings:** Sentence Transformers (all-MiniLM-L6-v2)
+- **Deployment:** Docker on Hugging Face Spaces

app.py

@@ -1,9 +1,17 @@
 import os
-from fastapi import FastAPI
+from fastapi import FastAPI, HTTPException
+from fastapi.responses import StreamingResponse
 import lancedb
 from sentence_transformers import SentenceTransformer
 from huggingface_hub import snapshot_download
 import shutil
+import requests
+import io
+
+HF_DATASET_BASE_URL = "https://huggingface.co/datasets/hkir-dev/ndl-core-structured-data"
+HF_API_BASE_URL = "https://huggingface.co/api/datasets/hkir-dev/ndl-core-structured-data"
+
+THIS_API_URL = "https://hkir-dev-ndl-core-data-api.hf.space/" 
 
 app = FastAPI()
 
@@ -16,6 +24,7 @@ index_path = snapshot_download(
     force_download=True  # ensure we get the latest version
 )
 
+# This i mandatory to avoid "file size is too small" errors from LanceDB
 dst = "/tmp/lancedb_search_index"
 shutil.copytree(f"{index_path}/lancedb_search_index", dst)
 
@@ -36,10 +45,154 @@ model = SentenceTransformer('all-MiniLM-L6-v2')
 def search(query: str, limit: int = 5):
     query_vector = model.encode(query)
     results = (
-        table.search(query_vector)      # Your vector search
-        .metric("cosine")               # Ensure metric matches your index
-        .select(columns_to_select)      # <--- The key step: explicit column selection
-        .limit(5)                       # Number of results
-        .to_pandas()                    # Convert to DataFrame
+        table.search(query_vector)      # vector search
+        .metric("cosine")               # Ensure metric matches index
+        .select(columns_to_select)      # explicit column selection
+        .limit(limit)                   
+        .to_pandas()                    
     )
+    
+    # Truncate text column to preview only
+    if "text" in results.columns:
+        results["text"] = results["text"].apply(truncate_text)
+    
     return results.to_dict(orient='records')
+
+
+@app.get("/download")
+def download(identifiers: str):
+    """
+    Download endpoint that returns data based on the identifiers.
+    
+    Args:
+        identifiers: Comma-separated list of identifiers
+    
+    Returns:
+        List of objects with:
+        - For text format: {"identifier": "...", "format": "text", "data": ["<text content>"]}
+        - For parquet format: {"identifier": "...", "format": "parquet", "data": ["<download links>"]}
+    """
+    identifier_list = [id.strip() for id in identifiers.split(",") if id.strip()]
+    return [process_single_identifier(identifier) for identifier in identifier_list]
+
+@app.get("/download/text/{identifier}")
+def download_text_file(identifier: str):
+    """
+    Stream text content as a downloadable file.
+    
+    Args:
+        identifier: The record identifier
+    
+    Returns:
+        StreamingResponse with the text content as a downloadable file
+    """
+    record = find_record_by_identifier(identifier)
+    
+    if record is None:
+        raise HTTPException(status_code=404, detail=f"No record found with identifier: {identifier}")
+    
+    record_format = record.get("format", "")
+    if record_format != "text":
+        raise HTTPException(status_code=400, detail=f"Record is not text format: {record_format}")
+    
+    text_data = record.get("text", "")
+    
+    # Create a file-like object from the text
+    file_stream = io.BytesIO(text_data.encode("utf-8"))
+    
+    return StreamingResponse(
+        file_stream,
+        media_type="text/plain",
+        headers={
+            "Content-Disposition": f"attachment; filename={identifier}.txt"
+        }
+    )
+
+def truncate_text(text: str, max_length: int = 100) -> str:
+    """Return first max_length characters of text with '...' if truncated, or empty string if no text."""
+    if not text:
+        return ""
+    if len(text) <= max_length:
+        return text
+    return text[:max_length] + "..."
+
+def get_folder_file_urls(folder_name: str) -> list:
+    """Fetch all file URLs from a folder in the HuggingFace dataset."""
+    api_url = f"{HF_API_BASE_URL}/tree/main/{folder_name}"
+    response = requests.get(api_url)
+    if response.status_code != 200:
+        return []
+    
+    files = response.json()
+    file_urls = []
+    for file_info in files:
+        if file_info.get("type") == "file":
+            file_path = file_info.get("path", "")
+            download_url = f"{HF_DATASET_BASE_URL}/resolve/main/{file_path}"
+            file_urls.append(download_url)
+    return file_urls
+
+
+def find_record_by_identifier(identifier: str):
+    """Search for a record in LanceDB by identifier."""
+    results = (
+        table.search()
+        .where(f"identifier = '{identifier}'")
+        .select(columns_to_select)
+        .limit(1)
+        .to_pandas()
+    )
+    return results.iloc[0] if not results.empty else None
+
+
+def build_error_response(identifier: str, message: str) -> dict:
+    """Build an error response object."""
+    return {"identifier": identifier, "error": message}
+
+
+def build_success_response(identifier: str, format_type: str, data: list) -> dict:
+    """Build a success response object."""
+    return {"identifier": identifier, "format": format_type, "data": data}
+
+
+def process_text_record(identifier: str, record) -> dict:
+    """Process a text format record and return response."""
+    download_url = f"{THIS_API_URL}/download/text/{identifier}"
+    return build_success_response(identifier, "text", [download_url])
+
+
+def process_parquet_record(identifier: str, record) -> dict:
+    """Process a parquet format record and return response."""
+    data_file = record.get("data_file", "")
+    
+    if not data_file:
+        return build_error_response(identifier, "No data_file found for this parquet record")
+    
+    if data_file.endswith(".parquet"):
+        download_url = f"{HF_DATASET_BASE_URL}/resolve/main/{data_file}"
+        return build_success_response(identifier, "parquet", [download_url])
+    
+    # It's a folder (UUID) - fetch all files in the folder
+    file_urls = get_folder_file_urls(data_file)
+    if not file_urls:
+        return build_error_response(identifier, f"No files found in folder: {data_file}")
+    
+    return build_success_response(identifier, "parquet", file_urls)
+
+
+def process_single_identifier(identifier: str) -> dict:
+    """Process a single identifier and return the appropriate download response based on its format."""
+    record = find_record_by_identifier(identifier)
+    
+    if record is None:
+        return build_error_response(identifier, f"No record found with identifier: {identifier}")
+    
+    record_format = record.get("format", "")
+    
+    if record_format == "text":
+        return process_text_record(identifier, record)
+    elif record_format == "parquet":
+        return process_parquet_record(identifier, record)
+    else:
+        return build_error_response(identifier, f"Unknown format: {record_format}")
+

requirements.txt

@@ -6,4 +6,5 @@ pandas
 huggingface-hub
 pyarrow
 torch
-numpy
+numpy
+requests