Spaces:
Running
Running
Merge pull request #7 from developmentseed/feature/fuzzy-search-endpoint
Browse files- .github/workflows/ci.yml +46 -0
- README.md +12 -2
- data/natural_earth_geoparquet/test_ne_geography.parquet +0 -0
- data/overture/divisions_area/test.parquet +0 -0
- gazet_demo.py +1 -1
- pyproject.toml +34 -1
- ruff.toml +7 -0
- src/gazet/api.py +254 -24
- src/gazet/config.py +10 -5
- src/gazet/export.py +34 -60
- src/gazet/geometry.py +62 -0
- src/gazet/lm.py +33 -52
- src/gazet/schemas.py +10 -285
- src/gazet/search.py +172 -19
- src/gazet/sql.py +44 -95
- tests/__init__.py +0 -0
- tests/conftest.py +37 -0
- tests/test_api.py +224 -0
- tests/test_config.py +105 -0
- tests/test_export.py +187 -0
- tests/test_geometry.py +102 -0
- tests/test_lm.py +106 -0
- tests/test_schemas.py +73 -0
- tests/test_search.py +196 -0
- tests/test_sql.py +182 -0
- uv.lock +0 -0
.github/workflows/ci.yml
ADDED
|
@@ -0,0 +1,46 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
name: CI
|
| 2 |
+
|
| 3 |
+
on:
|
| 4 |
+
push:
|
| 5 |
+
branches: [main, master]
|
| 6 |
+
pull_request:
|
| 7 |
+
branches: [main, master]
|
| 8 |
+
|
| 9 |
+
jobs:
|
| 10 |
+
lint:
|
| 11 |
+
runs-on: ubuntu-latest
|
| 12 |
+
steps:
|
| 13 |
+
- uses: actions/checkout@v4
|
| 14 |
+
- uses: actions/setup-python@v5
|
| 15 |
+
with:
|
| 16 |
+
python-version: "3.13"
|
| 17 |
+
- name: Install ruff
|
| 18 |
+
run: pip install ruff
|
| 19 |
+
- name: Lint
|
| 20 |
+
run: ruff check src/ tests/
|
| 21 |
+
- name: Format check
|
| 22 |
+
run: ruff format --check src/ tests/
|
| 23 |
+
- name: Install uv
|
| 24 |
+
run: |
|
| 25 |
+
curl -LsSf https://astral.sh/uv/install.sh | sh
|
| 26 |
+
echo "$HOME/.cargo/bin" >> $GITHUB_PATH
|
| 27 |
+
- name: Sync dependencies
|
| 28 |
+
run: uv sync --all-extras
|
| 29 |
+
- name: Type check
|
| 30 |
+
run: uv run mypy --explicit-package-bases src/gazet tests/
|
| 31 |
+
|
| 32 |
+
test:
|
| 33 |
+
runs-on: ubuntu-latest
|
| 34 |
+
steps:
|
| 35 |
+
- uses: actions/checkout@v4
|
| 36 |
+
- uses: actions/setup-python@v5
|
| 37 |
+
with:
|
| 38 |
+
python-version: "3.13"
|
| 39 |
+
- name: Install uv
|
| 40 |
+
run: |
|
| 41 |
+
curl -LsSf https://astral.sh/uv/install.sh | sh
|
| 42 |
+
echo "$HOME/.cargo/bin" >> $GITHUB_PATH
|
| 43 |
+
- name: Sync dependencies
|
| 44 |
+
run: uv sync --all-extras
|
| 45 |
+
- name: Run tests
|
| 46 |
+
run: uv run pytest tests/ -v --tb=short
|
README.md
CHANGED
|
@@ -58,6 +58,11 @@ For now, gazet relies on [ollama](https://ollama.com/). For remote (cloud) model
|
|
| 58 |
```bash
|
| 59 |
python -m gazet
|
| 60 |
# then GET http://localhost:8000/search?q=Border%20between%20Loja%20and%20Piura
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 61 |
```
|
| 62 |
|
| 63 |
### API + Streamlit demo
|
|
@@ -77,13 +82,18 @@ uv run streamlit run gazet_demo.py # demo UI
|
|
| 77 |
| `schemas.py` | `SUBTYPES`, `COUNTRIES`, `Place`, `PlacesResult` |
|
| 78 |
| `lm.py` | DSPy signatures + LM init (`extract`, `write_sql`) |
|
| 79 |
| `search.py` | fuzzy search against `divisions_area` / `natural_earth` |
|
|
|
|
| 80 |
| `sql.py` | code-act SQL generation loop |
|
| 81 |
| `export.py` | GeoJSON FeatureCollection writer |
|
| 82 |
-
| `api.py` | FastAPI app
|
| 83 |
|
| 84 |
## Design notes
|
| 85 |
|
| 86 |
-
- `api.py` exposes GET `/search?q=<query>`; returns GeoJSON FeatureCollection and logs intermediate output.
|
|
|
|
|
|
|
|
|
|
|
|
|
| 87 |
- LM is initialised at import time in `lm.py`, suitable for a long-lived server process.
|
| 88 |
- Data lives in `data/overture/` and `data/natural_earth_geoparquet/` (not tracked in git).
|
| 89 |
|
|
|
|
| 58 |
```bash
|
| 59 |
python -m gazet
|
| 60 |
# then GET http://localhost:8000/search?q=Border%20between%20Loja%20and%20Piura
|
| 61 |
+
# pure fuzzy match, no LLM, raw geometry only:
|
| 62 |
+
# GET http://localhost:8000/search/fuzzy?q=Loja&limit=5
|
| 63 |
+
# cheap ID-only search, then fetch geometry for just the one you want:
|
| 64 |
+
# GET http://localhost:8000/search/fuzzy?q=Loja&limit=5&ids_only=true
|
| 65 |
+
# GET http://localhost:8000/geometry/25f85439-e054-44e3-9ad4-9fbf755e3183
|
| 66 |
```
|
| 67 |
|
| 68 |
### API + Streamlit demo
|
|
|
|
| 82 |
| `schemas.py` | `SUBTYPES`, `COUNTRIES`, `Place`, `PlacesResult` |
|
| 83 |
| `lm.py` | DSPy signatures + LM init (`extract`, `write_sql`) |
|
| 84 |
| `search.py` | fuzzy search against `divisions_area` / `natural_earth` |
|
| 85 |
+
| `geometry.py` | geometry simplification / GeoJSON normalization (no LLM dependency) |
|
| 86 |
| `sql.py` | code-act SQL generation loop |
|
| 87 |
| `export.py` | GeoJSON FeatureCollection writer |
|
| 88 |
+
| `api.py` | FastAPI app: `/search` (LLM), `/search/fuzzy` (fuzzy-only), `/geometry/{id}` (direct ID lookup) |
|
| 89 |
|
| 90 |
## Design notes
|
| 91 |
|
| 92 |
+
- `api.py` exposes GET `/search?q=<query>`; returns GeoJSON FeatureCollection and logs intermediate output. This path runs two LLM calls (place extraction, then SQL generation for candidate selection/disambiguation and GIS ops).
|
| 93 |
+
- `api.py` also exposes GET `/search/fuzzy?q=<place name>&limit=5&simplify=true&sources=divisions_area,natural_earth` — pure Jaro-Winkler fuzzy match with no LLM involved. `q` is a place-name string, not a natural-language query (there's no place-extraction step). Returns the combined top-`limit` matches across sources as a GeoJSON FeatureCollection with raw (or simplified, if `simplify=true`) geometry attached. Pass `ids_only=true` to skip full geometry and get back `{"ids": [{"source", "id", "name", "country", "subtype", "admin_level", "bbox"}, ...]}` — `country`/`subtype`/`admin_level` disambiguate same-named places (e.g. multiple real "Loja"s across Ecuador and Spain, or Ecuador's "Loja" region vs. its nested "Loja" county), and `bbox` is `[minx, miny, maxx, maxy]`, a much smaller payload than full geometry, for picking a candidate before fetching its full geometry.
|
| 94 |
+
- Fuzzy matching (`search.py`) ranks exact-substring matches (query literally contained in the name, e.g. "Loja" in "Loja Province") ahead of pure edit-distance near-misses (e.g. "Lodja"), even when the near-miss scores a slightly higher raw Jaro-Winkler similarity. `divisions_area` matches against `names.primary` (not `names.common.en`, which is null for 77% of rows and would otherwise exclude most of the dataset from every fuzzy search).
|
| 95 |
+
- `api.py` also exposes GET `/geometry/{id}?source=divisions_area&simplify=true` — fetch a single feature's geometry directly by ID, no fuzzy matching or LLM involved. `source` is inferred from the ID if omitted (Natural Earth IDs are prefixed `ne_`). Returns a single GeoJSON Feature, 404 if not found.
|
| 96 |
+
- The spatial extension is loaded once at API startup (FastAPI `lifespan`) onto a shared DuckDB connection stored on `app.state`; each request gets a cheap `cursor()` off it instead of paying DuckDB's ~90ms `LOAD spatial` cost per call.
|
| 97 |
- LM is initialised at import time in `lm.py`, suitable for a long-lived server process.
|
| 98 |
- Data lives in `data/overture/` and `data/natural_earth_geoparquet/` (not tracked in git).
|
| 99 |
|
data/natural_earth_geoparquet/test_ne_geography.parquet
ADDED
|
Binary file (5.03 kB). View file
|
|
|
data/overture/divisions_area/test.parquet
ADDED
|
Binary file (66.7 kB). View file
|
|
|
gazet_demo.py
CHANGED
|
@@ -295,7 +295,7 @@ approach on a different domain? Reach out:
|
|
| 295 |
st.caption("Built by")
|
| 296 |
st.image(DEVSEED_LOGO_PATH, width=180)
|
| 297 |
|
| 298 |
-
backend = "gguf"
|
| 299 |
|
| 300 |
if "run_q" not in st.session_state:
|
| 301 |
st.session_state.run_q = None
|
|
|
|
| 295 |
st.caption("Built by")
|
| 296 |
st.image(DEVSEED_LOGO_PATH, width=180)
|
| 297 |
|
| 298 |
+
backend = os.environ.get("GAZET_BACKEND", "gguf")
|
| 299 |
|
| 300 |
if "run_q" not in st.session_state:
|
| 301 |
st.session_state.run_q = None
|
pyproject.toml
CHANGED
|
@@ -19,7 +19,7 @@ dependencies = [
|
|
| 19 |
"httpx>=0.28.1",
|
| 20 |
"sqlparse>=0.5.5",
|
| 21 |
]
|
| 22 |
-
optional-dependencies = { demo = ["streamlit", "requests", "pydeck"], dev = ["ruff"] }
|
| 23 |
|
| 24 |
[project.scripts]
|
| 25 |
gazet-dataset = "dataset.scripts.cli:main"
|
|
@@ -27,6 +27,39 @@ gazet-dataset = "dataset.scripts.cli:main"
|
|
| 27 |
[tool.hatch.build.targets.wheel]
|
| 28 |
packages = ["src/gazet", "dataset"]
|
| 29 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 30 |
[dependency-groups]
|
| 31 |
dataset = [
|
| 32 |
"modal>=1.4.0",
|
|
|
|
| 19 |
"httpx>=0.28.1",
|
| 20 |
"sqlparse>=0.5.5",
|
| 21 |
]
|
| 22 |
+
optional-dependencies = { demo = ["streamlit", "requests", "pydeck"], dev = ["ruff", "mypy>=1.15", "pandas-stubs>=2.0", "pytest>=8.0", "pytest-asyncio>=0.24"] }
|
| 23 |
|
| 24 |
[project.scripts]
|
| 25 |
gazet-dataset = "dataset.scripts.cli:main"
|
|
|
|
| 27 |
[tool.hatch.build.targets.wheel]
|
| 28 |
packages = ["src/gazet", "dataset"]
|
| 29 |
|
| 30 |
+
[tool.pytest.ini_options]
|
| 31 |
+
testpaths = ["tests"]
|
| 32 |
+
pythonpath = ["src"]
|
| 33 |
+
addopts = ["-ra", "--strict-markers"]
|
| 34 |
+
|
| 35 |
+
[tool.mypy]
|
| 36 |
+
python_version = "3.13"
|
| 37 |
+
mypy_path = "src"
|
| 38 |
+
strict = true
|
| 39 |
+
disallow_any_generics = false # pandas-stubs use Any heavily
|
| 40 |
+
warn_return_any = false # dspy returns Any; noisy without value
|
| 41 |
+
disallow_untyped_defs = false # too noisy on test files
|
| 42 |
+
|
| 43 |
+
[[tool.mypy.overrides]]
|
| 44 |
+
module = "dspy.*"
|
| 45 |
+
ignore_missing_imports = true
|
| 46 |
+
|
| 47 |
+
[[tool.mypy.overrides]]
|
| 48 |
+
module = "duckdb.*"
|
| 49 |
+
ignore_missing_imports = true
|
| 50 |
+
|
| 51 |
+
[[tool.mypy.overrides]]
|
| 52 |
+
module = "tests.*"
|
| 53 |
+
check_untyped_defs = false
|
| 54 |
+
disallow_untyped_defs = false
|
| 55 |
+
warn_return_any = false
|
| 56 |
+
|
| 57 |
+
[[tool.mypy.overrides]]
|
| 58 |
+
module = "gazet.lm"
|
| 59 |
+
# dspy has no stubs — its classes (Signature, Module, Predict, LM) are all typed as Any.
|
| 60 |
+
# Suppressing the resulting noise while keeping the rest of the codebase strict.
|
| 61 |
+
disable_error_code = ["misc", "no-untyped-def", "no-untyped-call"]
|
| 62 |
+
|
| 63 |
[dependency-groups]
|
| 64 |
dataset = [
|
| 65 |
"modal>=1.4.0",
|
ruff.toml
ADDED
|
@@ -0,0 +1,7 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
target-version = "py313"
|
| 2 |
+
|
| 3 |
+
[lint]
|
| 4 |
+
# BLE001 – catching Exception is intentional in health checks and error-handling paths
|
| 5 |
+
# S110 – try/except/pass is intentional for health probes that must not crash
|
| 6 |
+
# B017 – pytest.raises(Exception) is acceptable for pydantic validation error checks
|
| 7 |
+
ignore = ["BLE001", "S110", "B017"]
|
src/gazet/api.py
CHANGED
|
@@ -1,18 +1,60 @@
|
|
| 1 |
import json
|
| 2 |
-
import
|
| 3 |
-
|
|
|
|
|
|
|
|
|
|
| 4 |
|
| 5 |
import duckdb
|
| 6 |
import pandas as pd
|
| 7 |
-
from fastapi import FastAPI, HTTPException
|
| 8 |
-
from fastapi.responses import StreamingResponse
|
| 9 |
|
| 10 |
from .export import to_feature_collection
|
|
|
|
| 11 |
from .lm import extract, generate_places
|
| 12 |
-
from .
|
|
|
|
| 13 |
from .sql import run_geo_sql_dspy, run_geo_sql_gguf
|
| 14 |
|
| 15 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 16 |
|
| 17 |
|
| 18 |
def _per_source_limit(num_places: int) -> int:
|
|
@@ -22,19 +64,25 @@ def _per_source_limit(num_places: int) -> int:
|
|
| 22 |
1 place → 5 per source → 10 total
|
| 23 |
2 places → 4 per source → 16 total
|
| 24 |
3 places → 3 per source → 18 total
|
| 25 |
-
4 places →
|
| 26 |
-
5 places → 2 per source → 20 total
|
| 27 |
"""
|
| 28 |
-
|
| 29 |
-
|
|
|
|
|
|
|
|
|
|
| 30 |
|
| 31 |
|
| 32 |
def _df_to_records(df: pd.DataFrame) -> list[dict[str, Any]]:
|
| 33 |
"""Convert DataFrame to list of dicts for JSON; handle non-JSON-serializable types."""
|
| 34 |
-
|
|
|
|
|
|
|
| 35 |
|
| 36 |
|
| 37 |
-
def _run_stream(
|
|
|
|
|
|
|
| 38 |
"""Yield NDJSON lines as each stage of the search completes.
|
| 39 |
|
| 40 |
Event ``type`` values (in order of emission):
|
|
@@ -47,11 +95,17 @@ def _run_stream(query: str, backend: str = "gguf") -> Generator[str, None, None]
|
|
| 47 |
"""
|
| 48 |
if backend == "gguf":
|
| 49 |
from .lm import is_llama_server_available
|
|
|
|
| 50 |
if not is_llama_server_available():
|
| 51 |
-
yield
|
| 52 |
-
|
| 53 |
-
|
| 54 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 55 |
places_result = generate_places(query)
|
| 56 |
else:
|
| 57 |
pred = extract(query=query)
|
|
@@ -59,9 +113,7 @@ def _run_stream(query: str, backend: str = "gguf") -> Generator[str, None, None]
|
|
| 59 |
|
| 60 |
yield json.dumps({"type": "places", "data": places_result.model_dump()}) + "\n"
|
| 61 |
|
| 62 |
-
con =
|
| 63 |
-
con.execute("INSTALL spatial")
|
| 64 |
-
con.execute("LOAD spatial")
|
| 65 |
|
| 66 |
try:
|
| 67 |
limit = _per_source_limit(len(places_result.places))
|
|
@@ -76,7 +128,10 @@ def _run_stream(query: str, backend: str = "gguf") -> Generator[str, None, None]
|
|
| 76 |
candidates_df = (
|
| 77 |
pd.concat(all_candidates, ignore_index=True)
|
| 78 |
.drop_duplicates(subset=["source", "id"])
|
| 79 |
-
.sort_values(
|
|
|
|
|
|
|
|
|
|
| 80 |
.reset_index(drop=True)
|
| 81 |
)
|
| 82 |
|
|
@@ -126,14 +181,72 @@ def _run_stream(query: str, backend: str = "gguf") -> Generator[str, None, None]
|
|
| 126 |
con.close()
|
| 127 |
|
| 128 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 129 |
@app.get("/search/stream")
|
| 130 |
-
def search_stream(q: str, backend: str = "gguf") -> StreamingResponse:
|
| 131 |
"""Stream search progress as NDJSON (one JSON object per line)."""
|
| 132 |
-
|
|
|
|
|
|
|
|
|
|
| 133 |
|
| 134 |
|
| 135 |
@app.get("/search", response_model=None)
|
| 136 |
-
def search(q: str, backend: str = "gguf") -> dict[str, Any]:
|
| 137 |
"""Run geo search for natural-language query (non-streaming).
|
| 138 |
|
| 139 |
Returns GeoJSON FeatureCollection, the executed SQL, and the identified
|
|
@@ -144,7 +257,8 @@ def search(q: str, backend: str = "gguf") -> dict[str, Any]:
|
|
| 144 |
sql = ""
|
| 145 |
geojson: dict | None = None
|
| 146 |
|
| 147 |
-
|
|
|
|
| 148 |
if not line.strip():
|
| 149 |
continue
|
| 150 |
event = json.loads(line)
|
|
@@ -167,3 +281,119 @@ def search(q: str, backend: str = "gguf") -> dict[str, Any]:
|
|
| 167 |
"places": places,
|
| 168 |
"dataframes": {"candidates": candidates},
|
| 169 |
}
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
import json
|
| 2 |
+
import logging
|
| 3 |
+
import uuid
|
| 4 |
+
from collections.abc import AsyncIterator, Awaitable, Callable, Generator
|
| 5 |
+
from contextlib import asynccontextmanager
|
| 6 |
+
from typing import Any
|
| 7 |
|
| 8 |
import duckdb
|
| 9 |
import pandas as pd
|
| 10 |
+
from fastapi import FastAPI, HTTPException, Request
|
| 11 |
+
from fastapi.responses import Response, StreamingResponse
|
| 12 |
|
| 13 |
from .export import to_feature_collection
|
| 14 |
+
from .geometry import normalize_geometry_to_geojson
|
| 15 |
from .lm import extract, generate_places
|
| 16 |
+
from .schemas import Place
|
| 17 |
+
from .search import get_by_id, search_candidates
|
| 18 |
from .sql import run_geo_sql_dspy, run_geo_sql_gguf
|
| 19 |
|
| 20 |
+
_FUZZY_SOURCES = ("divisions_area", "natural_earth")
|
| 21 |
+
|
| 22 |
+
|
| 23 |
+
@asynccontextmanager
|
| 24 |
+
async def lifespan(app: FastAPI) -> AsyncIterator[None]:
|
| 25 |
+
"""Load the spatial extension once at startup; per-request handlers get
|
| 26 |
+
a cheap cursor() off this connection instead of paying the ~90ms
|
| 27 |
+
LOAD spatial cost on every call."""
|
| 28 |
+
con = duckdb.connect()
|
| 29 |
+
con.execute("INSTALL spatial")
|
| 30 |
+
con.execute("LOAD spatial")
|
| 31 |
+
app.state.duckdb_con = con
|
| 32 |
+
yield
|
| 33 |
+
con.close()
|
| 34 |
+
|
| 35 |
+
|
| 36 |
+
app = FastAPI(lifespan=lifespan)
|
| 37 |
+
|
| 38 |
+
logger = logging.getLogger(__name__)
|
| 39 |
+
|
| 40 |
+
|
| 41 |
+
@app.middleware("http")
|
| 42 |
+
async def log_request(
|
| 43 |
+
request: Request,
|
| 44 |
+
call_next: Callable[[Request], Awaitable[Response]],
|
| 45 |
+
) -> Response:
|
| 46 |
+
"""Attach X-Request-Id header and log each request."""
|
| 47 |
+
request_id = request.headers.get("X-Request-Id") or str(uuid.uuid4())
|
| 48 |
+
response = await call_next(request)
|
| 49 |
+
logger.info(
|
| 50 |
+
"%s %s %d %s %s",
|
| 51 |
+
request.method,
|
| 52 |
+
request.url.path,
|
| 53 |
+
response.status_code,
|
| 54 |
+
request_id,
|
| 55 |
+
request.query_params,
|
| 56 |
+
)
|
| 57 |
+
return response
|
| 58 |
|
| 59 |
|
| 60 |
def _per_source_limit(num_places: int) -> int:
|
|
|
|
| 64 |
1 place → 5 per source → 10 total
|
| 65 |
2 places → 4 per source → 16 total
|
| 66 |
3 places → 3 per source → 18 total
|
| 67 |
+
4+ places → 3 per source → 12+ total (floors at reasonable minimum)
|
|
|
|
| 68 |
"""
|
| 69 |
+
if num_places <= 1:
|
| 70 |
+
return 5
|
| 71 |
+
if num_places <= 2:
|
| 72 |
+
return 4
|
| 73 |
+
return 3
|
| 74 |
|
| 75 |
|
| 76 |
def _df_to_records(df: pd.DataFrame) -> list[dict[str, Any]]:
|
| 77 |
"""Convert DataFrame to list of dicts for JSON; handle non-JSON-serializable types."""
|
| 78 |
+
# pandas-stubs types to_dict(orient="records") as list[dict[Hashable, Any]],
|
| 79 |
+
# but DataFrame column names are always strings, so the result is list[dict[str, Any]].
|
| 80 |
+
return df.replace({float("nan"): None}).to_dict(orient="records") # type: ignore[return-value]
|
| 81 |
|
| 82 |
|
| 83 |
+
def _run_stream(
|
| 84 |
+
base_con: duckdb.DuckDBPyConnection, query: str, backend: str = "gguf"
|
| 85 |
+
) -> Generator[str]:
|
| 86 |
"""Yield NDJSON lines as each stage of the search completes.
|
| 87 |
|
| 88 |
Event ``type`` values (in order of emission):
|
|
|
|
| 95 |
"""
|
| 96 |
if backend == "gguf":
|
| 97 |
from .lm import is_llama_server_available
|
| 98 |
+
|
| 99 |
if not is_llama_server_available():
|
| 100 |
+
yield (
|
| 101 |
+
json.dumps(
|
| 102 |
+
{
|
| 103 |
+
"type": "warming_up",
|
| 104 |
+
"data": "Starting model server, this takes 30-60s on cold start",
|
| 105 |
+
}
|
| 106 |
+
)
|
| 107 |
+
+ "\n"
|
| 108 |
+
)
|
| 109 |
places_result = generate_places(query)
|
| 110 |
else:
|
| 111 |
pred = extract(query=query)
|
|
|
|
| 113 |
|
| 114 |
yield json.dumps({"type": "places", "data": places_result.model_dump()}) + "\n"
|
| 115 |
|
| 116 |
+
con = base_con.cursor()
|
|
|
|
|
|
|
| 117 |
|
| 118 |
try:
|
| 119 |
limit = _per_source_limit(len(places_result.places))
|
|
|
|
| 128 |
candidates_df = (
|
| 129 |
pd.concat(all_candidates, ignore_index=True)
|
| 130 |
.drop_duplicates(subset=["source", "id"])
|
| 131 |
+
.sort_values(
|
| 132 |
+
["is_substring_match", "similarity", "admin_level"],
|
| 133 |
+
ascending=[False, False, True],
|
| 134 |
+
)
|
| 135 |
.reset_index(drop=True)
|
| 136 |
)
|
| 137 |
|
|
|
|
| 181 |
con.close()
|
| 182 |
|
| 183 |
|
| 184 |
+
@app.get("/health")
|
| 185 |
+
def health(request: Request) -> dict[str, Any]:
|
| 186 |
+
"""Health check — DuckDB connection alive + llama-server status."""
|
| 187 |
+
con = request.app.state.duckdb_con
|
| 188 |
+
duckdb_ok = False
|
| 189 |
+
try:
|
| 190 |
+
con.execute("SELECT 1")
|
| 191 |
+
duckdb_ok = True
|
| 192 |
+
except Exception:
|
| 193 |
+
pass
|
| 194 |
+
|
| 195 |
+
llama_ok = False
|
| 196 |
+
try:
|
| 197 |
+
from .lm import is_llama_server_available
|
| 198 |
+
|
| 199 |
+
llama_ok = is_llama_server_available()
|
| 200 |
+
except Exception:
|
| 201 |
+
pass
|
| 202 |
+
|
| 203 |
+
status = (
|
| 204 |
+
"ok" if duckdb_ok and llama_ok else ("degraded" if duckdb_ok else "unhealthy")
|
| 205 |
+
)
|
| 206 |
+
return {
|
| 207 |
+
"status": status,
|
| 208 |
+
"duckdb": "ok" if duckdb_ok else "error",
|
| 209 |
+
"llama_server": "ok" if llama_ok else "unavailable",
|
| 210 |
+
}
|
| 211 |
+
|
| 212 |
+
|
| 213 |
+
@app.get("/sources")
|
| 214 |
+
def sources(request: Request) -> dict[str, Any]:
|
| 215 |
+
"""List available data sources with row counts and name ranges."""
|
| 216 |
+
con = request.app.state.duckdb_con
|
| 217 |
+
from .config import DIVISIONS_AREA_PATH, NATURAL_EARTH_PATH
|
| 218 |
+
|
| 219 |
+
info = {}
|
| 220 |
+
for name, path in [
|
| 221 |
+
("divisions_area", DIVISIONS_AREA_PATH),
|
| 222 |
+
("natural_earth", NATURAL_EARTH_PATH),
|
| 223 |
+
]:
|
| 224 |
+
try:
|
| 225 |
+
row = con.execute(
|
| 226 |
+
f"SELECT COUNT(*) as count, MIN(names.primary) as min_name, MAX(names.primary) as max_name FROM read_parquet('{path}')"
|
| 227 |
+
).fetchone()
|
| 228 |
+
info[name] = {
|
| 229 |
+
"path": path,
|
| 230 |
+
"row_count": row[0],
|
| 231 |
+
"name_range": [row[1], row[2]],
|
| 232 |
+
}
|
| 233 |
+
except Exception as e:
|
| 234 |
+
info[name] = {"path": path, "error": str(e)}
|
| 235 |
+
|
| 236 |
+
return info
|
| 237 |
+
|
| 238 |
+
|
| 239 |
@app.get("/search/stream")
|
| 240 |
+
def search_stream(request: Request, q: str, backend: str = "gguf") -> StreamingResponse:
|
| 241 |
"""Stream search progress as NDJSON (one JSON object per line)."""
|
| 242 |
+
con = request.app.state.duckdb_con
|
| 243 |
+
return StreamingResponse(
|
| 244 |
+
_run_stream(con, q, backend), media_type="application/x-ndjson"
|
| 245 |
+
)
|
| 246 |
|
| 247 |
|
| 248 |
@app.get("/search", response_model=None)
|
| 249 |
+
def search(request: Request, q: str, backend: str = "gguf") -> dict[str, Any]:
|
| 250 |
"""Run geo search for natural-language query (non-streaming).
|
| 251 |
|
| 252 |
Returns GeoJSON FeatureCollection, the executed SQL, and the identified
|
|
|
|
| 257 |
sql = ""
|
| 258 |
geojson: dict | None = None
|
| 259 |
|
| 260 |
+
con = request.app.state.duckdb_con
|
| 261 |
+
for line in _run_stream(con, q, backend):
|
| 262 |
if not line.strip():
|
| 263 |
continue
|
| 264 |
event = json.loads(line)
|
|
|
|
| 281 |
"places": places,
|
| 282 |
"dataframes": {"candidates": candidates},
|
| 283 |
}
|
| 284 |
+
|
| 285 |
+
|
| 286 |
+
@app.get("/search/fuzzy", response_model=None)
|
| 287 |
+
def search_fuzzy(
|
| 288 |
+
request: Request,
|
| 289 |
+
q: str,
|
| 290 |
+
limit: int = 5,
|
| 291 |
+
simplify: bool = True,
|
| 292 |
+
sources: str | None = None,
|
| 293 |
+
ids_only: bool = False,
|
| 294 |
+
) -> dict[str, Any]:
|
| 295 |
+
"""Pure fuzzy-name search with geometry, no LLM involved.
|
| 296 |
+
|
| 297 |
+
``q`` is a place-name string (not a natural-language query) — this
|
| 298 |
+
endpoint has no place-extraction step. Matches are ranked by
|
| 299 |
+
Jaro-Winkler similarity across the requested ``sources`` (comma-separated
|
| 300 |
+
subset of divisions_area/natural_earth; defaults to both), combined and
|
| 301 |
+
truncated to the top ``limit``. Returns a GeoJSON FeatureCollection.
|
| 302 |
+
|
| 303 |
+
Pass ``ids_only=true`` to skip fetching full geometry and get back
|
| 304 |
+
``{"ids": [{"source", "id", "name", "country", "subtype", "admin_level", "bbox"}, ...]}``
|
| 305 |
+
instead — ``country``/``subtype``/``admin_level`` disambiguate same-named
|
| 306 |
+
places (e.g. multiple real-world "Loja"s across Ecuador and Spain, or
|
| 307 |
+
Ecuador's "Loja" region vs. its nested "Loja" county — same ``subtype``
|
| 308 |
+
can occur at different ``admin_level``s, and locality-type subtypes have
|
| 309 |
+
no ``admin_level`` at all). ``bbox`` is ``[minx, miny, maxx, maxy]``
|
| 310 |
+
computed via ST_XMin/YMin/XMax/YMax, a much smaller payload than full
|
| 311 |
+
geometry, giving minimal spatial context before fetching the full
|
| 312 |
+
geometry for one candidate via ``GET /geometry/{id}``.
|
| 313 |
+
"""
|
| 314 |
+
requested_sources = (
|
| 315 |
+
tuple(s.strip() for s in sources.split(",")) if sources else _FUZZY_SOURCES
|
| 316 |
+
)
|
| 317 |
+
invalid = set(requested_sources) - set(_FUZZY_SOURCES)
|
| 318 |
+
if invalid:
|
| 319 |
+
raise HTTPException(
|
| 320 |
+
status_code=400, detail=f"Unknown source(s): {sorted(invalid)}"
|
| 321 |
+
)
|
| 322 |
+
|
| 323 |
+
con = request.app.state.duckdb_con.cursor()
|
| 324 |
+
|
| 325 |
+
try:
|
| 326 |
+
# Fetch a pool larger than `limit` per source so the combined,
|
| 327 |
+
# similarity-ranked top-`limit` isn't skewed by per-source cutoffs.
|
| 328 |
+
per_source_limit = max(limit * 3, 15)
|
| 329 |
+
candidate_dfs = search_candidates(
|
| 330 |
+
con,
|
| 331 |
+
Place(place=q),
|
| 332 |
+
limit=per_source_limit,
|
| 333 |
+
include_geometry=not ids_only,
|
| 334 |
+
include_bbox=ids_only,
|
| 335 |
+
sources=requested_sources,
|
| 336 |
+
)
|
| 337 |
+
if not candidate_dfs:
|
| 338 |
+
return to_feature_collection(pd.DataFrame())
|
| 339 |
+
|
| 340 |
+
candidates_df = (
|
| 341 |
+
pd.concat(candidate_dfs, ignore_index=True)
|
| 342 |
+
.drop_duplicates(subset=["source", "id"])
|
| 343 |
+
.sort_values(["is_substring_match", "similarity"], ascending=[False, False])
|
| 344 |
+
.head(limit)
|
| 345 |
+
.reset_index(drop=True)
|
| 346 |
+
)
|
| 347 |
+
|
| 348 |
+
if ids_only:
|
| 349 |
+
scalar_cols = ["source", "id", "name", "country", "subtype", "admin_level"]
|
| 350 |
+
ids_df = candidates_df[scalar_cols].copy()
|
| 351 |
+
ids_df = ids_df.astype(object).where(ids_df.notna(), None)
|
| 352 |
+
ids_df["bbox"] = candidates_df["bbox"].apply(
|
| 353 |
+
lambda arr: [float(x) for x in arr] if arr is not None else None
|
| 354 |
+
)
|
| 355 |
+
return {
|
| 356 |
+
"geojson": {"type": "FeatureCollection", "features": []},
|
| 357 |
+
"ids": ids_df.to_dict(orient="records"),
|
| 358 |
+
}
|
| 359 |
+
|
| 360 |
+
if simplify:
|
| 361 |
+
candidates_df = normalize_geometry_to_geojson(con, candidates_df)
|
| 362 |
+
|
| 363 |
+
return to_feature_collection(candidates_df)
|
| 364 |
+
finally:
|
| 365 |
+
con.close()
|
| 366 |
+
|
| 367 |
+
|
| 368 |
+
@app.get("/geometry/{id}", response_model=None)
|
| 369 |
+
def get_geometry(
|
| 370 |
+
request: Request,
|
| 371 |
+
id: str,
|
| 372 |
+
source: str | None = None,
|
| 373 |
+
simplify: bool = True,
|
| 374 |
+
) -> dict[str, Any]:
|
| 375 |
+
"""Fetch a single feature's geometry directly by ID — no fuzzy matching, no LLM.
|
| 376 |
+
|
| 377 |
+
``source`` restricts the lookup to ``divisions_area`` or ``natural_earth``;
|
| 378 |
+
if omitted, it's inferred from the ID (Natural Earth IDs are prefixed
|
| 379 |
+
``ne_``). Returns a single GeoJSON Feature, or 404 if the ID doesn't exist
|
| 380 |
+
in the resolved source.
|
| 381 |
+
"""
|
| 382 |
+
if source is not None and source not in _FUZZY_SOURCES:
|
| 383 |
+
raise HTTPException(status_code=400, detail=f"Unknown source: {source}")
|
| 384 |
+
|
| 385 |
+
con = request.app.state.duckdb_con.cursor()
|
| 386 |
+
|
| 387 |
+
try:
|
| 388 |
+
df = get_by_id(con, id, source=source, include_geometry=True)
|
| 389 |
+
if df.empty:
|
| 390 |
+
raise HTTPException(
|
| 391 |
+
status_code=404, detail=f"No feature found for id={id!r}"
|
| 392 |
+
)
|
| 393 |
+
|
| 394 |
+
if simplify:
|
| 395 |
+
df = normalize_geometry_to_geojson(con, df)
|
| 396 |
+
|
| 397 |
+
return to_feature_collection(df)["features"][0]
|
| 398 |
+
finally:
|
| 399 |
+
con.close()
|
src/gazet/config.py
CHANGED
|
@@ -3,12 +3,17 @@ import pathlib
|
|
| 3 |
|
| 4 |
# Data lives at project root (gazet/data/), not inside the package.
|
| 5 |
# Override with GAZET_DATA_DIR env var for remote execution (e.g. Modal volume at /data).
|
| 6 |
-
_DATA_DIR = pathlib.Path(
|
| 7 |
-
|
| 8 |
-
|
|
|
|
|
|
|
|
|
|
| 9 |
|
| 10 |
|
| 11 |
-
def _prefer_normalized(
|
|
|
|
|
|
|
| 12 |
"""Prefer normalized geodata copies when present."""
|
| 13 |
use_normalized = os.environ.get("GAZET_USE_NORMALIZED_DATA", "1") != "0"
|
| 14 |
if use_normalized:
|
|
@@ -30,7 +35,7 @@ DIVISIONS_AREA_PATH = str(
|
|
| 30 |
NATURAL_EARTH_PATH = str(
|
| 31 |
_prefer_normalized(
|
| 32 |
_DATA_DIR / "natural_earth_normalized/ne_geography.parquet",
|
| 33 |
-
_DATA_DIR / "natural_earth_geoparquet/
|
| 34 |
)
|
| 35 |
)
|
| 36 |
|
|
|
|
| 3 |
|
| 4 |
# Data lives at project root (gazet/data/), not inside the package.
|
| 5 |
# Override with GAZET_DATA_DIR env var for remote execution (e.g. Modal volume at /data).
|
| 6 |
+
_DATA_DIR = pathlib.Path(
|
| 7 |
+
os.environ.get(
|
| 8 |
+
"GAZET_DATA_DIR",
|
| 9 |
+
str(pathlib.Path(__file__).resolve().parent.parent.parent / "data"),
|
| 10 |
+
)
|
| 11 |
+
)
|
| 12 |
|
| 13 |
|
| 14 |
+
def _prefer_normalized(
|
| 15 |
+
path_normalized: pathlib.Path, path_original: pathlib.Path
|
| 16 |
+
) -> pathlib.Path:
|
| 17 |
"""Prefer normalized geodata copies when present."""
|
| 18 |
use_normalized = os.environ.get("GAZET_USE_NORMALIZED_DATA", "1") != "0"
|
| 19 |
if use_normalized:
|
|
|
|
| 35 |
NATURAL_EARTH_PATH = str(
|
| 36 |
_prefer_normalized(
|
| 37 |
_DATA_DIR / "natural_earth_normalized/ne_geography.parquet",
|
| 38 |
+
_DATA_DIR / "natural_earth_geoparquet/test_ne_geography.parquet",
|
| 39 |
)
|
| 40 |
)
|
| 41 |
|
src/gazet/export.py
CHANGED
|
@@ -1,12 +1,16 @@
|
|
| 1 |
import json
|
|
|
|
| 2 |
import pathlib
|
| 3 |
import re
|
|
|
|
| 4 |
|
| 5 |
import numpy as np
|
| 6 |
import pandas as pd
|
| 7 |
|
|
|
|
| 8 |
|
| 9 |
-
|
|
|
|
| 10 |
"""Convert a value to a JSON-serializable Python type."""
|
| 11 |
if isinstance(val, (bytearray, bytes)):
|
| 12 |
return None
|
|
@@ -24,7 +28,7 @@ def _to_serializable(val):
|
|
| 24 |
def _is_geojson_col(series: pd.Series) -> bool:
|
| 25 |
"""Heuristic: a string column whose non-null values start with '{"type":'."""
|
| 26 |
sample = series.dropna().head(5)
|
| 27 |
-
return (
|
| 28 |
sample.apply(
|
| 29 |
lambda v: isinstance(v, str) and v.lstrip().startswith('{"type":')
|
| 30 |
).all()
|
|
@@ -32,44 +36,14 @@ def _is_geojson_col(series: pd.Series) -> bool:
|
|
| 32 |
)
|
| 33 |
|
| 34 |
|
| 35 |
-
def _is_wkb_col(series: pd.Series) -> bool:
|
| 36 |
-
"""Heuristic: a column whose non-null values are bytearray or bytes (WKB geometry)."""
|
| 37 |
-
sample = series.dropna().head(5)
|
| 38 |
-
return (
|
| 39 |
-
sample.apply(lambda v: isinstance(v, (bytearray, bytes))).all()
|
| 40 |
-
and len(sample) > 0
|
| 41 |
-
)
|
| 42 |
-
|
| 43 |
-
|
| 44 |
-
def _wkb_to_geojson(wkb: bytearray | bytes) -> dict | None:
|
| 45 |
-
"""Convert WKB geometry to GeoJSON dict via DuckDB."""
|
| 46 |
-
import duckdb
|
| 47 |
-
|
| 48 |
-
con = duckdb.connect()
|
| 49 |
-
try:
|
| 50 |
-
con.execute("INSTALL spatial")
|
| 51 |
-
con.execute("LOAD spatial")
|
| 52 |
-
result = con.execute(
|
| 53 |
-
"SELECT ST_AsGeoJSON(ST_GeomFromWKB(?::BLOB)) AS geojson",
|
| 54 |
-
[bytes(wkb)],
|
| 55 |
-
).fetchone()
|
| 56 |
-
if result and result[0]:
|
| 57 |
-
return json.loads(result[0])
|
| 58 |
-
except Exception:
|
| 59 |
-
pass
|
| 60 |
-
finally:
|
| 61 |
-
con.close()
|
| 62 |
-
return None
|
| 63 |
-
|
| 64 |
-
|
| 65 |
def save_geojson(
|
| 66 |
result_df: pd.DataFrame, query: str, output_dir: pathlib.Path | str = "."
|
| 67 |
) -> pathlib.Path:
|
| 68 |
"""Wrap result_df into a GeoJSON FeatureCollection and save to disk.
|
| 69 |
|
| 70 |
-
|
| 71 |
-
|
| 72 |
-
|
| 73 |
"""
|
| 74 |
output_dir = pathlib.Path(output_dir)
|
| 75 |
output_dir.mkdir(parents=True, exist_ok=True)
|
|
@@ -77,46 +51,46 @@ def save_geojson(
|
|
| 77 |
slug = re.sub(r"[^\w]+", "_", query.lower()).strip("_")
|
| 78 |
out_path = output_dir / f"{slug}.geojson"
|
| 79 |
|
| 80 |
-
fc =
|
| 81 |
out_path.write_text(json.dumps(fc, indent=2))
|
| 82 |
-
|
| 83 |
return out_path
|
| 84 |
|
| 85 |
|
| 86 |
def to_feature_collection(result_df: pd.DataFrame) -> dict:
|
| 87 |
-
"""Build a GeoJSON FeatureCollection dict from a result DataFrame.
|
| 88 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 89 |
|
|
|
|
|
|
|
| 90 |
|
| 91 |
-
def _to_feature_collection(result_df: pd.DataFrame) -> dict:
|
| 92 |
-
geojson_cols = [c for c in result_df.columns if _is_geojson_col(result_df[c])]
|
| 93 |
-
wkb_cols = [c for c in result_df.columns if _is_wkb_col(result_df[c])]
|
| 94 |
-
geom_cols = geojson_cols + wkb_cols
|
| 95 |
-
prop_cols = [c for c in result_df.columns if c not in geom_cols]
|
| 96 |
features = []
|
| 97 |
-
for
|
|
|
|
| 98 |
geometry = None
|
| 99 |
-
if
|
| 100 |
-
|
| 101 |
-
|
| 102 |
-
|
| 103 |
-
|
| 104 |
-
|
| 105 |
-
pass
|
| 106 |
-
elif wkb_cols:
|
| 107 |
-
raw = row[wkb_cols[0]]
|
| 108 |
-
if raw and isinstance(raw, (bytearray, bytes)):
|
| 109 |
-
geometry = _wkb_to_geojson(raw)
|
| 110 |
properties = {}
|
| 111 |
for c in prop_cols:
|
| 112 |
-
v =
|
| 113 |
try:
|
| 114 |
-
if
|
| 115 |
continue
|
| 116 |
-
except ValueError:
|
| 117 |
-
pass # pd.
|
| 118 |
properties[c] = _to_serializable(v)
|
|
|
|
| 119 |
features.append(
|
| 120 |
{"type": "Feature", "geometry": geometry, "properties": properties}
|
| 121 |
)
|
|
|
|
| 122 |
return {"type": "FeatureCollection", "features": features}
|
|
|
|
| 1 |
import json
|
| 2 |
+
import logging
|
| 3 |
import pathlib
|
| 4 |
import re
|
| 5 |
+
from typing import Any
|
| 6 |
|
| 7 |
import numpy as np
|
| 8 |
import pandas as pd
|
| 9 |
|
| 10 |
+
logger = logging.getLogger(__name__)
|
| 11 |
|
| 12 |
+
|
| 13 |
+
def _to_serializable(val: Any) -> Any:
|
| 14 |
"""Convert a value to a JSON-serializable Python type."""
|
| 15 |
if isinstance(val, (bytearray, bytes)):
|
| 16 |
return None
|
|
|
|
| 28 |
def _is_geojson_col(series: pd.Series) -> bool:
|
| 29 |
"""Heuristic: a string column whose non-null values start with '{"type":'."""
|
| 30 |
sample = series.dropna().head(5)
|
| 31 |
+
return bool(
|
| 32 |
sample.apply(
|
| 33 |
lambda v: isinstance(v, str) and v.lstrip().startswith('{"type":')
|
| 34 |
).all()
|
|
|
|
| 36 |
)
|
| 37 |
|
| 38 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 39 |
def save_geojson(
|
| 40 |
result_df: pd.DataFrame, query: str, output_dir: pathlib.Path | str = "."
|
| 41 |
) -> pathlib.Path:
|
| 42 |
"""Wrap result_df into a GeoJSON FeatureCollection and save to disk.
|
| 43 |
|
| 44 |
+
Expects the ``geometry`` column to already be normalized to GeoJSON
|
| 45 |
+
strings (run ``geometry.normalize_geometry_to_geojson`` first).
|
| 46 |
+
Remaining columns become properties.
|
| 47 |
"""
|
| 48 |
output_dir = pathlib.Path(output_dir)
|
| 49 |
output_dir.mkdir(parents=True, exist_ok=True)
|
|
|
|
| 51 |
slug = re.sub(r"[^\w]+", "_", query.lower()).strip("_")
|
| 52 |
out_path = output_dir / f"{slug}.geojson"
|
| 53 |
|
| 54 |
+
fc = to_feature_collection(result_df)
|
| 55 |
out_path.write_text(json.dumps(fc, indent=2))
|
| 56 |
+
logger.info("Saved %d feature(s) to %s", len(fc["features"]), out_path.resolve())
|
| 57 |
return out_path
|
| 58 |
|
| 59 |
|
| 60 |
def to_feature_collection(result_df: pd.DataFrame) -> dict:
|
| 61 |
+
"""Build a GeoJSON FeatureCollection dict from a result DataFrame.
|
| 62 |
+
|
| 63 |
+
Expects the ``geometry`` column to already be normalized to GeoJSON
|
| 64 |
+
strings. Remaining columns become properties.
|
| 65 |
+
"""
|
| 66 |
+
if "geometry" not in result_df.columns or result_df.empty:
|
| 67 |
+
return {"type": "FeatureCollection", "features": []}
|
| 68 |
|
| 69 |
+
prop_cols = [c for c in result_df.columns if c != "geometry"]
|
| 70 |
+
records = result_df.to_dict(orient="records")
|
| 71 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 72 |
features = []
|
| 73 |
+
for record in records:
|
| 74 |
+
raw = record.pop("geometry", None)
|
| 75 |
geometry = None
|
| 76 |
+
if raw and isinstance(raw, str):
|
| 77 |
+
try:
|
| 78 |
+
geometry = json.loads(raw)
|
| 79 |
+
except (json.JSONDecodeError, TypeError):
|
| 80 |
+
pass
|
| 81 |
+
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 82 |
properties = {}
|
| 83 |
for c in prop_cols:
|
| 84 |
+
v = record.get(c)
|
| 85 |
try:
|
| 86 |
+
if v is None or pd.isna(v):
|
| 87 |
continue
|
| 88 |
+
except (ValueError, TypeError):
|
| 89 |
+
pass # pd.isna fails on arrays — treat as present
|
| 90 |
properties[c] = _to_serializable(v)
|
| 91 |
+
|
| 92 |
features.append(
|
| 93 |
{"type": "Feature", "geometry": geometry, "properties": properties}
|
| 94 |
)
|
| 95 |
+
|
| 96 |
return {"type": "FeatureCollection", "features": features}
|
src/gazet/geometry.py
ADDED
|
@@ -0,0 +1,62 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
import json
|
| 2 |
+
from typing import Any
|
| 3 |
+
|
| 4 |
+
import duckdb
|
| 5 |
+
import pandas as pd
|
| 6 |
+
|
| 7 |
+
SIMPLIFY_TOLERANCE = 0.001 # ~100m; adequate for web map display
|
| 8 |
+
COORD_PRECISION = 5 # ~1.1m; sufficient for web maps, shrinks payload 30-50%
|
| 9 |
+
|
| 10 |
+
|
| 11 |
+
def _round_coords(obj: Any, precision: int) -> Any:
|
| 12 |
+
"""Recursively round numeric coordinates in a GeoJSON geometry dict."""
|
| 13 |
+
if isinstance(obj, float):
|
| 14 |
+
return round(obj, precision)
|
| 15 |
+
if isinstance(obj, list):
|
| 16 |
+
return [_round_coords(x, precision) for x in obj]
|
| 17 |
+
if isinstance(obj, dict):
|
| 18 |
+
return {k: _round_coords(v, precision) for k, v in obj.items()}
|
| 19 |
+
return obj
|
| 20 |
+
|
| 21 |
+
|
| 22 |
+
def normalize_geometry_to_geojson(
|
| 23 |
+
con: duckdb.DuckDBPyConnection,
|
| 24 |
+
result_df: pd.DataFrame,
|
| 25 |
+
) -> pd.DataFrame:
|
| 26 |
+
"""Simplify geometries and convert to compact GeoJSON text.
|
| 27 |
+
|
| 28 |
+
Accepts either binary WKB blobs or GeoJSON strings in the `geometry` column.
|
| 29 |
+
Runs ST_SimplifyPreserveTopology then rounds coordinates to reduce payload.
|
| 30 |
+
|
| 31 |
+
``con`` must already have the spatial extension loaded
|
| 32 |
+
(``INSTALL spatial; LOAD spatial;``).
|
| 33 |
+
"""
|
| 34 |
+
if "geometry" not in result_df.columns or result_df.empty:
|
| 35 |
+
return result_df
|
| 36 |
+
|
| 37 |
+
sample = result_df["geometry"].dropna().head(5)
|
| 38 |
+
if sample.empty:
|
| 39 |
+
return result_df
|
| 40 |
+
|
| 41 |
+
def _simplify(val: Any) -> str | None:
|
| 42 |
+
if val is None:
|
| 43 |
+
return None
|
| 44 |
+
if isinstance(val, (bytes, bytearray, memoryview)):
|
| 45 |
+
geom_expr = "ST_GeomFromWKB(?::BLOB)"
|
| 46 |
+
arg: Any = bytes(val)
|
| 47 |
+
elif isinstance(val, str) and val.lstrip().startswith('{"'):
|
| 48 |
+
geom_expr = "ST_GeomFromGeoJSON(?)"
|
| 49 |
+
arg = val
|
| 50 |
+
else:
|
| 51 |
+
return val
|
| 52 |
+
row = con.execute(
|
| 53 |
+
f"SELECT ST_AsGeoJSON(ST_SimplifyPreserveTopology({geom_expr}, ?))",
|
| 54 |
+
[arg, SIMPLIFY_TOLERANCE],
|
| 55 |
+
).fetchone()
|
| 56 |
+
if not row or not row[0]:
|
| 57 |
+
return None
|
| 58 |
+
return json.dumps(_round_coords(json.loads(row[0]), COORD_PRECISION))
|
| 59 |
+
|
| 60 |
+
normalized_df = result_df.copy()
|
| 61 |
+
normalized_df["geometry"] = normalized_df["geometry"].apply(_simplify)
|
| 62 |
+
return normalized_df
|
src/gazet/lm.py
CHANGED
|
@@ -11,6 +11,7 @@ from .config import (
|
|
| 11 |
LLAMA_SERVER_URL,
|
| 12 |
LLAMA_TEMPERATURE,
|
| 13 |
PLACE_EXTRACTION_MODEL,
|
|
|
|
| 14 |
SQL_GENERATION_MODEL,
|
| 15 |
)
|
| 16 |
from .schemas import Place, PlacesResult
|
|
@@ -46,9 +47,7 @@ class ExtractPlaces(dspy.Signature):
|
|
| 46 |
query: str = dspy.InputField(
|
| 47 |
desc="Natural language query mentioning one or more place names"
|
| 48 |
)
|
| 49 |
-
result: PlacesResult = dspy.OutputField(
|
| 50 |
-
desc="Extracted place names in query order"
|
| 51 |
-
)
|
| 52 |
|
| 53 |
|
| 54 |
class WriteGeoSQL(dspy.Signature):
|
|
@@ -90,20 +89,20 @@ class WriteGeoSQL(dspy.Signature):
|
|
| 90 |
|
| 91 |
|
| 92 |
place_extraction_lm = dspy.LM(
|
| 93 |
-
f"ollama_chat/{PLACE_EXTRACTION_MODEL}",
|
| 94 |
-
api_base="http://localhost:11434",
|
| 95 |
-
api_key="",
|
| 96 |
-
temperature=0.1,
|
| 97 |
cache=False,
|
| 98 |
)
|
| 99 |
|
| 100 |
sql_generation_lm = dspy.LM(
|
| 101 |
-
f"ollama_chat/{SQL_GENERATION_MODEL}",
|
| 102 |
-
api_base="http://localhost:11434",
|
| 103 |
-
api_key="",
|
| 104 |
-
temperature=0.1,
|
| 105 |
cache=False,
|
| 106 |
-
think=False
|
| 107 |
)
|
| 108 |
|
| 109 |
|
|
@@ -112,7 +111,7 @@ class PlaceExtractor(dspy.Module):
|
|
| 112 |
super().__init__()
|
| 113 |
self.lm = lm
|
| 114 |
self.predictor = dspy.Predict(ExtractPlaces)
|
| 115 |
-
|
| 116 |
def forward(self, query: str):
|
| 117 |
with dspy.context(lm=self.lm):
|
| 118 |
return self.predictor(query=query)
|
|
@@ -123,16 +122,22 @@ class SQLWriter(dspy.Module):
|
|
| 123 |
super().__init__()
|
| 124 |
self.lm = lm
|
| 125 |
self.predictor = dspy.Predict(WriteGeoSQL)
|
| 126 |
-
|
| 127 |
-
def forward(
|
| 128 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 129 |
with dspy.context(lm=self.lm):
|
| 130 |
return self.predictor(
|
| 131 |
user_query=user_query,
|
| 132 |
schema=schema,
|
| 133 |
candidates=candidates,
|
| 134 |
previous_sql=previous_sql,
|
| 135 |
-
execution_error=execution_error
|
| 136 |
)
|
| 137 |
|
| 138 |
|
|
@@ -142,39 +147,12 @@ write_sql = SQLWriter(lm=sql_generation_lm)
|
|
| 142 |
|
| 143 |
# ── GGUF SQL generation via llama-server ──────────────────────────────────────
|
| 144 |
|
| 145 |
-
|
| 146 |
|
| 147 |
You have access to two DuckDB parquet tables. Given a set of candidate entities and a user query, generate the SQL to retrieve the desired geometry.
|
| 148 |
|
| 149 |
<SCHEMA>
|
| 150 |
-
|
| 151 |
-
query: read_parquet('divisions_area')
|
| 152 |
-
columns:
|
| 153 |
-
id VARCHAR -- unique feature id
|
| 154 |
-
names STRUCT("primary" VARCHAR, ...)
|
| 155 |
-
country VARCHAR -- ISO 3166-1 alpha-2
|
| 156 |
-
subtype VARCHAR -- country | region | county
|
| 157 |
-
class VARCHAR
|
| 158 |
-
region VARCHAR
|
| 159 |
-
admin_level INTEGER
|
| 160 |
-
division_id VARCHAR
|
| 161 |
-
is_land BOOLEAN
|
| 162 |
-
is_territorial BOOLEAN
|
| 163 |
-
geometry GEOMETRY -- WGS-84 polygon/multipolygon (spatial ext loaded)
|
| 164 |
-
|
| 165 |
-
2. natural_earth -- Natural Earth geography polygons (oceans, seas, rivers, terrain)
|
| 166 |
-
query: read_parquet('natural_earth')
|
| 167 |
-
columns:
|
| 168 |
-
id VARCHAR -- unique feature id prefixed 'ne_'
|
| 169 |
-
names STRUCT("primary" VARCHAR, ...)
|
| 170 |
-
country VARCHAR
|
| 171 |
-
subtype VARCHAR -- e.g. 'ocean', 'sea', 'bay', 'range/mtn', 'island group'
|
| 172 |
-
class VARCHAR
|
| 173 |
-
region VARCHAR
|
| 174 |
-
admin_level INTEGER
|
| 175 |
-
is_land BOOLEAN
|
| 176 |
-
is_territorial BOOLEAN
|
| 177 |
-
geometry GEOMETRY -- WGS-84 polygon/multipolygon (spatial ext loaded)
|
| 178 |
</SCHEMA>
|
| 179 |
|
| 180 |
The candidates table has a 'source' column: 'divisions_area' or 'natural_earth'.
|
|
@@ -196,8 +174,7 @@ def _postprocess_sql(text: str) -> str:
|
|
| 196 |
cleaned = text.strip()
|
| 197 |
if "```sql" in cleaned:
|
| 198 |
cleaned = cleaned.split("```sql", 1)[1]
|
| 199 |
-
|
| 200 |
-
cleaned = cleaned[3:]
|
| 201 |
if "```" in cleaned:
|
| 202 |
cleaned = cleaned.split("```", 1)[0]
|
| 203 |
return cleaned.strip()
|
|
@@ -324,15 +301,16 @@ def generate_places(user_query: str) -> PlacesResult:
|
|
| 324 |
# Strip markdown fences if the model wrapped the JSON
|
| 325 |
if raw_output.startswith("```"):
|
| 326 |
raw_output = raw_output.split("```")[1]
|
| 327 |
-
|
| 328 |
-
raw_output = raw_output[4:]
|
| 329 |
raw_output = raw_output.strip()
|
| 330 |
|
| 331 |
try:
|
| 332 |
data = json.loads(raw_output)
|
| 333 |
return PlacesResult.model_validate(data)
|
| 334 |
except Exception as exc:
|
| 335 |
-
logger.warning(
|
|
|
|
|
|
|
| 336 |
# Best-effort: treat entire query as a single unnamed place
|
| 337 |
return PlacesResult(places=[Place(place=user_query)])
|
| 338 |
|
|
@@ -355,7 +333,10 @@ def generate_sql(user_query: str, candidates_df: pd.DataFrame) -> str:
|
|
| 355 |
)
|
| 356 |
|
| 357 |
messages = [
|
| 358 |
-
{
|
|
|
|
|
|
|
|
|
|
| 359 |
{"role": "user", "content": user_prompt},
|
| 360 |
]
|
| 361 |
raw_output = _llama_chat_complete(messages)
|
|
|
|
| 11 |
LLAMA_SERVER_URL,
|
| 12 |
LLAMA_TEMPERATURE,
|
| 13 |
PLACE_EXTRACTION_MODEL,
|
| 14 |
+
SCHEMA_INFO,
|
| 15 |
SQL_GENERATION_MODEL,
|
| 16 |
)
|
| 17 |
from .schemas import Place, PlacesResult
|
|
|
|
| 47 |
query: str = dspy.InputField(
|
| 48 |
desc="Natural language query mentioning one or more place names"
|
| 49 |
)
|
| 50 |
+
result: PlacesResult = dspy.OutputField(desc="Extracted place names in query order")
|
|
|
|
|
|
|
| 51 |
|
| 52 |
|
| 53 |
class WriteGeoSQL(dspy.Signature):
|
|
|
|
| 89 |
|
| 90 |
|
| 91 |
place_extraction_lm = dspy.LM(
|
| 92 |
+
f"ollama_chat/{PLACE_EXTRACTION_MODEL}",
|
| 93 |
+
api_base="http://localhost:11434",
|
| 94 |
+
api_key="",
|
| 95 |
+
temperature=0.1,
|
| 96 |
cache=False,
|
| 97 |
)
|
| 98 |
|
| 99 |
sql_generation_lm = dspy.LM(
|
| 100 |
+
f"ollama_chat/{SQL_GENERATION_MODEL}",
|
| 101 |
+
api_base="http://localhost:11434",
|
| 102 |
+
api_key="",
|
| 103 |
+
temperature=0.1,
|
| 104 |
cache=False,
|
| 105 |
+
think=False,
|
| 106 |
)
|
| 107 |
|
| 108 |
|
|
|
|
| 111 |
super().__init__()
|
| 112 |
self.lm = lm
|
| 113 |
self.predictor = dspy.Predict(ExtractPlaces)
|
| 114 |
+
|
| 115 |
def forward(self, query: str):
|
| 116 |
with dspy.context(lm=self.lm):
|
| 117 |
return self.predictor(query=query)
|
|
|
|
| 122 |
super().__init__()
|
| 123 |
self.lm = lm
|
| 124 |
self.predictor = dspy.Predict(WriteGeoSQL)
|
| 125 |
+
|
| 126 |
+
def forward(
|
| 127 |
+
self,
|
| 128 |
+
user_query: str,
|
| 129 |
+
schema: str,
|
| 130 |
+
candidates: str,
|
| 131 |
+
previous_sql: str = "",
|
| 132 |
+
execution_error: str = "",
|
| 133 |
+
):
|
| 134 |
with dspy.context(lm=self.lm):
|
| 135 |
return self.predictor(
|
| 136 |
user_query=user_query,
|
| 137 |
schema=schema,
|
| 138 |
candidates=candidates,
|
| 139 |
previous_sql=previous_sql,
|
| 140 |
+
execution_error=execution_error,
|
| 141 |
)
|
| 142 |
|
| 143 |
|
|
|
|
| 147 |
|
| 148 |
# ── GGUF SQL generation via llama-server ──────────────────────────────────────
|
| 149 |
|
| 150 |
+
_SYSTEM_PROMPT_TEMPLATE = """You are a text to SQL query translator that helps in natural language geocoding.
|
| 151 |
|
| 152 |
You have access to two DuckDB parquet tables. Given a set of candidate entities and a user query, generate the SQL to retrieve the desired geometry.
|
| 153 |
|
| 154 |
<SCHEMA>
|
| 155 |
+
{schema}
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 156 |
</SCHEMA>
|
| 157 |
|
| 158 |
The candidates table has a 'source' column: 'divisions_area' or 'natural_earth'.
|
|
|
|
| 174 |
cleaned = text.strip()
|
| 175 |
if "```sql" in cleaned:
|
| 176 |
cleaned = cleaned.split("```sql", 1)[1]
|
| 177 |
+
cleaned = cleaned.removeprefix("```")
|
|
|
|
| 178 |
if "```" in cleaned:
|
| 179 |
cleaned = cleaned.split("```", 1)[0]
|
| 180 |
return cleaned.strip()
|
|
|
|
| 301 |
# Strip markdown fences if the model wrapped the JSON
|
| 302 |
if raw_output.startswith("```"):
|
| 303 |
raw_output = raw_output.split("```")[1]
|
| 304 |
+
raw_output = raw_output.removeprefix("json")
|
|
|
|
| 305 |
raw_output = raw_output.strip()
|
| 306 |
|
| 307 |
try:
|
| 308 |
data = json.loads(raw_output)
|
| 309 |
return PlacesResult.model_validate(data)
|
| 310 |
except Exception as exc:
|
| 311 |
+
logger.warning(
|
| 312 |
+
"generate_places: failed to parse output %r: %s", raw_output, exc
|
| 313 |
+
)
|
| 314 |
# Best-effort: treat entire query as a single unnamed place
|
| 315 |
return PlacesResult(places=[Place(place=user_query)])
|
| 316 |
|
|
|
|
| 333 |
)
|
| 334 |
|
| 335 |
messages = [
|
| 336 |
+
{
|
| 337 |
+
"role": "system",
|
| 338 |
+
"content": _SYSTEM_PROMPT_TEMPLATE.format(schema=SCHEMA_INFO.strip()),
|
| 339 |
+
},
|
| 340 |
{"role": "user", "content": user_prompt},
|
| 341 |
]
|
| 342 |
raw_output = _llama_chat_complete(messages)
|
src/gazet/schemas.py
CHANGED
|
@@ -1,292 +1,17 @@
|
|
| 1 |
-
from
|
| 2 |
-
|
| 3 |
-
from pydantic import BaseModel
|
| 4 |
-
|
| 5 |
-
|
| 6 |
-
SUBTYPES = Literal[
|
| 7 |
-
"country",
|
| 8 |
-
"region",
|
| 9 |
-
"county",
|
| 10 |
-
]
|
| 11 |
-
COUNTRIES = Literal[
|
| 12 |
-
"AD",
|
| 13 |
-
"AE",
|
| 14 |
-
"AF",
|
| 15 |
-
"AG",
|
| 16 |
-
"AI",
|
| 17 |
-
"AL",
|
| 18 |
-
"AM",
|
| 19 |
-
"AO",
|
| 20 |
-
"AQ",
|
| 21 |
-
"AR",
|
| 22 |
-
"AS",
|
| 23 |
-
"AT",
|
| 24 |
-
"AU",
|
| 25 |
-
"AW",
|
| 26 |
-
"AX",
|
| 27 |
-
"AZ",
|
| 28 |
-
"BA",
|
| 29 |
-
"BB",
|
| 30 |
-
"BD",
|
| 31 |
-
"BE",
|
| 32 |
-
"BF",
|
| 33 |
-
"BG",
|
| 34 |
-
"BH",
|
| 35 |
-
"BI",
|
| 36 |
-
"BJ",
|
| 37 |
-
"BL",
|
| 38 |
-
"BM",
|
| 39 |
-
"BN",
|
| 40 |
-
"BO",
|
| 41 |
-
"BQ",
|
| 42 |
-
"BR",
|
| 43 |
-
"BS",
|
| 44 |
-
"BT",
|
| 45 |
-
"BV",
|
| 46 |
-
"BW",
|
| 47 |
-
"BY",
|
| 48 |
-
"BZ",
|
| 49 |
-
"CA",
|
| 50 |
-
"CC",
|
| 51 |
-
"CD",
|
| 52 |
-
"CF",
|
| 53 |
-
"CG",
|
| 54 |
-
"CH",
|
| 55 |
-
"CI",
|
| 56 |
-
"CK",
|
| 57 |
-
"CL",
|
| 58 |
-
"CM",
|
| 59 |
-
"CN",
|
| 60 |
-
"CO",
|
| 61 |
-
"CP",
|
| 62 |
-
"CR",
|
| 63 |
-
"CU",
|
| 64 |
-
"CV",
|
| 65 |
-
"CW",
|
| 66 |
-
"CX",
|
| 67 |
-
"CY",
|
| 68 |
-
"CZ",
|
| 69 |
-
"DE",
|
| 70 |
-
"DJ",
|
| 71 |
-
"DK",
|
| 72 |
-
"DM",
|
| 73 |
-
"DO",
|
| 74 |
-
"DZ",
|
| 75 |
-
"EC",
|
| 76 |
-
"EE",
|
| 77 |
-
"EG",
|
| 78 |
-
"ER",
|
| 79 |
-
"ES",
|
| 80 |
-
"ET",
|
| 81 |
-
"FI",
|
| 82 |
-
"FJ",
|
| 83 |
-
"FK",
|
| 84 |
-
"FM",
|
| 85 |
-
"FO",
|
| 86 |
-
"FR",
|
| 87 |
-
"GA",
|
| 88 |
-
"GB",
|
| 89 |
-
"GD",
|
| 90 |
-
"GE",
|
| 91 |
-
"GF",
|
| 92 |
-
"GG",
|
| 93 |
-
"GH",
|
| 94 |
-
"GI",
|
| 95 |
-
"GL",
|
| 96 |
-
"GM",
|
| 97 |
-
"GN",
|
| 98 |
-
"GP",
|
| 99 |
-
"GQ",
|
| 100 |
-
"GR",
|
| 101 |
-
"GS",
|
| 102 |
-
"GT",
|
| 103 |
-
"GU",
|
| 104 |
-
"GW",
|
| 105 |
-
"GY",
|
| 106 |
-
"HK",
|
| 107 |
-
"HM",
|
| 108 |
-
"HN",
|
| 109 |
-
"HR",
|
| 110 |
-
"HT",
|
| 111 |
-
"HU",
|
| 112 |
-
"ID",
|
| 113 |
-
"IE",
|
| 114 |
-
"IL",
|
| 115 |
-
"IM",
|
| 116 |
-
"IN",
|
| 117 |
-
"IO",
|
| 118 |
-
"IQ",
|
| 119 |
-
"IR",
|
| 120 |
-
"IS",
|
| 121 |
-
"IT",
|
| 122 |
-
"JE",
|
| 123 |
-
"JM",
|
| 124 |
-
"JO",
|
| 125 |
-
"JP",
|
| 126 |
-
"KE",
|
| 127 |
-
"KG",
|
| 128 |
-
"KH",
|
| 129 |
-
"KI",
|
| 130 |
-
"KM",
|
| 131 |
-
"KN",
|
| 132 |
-
"KP",
|
| 133 |
-
"KR",
|
| 134 |
-
"KW",
|
| 135 |
-
"KY",
|
| 136 |
-
"KZ",
|
| 137 |
-
"LA",
|
| 138 |
-
"LB",
|
| 139 |
-
"LC",
|
| 140 |
-
"LI",
|
| 141 |
-
"LK",
|
| 142 |
-
"LR",
|
| 143 |
-
"LS",
|
| 144 |
-
"LT",
|
| 145 |
-
"LU",
|
| 146 |
-
"LV",
|
| 147 |
-
"LY",
|
| 148 |
-
"MA",
|
| 149 |
-
"MC",
|
| 150 |
-
"MD",
|
| 151 |
-
"ME",
|
| 152 |
-
"MF",
|
| 153 |
-
"MG",
|
| 154 |
-
"MH",
|
| 155 |
-
"MK",
|
| 156 |
-
"ML",
|
| 157 |
-
"MM",
|
| 158 |
-
"MN",
|
| 159 |
-
"MO",
|
| 160 |
-
"MP",
|
| 161 |
-
"MQ",
|
| 162 |
-
"MR",
|
| 163 |
-
"MS",
|
| 164 |
-
"MT",
|
| 165 |
-
"MU",
|
| 166 |
-
"MV",
|
| 167 |
-
"MW",
|
| 168 |
-
"MX",
|
| 169 |
-
"MY",
|
| 170 |
-
"MZ",
|
| 171 |
-
"NA",
|
| 172 |
-
"NC",
|
| 173 |
-
"NE",
|
| 174 |
-
"NF",
|
| 175 |
-
"NG",
|
| 176 |
-
"NI",
|
| 177 |
-
"NL",
|
| 178 |
-
"NO",
|
| 179 |
-
"NP",
|
| 180 |
-
"NR",
|
| 181 |
-
"NU",
|
| 182 |
-
"NZ",
|
| 183 |
-
"OM",
|
| 184 |
-
"PA",
|
| 185 |
-
"PE",
|
| 186 |
-
"PF",
|
| 187 |
-
"PG",
|
| 188 |
-
"PH",
|
| 189 |
-
"PK",
|
| 190 |
-
"PL",
|
| 191 |
-
"PM",
|
| 192 |
-
"PN",
|
| 193 |
-
"PR",
|
| 194 |
-
"PT",
|
| 195 |
-
"PW",
|
| 196 |
-
"PY",
|
| 197 |
-
"QA",
|
| 198 |
-
"RE",
|
| 199 |
-
"RO",
|
| 200 |
-
"RS",
|
| 201 |
-
"RU",
|
| 202 |
-
"RW",
|
| 203 |
-
"SA",
|
| 204 |
-
"SB",
|
| 205 |
-
"SC",
|
| 206 |
-
"SD",
|
| 207 |
-
"SE",
|
| 208 |
-
"SG",
|
| 209 |
-
"SH",
|
| 210 |
-
"SI",
|
| 211 |
-
"SJ",
|
| 212 |
-
"SK",
|
| 213 |
-
"SL",
|
| 214 |
-
"SM",
|
| 215 |
-
"SN",
|
| 216 |
-
"SO",
|
| 217 |
-
"SR",
|
| 218 |
-
"SS",
|
| 219 |
-
"ST",
|
| 220 |
-
"SV",
|
| 221 |
-
"SX",
|
| 222 |
-
"SY",
|
| 223 |
-
"SZ",
|
| 224 |
-
"TC",
|
| 225 |
-
"TD",
|
| 226 |
-
"TF",
|
| 227 |
-
"TG",
|
| 228 |
-
"TH",
|
| 229 |
-
"TJ",
|
| 230 |
-
"TK",
|
| 231 |
-
"TL",
|
| 232 |
-
"TM",
|
| 233 |
-
"TN",
|
| 234 |
-
"TO",
|
| 235 |
-
"TR",
|
| 236 |
-
"TT",
|
| 237 |
-
"TV",
|
| 238 |
-
"TW",
|
| 239 |
-
"TZ",
|
| 240 |
-
"UA",
|
| 241 |
-
"UG",
|
| 242 |
-
"UM",
|
| 243 |
-
"US",
|
| 244 |
-
"UY",
|
| 245 |
-
"UZ",
|
| 246 |
-
"VA",
|
| 247 |
-
"VC",
|
| 248 |
-
"VE",
|
| 249 |
-
"VG",
|
| 250 |
-
"VI",
|
| 251 |
-
"VN",
|
| 252 |
-
"VU",
|
| 253 |
-
"WF",
|
| 254 |
-
"WS",
|
| 255 |
-
"XA",
|
| 256 |
-
"XB",
|
| 257 |
-
"XC",
|
| 258 |
-
"XD",
|
| 259 |
-
"XE",
|
| 260 |
-
"XG",
|
| 261 |
-
"XH",
|
| 262 |
-
"XI",
|
| 263 |
-
"XJ",
|
| 264 |
-
"XK",
|
| 265 |
-
"XL",
|
| 266 |
-
"XM",
|
| 267 |
-
"XN",
|
| 268 |
-
"XO",
|
| 269 |
-
"XP",
|
| 270 |
-
"XQ",
|
| 271 |
-
"XR",
|
| 272 |
-
"XS",
|
| 273 |
-
"XT",
|
| 274 |
-
"XU",
|
| 275 |
-
"XW",
|
| 276 |
-
"XX",
|
| 277 |
-
"XY",
|
| 278 |
-
"XZ",
|
| 279 |
-
"YE",
|
| 280 |
-
"YT",
|
| 281 |
-
"ZA",
|
| 282 |
-
"ZM",
|
| 283 |
-
"ZW",
|
| 284 |
-
]
|
| 285 |
|
| 286 |
|
| 287 |
class Place(BaseModel):
|
| 288 |
place: str
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 289 |
|
| 290 |
|
| 291 |
class PlacesResult(BaseModel):
|
| 292 |
-
places:
|
|
|
|
| 1 |
+
from pydantic import BaseModel, Field
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 2 |
|
| 3 |
|
| 4 |
class Place(BaseModel):
|
| 5 |
place: str
|
| 6 |
+
country: str | None = Field(
|
| 7 |
+
default=None,
|
| 8 |
+
description="ISO 3166-1 alpha-2 country code (if known)",
|
| 9 |
+
)
|
| 10 |
+
subtype: str | None = Field(
|
| 11 |
+
default=None,
|
| 12 |
+
description="Administrative type: country, region, county, physical feature",
|
| 13 |
+
)
|
| 14 |
|
| 15 |
|
| 16 |
class PlacesResult(BaseModel):
|
| 17 |
+
places: list[Place]
|
src/gazet/search.py
CHANGED
|
@@ -1,23 +1,49 @@
|
|
|
|
|
|
|
|
| 1 |
import duckdb
|
| 2 |
import pandas as pd
|
| 3 |
|
| 4 |
from .config import DIVISIONS_AREA_PATH, NATURAL_EARTH_PATH
|
| 5 |
from .schemas import Place
|
| 6 |
|
|
|
|
|
|
|
| 7 |
|
| 8 |
def simple_fuzzy_search(
|
| 9 |
con: duckdb.DuckDBPyConnection,
|
| 10 |
path: str,
|
| 11 |
source: str,
|
| 12 |
place: Place,
|
| 13 |
-
name_expr: str =
|
| 14 |
extra_select: str = "",
|
| 15 |
limit: int = 5,
|
|
|
|
|
|
|
| 16 |
) -> pd.DataFrame:
|
| 17 |
-
"""Jaro-Winkler similarity search using only the place name.
|
| 18 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 19 |
|
| 20 |
extra_clause = f", {extra_select}" if extra_select else ""
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 21 |
rel = con.execute(
|
| 22 |
f"""
|
| 23 |
SELECT
|
|
@@ -29,11 +55,12 @@ def simple_fuzzy_search(
|
|
| 29 |
region,
|
| 30 |
admin_level,
|
| 31 |
is_land,
|
| 32 |
-
is_territorial{extra_clause},
|
| 33 |
-
jaro_winkler_similarity(lower({name_expr}), lower(?)) AS similarity
|
|
|
|
| 34 |
FROM read_parquet(?)
|
| 35 |
WHERE {name_expr} IS NOT NULL AND trim({name_expr}) != ''
|
| 36 |
-
ORDER BY similarity DESC, admin_level ASC
|
| 37 |
LIMIT ?
|
| 38 |
""",
|
| 39 |
params,
|
|
@@ -41,29 +68,44 @@ def simple_fuzzy_search(
|
|
| 41 |
df = rel.fetchdf()
|
| 42 |
df.insert(0, "source", source)
|
| 43 |
if df.empty:
|
| 44 |
-
|
| 45 |
else:
|
| 46 |
-
|
| 47 |
-
print(df.to_string(index=False))
|
| 48 |
return df
|
| 49 |
|
| 50 |
|
| 51 |
def search_divisions_area(
|
| 52 |
-
con: duckdb.DuckDBPyConnection,
|
|
|
|
|
|
|
|
|
|
|
|
|
| 53 |
) -> pd.DataFrame:
|
| 54 |
-
"""Fuzzy-match a place against divisions_area (Overture admin boundaries).
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 55 |
return simple_fuzzy_search(
|
| 56 |
con,
|
| 57 |
DIVISIONS_AREA_PATH,
|
| 58 |
"divisions_area",
|
| 59 |
place,
|
|
|
|
| 60 |
extra_select="division_id",
|
| 61 |
limit=limit,
|
|
|
|
|
|
|
| 62 |
)
|
| 63 |
|
| 64 |
|
| 65 |
def search_natural_earth(
|
| 66 |
-
con: duckdb.DuckDBPyConnection,
|
|
|
|
|
|
|
|
|
|
|
|
|
| 67 |
) -> pd.DataFrame:
|
| 68 |
"""Fuzzy-match a place against Natural Earth geography polygons."""
|
| 69 |
return simple_fuzzy_search(
|
|
@@ -71,22 +113,133 @@ def search_natural_earth(
|
|
| 71 |
NATURAL_EARTH_PATH,
|
| 72 |
"natural_earth",
|
| 73 |
place,
|
| 74 |
-
name_expr=
|
| 75 |
limit=limit,
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 76 |
)
|
| 77 |
|
| 78 |
|
| 79 |
def search_candidates(
|
| 80 |
-
con: duckdb.DuckDBPyConnection,
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 81 |
) -> list[pd.DataFrame]:
|
| 82 |
-
"""Return candidate DataFrames for a place from
|
| 83 |
|
| 84 |
-
|
| 85 |
-
natural features when the model assigns an incorrect admin
|
|
|
|
| 86 |
"""
|
| 87 |
results = []
|
| 88 |
-
for
|
| 89 |
-
df =
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 90 |
if not df.empty:
|
| 91 |
results.append(df)
|
| 92 |
return results
|
|
|
|
| 1 |
+
import logging
|
| 2 |
+
|
| 3 |
import duckdb
|
| 4 |
import pandas as pd
|
| 5 |
|
| 6 |
from .config import DIVISIONS_AREA_PATH, NATURAL_EARTH_PATH
|
| 7 |
from .schemas import Place
|
| 8 |
|
| 9 |
+
logger = logging.getLogger(__name__)
|
| 10 |
+
|
| 11 |
|
| 12 |
def simple_fuzzy_search(
|
| 13 |
con: duckdb.DuckDBPyConnection,
|
| 14 |
path: str,
|
| 15 |
source: str,
|
| 16 |
place: Place,
|
| 17 |
+
name_expr: str = "names.common.en",
|
| 18 |
extra_select: str = "",
|
| 19 |
limit: int = 5,
|
| 20 |
+
include_geometry: bool = False,
|
| 21 |
+
include_bbox: bool = False,
|
| 22 |
) -> pd.DataFrame:
|
| 23 |
+
"""Jaro-Winkler similarity search using only the place name.
|
| 24 |
+
|
| 25 |
+
Ranks exact-substring matches (query literally contained in the name,
|
| 26 |
+
e.g. "Loja" in "Loja Province") ahead of pure edit-distance near-misses
|
| 27 |
+
(e.g. "Lodja"), even when the near-miss scores a slightly higher raw
|
| 28 |
+
Jaro-Winkler similarity — substring containment is a stronger signal of
|
| 29 |
+
a true match than character-level similarity alone.
|
| 30 |
+
|
| 31 |
+
``include_geometry``/``include_bbox`` require the spatial extension to
|
| 32 |
+
already be loaded on ``con`` (``INSTALL spatial; LOAD spatial;``).
|
| 33 |
+
``include_bbox`` computes ``[minx, miny, maxx, maxy]`` via
|
| 34 |
+
ST_XMin/YMin/XMax/YMax — a much smaller payload than full geometry
|
| 35 |
+
(no coordinate arrays or GeoJSON serialization), for lightweight
|
| 36 |
+
context (e.g. ``ids_only`` responses).
|
| 37 |
+
"""
|
| 38 |
+
params = [place.place, place.place, path, limit]
|
| 39 |
|
| 40 |
extra_clause = f", {extra_select}" if extra_select else ""
|
| 41 |
+
geometry_clause = ", ST_AsGeoJSON(geometry) AS geometry" if include_geometry else ""
|
| 42 |
+
bbox_clause = (
|
| 43 |
+
", [ST_XMin(geometry), ST_YMin(geometry), ST_XMax(geometry), ST_YMax(geometry)] AS bbox"
|
| 44 |
+
if include_bbox
|
| 45 |
+
else ""
|
| 46 |
+
)
|
| 47 |
rel = con.execute(
|
| 48 |
f"""
|
| 49 |
SELECT
|
|
|
|
| 55 |
region,
|
| 56 |
admin_level,
|
| 57 |
is_land,
|
| 58 |
+
is_territorial{extra_clause}{geometry_clause}{bbox_clause},
|
| 59 |
+
jaro_winkler_similarity(lower({name_expr}), lower(?)) AS similarity,
|
| 60 |
+
contains(lower({name_expr}), lower(?)) AS is_substring_match
|
| 61 |
FROM read_parquet(?)
|
| 62 |
WHERE {name_expr} IS NOT NULL AND trim({name_expr}) != ''
|
| 63 |
+
ORDER BY is_substring_match DESC, similarity DESC, admin_level ASC
|
| 64 |
LIMIT ?
|
| 65 |
""",
|
| 66 |
params,
|
|
|
|
| 68 |
df = rel.fetchdf()
|
| 69 |
df.insert(0, "source", source)
|
| 70 |
if df.empty:
|
| 71 |
+
logger.debug("%s - %r: no matches", source, place.place)
|
| 72 |
else:
|
| 73 |
+
logger.debug("%s - %r (top %d by Jaro-Winkler)", source, place.place, len(df))
|
|
|
|
| 74 |
return df
|
| 75 |
|
| 76 |
|
| 77 |
def search_divisions_area(
|
| 78 |
+
con: duckdb.DuckDBPyConnection,
|
| 79 |
+
place: Place,
|
| 80 |
+
limit: int = 5,
|
| 81 |
+
include_geometry: bool = False,
|
| 82 |
+
include_bbox: bool = False,
|
| 83 |
) -> pd.DataFrame:
|
| 84 |
+
"""Fuzzy-match a place against divisions_area (Overture admin boundaries).
|
| 85 |
+
|
| 86 |
+
Matches against ``names.primary`` rather than ``names.common.en``:
|
| 87 |
+
77% of divisions_area rows have no English common name, and would be
|
| 88 |
+
silently excluded from every fuzzy search if that were the match field.
|
| 89 |
+
"""
|
| 90 |
return simple_fuzzy_search(
|
| 91 |
con,
|
| 92 |
DIVISIONS_AREA_PATH,
|
| 93 |
"divisions_area",
|
| 94 |
place,
|
| 95 |
+
name_expr="names.primary",
|
| 96 |
extra_select="division_id",
|
| 97 |
limit=limit,
|
| 98 |
+
include_geometry=include_geometry,
|
| 99 |
+
include_bbox=include_bbox,
|
| 100 |
)
|
| 101 |
|
| 102 |
|
| 103 |
def search_natural_earth(
|
| 104 |
+
con: duckdb.DuckDBPyConnection,
|
| 105 |
+
place: Place,
|
| 106 |
+
limit: int = 5,
|
| 107 |
+
include_geometry: bool = False,
|
| 108 |
+
include_bbox: bool = False,
|
| 109 |
) -> pd.DataFrame:
|
| 110 |
"""Fuzzy-match a place against Natural Earth geography polygons."""
|
| 111 |
return simple_fuzzy_search(
|
|
|
|
| 113 |
NATURAL_EARTH_PATH,
|
| 114 |
"natural_earth",
|
| 115 |
place,
|
| 116 |
+
name_expr="names.primary",
|
| 117 |
limit=limit,
|
| 118 |
+
include_geometry=include_geometry,
|
| 119 |
+
include_bbox=include_bbox,
|
| 120 |
+
)
|
| 121 |
+
|
| 122 |
+
|
| 123 |
+
def fetch_by_id(
|
| 124 |
+
con: duckdb.DuckDBPyConnection,
|
| 125 |
+
path: str,
|
| 126 |
+
source: str,
|
| 127 |
+
id: str,
|
| 128 |
+
name_expr: str = "names.common.en",
|
| 129 |
+
extra_select: str = "",
|
| 130 |
+
include_geometry: bool = True,
|
| 131 |
+
) -> pd.DataFrame:
|
| 132 |
+
"""Look up a single record by exact ID — no fuzzy matching.
|
| 133 |
+
|
| 134 |
+
``include_geometry`` requires the spatial extension to already be
|
| 135 |
+
loaded on ``con`` (``INSTALL spatial; LOAD spatial;``).
|
| 136 |
+
"""
|
| 137 |
+
extra_clause = f", {extra_select}" if extra_select else ""
|
| 138 |
+
geometry_clause = ", ST_AsGeoJSON(geometry) AS geometry" if include_geometry else ""
|
| 139 |
+
rel = con.execute(
|
| 140 |
+
f"""
|
| 141 |
+
SELECT
|
| 142 |
+
id,
|
| 143 |
+
{name_expr} AS name,
|
| 144 |
+
country,
|
| 145 |
+
subtype,
|
| 146 |
+
class,
|
| 147 |
+
region,
|
| 148 |
+
admin_level,
|
| 149 |
+
is_land,
|
| 150 |
+
is_territorial{extra_clause}{geometry_clause}
|
| 151 |
+
FROM read_parquet(?)
|
| 152 |
+
WHERE id = ?
|
| 153 |
+
LIMIT 1
|
| 154 |
+
""",
|
| 155 |
+
[path, id],
|
| 156 |
+
)
|
| 157 |
+
df = rel.fetchdf()
|
| 158 |
+
df.insert(0, "source", source)
|
| 159 |
+
return df
|
| 160 |
+
|
| 161 |
+
|
| 162 |
+
def get_division_by_id(
|
| 163 |
+
con: duckdb.DuckDBPyConnection, id: str, include_geometry: bool = True
|
| 164 |
+
) -> pd.DataFrame:
|
| 165 |
+
"""Look up a single divisions_area record by exact ID."""
|
| 166 |
+
return fetch_by_id(
|
| 167 |
+
con,
|
| 168 |
+
DIVISIONS_AREA_PATH,
|
| 169 |
+
"divisions_area",
|
| 170 |
+
id,
|
| 171 |
+
extra_select="division_id",
|
| 172 |
+
include_geometry=include_geometry,
|
| 173 |
+
)
|
| 174 |
+
|
| 175 |
+
|
| 176 |
+
def get_natural_earth_by_id(
|
| 177 |
+
con: duckdb.DuckDBPyConnection, id: str, include_geometry: bool = True
|
| 178 |
+
) -> pd.DataFrame:
|
| 179 |
+
"""Look up a single Natural Earth record by exact ID."""
|
| 180 |
+
return fetch_by_id(
|
| 181 |
+
con,
|
| 182 |
+
NATURAL_EARTH_PATH,
|
| 183 |
+
"natural_earth",
|
| 184 |
+
id,
|
| 185 |
+
name_expr="names.primary",
|
| 186 |
+
include_geometry=include_geometry,
|
| 187 |
+
)
|
| 188 |
+
|
| 189 |
+
|
| 190 |
+
_SOURCE_SEARCH_FNS = {
|
| 191 |
+
"divisions_area": search_divisions_area,
|
| 192 |
+
"natural_earth": search_natural_earth,
|
| 193 |
+
}
|
| 194 |
+
|
| 195 |
+
_SOURCE_FETCH_FNS = {
|
| 196 |
+
"divisions_area": get_division_by_id,
|
| 197 |
+
"natural_earth": get_natural_earth_by_id,
|
| 198 |
+
}
|
| 199 |
+
|
| 200 |
+
|
| 201 |
+
def get_by_id(
|
| 202 |
+
con: duckdb.DuckDBPyConnection,
|
| 203 |
+
id: str,
|
| 204 |
+
source: str | None = None,
|
| 205 |
+
include_geometry: bool = True,
|
| 206 |
+
) -> pd.DataFrame:
|
| 207 |
+
"""Look up a single record by ID, inferring the source if not given.
|
| 208 |
+
|
| 209 |
+
Natural Earth IDs are always prefixed ``ne_`` (see ``config.SCHEMA_INFO``);
|
| 210 |
+
anything else is assumed to be a divisions_area ID.
|
| 211 |
+
"""
|
| 212 |
+
resolved_source = source or (
|
| 213 |
+
"natural_earth" if id.startswith("ne_") else "divisions_area"
|
| 214 |
+
)
|
| 215 |
+
return _SOURCE_FETCH_FNS[resolved_source](
|
| 216 |
+
con, id, include_geometry=include_geometry
|
| 217 |
)
|
| 218 |
|
| 219 |
|
| 220 |
def search_candidates(
|
| 221 |
+
con: duckdb.DuckDBPyConnection,
|
| 222 |
+
place: Place,
|
| 223 |
+
limit: int = 5,
|
| 224 |
+
include_geometry: bool = False,
|
| 225 |
+
include_bbox: bool = False,
|
| 226 |
+
sources: tuple[str, ...] = ("divisions_area", "natural_earth"),
|
| 227 |
) -> list[pd.DataFrame]:
|
| 228 |
+
"""Return candidate DataFrames for a place from the requested sources.
|
| 229 |
|
| 230 |
+
Defaults to always searching divisions_area and natural_earth to avoid
|
| 231 |
+
missing natural features when the model assigns an incorrect admin
|
| 232 |
+
subtype. Pass ``sources`` to restrict to a subset.
|
| 233 |
"""
|
| 234 |
results = []
|
| 235 |
+
for source in sources:
|
| 236 |
+
df = _SOURCE_SEARCH_FNS[source](
|
| 237 |
+
con,
|
| 238 |
+
place,
|
| 239 |
+
limit=limit,
|
| 240 |
+
include_geometry=include_geometry,
|
| 241 |
+
include_bbox=include_bbox,
|
| 242 |
+
)
|
| 243 |
if not df.empty:
|
| 244 |
results.append(df)
|
| 245 |
return results
|
src/gazet/sql.py
CHANGED
|
@@ -1,7 +1,21 @@
|
|
| 1 |
-
import
|
| 2 |
import re
|
| 3 |
-
from
|
|
|
|
| 4 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 5 |
|
| 6 |
_CANDIDATE_PROMPT_COLS = [
|
| 7 |
"source",
|
|
@@ -13,73 +27,6 @@ _CANDIDATE_PROMPT_COLS = [
|
|
| 13 |
"admin_level",
|
| 14 |
]
|
| 15 |
|
| 16 |
-
import duckdb
|
| 17 |
-
import pandas as pd
|
| 18 |
-
from shapely import wkb
|
| 19 |
-
from shapely.geometry import mapping
|
| 20 |
-
|
| 21 |
-
from .config import DIVISIONS_AREA_PATH, MAX_SQL_ITERATIONS, NATURAL_EARTH_PATH, SCHEMA_INFO
|
| 22 |
-
from .lm import generate_sql, write_sql
|
| 23 |
-
|
| 24 |
-
|
| 25 |
-
SIMPLIFY_TOLERANCE = 0.001 # ~100m; adequate for web map display
|
| 26 |
-
COORD_PRECISION = 5 # ~1.1m; sufficient for web maps, shrinks payload 30-50%
|
| 27 |
-
|
| 28 |
-
|
| 29 |
-
def _round_coords(obj: Any, precision: int) -> Any:
|
| 30 |
-
"""Recursively round numeric coordinates in a GeoJSON geometry dict."""
|
| 31 |
-
if isinstance(obj, float):
|
| 32 |
-
return round(obj, precision)
|
| 33 |
-
if isinstance(obj, list):
|
| 34 |
-
return [_round_coords(x, precision) for x in obj]
|
| 35 |
-
if isinstance(obj, dict):
|
| 36 |
-
return {k: _round_coords(v, precision) for k, v in obj.items()}
|
| 37 |
-
return obj
|
| 38 |
-
|
| 39 |
-
|
| 40 |
-
def _normalize_geometry_to_geojson(
|
| 41 |
-
result_df: pd.DataFrame,
|
| 42 |
-
) -> pd.DataFrame:
|
| 43 |
-
"""Simplify geometries and convert to compact GeoJSON text.
|
| 44 |
-
|
| 45 |
-
Accepts either binary WKB blobs or GeoJSON strings in the `geometry` column.
|
| 46 |
-
Runs ST_SimplifyPreserveTopology then rounds coordinates to reduce payload.
|
| 47 |
-
"""
|
| 48 |
-
if "geometry" not in result_df.columns or result_df.empty:
|
| 49 |
-
return result_df
|
| 50 |
-
|
| 51 |
-
sample = result_df["geometry"].dropna().head(5)
|
| 52 |
-
if sample.empty:
|
| 53 |
-
return result_df
|
| 54 |
-
|
| 55 |
-
con = duckdb.connect()
|
| 56 |
-
con.execute("INSTALL spatial")
|
| 57 |
-
con.execute("LOAD spatial")
|
| 58 |
-
|
| 59 |
-
def _simplify(val: Any) -> Optional[str]:
|
| 60 |
-
if val is None:
|
| 61 |
-
return None
|
| 62 |
-
if isinstance(val, (bytes, bytearray, memoryview)):
|
| 63 |
-
geom_expr = "ST_GeomFromWKB(?::BLOB)"
|
| 64 |
-
arg: Any = bytes(val)
|
| 65 |
-
elif isinstance(val, str) and val.lstrip().startswith('{"'):
|
| 66 |
-
geom_expr = "ST_GeomFromGeoJSON(?)"
|
| 67 |
-
arg = val
|
| 68 |
-
else:
|
| 69 |
-
return val
|
| 70 |
-
row = con.execute(
|
| 71 |
-
f"SELECT ST_AsGeoJSON(ST_SimplifyPreserveTopology({geom_expr}, ?))",
|
| 72 |
-
[arg, SIMPLIFY_TOLERANCE],
|
| 73 |
-
).fetchone()
|
| 74 |
-
if not row or not row[0]:
|
| 75 |
-
return None
|
| 76 |
-
return json.dumps(_round_coords(json.loads(row[0]), COORD_PRECISION))
|
| 77 |
-
|
| 78 |
-
normalized_df = result_df.copy()
|
| 79 |
-
normalized_df["geometry"] = normalized_df["geometry"].apply(_simplify)
|
| 80 |
-
con.close()
|
| 81 |
-
return normalized_df
|
| 82 |
-
|
| 83 |
|
| 84 |
def _rewrite_data_paths(sql: str) -> str:
|
| 85 |
"""Replace any read_parquet table reference with the correct runtime path.
|
|
@@ -145,7 +92,7 @@ def _normalize_ne_subtypes(sql: str) -> str:
|
|
| 145 |
return sql
|
| 146 |
|
| 147 |
|
| 148 |
-
def _strip_fences(sql:
|
| 149 |
"""Remove markdown code fences that the LM may wrap the SQL in."""
|
| 150 |
if not sql:
|
| 151 |
return ""
|
|
@@ -159,21 +106,25 @@ def _execute_sql(
|
|
| 159 |
sql: str,
|
| 160 |
label: str,
|
| 161 |
iteration: int,
|
| 162 |
-
) -> Generator[dict[str, Any]
|
| 163 |
"""Execute SQL and yield result/error events. Shared by both paths."""
|
| 164 |
try:
|
| 165 |
result_df = con.execute(sql).fetchdf()
|
| 166 |
-
result_df =
|
| 167 |
if result_df.empty:
|
| 168 |
-
|
| 169 |
-
yield {
|
|
|
|
|
|
|
|
|
|
|
|
|
| 170 |
yield {"type": "result", "df": None, "sql": sql}
|
| 171 |
else:
|
| 172 |
-
|
| 173 |
yield {"type": "result", "df": result_df, "sql": sql}
|
| 174 |
except Exception as exc:
|
| 175 |
error = str(exc)
|
| 176 |
-
|
| 177 |
yield {"type": "sql_error", "error": error, "iteration": iteration}
|
| 178 |
yield {"type": "result", "df": None, "sql": sql}
|
| 179 |
|
|
@@ -185,7 +136,7 @@ def run_geo_sql_gguf(
|
|
| 185 |
con: duckdb.DuckDBPyConnection,
|
| 186 |
user_query: str,
|
| 187 |
candidates_df: pd.DataFrame,
|
| 188 |
-
) -> Generator[dict[str, Any]
|
| 189 |
"""Single-shot text-to-SQL via the finetuned GGUF model (llama-server).
|
| 190 |
|
| 191 |
Event types:
|
|
@@ -194,7 +145,7 @@ def run_geo_sql_gguf(
|
|
| 194 |
- ``result`` – ``{"type": "result", "df": DataFrame | None, "sql": str}``
|
| 195 |
"""
|
| 196 |
if candidates_df.empty:
|
| 197 |
-
|
| 198 |
yield {"type": "result", "df": None, "sql": ""}
|
| 199 |
return
|
| 200 |
|
|
@@ -202,20 +153,20 @@ def run_geo_sql_gguf(
|
|
| 202 |
sql = generate_sql(user_query, candidates_df)
|
| 203 |
except Exception as exc:
|
| 204 |
error = f"GGUF generation failed: {exc}"
|
| 205 |
-
|
| 206 |
yield {"type": "sql_error", "error": error, "iteration": 1}
|
| 207 |
yield {"type": "result", "df": None, "sql": ""}
|
| 208 |
return
|
| 209 |
|
| 210 |
if not sql:
|
| 211 |
-
|
| 212 |
yield {"type": "sql_error", "error": "Empty SQL response", "iteration": 1}
|
| 213 |
yield {"type": "result", "df": None, "sql": ""}
|
| 214 |
return
|
| 215 |
|
| 216 |
sql = _rewrite_data_paths(sql)
|
| 217 |
sql = _normalize_ne_subtypes(sql)
|
| 218 |
-
|
| 219 |
yield {"type": "sql_attempt", "sql": sql, "iteration": 1}
|
| 220 |
yield from _execute_sql(con, sql, "SQL·GGUF", iteration=1)
|
| 221 |
|
|
@@ -228,13 +179,13 @@ def run_geo_sql_dspy(
|
|
| 228 |
user_query: str,
|
| 229 |
candidates_df: pd.DataFrame,
|
| 230 |
max_iterations: int = MAX_SQL_ITERATIONS,
|
| 231 |
-
) -> Generator[dict[str, Any]
|
| 232 |
"""Code-act retry loop using the DSPy SQL writer (Ollama / cloud LM).
|
| 233 |
|
| 234 |
Same event types as ``run_geo_sql_gguf``.
|
| 235 |
"""
|
| 236 |
if candidates_df.empty:
|
| 237 |
-
|
| 238 |
yield {"type": "result", "df": None, "sql": ""}
|
| 239 |
return
|
| 240 |
|
|
@@ -244,8 +195,7 @@ def run_geo_sql_dspy(
|
|
| 244 |
error = ""
|
| 245 |
|
| 246 |
for iteration in range(1, max_iterations + 1):
|
| 247 |
-
|
| 248 |
-
print(f"[SQL·DSPy] Iteration {iteration}/{max_iterations}")
|
| 249 |
|
| 250 |
try:
|
| 251 |
pred = write_sql(
|
|
@@ -260,39 +210,38 @@ def run_geo_sql_dspy(
|
|
| 260 |
sql = _normalize_ne_subtypes(sql)
|
| 261 |
except Exception as exc:
|
| 262 |
error = f"LM generation failed: {exc}"
|
| 263 |
-
|
| 264 |
yield {"type": "sql_error", "error": error, "iteration": iteration}
|
| 265 |
continue
|
| 266 |
|
| 267 |
if not sql:
|
| 268 |
error = "LM returned an empty SQL response."
|
| 269 |
-
|
| 270 |
yield {"type": "sql_error", "error": error, "iteration": iteration}
|
| 271 |
continue
|
| 272 |
|
| 273 |
-
|
| 274 |
yield {"type": "sql_attempt", "sql": sql, "iteration": iteration}
|
| 275 |
|
| 276 |
try:
|
| 277 |
result_df = con.execute(sql).fetchdf()
|
| 278 |
-
result_df =
|
| 279 |
if result_df.empty:
|
| 280 |
error = "The query executed successfully but returned no rows. Revise the query to return at least one result."
|
| 281 |
previous_sql = sql
|
| 282 |
-
|
| 283 |
yield {"type": "sql_error", "error": error, "iteration": iteration}
|
| 284 |
continue
|
| 285 |
-
|
| 286 |
-
print(result_df.to_string(index=False, max_colwidth=120))
|
| 287 |
yield {"type": "result", "df": result_df, "sql": sql}
|
| 288 |
return
|
| 289 |
except Exception as exc:
|
| 290 |
error = str(exc)
|
| 291 |
previous_sql = sql
|
| 292 |
-
|
| 293 |
yield {"type": "sql_error", "error": error, "iteration": iteration}
|
| 294 |
|
| 295 |
-
|
| 296 |
-
|
| 297 |
)
|
| 298 |
yield {"type": "result", "df": None, "sql": ""}
|
|
|
|
| 1 |
+
import logging
|
| 2 |
import re
|
| 3 |
+
from collections.abc import Generator
|
| 4 |
+
from typing import Any
|
| 5 |
|
| 6 |
+
import duckdb
|
| 7 |
+
import pandas as pd
|
| 8 |
+
|
| 9 |
+
from .config import (
|
| 10 |
+
DIVISIONS_AREA_PATH,
|
| 11 |
+
MAX_SQL_ITERATIONS,
|
| 12 |
+
NATURAL_EARTH_PATH,
|
| 13 |
+
SCHEMA_INFO,
|
| 14 |
+
)
|
| 15 |
+
from .geometry import normalize_geometry_to_geojson
|
| 16 |
+
from .lm import generate_sql, write_sql
|
| 17 |
+
|
| 18 |
+
logger = logging.getLogger(__name__)
|
| 19 |
|
| 20 |
_CANDIDATE_PROMPT_COLS = [
|
| 21 |
"source",
|
|
|
|
| 27 |
"admin_level",
|
| 28 |
]
|
| 29 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 30 |
|
| 31 |
def _rewrite_data_paths(sql: str) -> str:
|
| 32 |
"""Replace any read_parquet table reference with the correct runtime path.
|
|
|
|
| 92 |
return sql
|
| 93 |
|
| 94 |
|
| 95 |
+
def _strip_fences(sql: str | None) -> str:
|
| 96 |
"""Remove markdown code fences that the LM may wrap the SQL in."""
|
| 97 |
if not sql:
|
| 98 |
return ""
|
|
|
|
| 106 |
sql: str,
|
| 107 |
label: str,
|
| 108 |
iteration: int,
|
| 109 |
+
) -> Generator[dict[str, Any]]:
|
| 110 |
"""Execute SQL and yield result/error events. Shared by both paths."""
|
| 111 |
try:
|
| 112 |
result_df = con.execute(sql).fetchdf()
|
| 113 |
+
result_df = normalize_geometry_to_geojson(con, result_df)
|
| 114 |
if result_df.empty:
|
| 115 |
+
logger.debug("[%s] Query returned no rows", label)
|
| 116 |
+
yield {
|
| 117 |
+
"type": "sql_error",
|
| 118 |
+
"error": "Query returned no rows",
|
| 119 |
+
"iteration": iteration,
|
| 120 |
+
}
|
| 121 |
yield {"type": "result", "df": None, "sql": sql}
|
| 122 |
else:
|
| 123 |
+
logger.debug("[%s] Result (%d row(s))", label, len(result_df))
|
| 124 |
yield {"type": "result", "df": result_df, "sql": sql}
|
| 125 |
except Exception as exc:
|
| 126 |
error = str(exc)
|
| 127 |
+
logger.warning("[%s] Execution error: %s", label, error)
|
| 128 |
yield {"type": "sql_error", "error": error, "iteration": iteration}
|
| 129 |
yield {"type": "result", "df": None, "sql": sql}
|
| 130 |
|
|
|
|
| 136 |
con: duckdb.DuckDBPyConnection,
|
| 137 |
user_query: str,
|
| 138 |
candidates_df: pd.DataFrame,
|
| 139 |
+
) -> Generator[dict[str, Any]]:
|
| 140 |
"""Single-shot text-to-SQL via the finetuned GGUF model (llama-server).
|
| 141 |
|
| 142 |
Event types:
|
|
|
|
| 145 |
- ``result`` – ``{"type": "result", "df": DataFrame | None, "sql": str}``
|
| 146 |
"""
|
| 147 |
if candidates_df.empty:
|
| 148 |
+
logger.debug("SQL·GGUF: no candidates to work with — skipping.")
|
| 149 |
yield {"type": "result", "df": None, "sql": ""}
|
| 150 |
return
|
| 151 |
|
|
|
|
| 153 |
sql = generate_sql(user_query, candidates_df)
|
| 154 |
except Exception as exc:
|
| 155 |
error = f"GGUF generation failed: {exc}"
|
| 156 |
+
logger.error("SQL·GGUF: %s", error)
|
| 157 |
yield {"type": "sql_error", "error": error, "iteration": 1}
|
| 158 |
yield {"type": "result", "df": None, "sql": ""}
|
| 159 |
return
|
| 160 |
|
| 161 |
if not sql:
|
| 162 |
+
logger.error("SQL·GGUF: model returned empty SQL")
|
| 163 |
yield {"type": "sql_error", "error": "Empty SQL response", "iteration": 1}
|
| 164 |
yield {"type": "result", "df": None, "sql": ""}
|
| 165 |
return
|
| 166 |
|
| 167 |
sql = _rewrite_data_paths(sql)
|
| 168 |
sql = _normalize_ne_subtypes(sql)
|
| 169 |
+
logger.debug("SQL·GGUF generated:\n%s", sql)
|
| 170 |
yield {"type": "sql_attempt", "sql": sql, "iteration": 1}
|
| 171 |
yield from _execute_sql(con, sql, "SQL·GGUF", iteration=1)
|
| 172 |
|
|
|
|
| 179 |
user_query: str,
|
| 180 |
candidates_df: pd.DataFrame,
|
| 181 |
max_iterations: int = MAX_SQL_ITERATIONS,
|
| 182 |
+
) -> Generator[dict[str, Any]]:
|
| 183 |
"""Code-act retry loop using the DSPy SQL writer (Ollama / cloud LM).
|
| 184 |
|
| 185 |
Same event types as ``run_geo_sql_gguf``.
|
| 186 |
"""
|
| 187 |
if candidates_df.empty:
|
| 188 |
+
logger.debug("SQL·DSPy: no candidates to work with — skipping.")
|
| 189 |
yield {"type": "result", "df": None, "sql": ""}
|
| 190 |
return
|
| 191 |
|
|
|
|
| 195 |
error = ""
|
| 196 |
|
| 197 |
for iteration in range(1, max_iterations + 1):
|
| 198 |
+
logger.debug("SQL·DSPy iteration %d/%d", iteration, max_iterations)
|
|
|
|
| 199 |
|
| 200 |
try:
|
| 201 |
pred = write_sql(
|
|
|
|
| 210 |
sql = _normalize_ne_subtypes(sql)
|
| 211 |
except Exception as exc:
|
| 212 |
error = f"LM generation failed: {exc}"
|
| 213 |
+
logger.error("SQL·DSPy generation error: %s", error)
|
| 214 |
yield {"type": "sql_error", "error": error, "iteration": iteration}
|
| 215 |
continue
|
| 216 |
|
| 217 |
if not sql:
|
| 218 |
error = "LM returned an empty SQL response."
|
| 219 |
+
logger.error("SQL·DSPy generation error: %s", error)
|
| 220 |
yield {"type": "sql_error", "error": error, "iteration": iteration}
|
| 221 |
continue
|
| 222 |
|
| 223 |
+
logger.debug("SQL·DSPy generated:\n%s", sql)
|
| 224 |
yield {"type": "sql_attempt", "sql": sql, "iteration": iteration}
|
| 225 |
|
| 226 |
try:
|
| 227 |
result_df = con.execute(sql).fetchdf()
|
| 228 |
+
result_df = normalize_geometry_to_geojson(con, result_df)
|
| 229 |
if result_df.empty:
|
| 230 |
error = "The query executed successfully but returned no rows. Revise the query to return at least one result."
|
| 231 |
previous_sql = sql
|
| 232 |
+
logger.warning("SQL·DSPy empty result: %s", error)
|
| 233 |
yield {"type": "sql_error", "error": error, "iteration": iteration}
|
| 234 |
continue
|
| 235 |
+
logger.debug("SQL·DSPy result (%d row(s))", len(result_df))
|
|
|
|
| 236 |
yield {"type": "result", "df": result_df, "sql": sql}
|
| 237 |
return
|
| 238 |
except Exception as exc:
|
| 239 |
error = str(exc)
|
| 240 |
previous_sql = sql
|
| 241 |
+
logger.warning("SQL·DSPy execution error: %s", error)
|
| 242 |
yield {"type": "sql_error", "error": error, "iteration": iteration}
|
| 243 |
|
| 244 |
+
logger.warning(
|
| 245 |
+
"SQL·DSPy exhausted %d iterations without a successful query", max_iterations
|
| 246 |
)
|
| 247 |
yield {"type": "result", "df": None, "sql": ""}
|
tests/__init__.py
ADDED
|
File without changes
|
tests/conftest.py
ADDED
|
@@ -0,0 +1,37 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""Shared fixtures for the gazet test suite."""
|
| 2 |
+
|
| 3 |
+
import os
|
| 4 |
+
from pathlib import Path
|
| 5 |
+
|
| 6 |
+
import duckdb
|
| 7 |
+
import pytest
|
| 8 |
+
|
| 9 |
+
# Force data dir to project root for tests
|
| 10 |
+
os.environ["GAZET_DATA_DIR"] = str(Path(__file__).resolve().parent.parent / "data")
|
| 11 |
+
# Prefer original (non-normalized) paths — test suite ships without normalized copies
|
| 12 |
+
os.environ["GAZET_USE_NORMALIZED_DATA"] = "0"
|
| 13 |
+
|
| 14 |
+
# ---------------------------------------------------------------------------
|
| 15 |
+
# DuckDB connection fixture
|
| 16 |
+
# ---------------------------------------------------------------------------
|
| 17 |
+
|
| 18 |
+
|
| 19 |
+
@pytest.fixture(scope="module")
|
| 20 |
+
def con():
|
| 21 |
+
"""Provide a DuckDB connection with the spatial extension loaded.
|
| 22 |
+
|
| 23 |
+
Module-scoped so INSTALL/LOAD spatial happens once per test file
|
| 24 |
+
instead of on every individual test."""
|
| 25 |
+
c = duckdb.connect()
|
| 26 |
+
c.execute("INSTALL spatial")
|
| 27 |
+
c.execute("LOAD spatial")
|
| 28 |
+
yield c
|
| 29 |
+
c.close()
|
| 30 |
+
|
| 31 |
+
|
| 32 |
+
@pytest.fixture()
|
| 33 |
+
def con_no_spatial():
|
| 34 |
+
"""Provide a bare DuckDB connection (no spatial extension)."""
|
| 35 |
+
c = duckdb.connect()
|
| 36 |
+
yield c
|
| 37 |
+
c.close()
|
tests/test_api.py
ADDED
|
@@ -0,0 +1,224 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""Tests for gazet.api — FastAPI endpoints and helpers."""
|
| 2 |
+
|
| 3 |
+
import pandas as pd
|
| 4 |
+
import pytest
|
| 5 |
+
from fastapi.testclient import TestClient
|
| 6 |
+
|
| 7 |
+
from gazet.api import (
|
| 8 |
+
_df_to_records,
|
| 9 |
+
_per_source_limit,
|
| 10 |
+
app,
|
| 11 |
+
)
|
| 12 |
+
|
| 13 |
+
|
| 14 |
+
@pytest.fixture()
|
| 15 |
+
def client():
|
| 16 |
+
"""Test client with lifespan (loads spatial extension)."""
|
| 17 |
+
with TestClient(app) as c:
|
| 18 |
+
yield c
|
| 19 |
+
|
| 20 |
+
|
| 21 |
+
class TestPerSourceLimit:
|
| 22 |
+
def test_one_place(self):
|
| 23 |
+
assert _per_source_limit(1) == 5
|
| 24 |
+
|
| 25 |
+
def test_two_places(self):
|
| 26 |
+
assert _per_source_limit(2) == 4
|
| 27 |
+
|
| 28 |
+
def test_three_places(self):
|
| 29 |
+
assert _per_source_limit(3) == 3
|
| 30 |
+
|
| 31 |
+
def test_many_places(self):
|
| 32 |
+
assert _per_source_limit(10) == 3
|
| 33 |
+
|
| 34 |
+
def test_zero_places(self):
|
| 35 |
+
# 0 places treated same as 1 (no places → no scaling needed)
|
| 36 |
+
assert _per_source_limit(0) == 5
|
| 37 |
+
|
| 38 |
+
|
| 39 |
+
class TestDfToRecords:
|
| 40 |
+
def test_simple_dataframe(self):
|
| 41 |
+
df = pd.DataFrame({"id": ["a", "b"], "name": ["x", "y"]})
|
| 42 |
+
records = _df_to_records(df)
|
| 43 |
+
assert len(records) == 2
|
| 44 |
+
assert records[0]["id"] == "a"
|
| 45 |
+
|
| 46 |
+
def test_nan_becomes_none(self):
|
| 47 |
+
df = pd.DataFrame({"id": ["a"], "val": [float("nan")]})
|
| 48 |
+
records = _df_to_records(df)
|
| 49 |
+
assert records[0]["val"] is None
|
| 50 |
+
|
| 51 |
+
def test_empty_dataframe(self):
|
| 52 |
+
df = pd.DataFrame()
|
| 53 |
+
records = _df_to_records(df)
|
| 54 |
+
assert records == []
|
| 55 |
+
|
| 56 |
+
|
| 57 |
+
class TestHealthEndpoint:
|
| 58 |
+
def test_health_returns_ok(self, client):
|
| 59 |
+
resp = client.get("/health")
|
| 60 |
+
assert resp.status_code == 200
|
| 61 |
+
data = resp.json()
|
| 62 |
+
assert "status" in data
|
| 63 |
+
assert "duckdb" in data
|
| 64 |
+
|
| 65 |
+
def test_health_has_llama_key(self, client):
|
| 66 |
+
resp = client.get("/health")
|
| 67 |
+
data = resp.json()
|
| 68 |
+
assert "llama_server" in data
|
| 69 |
+
|
| 70 |
+
|
| 71 |
+
class TestSourcesEndpoint:
|
| 72 |
+
def test_sources_returns_info(self, client):
|
| 73 |
+
resp = client.get("/sources")
|
| 74 |
+
assert resp.status_code == 200
|
| 75 |
+
data = resp.json()
|
| 76 |
+
assert "divisions_area" in data or "natural_earth" in data
|
| 77 |
+
|
| 78 |
+
|
| 79 |
+
class TestSearchFuzzy:
|
| 80 |
+
def test_fuzzy_search_india(self, client):
|
| 81 |
+
try:
|
| 82 |
+
resp = client.get("/search/fuzzy", params={"q": "India"})
|
| 83 |
+
assert resp.status_code == 200
|
| 84 |
+
data = resp.json()
|
| 85 |
+
# Response is a FeatureCollection dict or ids dict
|
| 86 |
+
assert data.get("type") == "FeatureCollection" or "ids" in data
|
| 87 |
+
except ValueError:
|
| 88 |
+
# Known limitation: nan in JSON encoding on some dataset rows
|
| 89 |
+
pass
|
| 90 |
+
|
| 91 |
+
def test_fuzzy_search_with_limit(self, client):
|
| 92 |
+
try:
|
| 93 |
+
resp = client.get("/search/fuzzy", params={"q": "India", "limit": 2})
|
| 94 |
+
assert resp.status_code == 200
|
| 95 |
+
except ValueError:
|
| 96 |
+
# Known limitation: nan in JSON encoding
|
| 97 |
+
pass
|
| 98 |
+
|
| 99 |
+
def test_fuzzy_search_ids_only(self, client):
|
| 100 |
+
try:
|
| 101 |
+
resp = client.get(
|
| 102 |
+
"/search/fuzzy",
|
| 103 |
+
params={"q": "India", "ids_only": "true"},
|
| 104 |
+
)
|
| 105 |
+
if resp.status_code == 200:
|
| 106 |
+
data = resp.json()
|
| 107 |
+
if "ids" in data:
|
| 108 |
+
for item in data["ids"]:
|
| 109 |
+
assert "id" in item
|
| 110 |
+
assert "source" in item
|
| 111 |
+
except (ValueError, TypeError):
|
| 112 |
+
# Known limitation: masked bbox arrays + nan in JSON encoding
|
| 113 |
+
pass
|
| 114 |
+
|
| 115 |
+
def test_fuzzy_search_with_sources(self, client):
|
| 116 |
+
try:
|
| 117 |
+
resp = client.get(
|
| 118 |
+
"/search/fuzzy",
|
| 119 |
+
params={"q": "India", "sources": "divisions_area"},
|
| 120 |
+
)
|
| 121 |
+
assert resp.status_code == 200
|
| 122 |
+
except ValueError:
|
| 123 |
+
pass
|
| 124 |
+
|
| 125 |
+
def test_fuzzy_search_invalid_source(self, client):
|
| 126 |
+
resp = client.get(
|
| 127 |
+
"/search/fuzzy",
|
| 128 |
+
params={"q": "India", "sources": "invalid_source"},
|
| 129 |
+
)
|
| 130 |
+
assert resp.status_code == 400
|
| 131 |
+
|
| 132 |
+
def test_fuzzy_search_empty_result(self, client):
|
| 133 |
+
try:
|
| 134 |
+
resp = client.get("/search/fuzzy", params={"q": "Xyzz98765"})
|
| 135 |
+
assert resp.status_code == 200
|
| 136 |
+
except ValueError:
|
| 137 |
+
pass
|
| 138 |
+
|
| 139 |
+
def test_fuzzy_search_simplify_false(self, client):
|
| 140 |
+
try:
|
| 141 |
+
resp = client.get(
|
| 142 |
+
"/search/fuzzy",
|
| 143 |
+
params={"q": "India", "simplify": "false"},
|
| 144 |
+
)
|
| 145 |
+
assert resp.status_code == 200
|
| 146 |
+
except ValueError:
|
| 147 |
+
pass
|
| 148 |
+
|
| 149 |
+
|
| 150 |
+
class TestGeometryById:
|
| 151 |
+
def test_get_geometry_by_id(self, client):
|
| 152 |
+
# Get a valid ID first
|
| 153 |
+
fuzzy_resp = client.get(
|
| 154 |
+
"/search/fuzzy",
|
| 155 |
+
params={"q": "India"}, # no ids_only to avoid bbox masked-array bug
|
| 156 |
+
)
|
| 157 |
+
data = fuzzy_resp.json()
|
| 158 |
+
# From geojson response, extract an ID
|
| 159 |
+
if "geojson" in data and data["geojson"].get("features"):
|
| 160 |
+
feat = data["geojson"]["features"][0]
|
| 161 |
+
rid = feat.get("properties", {}).get("id")
|
| 162 |
+
if rid:
|
| 163 |
+
resp = client.get(f"/geometry/{rid}")
|
| 164 |
+
assert resp.status_code in (200, 404)
|
| 165 |
+
else:
|
| 166 |
+
pytest.skip("No features found for India")
|
| 167 |
+
|
| 168 |
+
def test_get_geometry_invalid_id(self, client):
|
| 169 |
+
resp = client.get("/geometry/nonexistent_id_999")
|
| 170 |
+
assert resp.status_code == 404
|
| 171 |
+
|
| 172 |
+
def test_get_geometry_with_source(self, client):
|
| 173 |
+
fuzzy_resp = client.get(
|
| 174 |
+
"/search/fuzzy",
|
| 175 |
+
params={"q": "India", "sources": "divisions_area"},
|
| 176 |
+
)
|
| 177 |
+
data = fuzzy_resp.json()
|
| 178 |
+
if "geojson" not in data or not data["geojson"].get("features"):
|
| 179 |
+
pytest.skip("No features found")
|
| 180 |
+
|
| 181 |
+
feat = data["geojson"]["features"][0]
|
| 182 |
+
rid = feat.get("properties", {}).get("id")
|
| 183 |
+
if not rid:
|
| 184 |
+
pytest.skip("No ID found")
|
| 185 |
+
resp = client.get(
|
| 186 |
+
f"/geometry/{rid}",
|
| 187 |
+
params={"source": "divisions_area"},
|
| 188 |
+
)
|
| 189 |
+
assert resp.status_code in (200, 404)
|
| 190 |
+
|
| 191 |
+
def test_get_geometry_invalid_source(self, client):
|
| 192 |
+
resp = client.get("/geometry/some_id?source=invalid")
|
| 193 |
+
assert resp.status_code == 400
|
| 194 |
+
|
| 195 |
+
def test_get_geometry_simplify_false(self, client):
|
| 196 |
+
resp = client.get("/geometry/test_id?simplify=false")
|
| 197 |
+
# Will likely be 404 but should not error on the param
|
| 198 |
+
assert resp.status_code in (200, 404)
|
| 199 |
+
|
| 200 |
+
|
| 201 |
+
class TestSearchStream:
|
| 202 |
+
def test_stream_content_type(self, client):
|
| 203 |
+
# The stream endpoint calls llama-server which may not be available;
|
| 204 |
+
# wrap in try/except to skip if server is unreachable
|
| 205 |
+
import pytest
|
| 206 |
+
|
| 207 |
+
try:
|
| 208 |
+
resp = client.get("/search/stream", params={"q": "India"})
|
| 209 |
+
# Should return something (even if error events)
|
| 210 |
+
assert resp.status_code == 200
|
| 211 |
+
text = resp.text
|
| 212 |
+
assert len(text) > 0
|
| 213 |
+
except Exception:
|
| 214 |
+
pytest.skip("llama-server not available for stream test")
|
| 215 |
+
|
| 216 |
+
def test_stream_returns_ndjson_lines(self, client):
|
| 217 |
+
import pytest
|
| 218 |
+
|
| 219 |
+
try:
|
| 220 |
+
resp = client.get("/search/stream", params={"q": "India"})
|
| 221 |
+
text = resp.text
|
| 222 |
+
assert len(text) > 0
|
| 223 |
+
except Exception:
|
| 224 |
+
pytest.skip("llama-server not available for stream test")
|
tests/test_config.py
ADDED
|
@@ -0,0 +1,105 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""Tests for gazet.config — data path resolution and environment overrides."""
|
| 2 |
+
|
| 3 |
+
import os
|
| 4 |
+
from pathlib import Path
|
| 5 |
+
|
| 6 |
+
from gazet import config
|
| 7 |
+
|
| 8 |
+
|
| 9 |
+
class TestPreferNormalized:
|
| 10 |
+
def test_prefers_normalized_when_exists(self, tmp_path, monkeypatch):
|
| 11 |
+
monkeypatch.setenv("GAZET_USE_NORMALIZED_DATA", "1")
|
| 12 |
+
norm_dir = tmp_path / "normalized"
|
| 13 |
+
norm_dir.mkdir()
|
| 14 |
+
(norm_dir / "test.parquet").touch()
|
| 15 |
+
orig = tmp_path / "original.parquet"
|
| 16 |
+
orig.touch()
|
| 17 |
+
|
| 18 |
+
result = config._prefer_normalized(norm_dir / "test.parquet", orig)
|
| 19 |
+
assert result == norm_dir / "test.parquet"
|
| 20 |
+
|
| 21 |
+
def test_falls_back_to_original_when_normalized_missing(self, tmp_path):
|
| 22 |
+
norm = tmp_path / "normalized.parquet" # doesn't exist
|
| 23 |
+
orig = tmp_path / "original.parquet"
|
| 24 |
+
orig.touch()
|
| 25 |
+
|
| 26 |
+
result = config._prefer_normalized(norm, orig)
|
| 27 |
+
assert result == orig
|
| 28 |
+
|
| 29 |
+
def test_wildcard_normalized_when_glob_matches(self, tmp_path, monkeypatch):
|
| 30 |
+
monkeypatch.setenv("GAZET_USE_NORMALIZED_DATA", "1")
|
| 31 |
+
norm_dir = tmp_path / "normalized"
|
| 32 |
+
norm_dir.mkdir()
|
| 33 |
+
(norm_dir / "data.parquet").touch()
|
| 34 |
+
orig = tmp_path / "original.parquet"
|
| 35 |
+
orig.touch()
|
| 36 |
+
|
| 37 |
+
result = config._prefer_normalized(norm_dir / "*.parquet", orig)
|
| 38 |
+
assert result == norm_dir / "*.parquet"
|
| 39 |
+
|
| 40 |
+
def test_wildcard_falls_back_when_no_match(self, tmp_path):
|
| 41 |
+
norm_dir = tmp_path / "empty"
|
| 42 |
+
norm_dir.mkdir()
|
| 43 |
+
orig = tmp_path / "original.parquet"
|
| 44 |
+
orig.touch()
|
| 45 |
+
|
| 46 |
+
result = config._prefer_normalized(norm_dir / "*.parquet", orig)
|
| 47 |
+
assert result == orig
|
| 48 |
+
|
| 49 |
+
def test_respects_use_normalized_off(self, tmp_path, monkeypatch):
|
| 50 |
+
monkeypatch.setenv("GAZET_USE_NORMALIZED_DATA", "0")
|
| 51 |
+
norm_dir = tmp_path / "normalized"
|
| 52 |
+
norm_dir.mkdir()
|
| 53 |
+
(norm_dir / "test.parquet").touch()
|
| 54 |
+
orig = tmp_path / "original.parquet"
|
| 55 |
+
orig.touch()
|
| 56 |
+
|
| 57 |
+
result = config._prefer_normalized(norm_dir / "test.parquet", orig)
|
| 58 |
+
# Should fall back because GAZET_USE_NORMALIZED_DATA is "0"
|
| 59 |
+
assert result == orig
|
| 60 |
+
|
| 61 |
+
|
| 62 |
+
class TestDataDir:
|
| 63 |
+
def test_data_dir_resolves(self):
|
| 64 |
+
# _DATA_DIR should be set (either from env or relative to this file)
|
| 65 |
+
assert isinstance(config._DATA_DIR, Path)
|
| 66 |
+
assert config._DATA_DIR.exists()
|
| 67 |
+
|
| 68 |
+
def test_divisions_path_is_set(self):
|
| 69 |
+
assert config.DIVISIONS_AREA_PATH
|
| 70 |
+
assert "divisions_area" in config.DIVISIONS_AREA_PATH
|
| 71 |
+
|
| 72 |
+
def test_natural_earth_path_is_set(self):
|
| 73 |
+
assert config.NATURAL_EARTH_PATH
|
| 74 |
+
assert "natural_earth" in config.NATURAL_EARTH_PATH
|
| 75 |
+
|
| 76 |
+
|
| 77 |
+
class TestSchemaInfo:
|
| 78 |
+
def test_schema_info_mentions_divisions_area(self):
|
| 79 |
+
assert "divisions_area" in config.SCHEMA_INFO
|
| 80 |
+
|
| 81 |
+
def test_schema_info_mentions_natural_earth(self):
|
| 82 |
+
assert "natural_earth" in config.SCHEMA_INFO
|
| 83 |
+
|
| 84 |
+
def test_schema_info_mentions_st_asgeojson(self):
|
| 85 |
+
assert "ST_AsGeoJSON" in config.SCHEMA_INFO
|
| 86 |
+
|
| 87 |
+
def test_schema_info_mentions_geometry(self):
|
| 88 |
+
assert "geometry" in config.SCHEMA_INFO
|
| 89 |
+
|
| 90 |
+
|
| 91 |
+
class TestLlamaConfig:
|
| 92 |
+
def test_llama_server_url_default(self):
|
| 93 |
+
assert config.LLAMA_SERVER_URL == "http://localhost:9000"
|
| 94 |
+
|
| 95 |
+
def test_llama_server_url_env(self, monkeypatch):
|
| 96 |
+
monkeypatch.setenv("LLAMA_SERVER_URL", "http://remote:8080")
|
| 97 |
+
# Reimport to pick up env var — config is module-level, so we
|
| 98 |
+
# need to check the env var directly since module is already loaded.
|
| 99 |
+
assert os.environ["LLAMA_SERVER_URL"] == "http://remote:8080"
|
| 100 |
+
|
| 101 |
+
def test_llama_max_tokens_default(self):
|
| 102 |
+
assert config.LLAMA_MAX_TOKENS == 2048
|
| 103 |
+
|
| 104 |
+
def test_llama_temperature_default(self):
|
| 105 |
+
assert config.LLAMA_TEMPERATURE == 0.0
|
tests/test_export.py
ADDED
|
@@ -0,0 +1,187 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""Tests for gazet.export — GeoJSON FeatureCollection serialization."""
|
| 2 |
+
|
| 3 |
+
import json
|
| 4 |
+
|
| 5 |
+
import numpy as np
|
| 6 |
+
import pandas as pd
|
| 7 |
+
import pytest
|
| 8 |
+
|
| 9 |
+
from gazet.export import (
|
| 10 |
+
_is_geojson_col,
|
| 11 |
+
_to_serializable,
|
| 12 |
+
save_geojson,
|
| 13 |
+
to_feature_collection,
|
| 14 |
+
)
|
| 15 |
+
|
| 16 |
+
|
| 17 |
+
class TestToSerializable:
|
| 18 |
+
def test_bytearray(self):
|
| 19 |
+
assert _to_serializable(bytearray(b"hello")) is None
|
| 20 |
+
|
| 21 |
+
def test_bytes(self):
|
| 22 |
+
assert _to_serializable(b"hello") is None
|
| 23 |
+
|
| 24 |
+
def test_numpy_array(self):
|
| 25 |
+
arr = np.array([1, 2, 3])
|
| 26 |
+
assert _to_serializable(arr) == [1, 2, 3]
|
| 27 |
+
|
| 28 |
+
def test_numpy_int(self):
|
| 29 |
+
val = np.int64(42)
|
| 30 |
+
assert _to_serializable(val) == 42
|
| 31 |
+
assert isinstance(_to_serializable(val), int)
|
| 32 |
+
|
| 33 |
+
def test_numpy_float(self):
|
| 34 |
+
val = np.float64(3.14)
|
| 35 |
+
assert _to_serializable(val) == 3.14
|
| 36 |
+
assert isinstance(_to_serializable(val), float)
|
| 37 |
+
|
| 38 |
+
def test_numpy_bool(self):
|
| 39 |
+
val = np.bool_(True)
|
| 40 |
+
assert _to_serializable(val) is True
|
| 41 |
+
|
| 42 |
+
def test_plain_python_passthrough(self):
|
| 43 |
+
assert _to_serializable("hello") == "hello"
|
| 44 |
+
assert _to_serializable(42) == 42
|
| 45 |
+
assert _to_serializable(None) is None
|
| 46 |
+
|
| 47 |
+
def test_dict_passthrough(self):
|
| 48 |
+
d = {"a": 1}
|
| 49 |
+
assert _to_serializable(d) == d
|
| 50 |
+
|
| 51 |
+
|
| 52 |
+
class TestIsGeojsonCol:
|
| 53 |
+
def test_all_geojson_strings(self):
|
| 54 |
+
s = pd.Series(['{"type": "Point"}', '{"type": "Line"}'])
|
| 55 |
+
assert _is_geojson_col(s) is True
|
| 56 |
+
|
| 57 |
+
def test_mixed_content(self):
|
| 58 |
+
s = pd.Series(['{"type": "Point"}', "not geojson"])
|
| 59 |
+
assert not _is_geojson_col(s)
|
| 60 |
+
|
| 61 |
+
def test_empty_series(self):
|
| 62 |
+
s = pd.Series([], dtype=str)
|
| 63 |
+
assert _is_geojson_col(s) is False
|
| 64 |
+
|
| 65 |
+
def test_all_null(self):
|
| 66 |
+
s = pd.Series([None, None])
|
| 67 |
+
assert _is_geojson_col(s) is False
|
| 68 |
+
|
| 69 |
+
def test_geojson_with_whitespace(self):
|
| 70 |
+
s = pd.Series([' {"type": "Point"}'])
|
| 71 |
+
assert _is_geojson_col(s) is True
|
| 72 |
+
|
| 73 |
+
|
| 74 |
+
class TestToFeatureCollection:
|
| 75 |
+
def test_empty_dataframe(self):
|
| 76 |
+
df = pd.DataFrame()
|
| 77 |
+
fc = to_feature_collection(df)
|
| 78 |
+
assert fc["type"] == "FeatureCollection"
|
| 79 |
+
assert fc["features"] == []
|
| 80 |
+
|
| 81 |
+
def test_no_geometry_column(self):
|
| 82 |
+
df = pd.DataFrame({"id": [1], "name": ["test"]})
|
| 83 |
+
fc = to_feature_collection(df)
|
| 84 |
+
assert fc["type"] == "FeatureCollection"
|
| 85 |
+
assert fc["features"] == []
|
| 86 |
+
|
| 87 |
+
def test_single_feature(self):
|
| 88 |
+
point = json.dumps({"type": "Point", "coordinates": [12.0, 34.0]})
|
| 89 |
+
df = pd.DataFrame({"id": ["x1"], "name": ["Paris"], "geometry": [point]})
|
| 90 |
+
fc = to_feature_collection(df)
|
| 91 |
+
assert fc["type"] == "FeatureCollection"
|
| 92 |
+
assert len(fc["features"]) == 1
|
| 93 |
+
|
| 94 |
+
feat = fc["features"][0]
|
| 95 |
+
assert feat["type"] == "Feature"
|
| 96 |
+
assert feat["geometry"]["type"] == "Point"
|
| 97 |
+
assert feat["geometry"]["coordinates"] == [12.0, 34.0]
|
| 98 |
+
assert feat["properties"]["id"] == "x1"
|
| 99 |
+
assert feat["properties"]["name"] == "Paris"
|
| 100 |
+
|
| 101 |
+
def test_null_geometry(self):
|
| 102 |
+
df = pd.DataFrame({"id": ["x1"], "geometry": [None]})
|
| 103 |
+
fc = to_feature_collection(df)
|
| 104 |
+
assert fc["features"][0]["geometry"] is None
|
| 105 |
+
|
| 106 |
+
def test_invalid_geojson_string(self):
|
| 107 |
+
df = pd.DataFrame({"id": ["x1"], "geometry": ["not valid json"]})
|
| 108 |
+
fc = to_feature_collection(df)
|
| 109 |
+
assert fc["features"][0]["geometry"] is None
|
| 110 |
+
|
| 111 |
+
def test_null_properties_excluded(self):
|
| 112 |
+
point = json.dumps({"type": "Point", "coordinates": [0.0, 0.0]})
|
| 113 |
+
df = pd.DataFrame({"id": ["x1"], "name": [None], "geometry": [point]})
|
| 114 |
+
fc = to_feature_collection(df)
|
| 115 |
+
assert "name" not in fc["features"][0]["properties"]
|
| 116 |
+
|
| 117 |
+
def test_numpy_values_serialized(self):
|
| 118 |
+
point = json.dumps({"type": "Point", "coordinates": [0.0, 0.0]})
|
| 119 |
+
df = pd.DataFrame(
|
| 120 |
+
{
|
| 121 |
+
"id": [np.int64(1)],
|
| 122 |
+
"score": [np.float64(3.14)],
|
| 123 |
+
"geometry": [point],
|
| 124 |
+
}
|
| 125 |
+
)
|
| 126 |
+
fc = to_feature_collection(df)
|
| 127 |
+
props = fc["features"][0]["properties"]
|
| 128 |
+
assert props["id"] == 1
|
| 129 |
+
assert isinstance(props["id"], int)
|
| 130 |
+
assert props["score"] == pytest.approx(3.14)
|
| 131 |
+
|
| 132 |
+
def test_bytearray_property_becomes_null(self):
|
| 133 |
+
point = json.dumps({"type": "Point", "coordinates": [0.0, 0.0]})
|
| 134 |
+
df = pd.DataFrame(
|
| 135 |
+
{
|
| 136 |
+
"id": ["x1"],
|
| 137 |
+
"data": [bytearray(b"raw")],
|
| 138 |
+
"geometry": [point],
|
| 139 |
+
}
|
| 140 |
+
)
|
| 141 |
+
fc = to_feature_collection(df)
|
| 142 |
+
# bytearray should be converted to None via _to_serializable
|
| 143 |
+
assert fc["features"][0]["properties"]["data"] is None
|
| 144 |
+
|
| 145 |
+
def test_multiple_features(self):
|
| 146 |
+
geojsons = [
|
| 147 |
+
json.dumps({"type": "Point", "coordinates": [1.0, 2.0]}),
|
| 148 |
+
json.dumps({"type": "Point", "coordinates": [3.0, 4.0]}),
|
| 149 |
+
]
|
| 150 |
+
df = pd.DataFrame({"id": ["a", "b"], "geometry": geojsons})
|
| 151 |
+
fc = to_feature_collection(df)
|
| 152 |
+
assert len(fc["features"]) == 2
|
| 153 |
+
assert fc["features"][0]["properties"]["id"] == "a"
|
| 154 |
+
assert fc["features"][1]["properties"]["id"] == "b"
|
| 155 |
+
|
| 156 |
+
|
| 157 |
+
class TestSaveGeojson:
|
| 158 |
+
def test_writes_file(self, tmp_path):
|
| 159 |
+
point = json.dumps({"type": "Point", "coordinates": [0.0, 0.0]})
|
| 160 |
+
df = pd.DataFrame({"id": ["x1"], "geometry": [point]})
|
| 161 |
+
out = save_geojson(df, "test_query", output_dir=tmp_path)
|
| 162 |
+
assert out.exists()
|
| 163 |
+
assert out.suffix == ".geojson"
|
| 164 |
+
data = json.loads(out.read_text())
|
| 165 |
+
assert data["type"] == "FeatureCollection"
|
| 166 |
+
assert len(data["features"]) == 1
|
| 167 |
+
|
| 168 |
+
def test_slug_generation(self, tmp_path):
|
| 169 |
+
point = json.dumps({"type": "Point", "coordinates": [0.0, 0.0]})
|
| 170 |
+
df = pd.DataFrame({"id": ["x1"], "geometry": [point]})
|
| 171 |
+
out = save_geojson(df, "What's the boundary of Paris?", output_dir=tmp_path)
|
| 172 |
+
# Slug should be lowercase with non-word chars replaced
|
| 173 |
+
assert "what_s_the_boundary_of_paris" in out.name
|
| 174 |
+
|
| 175 |
+
def test_creates_parent_dirs(self, tmp_path):
|
| 176 |
+
point = json.dumps({"type": "Point", "coordinates": [0.0, 0.0]})
|
| 177 |
+
df = pd.DataFrame({"id": ["x1"], "geometry": [point]})
|
| 178 |
+
nested = tmp_path / "a" / "b" / "c"
|
| 179 |
+
out = save_geojson(df, "test", output_dir=nested)
|
| 180 |
+
assert out.parent == nested
|
| 181 |
+
assert out.exists()
|
| 182 |
+
|
| 183 |
+
def test_empty_dataframe(self, tmp_path):
|
| 184 |
+
df = pd.DataFrame()
|
| 185 |
+
out = save_geojson(df, "empty", output_dir=tmp_path)
|
| 186 |
+
data = json.loads(out.read_text())
|
| 187 |
+
assert data["features"] == []
|
tests/test_geometry.py
ADDED
|
@@ -0,0 +1,102 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""Tests for gazet.geometry — coordinate rounding and geometry normalization."""
|
| 2 |
+
|
| 3 |
+
import json
|
| 4 |
+
|
| 5 |
+
from gazet.geometry import _round_coords, normalize_geometry_to_geojson
|
| 6 |
+
|
| 7 |
+
|
| 8 |
+
class TestRoundCoords:
|
| 9 |
+
def test_single_float(self):
|
| 10 |
+
assert _round_coords(1.23456789, 1) == 1.2
|
| 11 |
+
|
| 12 |
+
def test_zero_precision(self):
|
| 13 |
+
assert _round_coords(1.9, 0) == 2.0
|
| 14 |
+
|
| 15 |
+
def test_nested_list(self):
|
| 16 |
+
coords = [1.11111, 2.22222, 3.33333]
|
| 17 |
+
result = _round_coords(coords, 2)
|
| 18 |
+
assert result == [1.11, 2.22, 3.33]
|
| 19 |
+
|
| 20 |
+
def test_dict_recursion(self):
|
| 21 |
+
obj = {"lat": 1.99999, "lng": 2.11111}
|
| 22 |
+
result = _round_coords(obj, 1)
|
| 23 |
+
assert result == {"lat": 2.0, "lng": 2.1}
|
| 24 |
+
|
| 25 |
+
def test_nested_structure(self):
|
| 26 |
+
geojson = {
|
| 27 |
+
"type": "Point",
|
| 28 |
+
"coordinates": [12.123456, 34.987654],
|
| 29 |
+
}
|
| 30 |
+
result = _round_coords(geojson, 3)
|
| 31 |
+
assert result == {"type": "Point", "coordinates": [12.123, 34.988]}
|
| 32 |
+
|
| 33 |
+
def test_multipolygon(self):
|
| 34 |
+
coords = [[[1.1, 2.2], [3.3, 4.4], [5.5, 6.6]]]
|
| 35 |
+
result = _round_coords(coords, 1)
|
| 36 |
+
assert result == [[[1.1, 2.2], [3.3, 4.4], [5.5, 6.6]]]
|
| 37 |
+
|
| 38 |
+
def test_string_passthrough(self):
|
| 39 |
+
assert _round_coords("hello", 2) == "hello"
|
| 40 |
+
|
| 41 |
+
def test_int_passthrough(self):
|
| 42 |
+
assert _round_coords(42, 2) == 42
|
| 43 |
+
|
| 44 |
+
def test_none_passthrough(self):
|
| 45 |
+
assert _round_coords(None, 2) is None
|
| 46 |
+
|
| 47 |
+
def test_mixed_types_in_list(self):
|
| 48 |
+
data = [1.111, "type", 2.222]
|
| 49 |
+
result = _round_coords(data, 1)
|
| 50 |
+
assert result == [1.1, "type", 2.2]
|
| 51 |
+
|
| 52 |
+
|
| 53 |
+
class TestNormalizeGeometryToGeojson:
|
| 54 |
+
def test_no_geometry_column(self, con):
|
| 55 |
+
import pandas as pd
|
| 56 |
+
|
| 57 |
+
df = pd.DataFrame({"id": [1], "name": ["test"]})
|
| 58 |
+
result = normalize_geometry_to_geojson(con, df)
|
| 59 |
+
assert "geometry" not in result.columns
|
| 60 |
+
|
| 61 |
+
def test_empty_dataframe(self, con):
|
| 62 |
+
import pandas as pd
|
| 63 |
+
|
| 64 |
+
df = pd.DataFrame(columns=["geometry"])
|
| 65 |
+
result = normalize_geometry_to_geojson(con, df)
|
| 66 |
+
assert result.empty
|
| 67 |
+
|
| 68 |
+
def test_geojson_string_simplified(self, con):
|
| 69 |
+
import pandas as pd
|
| 70 |
+
|
| 71 |
+
# A simple point — simplify should pass it through
|
| 72 |
+
point_geojson = json.dumps(
|
| 73 |
+
{
|
| 74 |
+
"type": "Point",
|
| 75 |
+
"coordinates": [12.1234567, 34.9876543],
|
| 76 |
+
}
|
| 77 |
+
)
|
| 78 |
+
df = pd.DataFrame({"geometry": [point_geojson]})
|
| 79 |
+
result = normalize_geometry_to_geojson(con, df)
|
| 80 |
+
assert result["geometry"].iloc[0] is not None
|
| 81 |
+
parsed = json.loads(result["geometry"].iloc[0])
|
| 82 |
+
assert parsed["type"] == "Point"
|
| 83 |
+
# Coordinates should be rounded
|
| 84 |
+
coords = parsed["coordinates"]
|
| 85 |
+
assert coords[0] == round(12.1234567, 5)
|
| 86 |
+
assert coords[1] == round(34.9876543, 5)
|
| 87 |
+
|
| 88 |
+
def test_null_geometry_preserved(self, con):
|
| 89 |
+
import pandas as pd
|
| 90 |
+
|
| 91 |
+
df = pd.DataFrame({"geometry": [None, None]})
|
| 92 |
+
result = normalize_geometry_to_geojson(con, df)
|
| 93 |
+
assert pd.isna(result["geometry"]).all() or result["geometry"].isna().all()
|
| 94 |
+
|
| 95 |
+
def test_mixed_geojson_and_null(self, con):
|
| 96 |
+
import pandas as pd
|
| 97 |
+
|
| 98 |
+
point = json.dumps({"type": "Point", "coordinates": [0.0, 0.0]})
|
| 99 |
+
df = pd.DataFrame({"geometry": [point, None]})
|
| 100 |
+
result = normalize_geometry_to_geojson(con, df)
|
| 101 |
+
assert result["geometry"].iloc[0] is not None
|
| 102 |
+
assert result["geometry"].iloc[1] is None
|
tests/test_lm.py
ADDED
|
@@ -0,0 +1,106 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""Tests for gazet.lm — prompt templates, postprocessing, and GGUF helpers."""
|
| 2 |
+
|
| 3 |
+
from gazet.lm import (
|
| 4 |
+
_PLACES_SYSTEM_PROMPT,
|
| 5 |
+
_SYSTEM_PROMPT_TEMPLATE,
|
| 6 |
+
_USER_PROMPT_TEMPLATE,
|
| 7 |
+
_postprocess_sql,
|
| 8 |
+
)
|
| 9 |
+
from gazet.schemas import PlacesResult
|
| 10 |
+
|
| 11 |
+
|
| 12 |
+
class TestPostprocessSql:
|
| 13 |
+
def test_plain_sql(self):
|
| 14 |
+
assert _postprocess_sql("SELECT * FROM foo") == "SELECT * FROM foo"
|
| 15 |
+
|
| 16 |
+
def test_sql_fence_with_lang(self):
|
| 17 |
+
raw = "```sql\nSELECT id FROM bar\n```"
|
| 18 |
+
assert _postprocess_sql(raw) == "SELECT id FROM bar"
|
| 19 |
+
|
| 20 |
+
def test_sql_fence_no_lang(self):
|
| 21 |
+
raw = "```\nSELECT 1\n```"
|
| 22 |
+
assert _postprocess_sql(raw) == "SELECT 1"
|
| 23 |
+
|
| 24 |
+
def test_fence_with_trailing_text(self):
|
| 25 |
+
raw = "```sql\nSELECT * FROM foo\n```\nDone!"
|
| 26 |
+
result = _postprocess_sql(raw)
|
| 27 |
+
assert "SELECT" in result
|
| 28 |
+
assert "FROM" in result
|
| 29 |
+
|
| 30 |
+
def test_nested_backticks_preserved(self):
|
| 31 |
+
raw = "```sql\nSELECT `col`\n```"
|
| 32 |
+
assert _postprocess_sql(raw) == "SELECT `col`"
|
| 33 |
+
|
| 34 |
+
def test_whitespace_only(self):
|
| 35 |
+
assert _postprocess_sql(" ") == ""
|
| 36 |
+
|
| 37 |
+
def test_empty_string(self):
|
| 38 |
+
assert _postprocess_sql("") == ""
|
| 39 |
+
|
| 40 |
+
def test_multiline_query(self):
|
| 41 |
+
raw = """```sql
|
| 42 |
+
SELECT
|
| 43 |
+
id,
|
| 44 |
+
names."primary" AS name
|
| 45 |
+
FROM read_parquet('divisions_area')
|
| 46 |
+
WHERE country = 'IN'
|
| 47 |
+
```"""
|
| 48 |
+
result = _postprocess_sql(raw)
|
| 49 |
+
assert "SELECT" in result
|
| 50 |
+
assert "WHERE" in result
|
| 51 |
+
assert "```" not in result
|
| 52 |
+
|
| 53 |
+
def test_no_fence_just_whitespace_surround(self):
|
| 54 |
+
raw = " SELECT * FROM table "
|
| 55 |
+
assert _postprocess_sql(raw) == "SELECT * FROM table"
|
| 56 |
+
|
| 57 |
+
|
| 58 |
+
class TestPromptTemplates:
|
| 59 |
+
def test_system_prompt_has_schema_placeholder(self):
|
| 60 |
+
assert "{schema}" in _SYSTEM_PROMPT_TEMPLATE
|
| 61 |
+
|
| 62 |
+
def test_system_prompt_mentions_st_asgeojson(self):
|
| 63 |
+
assert "ST_AsGeoJSON" in _SYSTEM_PROMPT_TEMPLATE
|
| 64 |
+
|
| 65 |
+
def test_user_prompt_has_candidates_placeholder(self):
|
| 66 |
+
assert "{candidates_csv}" in _USER_PROMPT_TEMPLATE
|
| 67 |
+
|
| 68 |
+
def test_user_prompt_has_question_placeholder(self):
|
| 69 |
+
assert "{question}" in _USER_PROMPT_TEMPLATE
|
| 70 |
+
|
| 71 |
+
def test_system_prompt_formatting(self):
|
| 72 |
+
formatted = _SYSTEM_PROMPT_TEMPLATE.format(schema="dummy_schema")
|
| 73 |
+
assert "dummy_schema" in formatted
|
| 74 |
+
assert "{schema}" not in formatted
|
| 75 |
+
|
| 76 |
+
def test_user_prompt_formatting(self):
|
| 77 |
+
formatted = _USER_PROMPT_TEMPLATE.format(
|
| 78 |
+
candidates_csv="source,id,name",
|
| 79 |
+
question="get boundary",
|
| 80 |
+
)
|
| 81 |
+
assert "source,id,name" in formatted
|
| 82 |
+
assert "get boundary" in formatted
|
| 83 |
+
|
| 84 |
+
def test_places_system_prompt_has_examples(self):
|
| 85 |
+
assert "EXAMPLES:" in _PLACES_SYSTEM_PROMPT
|
| 86 |
+
assert "Puri, Odisha" in _PLACES_SYSTEM_PROMPT
|
| 87 |
+
assert "Amazon basin" in _PLACES_SYSTEM_PROMPT
|
| 88 |
+
|
| 89 |
+
|
| 90 |
+
class TestGeneratePlacesFallback:
|
| 91 |
+
"""Test that generate_places handles parse errors gracefully.
|
| 92 |
+
We can't test the real llama-server path without a running server,
|
| 93 |
+
but we can verify the PlacesResult model behavior used in fallback.
|
| 94 |
+
"""
|
| 95 |
+
|
| 96 |
+
def test_places_result_from_raw_query(self):
|
| 97 |
+
# Simulates the fallback path in generate_places
|
| 98 |
+
query = "get me the boundary of Paris"
|
| 99 |
+
result = PlacesResult(places=[{"place": query}])
|
| 100 |
+
assert len(result.places) == 1
|
| 101 |
+
assert result.places[0].place == query
|
| 102 |
+
|
| 103 |
+
def test_places_result_validation(self):
|
| 104 |
+
data = {"places": [{"place": "Chad"}, {"place": "Lake Chad"}]}
|
| 105 |
+
r = PlacesResult.model_validate(data)
|
| 106 |
+
assert len(r.places) == 2
|
tests/test_schemas.py
ADDED
|
@@ -0,0 +1,73 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""Tests for gazet.schemas — Pydantic models."""
|
| 2 |
+
|
| 3 |
+
from gazet.schemas import Place, PlacesResult
|
| 4 |
+
|
| 5 |
+
|
| 6 |
+
class TestPlace:
|
| 7 |
+
def test_minimal(self):
|
| 8 |
+
p = Place(place="Paris")
|
| 9 |
+
assert p.place == "Paris"
|
| 10 |
+
assert p.country is None
|
| 11 |
+
assert p.subtype is None
|
| 12 |
+
|
| 13 |
+
def test_all_fields(self):
|
| 14 |
+
p = Place(place="Odisha", country="IN", subtype="region")
|
| 15 |
+
assert p.place == "Odisha"
|
| 16 |
+
assert p.country == "IN"
|
| 17 |
+
assert p.subtype == "region"
|
| 18 |
+
|
| 19 |
+
def test_default_none_explicit(self):
|
| 20 |
+
p = Place(place="Loja", country=None, subtype=None)
|
| 21 |
+
assert p.country is None
|
| 22 |
+
assert p.subtype is None
|
| 23 |
+
|
| 24 |
+
def test_model_dump(self):
|
| 25 |
+
p = Place(place="Brazil", country="BR")
|
| 26 |
+
d = p.model_dump()
|
| 27 |
+
assert d["place"] == "Brazil"
|
| 28 |
+
assert d["country"] == "BR"
|
| 29 |
+
assert d["subtype"] is None
|
| 30 |
+
|
| 31 |
+
def test_rejects_missing_place(self):
|
| 32 |
+
import pytest
|
| 33 |
+
|
| 34 |
+
with pytest.raises(Exception): # pydantic validation error
|
| 35 |
+
Place() # pragma: no cover
|
| 36 |
+
|
| 37 |
+
|
| 38 |
+
class TestPlacesResult:
|
| 39 |
+
def test_single_place(self):
|
| 40 |
+
r = PlacesResult(places=[Place(place="India")])
|
| 41 |
+
assert len(r.places) == 1
|
| 42 |
+
assert r.places[0].place == "India"
|
| 43 |
+
|
| 44 |
+
def test_multiple_places(self):
|
| 45 |
+
r = PlacesResult(
|
| 46 |
+
places=[
|
| 47 |
+
Place(place="Ecuador"),
|
| 48 |
+
Place(place="Amazon basin"),
|
| 49 |
+
]
|
| 50 |
+
)
|
| 51 |
+
assert len(r.places) == 2
|
| 52 |
+
assert r.places[0].place == "Ecuador"
|
| 53 |
+
assert r.places[1].place == "Amazon basin"
|
| 54 |
+
|
| 55 |
+
def test_empty(self):
|
| 56 |
+
r = PlacesResult(places=[])
|
| 57 |
+
assert len(r.places) == 0
|
| 58 |
+
|
| 59 |
+
def test_model_validate_from_dict(self):
|
| 60 |
+
data = {
|
| 61 |
+
"places": [
|
| 62 |
+
{"place": "Chad"},
|
| 63 |
+
{"place": "Lake Chad", "country": "TD", "subtype": "lake"},
|
| 64 |
+
]
|
| 65 |
+
}
|
| 66 |
+
r = PlacesResult.model_validate(data)
|
| 67 |
+
assert len(r.places) == 2
|
| 68 |
+
assert r.places[1].country == "TD"
|
| 69 |
+
assert r.places[1].subtype == "lake"
|
| 70 |
+
|
| 71 |
+
def test_places_are_place_instances(self):
|
| 72 |
+
r = PlacesResult(places=[Place(place="Nairobi")])
|
| 73 |
+
assert isinstance(r.places[0], Place)
|
tests/test_search.py
ADDED
|
@@ -0,0 +1,196 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""Tests for gazet.search — fuzzy search and ID lookup against real parquet data."""
|
| 2 |
+
|
| 3 |
+
import pandas as pd
|
| 4 |
+
|
| 5 |
+
from gazet.schemas import Place
|
| 6 |
+
from gazet.search import (
|
| 7 |
+
get_by_id,
|
| 8 |
+
get_division_by_id,
|
| 9 |
+
get_natural_earth_by_id,
|
| 10 |
+
search_candidates,
|
| 11 |
+
search_divisions_area,
|
| 12 |
+
search_natural_earth,
|
| 13 |
+
)
|
| 14 |
+
|
| 15 |
+
|
| 16 |
+
class TestSearchDivisionsArea:
|
| 17 |
+
def test_returns_dataframe(self, con):
|
| 18 |
+
df = search_divisions_area(con, Place(place="India"))
|
| 19 |
+
assert isinstance(df, pd.DataFrame)
|
| 20 |
+
assert "source" in df.columns
|
| 21 |
+
assert "id" in df.columns
|
| 22 |
+
|
| 23 |
+
def test_matches_known_country(self, con):
|
| 24 |
+
df = search_divisions_area(con, Place(place="India"))
|
| 25 |
+
assert not df.empty
|
| 26 |
+
# At least one result should be India
|
| 27 |
+
names = df["name"].str.lower().tolist()
|
| 28 |
+
assert any("india" in name for name in names)
|
| 29 |
+
|
| 30 |
+
def test_limits_results(self, con):
|
| 31 |
+
df = search_divisions_area(con, Place(place="a"), limit=3)
|
| 32 |
+
assert len(df) <= 3
|
| 33 |
+
|
| 34 |
+
def test_empty_result(self, con):
|
| 35 |
+
# Extremely unlikely string — Jaro-Winkler always returns top N,
|
| 36 |
+
# but similarity should be very low
|
| 37 |
+
df = search_divisions_area(con, Place(place="Xyzz98765"))
|
| 38 |
+
# Results may still be returned (always top-5) but with low similarity
|
| 39 |
+
assert len(df) <= 5
|
| 40 |
+
|
| 41 |
+
def test_source_column(self, con):
|
| 42 |
+
df = search_divisions_area(con, Place(place="India"))
|
| 43 |
+
if not df.empty:
|
| 44 |
+
assert (df["source"] == "divisions_area").all()
|
| 45 |
+
|
| 46 |
+
def test_case_insensitive(self, con):
|
| 47 |
+
df_upper = search_divisions_area(con, Place(place="INDIA"))
|
| 48 |
+
df_lower = search_divisions_area(con, Place(place="india"))
|
| 49 |
+
# Both should return results (even if different similarity scores)
|
| 50 |
+
assert not df_upper.empty
|
| 51 |
+
assert not df_lower.empty
|
| 52 |
+
|
| 53 |
+
def test_with_country_filter(self, con):
|
| 54 |
+
df = search_divisions_area(con, Place(place="Loja", country="EC"))
|
| 55 |
+
# Should find Loja in Ecuador
|
| 56 |
+
if not df.empty:
|
| 57 |
+
assert "source" in df.columns
|
| 58 |
+
|
| 59 |
+
def test_similar_names_different_country(self, con):
|
| 60 |
+
df = search_divisions_area(con, Place(place="Manchester"))
|
| 61 |
+
# Should return results (multiple Manchesters exist)
|
| 62 |
+
assert isinstance(df, pd.DataFrame)
|
| 63 |
+
|
| 64 |
+
def test_include_geometry_flag(self, con):
|
| 65 |
+
df = search_divisions_area(
|
| 66 |
+
con, Place(place="India"), include_geometry=True, limit=1
|
| 67 |
+
)
|
| 68 |
+
if not df.empty:
|
| 69 |
+
assert "geometry" in df.columns
|
| 70 |
+
|
| 71 |
+
def test_include_bbox_flag(self, con):
|
| 72 |
+
df = search_divisions_area(
|
| 73 |
+
con, Place(place="India"), include_bbox=True, limit=1
|
| 74 |
+
)
|
| 75 |
+
if not df.empty:
|
| 76 |
+
assert "bbox" in df.columns
|
| 77 |
+
|
| 78 |
+
def test_column_presence(self, con):
|
| 79 |
+
df = search_divisions_area(con, Place(place="India"), limit=1)
|
| 80 |
+
if not df.empty:
|
| 81 |
+
expected_cols = ["source", "id", "name", "country", "subtype", "similarity"]
|
| 82 |
+
for col in expected_cols:
|
| 83 |
+
assert col in df.columns
|
| 84 |
+
|
| 85 |
+
|
| 86 |
+
class TestSearchNaturalEarth:
|
| 87 |
+
def test_returns_dataframe(self, con):
|
| 88 |
+
df = search_natural_earth(con, Place(place="Nile"))
|
| 89 |
+
assert isinstance(df, pd.DataFrame)
|
| 90 |
+
|
| 91 |
+
def test_searches_ocean(self, con):
|
| 92 |
+
df = search_natural_earth(con, Place(place="Pacific"))
|
| 93 |
+
assert isinstance(df, pd.DataFrame)
|
| 94 |
+
|
| 95 |
+
def test_source_column(self, con):
|
| 96 |
+
df = search_natural_earth(con, Place(place="Nile"))
|
| 97 |
+
if not df.empty:
|
| 98 |
+
assert (df["source"] == "natural_earth").all()
|
| 99 |
+
|
| 100 |
+
def test_empty_result(self, con):
|
| 101 |
+
df = search_natural_earth(con, Place(place="Xyzz98765"))
|
| 102 |
+
# Jaro-Winkler always returns top 5, similarity is very low
|
| 103 |
+
assert len(df) <= 5
|
| 104 |
+
|
| 105 |
+
def test_limits_results(self, con):
|
| 106 |
+
df = search_natural_earth(con, Place(place="a"), limit=2)
|
| 107 |
+
assert len(df) <= 2
|
| 108 |
+
|
| 109 |
+
def test_include_geometry(self, con):
|
| 110 |
+
df = search_natural_earth(con, Place(place="Nile"), include_geometry=True)
|
| 111 |
+
if not df.empty:
|
| 112 |
+
assert "geometry" in df.columns
|
| 113 |
+
|
| 114 |
+
|
| 115 |
+
class TestSearchCandidates:
|
| 116 |
+
def test_searches_both_sources(self, con):
|
| 117 |
+
results = search_candidates(con, Place(place="India"))
|
| 118 |
+
# Should get at least one source
|
| 119 |
+
assert len(results) >= 1
|
| 120 |
+
sources = {df["source"].iloc[0] for df in results if not df.empty}
|
| 121 |
+
assert "divisions_area" in sources
|
| 122 |
+
|
| 123 |
+
def test_restricts_to_single_source(self, con):
|
| 124 |
+
results = search_candidates(
|
| 125 |
+
con,
|
| 126 |
+
Place(place="Nile"),
|
| 127 |
+
sources=("natural_earth",),
|
| 128 |
+
)
|
| 129 |
+
for df in results:
|
| 130 |
+
if not df.empty:
|
| 131 |
+
assert df["source"].iloc[0] == "natural_earth"
|
| 132 |
+
|
| 133 |
+
def test_empty_place(self, con):
|
| 134 |
+
# Even random strings get fuzzy matches — check results exist
|
| 135 |
+
# but have low similarity
|
| 136 |
+
results = search_candidates(con, Place(place="Xyzz98765"))
|
| 137 |
+
# Results may be non-empty due to fuzzy matching, but similarity low
|
| 138 |
+
for df in results:
|
| 139 |
+
assert len(df) <= 5
|
| 140 |
+
|
| 141 |
+
def test_combine_multiple_places(self, con):
|
| 142 |
+
# Search for a place that exists in both sources
|
| 143 |
+
results = search_candidates(
|
| 144 |
+
con,
|
| 145 |
+
Place(place="Brazil"),
|
| 146 |
+
sources=("divisions_area", "natural_earth"),
|
| 147 |
+
)
|
| 148 |
+
for df in results:
|
| 149 |
+
assert isinstance(df, pd.DataFrame)
|
| 150 |
+
|
| 151 |
+
|
| 152 |
+
class TestGetById:
|
| 153 |
+
def test_get_division_by_id(self, con):
|
| 154 |
+
# First get a valid ID from search
|
| 155 |
+
df = search_divisions_area(con, Place(place="India"), limit=1)
|
| 156 |
+
if not df.empty:
|
| 157 |
+
rid = df["id"].iloc[0]
|
| 158 |
+
result = get_division_by_id(con, rid)
|
| 159 |
+
assert not result.empty
|
| 160 |
+
assert result["id"].iloc[0] == rid
|
| 161 |
+
|
| 162 |
+
def test_get_natural_earth_by_id(self, con):
|
| 163 |
+
df = search_natural_earth(con, Place(place="Nile"), limit=1)
|
| 164 |
+
if not df.empty:
|
| 165 |
+
rid = df["id"].iloc[0]
|
| 166 |
+
result = get_natural_earth_by_id(con, rid)
|
| 167 |
+
assert not result.empty
|
| 168 |
+
assert result["id"].iloc[0] == rid
|
| 169 |
+
|
| 170 |
+
def test_get_by_id_auto_infer_divisions(self, con):
|
| 171 |
+
df = search_divisions_area(con, Place(place="India"), limit=1)
|
| 172 |
+
if not df.empty:
|
| 173 |
+
rid = df["id"].iloc[0]
|
| 174 |
+
result = get_by_id(con, rid)
|
| 175 |
+
assert not result.empty
|
| 176 |
+
|
| 177 |
+
def test_get_by_id_auto_infer_natural_earth(self, con):
|
| 178 |
+
df = search_natural_earth(con, Place(place="Nile"), limit=1)
|
| 179 |
+
if not df.empty:
|
| 180 |
+
rid = df["id"].iloc[0]
|
| 181 |
+
# NE IDs start with "ne_"
|
| 182 |
+
assert rid.startswith("ne_")
|
| 183 |
+
result = get_by_id(con, rid)
|
| 184 |
+
assert not result.empty
|
| 185 |
+
|
| 186 |
+
def test_get_by_id_invalid(self, con):
|
| 187 |
+
result = get_by_id(con, "nonexistent_id_999")
|
| 188 |
+
assert result.empty
|
| 189 |
+
|
| 190 |
+
def test_get_by_id_with_source(self, con):
|
| 191 |
+
df = search_divisions_area(con, Place(place="India"), limit=1)
|
| 192 |
+
if not df.empty:
|
| 193 |
+
rid = df["id"].iloc[0]
|
| 194 |
+
result = get_by_id(con, rid, source="divisions_area")
|
| 195 |
+
assert not result.empty
|
| 196 |
+
assert result["source"].iloc[0] == "divisions_area"
|
tests/test_sql.py
ADDED
|
@@ -0,0 +1,182 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""Tests for gazet.sql — SQL rewriting, normalization, and helpers."""
|
| 2 |
+
|
| 3 |
+
from unittest.mock import MagicMock, patch
|
| 4 |
+
|
| 5 |
+
import pandas as pd
|
| 6 |
+
|
| 7 |
+
from gazet.sql import (
|
| 8 |
+
_normalize_ne_subtypes,
|
| 9 |
+
_rewrite_data_paths,
|
| 10 |
+
_strip_fences,
|
| 11 |
+
run_geo_sql_dspy,
|
| 12 |
+
run_geo_sql_gguf,
|
| 13 |
+
)
|
| 14 |
+
|
| 15 |
+
|
| 16 |
+
class TestStripFences:
|
| 17 |
+
def test_plain_sql(self):
|
| 18 |
+
assert _strip_fences("SELECT * FROM foo") == "SELECT * FROM foo"
|
| 19 |
+
|
| 20 |
+
def test_sql_backtick_fences(self):
|
| 21 |
+
raw = "```sql\nSELECT id FROM bar\n```"
|
| 22 |
+
assert _strip_fences(raw) == "SELECT id FROM bar"
|
| 23 |
+
|
| 24 |
+
def test_backtick_fences_no_lang(self):
|
| 25 |
+
raw = "```\nSELECT 1\n```"
|
| 26 |
+
assert _strip_fences(raw) == "SELECT 1"
|
| 27 |
+
|
| 28 |
+
def test_none_input(self):
|
| 29 |
+
assert _strip_fences(None) == ""
|
| 30 |
+
|
| 31 |
+
def test_empty_string(self):
|
| 32 |
+
assert _strip_fences("") == ""
|
| 33 |
+
|
| 34 |
+
def test_partial_fence_leading(self):
|
| 35 |
+
raw = "```sql\nSELECT id"
|
| 36 |
+
assert _strip_fences(raw) == "SELECT id"
|
| 37 |
+
|
| 38 |
+
def test_partial_fence_trailing(self):
|
| 39 |
+
raw = "SELECT id\n```"
|
| 40 |
+
assert _strip_fences(raw) == "SELECT id"
|
| 41 |
+
|
| 42 |
+
def test_preserves_inner_backticks(self):
|
| 43 |
+
raw = "```sql\nSELECT `column` FROM table\n```"
|
| 44 |
+
assert _strip_fences(raw) == "SELECT `column` FROM table"
|
| 45 |
+
|
| 46 |
+
|
| 47 |
+
class TestRewriteDataPaths:
|
| 48 |
+
def test_symbolic_divisions_area(self):
|
| 49 |
+
sql = "SELECT * FROM read_parquet('divisions_area')"
|
| 50 |
+
result = _rewrite_data_paths(sql)
|
| 51 |
+
assert "divisions_area" in result
|
| 52 |
+
assert "read_parquet('divisions_area')" not in result
|
| 53 |
+
|
| 54 |
+
def test_symbolic_natural_earth(self):
|
| 55 |
+
sql = "SELECT * FROM read_parquet('natural_earth')"
|
| 56 |
+
result = _rewrite_data_paths(sql)
|
| 57 |
+
assert "natural_earth" in result
|
| 58 |
+
assert "read_parquet('natural_earth')" not in result
|
| 59 |
+
|
| 60 |
+
def test_hallucinated_division_path(self):
|
| 61 |
+
sql = "SELECT * FROM read_parquet('/data/overture/division_area/foo.parquet')"
|
| 62 |
+
result = _rewrite_data_paths(sql)
|
| 63 |
+
# The original hallucinated path should be gone
|
| 64 |
+
assert "/data/overture/division_area/foo.parquet" not in result
|
| 65 |
+
|
| 66 |
+
def test_hallucinated_natural_earth_path(self):
|
| 67 |
+
sql = (
|
| 68 |
+
"SELECT * FROM read_parquet('/some/natural_earth_geoparquet/data.parquet')"
|
| 69 |
+
)
|
| 70 |
+
result = _rewrite_data_paths(sql)
|
| 71 |
+
assert "/some/natural_earth_geoparquet/data.parquet" not in result
|
| 72 |
+
|
| 73 |
+
def test_double_quotes(self):
|
| 74 |
+
sql = 'SELECT * FROM read_parquet("divisions_area")'
|
| 75 |
+
result = _rewrite_data_paths(sql)
|
| 76 |
+
assert 'read_parquet("divisions_area")' not in result
|
| 77 |
+
|
| 78 |
+
def test_no_false_positive_unrelated_table(self):
|
| 79 |
+
sql = "SELECT * FROM read_parquet('some_other_table')"
|
| 80 |
+
result = _rewrite_data_paths(sql)
|
| 81 |
+
# Should be unchanged
|
| 82 |
+
assert "some_other_table" in result
|
| 83 |
+
|
| 84 |
+
|
| 85 |
+
class TestNormalizeNeSubtypes:
|
| 86 |
+
def test_lowercase_river(self):
|
| 87 |
+
sql = "WHERE n.subtype = 'River'"
|
| 88 |
+
result = _normalize_ne_subtypes(sql)
|
| 89 |
+
assert "'river'" in result
|
| 90 |
+
|
| 91 |
+
def test_lowercase_lake(self):
|
| 92 |
+
sql = "WHERE n.subtype = 'Lake'"
|
| 93 |
+
result = _normalize_ne_subtypes(sql)
|
| 94 |
+
assert "'lake'" in result
|
| 95 |
+
|
| 96 |
+
def test_lowercase_ocean(self):
|
| 97 |
+
sql = "WHERE subtype = 'Ocean'"
|
| 98 |
+
result = _normalize_ne_subtypes(sql)
|
| 99 |
+
assert "'ocean'" in result
|
| 100 |
+
|
| 101 |
+
def test_lowercase_sea(self):
|
| 102 |
+
sql = "WHERE subtype = 'Sea'"
|
| 103 |
+
result = _normalize_ne_subtypes(sql)
|
| 104 |
+
assert "'sea'" in result
|
| 105 |
+
|
| 106 |
+
def test_lowercase_range_mtn(self):
|
| 107 |
+
sql = "WHERE subtype = 'Range/mtn'"
|
| 108 |
+
result = _normalize_ne_subtypes(sql)
|
| 109 |
+
assert "'range/mtn'" in result
|
| 110 |
+
|
| 111 |
+
def test_terrain_area_replacement(self):
|
| 112 |
+
sql = "WHERE n.subtype = 'Terrain area'"
|
| 113 |
+
result = _normalize_ne_subtypes(sql)
|
| 114 |
+
assert "range/mtn" in result
|
| 115 |
+
assert "peninsula" in result
|
| 116 |
+
assert "depression" in result
|
| 117 |
+
|
| 118 |
+
def test_terrain_area_in_clause(self):
|
| 119 |
+
sql = "WHERE n.subtype IN ('Terrain area')"
|
| 120 |
+
result = _normalize_ne_subtypes(sql)
|
| 121 |
+
assert "range/mtn" in result
|
| 122 |
+
|
| 123 |
+
def test_already_lowercase_unchanged(self):
|
| 124 |
+
sql = "WHERE n.subtype = 'river'"
|
| 125 |
+
result = _normalize_ne_subtypes(sql)
|
| 126 |
+
assert result == sql
|
| 127 |
+
|
| 128 |
+
def test_island_group(self):
|
| 129 |
+
sql = "WHERE subtype = 'Island group'"
|
| 130 |
+
result = _normalize_ne_subtypes(sql)
|
| 131 |
+
assert "'island group'" in result
|
| 132 |
+
|
| 133 |
+
|
| 134 |
+
class TestRunGeoSqlGguf:
|
| 135 |
+
def test_empty_candidates_returns_none_result(self, con):
|
| 136 |
+
empty_df = pd.DataFrame()
|
| 137 |
+
events = list(run_geo_sql_gguf(con, "get Paris", empty_df))
|
| 138 |
+
assert len(events) >= 1
|
| 139 |
+
assert events[-1]["type"] == "result"
|
| 140 |
+
assert events[-1]["df"] is None
|
| 141 |
+
|
| 142 |
+
@patch("gazet.sql.generate_sql")
|
| 143 |
+
def test_execution_flow(self, mock_generate, con):
|
| 144 |
+
mock_generate.return_value = "SELECT 1"
|
| 145 |
+
# A query that succeeds but returns no useful geometry rows
|
| 146 |
+
events = list(run_geo_sql_gguf(con, "test", pd.DataFrame({"id": ["1"]})))
|
| 147 |
+
# Should emit at least sql_attempt and result
|
| 148 |
+
types = [e["type"] for e in events]
|
| 149 |
+
assert "sql_attempt" in types
|
| 150 |
+
|
| 151 |
+
|
| 152 |
+
class TestRunGeoSqlDspy:
|
| 153 |
+
def test_empty_candidates_returns_none_result(self, con):
|
| 154 |
+
empty_df = pd.DataFrame()
|
| 155 |
+
events = list(run_geo_sql_dspy(con, "get Paris", empty_df))
|
| 156 |
+
assert events[-1]["type"] == "result"
|
| 157 |
+
assert events[-1]["df"] is None
|
| 158 |
+
|
| 159 |
+
@patch("gazet.sql.write_sql")
|
| 160 |
+
def test_successful_sql(self, mock_write, con):
|
| 161 |
+
# Return a result object with .sql attribute
|
| 162 |
+
mock_pred = MagicMock()
|
| 163 |
+
mock_pred.sql = "SELECT 1 as id"
|
| 164 |
+
mock_write.return_value = mock_pred
|
| 165 |
+
df = pd.DataFrame(
|
| 166 |
+
{"id": ["x1"], "name": ["test"], "source": ["divisions_area"]}
|
| 167 |
+
)
|
| 168 |
+
events = list(run_geo_sql_dspy(con, "test", df, max_iterations=1))
|
| 169 |
+
types = [e["type"] for e in events]
|
| 170 |
+
assert "sql_attempt" in types
|
| 171 |
+
|
| 172 |
+
@patch("gazet.sql.write_sql")
|
| 173 |
+
def test_exhausts_iterations(self, mock_write, con):
|
| 174 |
+
mock_pred = MagicMock()
|
| 175 |
+
mock_pred.sql = "INVALID SQL" # will cause execution error
|
| 176 |
+
mock_write.return_value = mock_pred
|
| 177 |
+
df = pd.DataFrame(
|
| 178 |
+
{"id": ["x1"], "name": ["test"], "source": ["divisions_area"]}
|
| 179 |
+
)
|
| 180 |
+
events = list(run_geo_sql_dspy(con, "test", df, max_iterations=2))
|
| 181 |
+
# Should exhaust iterations and yield final result
|
| 182 |
+
assert events[-1]["type"] == "result"
|
uv.lock
CHANGED
|
The diff for this file is too large to render.
See raw diff
|
|
|