Spaces:
Running
Running
Add: Bert
Browse filesThis view is limited to 50 files because it contains too many changes. See raw diff
- .env.example +41 -0
- Dockerfile +9 -6
- README.md +73 -25
- app/jobs/sync_place_embeddings.py +19 -8
- app/modules/places/api/dependencies.py +70 -12
- app/modules/places/api/internal_chat_schemas.py +39 -2
- app/modules/places/api/router.py +170 -4
- app/modules/places/api/schemas.py +24 -1
- app/modules/places/application/use_cases/chat_place_recommendations.py +379 -34
- app/modules/places/application/use_cases/chat_places.py +13 -2
- app/modules/places/application/use_cases/search_places.py +5 -1
- app/modules/places/domain/chat_intent.py +6 -0
- app/modules/places/domain/clarifications.py +39 -16
- app/modules/places/infrastructure/aws_pgvector_place_repository.py +35 -1
- app/modules/places/infrastructure/bert_intent_extractor.py +595 -0
- app/modules/places/infrastructure/deterministic_intent_parser.py +440 -69
- app/modules/places/infrastructure/hybrid_chat_retriever.py +214 -66
- app/modules/places/infrastructure/open_vocabulary_category_classifier.py +368 -0
- app/modules/places/infrastructure/place_category_catalog.py +87 -0
- app/modules/places/infrastructure/place_semantic_document.py +19 -43
- app/modules/places/infrastructure/semantic_activity_classifier.py +36 -184
- app/modules/places/infrastructure/semantic_place_ranker.py +7 -2
- app/shared/config/settings.py +174 -0
- app/shared/dependencies.py +30 -1
- app/shared/nlp/embeddings/cached.py +29 -1
- app/shared/nlp/embeddings/factory.py +50 -0
- app/shared/nlp/embeddings/sentence_transformer.py +288 -0
- app/shared/vector_store/aws_pgvector.py +66 -8
- requirements-training.txt +3 -0
- requirements.txt +2 -0
- scripts/train_place_intent_bert.py +501 -0
- scripts/train_place_retriever.py +147 -0
- sql/migrations/20260716_02_places_semantic_v1.sql +255 -0
- sql/verify_places_semantic_v1.sql +46 -0
- tests/conftest.py +4 -0
- tests/test_api_endpoints.py +73 -7
- tests/test_bert_place_intent_extractor.py +277 -0
- tests/test_embeddings.py +27 -0
- tests/test_hybrid_chat_retriever.py +232 -0
- tests/test_internal_places_chat.py +10 -9
- tests/test_main_api_place_source.py +10 -8
- tests/test_open_vocabulary_category_classifier.py +228 -0
- tests/test_pgvector_readiness.py +22 -1
- tests/test_place_chat_intent_parser.py +356 -3
- tests/test_place_chat_recommendations_use_case.py +389 -16
- tests/test_place_embedding_configuration.py +182 -0
- tests/test_places_use_cases.py +6 -2
- tests/test_semantic_activity_classifier.py +22 -0
- tests/test_sentence_transformer_embeddings.py +223 -0
- tests/test_sql_contract.py +26 -0
.env.example
CHANGED
|
@@ -57,6 +57,37 @@ FASTTEXT_MODEL_REPO_ID=facebook/fasttext-es-vectors
|
|
| 57 |
FASTTEXT_MODEL_FILENAME=model.bin
|
| 58 |
FASTTEXT_AUTO_DOWNLOAD=true
|
| 59 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 60 |
BM25_K1=1.5
|
| 61 |
BM25_B=0.75
|
| 62 |
BM25_RELEVANCE_THRESHOLD=3.0
|
|
@@ -70,7 +101,17 @@ PLACES_CHAT_CANDIDATE_LIMIT=30
|
|
| 70 |
PLACES_CHAT_MIN_CONTENT_SCORE=0.20
|
| 71 |
PLACES_CHAT_INTENT_MIN_CONFIDENCE=0.70
|
| 72 |
PLACES_CHAT_AMBIGUITY_DELTA=0.15
|
|
|
|
|
|
|
|
|
|
| 73 |
PLACES_CHAT_RANKING_VERSION=places-chat-v2
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 74 |
PLACES_CHAT_TAXONOMY_VERSION=places-taxonomy-v1
|
| 75 |
|
| 76 |
LOG_LEVEL=INFO
|
|
|
|
| 57 |
FASTTEXT_MODEL_FILENAME=model.bin
|
| 58 |
FASTTEXT_AUTO_DOWNLOAD=true
|
| 59 |
|
| 60 |
+
# Embeddings exclusivos de Places. Mantener FastText hasta aplicar/backfillear
|
| 61 |
+
# sql/migrations/20260716_02_places_semantic_v1.sql.
|
| 62 |
+
PLACES_EMBEDDING_PROVIDER=fasttext
|
| 63 |
+
PLACES_EMBEDDING_DIMENSION=300
|
| 64 |
+
PLACES_EMBEDDING_MODEL=facebook/fasttext-es-vectors
|
| 65 |
+
PLACES_EMBEDDING_VERSION=common-crawl-300-v1
|
| 66 |
+
PLACES_EMBEDDING_QUERY_PREFIX=
|
| 67 |
+
PLACES_EMBEDDING_PASSAGE_PREFIX=
|
| 68 |
+
PLACES_EMBEDDING_BATCH_SIZE=32
|
| 69 |
+
PLACES_EMBEDDING_DEVICE=
|
| 70 |
+
PLACES_CATEGORY_CATALOG_PATH=
|
| 71 |
+
PLACES_CATEGORY_MIN_SIMILARITY=0.44
|
| 72 |
+
PLACES_CATEGORY_MIN_MARGIN=0.04
|
| 73 |
+
PLACES_PGVECTOR_MATCH_FUNCTION=match_places
|
| 74 |
+
PLACES_PGVECTOR_HYBRID_FUNCTION=
|
| 75 |
+
PLACES_PGVECTOR_UPSERT_FUNCTION=upsert_place_embedding
|
| 76 |
+
PLACES_PGVECTOR_HASH_FUNCTION=get_place_content_hashes
|
| 77 |
+
|
| 78 |
+
# Perfil BERT recomendado despues del backfill:
|
| 79 |
+
# PLACES_EMBEDDING_PROVIDER=sentence_transformer
|
| 80 |
+
# PLACES_EMBEDDING_DIMENSION=768
|
| 81 |
+
# PLACES_EMBEDDING_MODEL=intfloat/multilingual-e5-base
|
| 82 |
+
# PLACES_EMBEDDING_VERSION=places-e5-domain-v1
|
| 83 |
+
# Quoting preserves the significant trailing space used by E5.
|
| 84 |
+
# PLACES_EMBEDDING_QUERY_PREFIX="query: "
|
| 85 |
+
# PLACES_EMBEDDING_PASSAGE_PREFIX="passage: "
|
| 86 |
+
# PLACES_PGVECTOR_MATCH_FUNCTION=match_places_semantic_v1
|
| 87 |
+
# PLACES_PGVECTOR_HYBRID_FUNCTION=search_places_semantic_v1
|
| 88 |
+
# PLACES_PGVECTOR_UPSERT_FUNCTION=upsert_place_embedding_semantic_v1
|
| 89 |
+
# PLACES_PGVECTOR_HASH_FUNCTION=get_place_content_hashes_semantic_v1
|
| 90 |
+
|
| 91 |
BM25_K1=1.5
|
| 92 |
BM25_B=0.75
|
| 93 |
BM25_RELEVANCE_THRESHOLD=3.0
|
|
|
|
| 101 |
PLACES_CHAT_MIN_CONTENT_SCORE=0.20
|
| 102 |
PLACES_CHAT_INTENT_MIN_CONFIDENCE=0.70
|
| 103 |
PLACES_CHAT_AMBIGUITY_DELTA=0.15
|
| 104 |
+
PLACES_CHAT_HYPOTHESIS_MIN_CONFIDENCE=0.60
|
| 105 |
+
PLACES_CHAT_HYPOTHESIS_MAX_GAP=0.15
|
| 106 |
+
PLACES_CHAT_DEFAULT_RADIUS_METERS=5000
|
| 107 |
PLACES_CHAT_RANKING_VERSION=places-chat-v2
|
| 108 |
+
PLACES_CHAT_INTENT_PROVIDER=deterministic
|
| 109 |
+
# Required only when PLACES_CHAT_INTENT_PROVIDER=bert. The model must expose
|
| 110 |
+
# IOB token labels for CATEGORY/PREFERENCE/EXCLUSION/LOCATION/REFERENCE/RADIUS.
|
| 111 |
+
PLACES_CHAT_BERT_MODEL_PATH=
|
| 112 |
+
PLACES_CHAT_BERT_MODEL_VERSION=
|
| 113 |
+
PLACES_CHAT_BERT_DEVICE=cpu
|
| 114 |
+
PLACES_CHAT_BERT_MIN_TOKEN_CONFIDENCE=0.60
|
| 115 |
PLACES_CHAT_TAXONOMY_VERSION=places-taxonomy-v1
|
| 116 |
|
| 117 |
LOG_LEVEL=INFO
|
Dockerfile
CHANGED
|
@@ -4,6 +4,7 @@ ENV PYTHONDONTWRITEBYTECODE=1
|
|
| 4 |
ENV PYTHONUNBUFFERED=1
|
| 5 |
ENV PORT=7860
|
| 6 |
ENV FASTTEXT_MODEL_PATH=/opt/models/fasttext-es/model.bin
|
|
|
|
| 7 |
|
| 8 |
WORKDIR /app
|
| 9 |
|
|
@@ -15,13 +16,15 @@ COPY requirements.txt .
|
|
| 15 |
RUN pip install --no-cache-dir --upgrade pip \
|
| 16 |
&& pip install --no-cache-dir -r requirements.txt
|
| 17 |
|
| 18 |
-
# Keep the
|
| 19 |
-
#
|
| 20 |
COPY app/shared/nlp/embeddings/download_fasttext_model.py /tmp/download_fasttext_model.py
|
| 21 |
-
RUN
|
| 22 |
-
|
| 23 |
-
|
| 24 |
-
|
|
|
|
|
|
|
| 25 |
&& rm -rf /tmp/hf-cache /tmp/download_fasttext_model.py
|
| 26 |
|
| 27 |
COPY app ./app
|
|
|
|
| 4 |
ENV PYTHONUNBUFFERED=1
|
| 5 |
ENV PORT=7860
|
| 6 |
ENV FASTTEXT_MODEL_PATH=/opt/models/fasttext-es/model.bin
|
| 7 |
+
ARG DOWNLOAD_FASTTEXT_MODEL=true
|
| 8 |
|
| 9 |
WORKDIR /app
|
| 10 |
|
|
|
|
| 16 |
RUN pip install --no-cache-dir --upgrade pip \
|
| 17 |
&& pip install --no-cache-dir -r requirements.txt
|
| 18 |
|
| 19 |
+
# Keep the rollback FastText model in a cached layer unless a BERT-only image
|
| 20 |
+
# is requested with --build-arg DOWNLOAD_FASTTEXT_MODEL=false.
|
| 21 |
COPY app/shared/nlp/embeddings/download_fasttext_model.py /tmp/download_fasttext_model.py
|
| 22 |
+
RUN if [ "${DOWNLOAD_FASTTEXT_MODEL}" = "true" ]; then \
|
| 23 |
+
HF_HOME=/tmp/hf-cache python /tmp/download_fasttext_model.py \
|
| 24 |
+
--repo-id facebook/fasttext-es-vectors \
|
| 25 |
+
--filename model.bin \
|
| 26 |
+
--destination ${FASTTEXT_MODEL_PATH}; \
|
| 27 |
+
fi \
|
| 28 |
&& rm -rf /tmp/hf-cache /tmp/download_fasttext_model.py
|
| 29 |
|
| 30 |
COPY app ./app
|
README.md
CHANGED
|
@@ -197,25 +197,28 @@ y ubicacion actual a la API principal; Go llama `POST /internal/places/chat`, hi
|
|
| 197 |
IDs devueltos, calcula distancias con PostGIS y aplica el orden geografico final.
|
| 198 |
|
| 199 |
NLP separa categoria, preferencias, exclusiones, referencia y alcance geografico antes
|
| 200 |
-
de buscar.
|
| 201 |
-
|
|
|
|
|
|
|
| 202 |
Las ambiguedades que cambiarian los resultados devuelven `action=clarification` y un
|
| 203 |
`state_patch` con `pending_clarification`; el siguiente turno puede resolverlo con frases
|
| 204 |
como "la primera opcion" o "la segunda, cerca de mi".
|
| 205 |
|
| 206 |
-
El chat no exige que el usuario nombre siempre una categoria.
|
| 207 |
-
|
| 208 |
-
|
| 209 |
-
|
| 210 |
-
|
| 211 |
-
|
| 212 |
-
|
| 213 |
-
|
| 214 |
-
|
| 215 |
-
|
| 216 |
-
|
| 217 |
-
|
| 218 |
-
|
|
|
|
| 219 |
`PLACES_CHAT_V2_ENABLED=false` y debe activarse despues de desplegar en Go tanto el proxy
|
| 220 |
de chat como `/api/v1/internal/places/resolve-anchor`.
|
| 221 |
|
|
@@ -346,23 +349,68 @@ Si el score maximo no supera `SEMANTIC_NO_MATCH_THRESHOLD`, envia a Llama el mod
|
|
| 346 |
`SEMANTIC_RELEVANCE_THRESHOLD` usa `low_confidence`; por encima usa `confident`.
|
| 347 |
Llama solo embellece el tono y recibe exclusivamente los lugares seleccionados.
|
| 348 |
|
| 349 |
-
### Documento Semantico
|
| 350 |
|
| 351 |
Los IDs numericos de tags devueltos por la API principal se resuelven mediante el
|
| 352 |
catalogo versionado en `app/modules/places/infrastructure/place_tag_catalog.json`.
|
| 353 |
-
El documento
|
| 354 |
-
|
| 355 |
|
| 356 |
```text
|
| 357 |
-
|
| 358 |
```
|
| 359 |
|
| 360 |
-
|
| 361 |
-
|
| 362 |
-
|
| 363 |
-
|
| 364 |
-
|
| 365 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 366 |
|
| 367 |
`GET` o `POST /places/search/metrics?k=5` conserva un benchmark offline separado llamado `built_in_places_v3_bm25`. Contiene doce lugares controlados, diez consultas y qrels graduados para calcular honestamente `Precision@k`, `Recall@k`, `MRR`, `MAP` y `nDCG@k`. Estas metricas requieren juicios de relevancia y por eso no se presentan como si midieran una consulta arbitraria de produccion.
|
| 368 |
|
|
|
|
| 197 |
IDs devueltos, calcula distancias con PostGIS y aplica el orden geografico final.
|
| 198 |
|
| 199 |
NLP separa categoria, preferencias, exclusiones, referencia y alcance geografico antes
|
| 200 |
+
de buscar. `is_active=true`, ciudad/estado confirmados y los IDs obtenidos por un filtro
|
| 201 |
+
geografico explicito son restricciones duras; la categoria es una hipotesis de ranking.
|
| 202 |
+
Esto permite recuperar vocabulario nuevo o categorias distintas entre sistemas sin
|
| 203 |
+
confundir una referencia como "cerca del parque" con el tipo de resultado solicitado.
|
| 204 |
Las ambiguedades que cambiarian los resultados devuelven `action=clarification` y un
|
| 205 |
`state_patch` con `pending_clarification`; el siguiente turno puede resolverlo con frases
|
| 206 |
como "la primera opcion" o "la segunda, cerca de mi".
|
| 207 |
|
| 208 |
+
El chat no exige que el usuario nombre siempre una categoria. Puede habilitar un BERT
|
| 209 |
+
fine-tuneado de token classification para extraer valores abiertos de categoria,
|
| 210 |
+
preferencia, exclusion, ubicacion, referencia y radio. Esos textos se alinean despues
|
| 211 |
+
contra un catalogo dinamico mediante embeddings; no se convierten con aliases dentro del
|
| 212 |
+
adaptador BERT. Las reglas lexicas existentes quedan como fallback de despliegue y no
|
| 213 |
+
bloquean retrieval. El clasificador se abstiene si la similitud es baja o dos conceptos
|
| 214 |
+
quedan demasiado cerca. Las aclaraciones usan hipotesis con evidencia o facetas de los
|
| 215 |
+
candidatos recuperados, no un menu fijo.
|
| 216 |
+
|
| 217 |
+
La recuperacion combina dense retrieval (FastText de rollback o SentenceTransformer),
|
| 218 |
+
BM25 y coincidencias de facetas. La categoria y las exclusiones aportan señales positivas
|
| 219 |
+
o negativas; no eliminan candidatos por una coincidencia textual aislada. NLP devuelve
|
| 220 |
+
candidatos tecnicos y `content_score`; el GPS se aplica como filtro explicito de IDs antes
|
| 221 |
+
del ranking cuando el proveedor de lugares cercanos esta configurado. El flag inicial es
|
| 222 |
`PLACES_CHAT_V2_ENABLED=false` y debe activarse despues de desplegar en Go tanto el proxy
|
| 223 |
de chat como `/api/v1/internal/places/resolve-anchor`.
|
| 224 |
|
|
|
|
| 349 |
`SEMANTIC_RELEVANCE_THRESHOLD` usa `low_confidence`; por encima usa `confident`.
|
| 350 |
Llama solo embellece el tono y recibe exclusivamente los lugares seleccionados.
|
| 351 |
|
| 352 |
+
### Documento Semantico Estructurado De Lugares
|
| 353 |
|
| 354 |
Los IDs numericos de tags devueltos por la API principal se resuelven mediante el
|
| 355 |
catalogo versionado en `app/modules/places/infrastructure/place_tag_catalog.json`.
|
| 356 |
+
El documento de Places contiene cada señal una sola vez y explicita el rol de cada
|
| 357 |
+
campo. Esto evita que la repeticion manual distorsione un encoder BERT:
|
| 358 |
|
| 359 |
```text
|
| 360 |
+
Nombre: ... Tipo registrado: ... Descripcion: ... Etiquetas: ...
|
| 361 |
```
|
| 362 |
|
| 363 |
+
No se expanden categorias mediante diccionarios de sinonimos. Direccion, ciudad,
|
| 364 |
+
estado, `source`, precio e IDs desconocidos permanecen fuera del embedding; siguen
|
| 365 |
+
disponibles como metadatos o filtros. La version `structured-place-v3` forma parte del
|
| 366 |
+
hash y fuerza un re-embedding seguro cuando cambia el documento.
|
| 367 |
+
|
| 368 |
+
### Migracion BERT/Sentence-Transformer exclusiva de Places
|
| 369 |
+
|
| 370 |
+
La migracion es aditiva y no cambia los vectores de posts, perfiles o feed:
|
| 371 |
+
|
| 372 |
+
1. Ejecuta `sql/migrations/20260716_02_places_semantic_v1.sql` con el rol DBA.
|
| 373 |
+
2. Configura temporalmente el perfil BERT mostrado en `.env.example`.
|
| 374 |
+
3. Ejecuta `python -m app.jobs.sync_place_embeddings` para backfill de la tabla
|
| 375 |
+
`place_embeddings_semantic_v1`.
|
| 376 |
+
4. Ejecuta `sql/verify_places_semantic_v1.sql` y revisa que el plan use HNSW con
|
| 377 |
+
un volumen representativo.
|
| 378 |
+
5. Activa `match_places_semantic_v1` y `search_places_semantic_v1` primero en shadow.
|
| 379 |
+
6. Conserva `match_places` y la tabla FastText para rollback.
|
| 380 |
+
|
| 381 |
+
`/places/chat` ya no requiere una categoria canonica para recuperar candidatos. La
|
| 382 |
+
categoria inferida solo aporta afinidad al ranking; la union SQL obtiene pools dense y
|
| 383 |
+
lexical independientes. Un cliente puede optar al contrato conversacional estructurado
|
| 384 |
+
enviando `conversation_id`, `conversation_state`, `clarification_choice` o
|
| 385 |
+
`user_location` mientras `PLACES_CHAT_V2_ENABLED=true`.
|
| 386 |
+
|
| 387 |
+
Para fine-tuning, `scripts/train_place_retriever.py` acepta JSONL con `query`,
|
| 388 |
+
`positive` y `hard_negatives`. El artefacto resultante se configura mediante
|
| 389 |
+
`PLACES_EMBEDDING_MODEL`; no se incluye un modelo ficticio preentrenado en el repo.
|
| 390 |
+
|
| 391 |
+
El extractor de intencion se entrena por separado con
|
| 392 |
+
`scripts/train_place_intent_bert.py`. Su JSONL contiene `text` y spans abiertos
|
| 393 |
+
`{start, end, slot}`; los slots permitidos son `CATEGORY`, `PREFERENCE`,
|
| 394 |
+
`EXCLUSION`, `LOCATION`, `REFERENCE` y `RADIUS`. Los valores concretos (por ejemplo
|
| 395 |
+
"donas artesanales") nunca se convierten en labels del modelo:
|
| 396 |
+
|
| 397 |
+
```powershell
|
| 398 |
+
python -m pip install -r requirements-training.txt
|
| 399 |
+
python scripts/train_place_intent_bert.py `
|
| 400 |
+
--train-file data/places-intent-train.jsonl `
|
| 401 |
+
--validation-file data/places-intent-validation.jsonl `
|
| 402 |
+
--output-dir .models/places-intent-bert
|
| 403 |
+
```
|
| 404 |
+
|
| 405 |
+
Para activarlo, configura `PLACES_CHAT_INTENT_PROVIDER=bert`,
|
| 406 |
+
`PLACES_CHAT_BERT_MODEL_PATH=.models/places-intent-bert` y una version inmutable en
|
| 407 |
+
`PLACES_CHAT_BERT_MODEL_VERSION`. El parser determinista queda como fallback si el
|
| 408 |
+
modelo no puede cargarse; cuando BERT responde, no se vuelven a aplicar aliases de
|
| 409 |
+
categoria ni implicaciones manuales sobre sus spans.
|
| 410 |
+
|
| 411 |
+
Una imagen que ya no necesite el artefacto de rollback FastText puede construirse con
|
| 412 |
+
`docker build --build-arg DOWNLOAD_FASTTEXT_MODEL=false .`. Conserva el valor por
|
| 413 |
+
defecto durante el shadow/canary para permitir rollback inmediato.
|
| 414 |
|
| 415 |
`GET` o `POST /places/search/metrics?k=5` conserva un benchmark offline separado llamado `built_in_places_v3_bm25`. Contiene doce lugares controlados, diez consultas y qrels graduados para calcular honestamente `Precision@k`, `Recall@k`, `MRR`, `MAP` y `nDCG@k`. Estas metricas requieren juicios de relevancia y por eso no se presentan como si midieran una consulta arbitraria de produccion.
|
| 416 |
|
app/jobs/sync_place_embeddings.py
CHANGED
|
@@ -9,7 +9,7 @@ from app.modules.places.infrastructure.main_api_place_source import (
|
|
| 9 |
from app.shared.config.settings import Settings, get_settings
|
| 10 |
from app.shared.logging.config import configure_logging, get_logger
|
| 11 |
from app.shared.nlp.embeddings.base import EmbeddingProvider
|
| 12 |
-
from app.shared.nlp.embeddings.factory import
|
| 13 |
from app.shared.nlp.embeddings.versioning import versioned_embedding_hash
|
| 14 |
from app.shared.vector_store.aws_pgvector import AwsPgvectorClient
|
| 15 |
from app.shared.vector_store.models import VectorUpsertRecord
|
|
@@ -35,12 +35,17 @@ async def main() -> None:
|
|
| 35 |
|
| 36 |
source = MainApiPlacesClient(settings)
|
| 37 |
vector_client = AwsPgvectorClient(settings, role="writer")
|
| 38 |
-
embedding_provider =
|
| 39 |
counters = SyncCounters()
|
| 40 |
batch: list[PlaceSourceRecord] = []
|
| 41 |
|
| 42 |
logger.info("Starting place embedding sync")
|
| 43 |
-
logger.info(
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 44 |
|
| 45 |
async for place in source.iter_places(
|
| 46 |
page_limit=args.page_limit,
|
|
@@ -93,14 +98,15 @@ async def _flush_batch(
|
|
| 93 |
counters.processed += len(batch)
|
| 94 |
try:
|
| 95 |
existing_hashes = await vector_client.fetch_place_content_hashes(
|
| 96 |
-
[record.id for record in batch]
|
|
|
|
| 97 |
)
|
| 98 |
expected_hashes = {
|
| 99 |
record.id: versioned_embedding_hash(
|
| 100 |
source_content_hash=record.content_hash,
|
| 101 |
-
model=settings.
|
| 102 |
-
version=settings.
|
| 103 |
-
dimension=settings.
|
| 104 |
)
|
| 105 |
for record in batch
|
| 106 |
}
|
|
@@ -130,7 +136,12 @@ async def _flush_batch(
|
|
| 130 |
if dry_run:
|
| 131 |
logger.info("Dry run: prepared %s place upserts", len(upserts))
|
| 132 |
else:
|
| 133 |
-
await vector_client.upsert_place_embeddings(
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 134 |
counters.upserted += len(upserts)
|
| 135 |
except Exception:
|
| 136 |
counters.errors += len(batch)
|
|
|
|
| 9 |
from app.shared.config.settings import Settings, get_settings
|
| 10 |
from app.shared.logging.config import configure_logging, get_logger
|
| 11 |
from app.shared.nlp.embeddings.base import EmbeddingProvider
|
| 12 |
+
from app.shared.nlp.embeddings.factory import create_place_embedding_provider
|
| 13 |
from app.shared.nlp.embeddings.versioning import versioned_embedding_hash
|
| 14 |
from app.shared.vector_store.aws_pgvector import AwsPgvectorClient
|
| 15 |
from app.shared.vector_store.models import VectorUpsertRecord
|
|
|
|
| 35 |
|
| 36 |
source = MainApiPlacesClient(settings)
|
| 37 |
vector_client = AwsPgvectorClient(settings, role="writer")
|
| 38 |
+
embedding_provider = create_place_embedding_provider(settings, text_role="passage")
|
| 39 |
counters = SyncCounters()
|
| 40 |
batch: list[PlaceSourceRecord] = []
|
| 41 |
|
| 42 |
logger.info("Starting place embedding sync")
|
| 43 |
+
logger.info(
|
| 44 |
+
"Places embedding model=%s version=%s dimension=%s",
|
| 45 |
+
settings.places_embedding_model,
|
| 46 |
+
settings.places_embedding_version,
|
| 47 |
+
settings.places_embedding_dimension,
|
| 48 |
+
)
|
| 49 |
|
| 50 |
async for place in source.iter_places(
|
| 51 |
page_limit=args.page_limit,
|
|
|
|
| 98 |
counters.processed += len(batch)
|
| 99 |
try:
|
| 100 |
existing_hashes = await vector_client.fetch_place_content_hashes(
|
| 101 |
+
[record.id for record in batch],
|
| 102 |
+
function_name=settings.places_pgvector_hash_function,
|
| 103 |
)
|
| 104 |
expected_hashes = {
|
| 105 |
record.id: versioned_embedding_hash(
|
| 106 |
source_content_hash=record.content_hash,
|
| 107 |
+
model=settings.places_embedding_model,
|
| 108 |
+
version=settings.places_embedding_version,
|
| 109 |
+
dimension=settings.places_embedding_dimension,
|
| 110 |
)
|
| 111 |
for record in batch
|
| 112 |
}
|
|
|
|
| 136 |
if dry_run:
|
| 137 |
logger.info("Dry run: prepared %s place upserts", len(upserts))
|
| 138 |
else:
|
| 139 |
+
await vector_client.upsert_place_embeddings(
|
| 140 |
+
upserts,
|
| 141 |
+
function_name=settings.places_pgvector_upsert_function,
|
| 142 |
+
embedding_model=settings.places_embedding_model,
|
| 143 |
+
embedding_version=settings.places_embedding_version,
|
| 144 |
+
)
|
| 145 |
counters.upserted += len(upserts)
|
| 146 |
except Exception:
|
| 147 |
counters.errors += len(batch)
|
app/modules/places/api/dependencies.py
CHANGED
|
@@ -13,6 +13,9 @@ from app.modules.places.infrastructure.aws_pgvector_place_repository import (
|
|
| 13 |
AwsPgvectorPlaceRepository,
|
| 14 |
)
|
| 15 |
from app.modules.places.infrastructure.bm25_place_ranker import Bm25PlaceRanker
|
|
|
|
|
|
|
|
|
|
| 16 |
from app.modules.places.infrastructure.main_api_nearby_place_provider import (
|
| 17 |
MainApiNearbyPlaceProvider,
|
| 18 |
)
|
|
@@ -33,13 +36,21 @@ from app.modules.places.infrastructure.place_search_benchmark import (
|
|
| 33 |
QRELS_SOURCE,
|
| 34 |
get_default_place_search_benchmark,
|
| 35 |
)
|
| 36 |
-
from app.modules.places.infrastructure.
|
| 37 |
-
|
| 38 |
-
SemanticPlaceActivityClassifier,
|
| 39 |
)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 40 |
from app.shared.cache.memory import SimpleTTLCache
|
| 41 |
from app.shared.config.settings import get_settings
|
| 42 |
-
from app.shared.dependencies import
|
|
|
|
|
|
|
|
|
|
|
|
|
| 43 |
from app.shared.nlp.embeddings.mock import MockEmbeddingProvider
|
| 44 |
from app.shared.nlp.llm.output_guard import PlaceChatOutputGuard
|
| 45 |
from app.shared.vector_store.aws_pgvector import AwsPgvectorClient
|
|
@@ -49,14 +60,21 @@ from app.shared.vector_store.aws_pgvector import AwsPgvectorClient
|
|
| 49 |
def get_place_repository() -> MockPlaceVectorRepository | AwsPgvectorPlaceRepository:
|
| 50 |
settings = get_settings()
|
| 51 |
if settings.vector_store_provider == "aws_pgvector":
|
| 52 |
-
return AwsPgvectorPlaceRepository(
|
| 53 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
| 54 |
|
| 55 |
|
| 56 |
@lru_cache
|
| 57 |
def get_place_ranker() -> SemanticPlaceRanker:
|
| 58 |
settings = get_settings()
|
| 59 |
-
return SemanticPlaceRanker(
|
|
|
|
|
|
|
|
|
|
| 60 |
|
| 61 |
|
| 62 |
@lru_cache
|
|
@@ -79,7 +97,7 @@ def get_nearby_place_provider() -> MainApiNearbyPlaceProvider:
|
|
| 79 |
@lru_cache
|
| 80 |
def get_search_places_use_case() -> SearchPlacesUseCase:
|
| 81 |
return SearchPlacesUseCase(
|
| 82 |
-
embedding_provider=
|
| 83 |
place_repository=get_place_repository(),
|
| 84 |
ranker=get_place_ranker(),
|
| 85 |
cache=get_place_search_cache(),
|
|
@@ -101,7 +119,9 @@ def get_recommend_places_use_case() -> RecommendPlacesUseCase:
|
|
| 101 |
@lru_cache
|
| 102 |
def get_evaluate_place_search_use_case() -> EvaluatePlaceSearchUseCase:
|
| 103 |
settings = get_settings()
|
| 104 |
-
embedding_provider = MockEmbeddingProvider(
|
|
|
|
|
|
|
| 105 |
benchmark_search = SearchPlacesUseCase(
|
| 106 |
embedding_provider=embedding_provider,
|
| 107 |
place_repository=MockPlaceVectorRepository(embedding_provider),
|
|
@@ -133,14 +153,42 @@ def get_place_chat_intent_parser() -> DeterministicPlaceChatIntentParser:
|
|
| 133 |
raise RuntimeError(
|
| 134 |
"PLACES_CHAT_TAXONOMY_VERSION does not match the bundled taxonomy"
|
| 135 |
)
|
|
|
|
|
|
|
|
|
|
|
|
|
| 136 |
activity_classifier = (
|
| 137 |
-
|
| 138 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 139 |
else None
|
| 140 |
)
|
| 141 |
return DeterministicPlaceChatIntentParser(
|
| 142 |
taxonomy=taxonomy,
|
| 143 |
activity_classifier=activity_classifier,
|
|
|
|
| 144 |
)
|
| 145 |
|
| 146 |
|
|
@@ -156,7 +204,7 @@ def get_place_anchor_resolver() -> MainApiPlaceAnchorResolver | MockPlaceAnchorR
|
|
| 156 |
def get_hybrid_place_chat_retriever() -> HybridContentPlaceChatRetriever:
|
| 157 |
settings = get_settings()
|
| 158 |
return HybridContentPlaceChatRetriever(
|
| 159 |
-
embedding_provider=
|
| 160 |
place_repository=get_place_repository(),
|
| 161 |
minimum_content_score=settings.places_chat_min_content_score,
|
| 162 |
k1=settings.bm25_k1,
|
|
@@ -179,4 +227,14 @@ def get_chat_place_recommendations_use_case() -> ChatPlaceRecommendationsUseCase
|
|
| 179 |
llm_enabled=settings.places_chat_llm_enabled,
|
| 180 |
anchor_ambiguity_delta=settings.places_chat_ambiguity_delta,
|
| 181 |
minimum_intent_confidence=settings.places_chat_intent_min_confidence,
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 182 |
)
|
|
|
|
| 13 |
AwsPgvectorPlaceRepository,
|
| 14 |
)
|
| 15 |
from app.modules.places.infrastructure.bm25_place_ranker import Bm25PlaceRanker
|
| 16 |
+
from app.modules.places.infrastructure.bert_intent_extractor import (
|
| 17 |
+
BertPlaceIntentExtractor,
|
| 18 |
+
)
|
| 19 |
from app.modules.places.infrastructure.main_api_nearby_place_provider import (
|
| 20 |
MainApiNearbyPlaceProvider,
|
| 21 |
)
|
|
|
|
| 36 |
QRELS_SOURCE,
|
| 37 |
get_default_place_search_benchmark,
|
| 38 |
)
|
| 39 |
+
from app.modules.places.infrastructure.open_vocabulary_category_classifier import (
|
| 40 |
+
OpenVocabularyPlaceCategoryClassifier,
|
|
|
|
| 41 |
)
|
| 42 |
+
from app.modules.places.infrastructure.place_category_catalog import (
|
| 43 |
+
load_place_category_concepts,
|
| 44 |
+
)
|
| 45 |
+
from app.modules.places.infrastructure.place_semantic_document import place_tag_catalog
|
| 46 |
+
from app.modules.places.infrastructure.semantic_place_ranker import SemanticPlaceRanker
|
| 47 |
from app.shared.cache.memory import SimpleTTLCache
|
| 48 |
from app.shared.config.settings import get_settings
|
| 49 |
+
from app.shared.dependencies import (
|
| 50 |
+
get_llm_provider,
|
| 51 |
+
get_place_embedding_provider,
|
| 52 |
+
get_place_passage_embedding_provider,
|
| 53 |
+
)
|
| 54 |
from app.shared.nlp.embeddings.mock import MockEmbeddingProvider
|
| 55 |
from app.shared.nlp.llm.output_guard import PlaceChatOutputGuard
|
| 56 |
from app.shared.vector_store.aws_pgvector import AwsPgvectorClient
|
|
|
|
| 60 |
def get_place_repository() -> MockPlaceVectorRepository | AwsPgvectorPlaceRepository:
|
| 61 |
settings = get_settings()
|
| 62 |
if settings.vector_store_provider == "aws_pgvector":
|
| 63 |
+
return AwsPgvectorPlaceRepository(
|
| 64 |
+
vector_client=AwsPgvectorClient(settings, role="reader"),
|
| 65 |
+
match_function=settings.places_pgvector_match_function,
|
| 66 |
+
hybrid_function=settings.places_pgvector_hybrid_function,
|
| 67 |
+
)
|
| 68 |
+
return MockPlaceVectorRepository(embedding_provider=get_place_embedding_provider())
|
| 69 |
|
| 70 |
|
| 71 |
@lru_cache
|
| 72 |
def get_place_ranker() -> SemanticPlaceRanker:
|
| 73 |
settings = get_settings()
|
| 74 |
+
return SemanticPlaceRanker(
|
| 75 |
+
dimension=settings.places_embedding_dimension,
|
| 76 |
+
model_name=settings.places_embedding_model,
|
| 77 |
+
)
|
| 78 |
|
| 79 |
|
| 80 |
@lru_cache
|
|
|
|
| 97 |
@lru_cache
|
| 98 |
def get_search_places_use_case() -> SearchPlacesUseCase:
|
| 99 |
return SearchPlacesUseCase(
|
| 100 |
+
embedding_provider=get_place_embedding_provider(),
|
| 101 |
place_repository=get_place_repository(),
|
| 102 |
ranker=get_place_ranker(),
|
| 103 |
cache=get_place_search_cache(),
|
|
|
|
| 119 |
@lru_cache
|
| 120 |
def get_evaluate_place_search_use_case() -> EvaluatePlaceSearchUseCase:
|
| 121 |
settings = get_settings()
|
| 122 |
+
embedding_provider = MockEmbeddingProvider(
|
| 123 |
+
dimension=settings.places_embedding_dimension
|
| 124 |
+
)
|
| 125 |
benchmark_search = SearchPlacesUseCase(
|
| 126 |
embedding_provider=embedding_provider,
|
| 127 |
place_repository=MockPlaceVectorRepository(embedding_provider),
|
|
|
|
| 153 |
raise RuntimeError(
|
| 154 |
"PLACES_CHAT_TAXONOMY_VERSION does not match the bundled taxonomy"
|
| 155 |
)
|
| 156 |
+
concepts = load_place_category_concepts(
|
| 157 |
+
settings.places_category_catalog_path,
|
| 158 |
+
fallback_tags=place_tag_catalog().values(),
|
| 159 |
+
)
|
| 160 |
activity_classifier = (
|
| 161 |
+
OpenVocabularyPlaceCategoryClassifier(
|
| 162 |
+
concepts=concepts,
|
| 163 |
+
embedding_provider=get_place_embedding_provider(),
|
| 164 |
+
concept_embedding_provider=(
|
| 165 |
+
get_place_passage_embedding_provider()
|
| 166 |
+
if settings.places_embedding_provider
|
| 167 |
+
in {"sentence_transformer", "bert"}
|
| 168 |
+
else get_place_embedding_provider()
|
| 169 |
+
),
|
| 170 |
+
minimum_similarity=settings.places_category_min_similarity,
|
| 171 |
+
minimum_margin=settings.places_category_min_margin,
|
| 172 |
+
)
|
| 173 |
+
if settings.places_embedding_provider.casefold() != "mock" and concepts
|
| 174 |
+
else None
|
| 175 |
+
)
|
| 176 |
+
contextual_extractor = (
|
| 177 |
+
BertPlaceIntentExtractor(
|
| 178 |
+
settings.places_chat_bert_model_path or "",
|
| 179 |
+
model_version=settings.places_chat_bert_model_version,
|
| 180 |
+
device=settings.places_chat_bert_device,
|
| 181 |
+
minimum_token_confidence=(
|
| 182 |
+
settings.places_chat_bert_min_token_confidence
|
| 183 |
+
),
|
| 184 |
+
)
|
| 185 |
+
if settings.places_chat_intent_provider == "bert"
|
| 186 |
else None
|
| 187 |
)
|
| 188 |
return DeterministicPlaceChatIntentParser(
|
| 189 |
taxonomy=taxonomy,
|
| 190 |
activity_classifier=activity_classifier,
|
| 191 |
+
contextual_extractor=contextual_extractor,
|
| 192 |
)
|
| 193 |
|
| 194 |
|
|
|
|
| 204 |
def get_hybrid_place_chat_retriever() -> HybridContentPlaceChatRetriever:
|
| 205 |
settings = get_settings()
|
| 206 |
return HybridContentPlaceChatRetriever(
|
| 207 |
+
embedding_provider=get_place_embedding_provider(),
|
| 208 |
place_repository=get_place_repository(),
|
| 209 |
minimum_content_score=settings.places_chat_min_content_score,
|
| 210 |
k1=settings.bm25_k1,
|
|
|
|
| 227 |
llm_enabled=settings.places_chat_llm_enabled,
|
| 228 |
anchor_ambiguity_delta=settings.places_chat_ambiguity_delta,
|
| 229 |
minimum_intent_confidence=settings.places_chat_intent_min_confidence,
|
| 230 |
+
minimum_hypothesis_confidence=(
|
| 231 |
+
settings.places_chat_hypothesis_min_confidence
|
| 232 |
+
),
|
| 233 |
+
maximum_hypothesis_gap=settings.places_chat_hypothesis_max_gap,
|
| 234 |
+
nearby_place_provider=(
|
| 235 |
+
get_nearby_place_provider()
|
| 236 |
+
if settings.vector_store_provider != "mock"
|
| 237 |
+
else None
|
| 238 |
+
),
|
| 239 |
+
default_radius_meters=settings.places_chat_default_radius_meters,
|
| 240 |
)
|
app/modules/places/api/internal_chat_schemas.py
CHANGED
|
@@ -65,7 +65,7 @@ class PendingClarificationOptionStateSchema(BaseModel):
|
|
| 65 |
model_config = ConfigDict(extra="forbid")
|
| 66 |
|
| 67 |
id: str = Field(..., min_length=1, max_length=64, pattern=r"^[a-zA-Z0-9_-]+$")
|
| 68 |
-
value: str = Field(..., min_length=1, max_length=
|
| 69 |
label: str = Field(..., min_length=1, max_length=160)
|
| 70 |
place_id: str | None = Field(default=None, max_length=100)
|
| 71 |
attributes: list[str] = Field(default_factory=list, max_length=30)
|
|
@@ -114,7 +114,7 @@ class PendingClarificationStateSchema(BaseModel):
|
|
| 114 |
class ConversationStateSchema(BaseModel):
|
| 115 |
model_config = ConfigDict(extra="forbid")
|
| 116 |
|
| 117 |
-
target_category: str | None = Field(default=None, max_length=
|
| 118 |
hard_filters: dict[str, Any] = Field(default_factory=dict)
|
| 119 |
soft_preferences: list[str] = Field(default_factory=list, max_length=30)
|
| 120 |
exclusions: list[str] = Field(default_factory=list, max_length=30)
|
|
@@ -240,6 +240,7 @@ class InternalPlaceChatResponse(BaseModel):
|
|
| 240 |
ranking_version: str
|
| 241 |
taxonomy_version: str
|
| 242 |
trace_id: str
|
|
|
|
| 243 |
metadata: dict[str, Any] = Field(default_factory=dict)
|
| 244 |
|
| 245 |
|
|
@@ -247,6 +248,16 @@ def internal_chat_result_to_schema(
|
|
| 247 |
result: ChatPlaceRecommendationsResult,
|
| 248 |
) -> InternalPlaceChatResponse:
|
| 249 |
directive = result.location_directive
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 250 |
return InternalPlaceChatResponse(
|
| 251 |
action=result.action,
|
| 252 |
message=result.message,
|
|
@@ -292,9 +303,35 @@ def internal_chat_result_to_schema(
|
|
| 292 |
ranking_version=result.ranking_version,
|
| 293 |
taxonomy_version=result.taxonomy_version,
|
| 294 |
trace_id=result.trace_id,
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 295 |
metadata={
|
| 296 |
"used_llm": result.used_llm,
|
| 297 |
"guard_reason": result.guard_reason,
|
| 298 |
"category_source": result.category_source,
|
|
|
|
|
|
|
| 299 |
},
|
| 300 |
)
|
|
|
|
| 65 |
model_config = ConfigDict(extra="forbid")
|
| 66 |
|
| 67 |
id: str = Field(..., min_length=1, max_length=64, pattern=r"^[a-zA-Z0-9_-]+$")
|
| 68 |
+
value: str = Field(..., min_length=1, max_length=500)
|
| 69 |
label: str = Field(..., min_length=1, max_length=160)
|
| 70 |
place_id: str | None = Field(default=None, max_length=100)
|
| 71 |
attributes: list[str] = Field(default_factory=list, max_length=30)
|
|
|
|
| 114 |
class ConversationStateSchema(BaseModel):
|
| 115 |
model_config = ConfigDict(extra="forbid")
|
| 116 |
|
| 117 |
+
target_category: str | None = Field(default=None, max_length=500)
|
| 118 |
hard_filters: dict[str, Any] = Field(default_factory=dict)
|
| 119 |
soft_preferences: list[str] = Field(default_factory=list, max_length=30)
|
| 120 |
exclusions: list[str] = Field(default_factory=list, max_length=30)
|
|
|
|
| 240 |
ranking_version: str
|
| 241 |
taxonomy_version: str
|
| 242 |
trace_id: str
|
| 243 |
+
uncertainty: dict[str, Any]
|
| 244 |
metadata: dict[str, Any] = Field(default_factory=dict)
|
| 245 |
|
| 246 |
|
|
|
|
| 248 |
result: ChatPlaceRecommendationsResult,
|
| 249 |
) -> InternalPlaceChatResponse:
|
| 250 |
directive = result.location_directive
|
| 251 |
+
hypotheses = list(result.category_hypotheses)
|
| 252 |
+
margin = (
|
| 253 |
+
max(
|
| 254 |
+
0.0,
|
| 255 |
+
float(hypotheses[0]["probability"])
|
| 256 |
+
- float(hypotheses[1]["probability"]),
|
| 257 |
+
)
|
| 258 |
+
if len(hypotheses) >= 2
|
| 259 |
+
else None
|
| 260 |
+
)
|
| 261 |
return InternalPlaceChatResponse(
|
| 262 |
action=result.action,
|
| 263 |
message=result.message,
|
|
|
|
| 303 |
ranking_version=result.ranking_version,
|
| 304 |
taxonomy_version=result.taxonomy_version,
|
| 305 |
trace_id=result.trace_id,
|
| 306 |
+
uncertainty={
|
| 307 |
+
"decision": (
|
| 308 |
+
"review"
|
| 309 |
+
if result.action == "recommendations" and result.unresolved
|
| 310 |
+
else {
|
| 311 |
+
"recommendations": "auto",
|
| 312 |
+
"clarification": "clarify",
|
| 313 |
+
"no_match": "abstain",
|
| 314 |
+
}[result.action]
|
| 315 |
+
),
|
| 316 |
+
"reason": (
|
| 317 |
+
result.unresolved[0]
|
| 318 |
+
if result.unresolved
|
| 319 |
+
else (
|
| 320 |
+
"sufficient_evidence"
|
| 321 |
+
if result.action == "recommendations"
|
| 322 |
+
else "catalog_exhausted"
|
| 323 |
+
)
|
| 324 |
+
),
|
| 325 |
+
"top_probability": round(result.intent_confidence, 6),
|
| 326 |
+
"category_hypotheses": hypotheses,
|
| 327 |
+
"category_margin": round(margin, 6) if margin is not None else None,
|
| 328 |
+
"calibration_version": "uncalibrated-shadow-v1",
|
| 329 |
+
},
|
| 330 |
metadata={
|
| 331 |
"used_llm": result.used_llm,
|
| 332 |
"guard_reason": result.guard_reason,
|
| 333 |
"category_source": result.category_source,
|
| 334 |
+
"raw_category_phrase": result.raw_category_phrase,
|
| 335 |
+
"intent_model_version": result.intent_model_version,
|
| 336 |
},
|
| 337 |
)
|
app/modules/places/api/router.py
CHANGED
|
@@ -1,16 +1,24 @@
|
|
| 1 |
-
|
|
|
|
|
|
|
|
|
|
| 2 |
|
| 3 |
from app.modules.places.api.dependencies import (
|
| 4 |
get_chat_places_use_case,
|
|
|
|
| 5 |
get_evaluate_place_search_use_case,
|
| 6 |
get_recommend_places_use_case,
|
| 7 |
get_search_places_use_case,
|
| 8 |
)
|
|
|
|
|
|
|
|
|
|
| 9 |
from app.modules.places.api.schemas import (
|
| 10 |
PlaceChatRequest,
|
| 11 |
PlaceChatResponse,
|
| 12 |
PlaceRecommendationRequest,
|
| 13 |
PlaceRecommendationResponse,
|
|
|
|
| 14 |
PlaceSearchMetricsResponse,
|
| 15 |
PlaceSearchRequest,
|
| 16 |
PlaceSearchResponse,
|
|
@@ -19,12 +27,19 @@ from app.modules.places.api.schemas import (
|
|
| 19 |
search_metrics_result_to_schema,
|
| 20 |
)
|
| 21 |
from app.modules.places.application.use_cases.chat_places import ChatPlacesUseCase
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 22 |
from app.modules.places.application.use_cases.evaluate_place_search import (
|
| 23 |
EvaluatePlaceSearchUseCase,
|
| 24 |
)
|
| 25 |
from app.modules.places.application.use_cases.recommend_places import RecommendPlacesUseCase
|
| 26 |
from app.modules.places.application.use_cases.search_places import SearchPlacesUseCase
|
| 27 |
from app.shared.security.rate_limit import rate_limit_placeholder
|
|
|
|
|
|
|
| 28 |
|
| 29 |
router = APIRouter(
|
| 30 |
prefix="/places",
|
|
@@ -85,12 +100,163 @@ async def recommend_places(
|
|
| 85 |
)
|
| 86 |
|
| 87 |
|
| 88 |
-
@router.post(
|
|
|
|
|
|
|
|
|
|
|
|
|
| 89 |
async def chat_places(
|
| 90 |
payload: PlaceChatRequest,
|
| 91 |
-
|
|
|
|
|
|
|
|
|
|
| 92 |
) -> PlaceChatResponse:
|
| 93 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 94 |
message=payload.message,
|
| 95 |
filters=payload.to_domain_filters(),
|
| 96 |
limit=payload.limit,
|
|
|
|
| 1 |
+
import asyncio
|
| 2 |
+
from dataclasses import replace
|
| 3 |
+
|
| 4 |
+
from fastapi import APIRouter, Depends, HTTPException, Query
|
| 5 |
|
| 6 |
from app.modules.places.api.dependencies import (
|
| 7 |
get_chat_places_use_case,
|
| 8 |
+
get_chat_place_recommendations_use_case,
|
| 9 |
get_evaluate_place_search_use_case,
|
| 10 |
get_recommend_places_use_case,
|
| 11 |
get_search_places_use_case,
|
| 12 |
)
|
| 13 |
+
from app.modules.places.api.internal_chat_schemas import (
|
| 14 |
+
internal_chat_result_to_schema,
|
| 15 |
+
)
|
| 16 |
from app.modules.places.api.schemas import (
|
| 17 |
PlaceChatRequest,
|
| 18 |
PlaceChatResponse,
|
| 19 |
PlaceRecommendationRequest,
|
| 20 |
PlaceRecommendationResponse,
|
| 21 |
+
PlaceResultSchema,
|
| 22 |
PlaceSearchMetricsResponse,
|
| 23 |
PlaceSearchRequest,
|
| 24 |
PlaceSearchResponse,
|
|
|
|
| 27 |
search_metrics_result_to_schema,
|
| 28 |
)
|
| 29 |
from app.modules.places.application.use_cases.chat_places import ChatPlacesUseCase
|
| 30 |
+
from app.modules.places.application.use_cases.chat_place_recommendations import (
|
| 31 |
+
ChatPlaceRecommendationsUseCase,
|
| 32 |
+
)
|
| 33 |
+
from app.modules.places.domain.chat_intent import ConversationState
|
| 34 |
+
from app.modules.places.domain.errors import ClarificationStateMismatchError
|
| 35 |
from app.modules.places.application.use_cases.evaluate_place_search import (
|
| 36 |
EvaluatePlaceSearchUseCase,
|
| 37 |
)
|
| 38 |
from app.modules.places.application.use_cases.recommend_places import RecommendPlacesUseCase
|
| 39 |
from app.modules.places.application.use_cases.search_places import SearchPlacesUseCase
|
| 40 |
from app.shared.security.rate_limit import rate_limit_placeholder
|
| 41 |
+
from app.shared.config.settings import get_settings
|
| 42 |
+
from app.shared.tracing import new_response_id
|
| 43 |
|
| 44 |
router = APIRouter(
|
| 45 |
prefix="/places",
|
|
|
|
| 100 |
)
|
| 101 |
|
| 102 |
|
| 103 |
+
@router.post(
|
| 104 |
+
"/chat",
|
| 105 |
+
response_model=PlaceChatResponse,
|
| 106 |
+
response_model_exclude_none=True,
|
| 107 |
+
)
|
| 108 |
async def chat_places(
|
| 109 |
payload: PlaceChatRequest,
|
| 110 |
+
legacy_use_case: ChatPlacesUseCase = Depends(get_chat_places_use_case),
|
| 111 |
+
semantic_use_case: ChatPlaceRecommendationsUseCase = Depends(
|
| 112 |
+
get_chat_place_recommendations_use_case
|
| 113 |
+
),
|
| 114 |
) -> PlaceChatResponse:
|
| 115 |
+
settings = get_settings()
|
| 116 |
+
use_semantic_chat = settings.places_chat_v2_enabled and any(
|
| 117 |
+
(
|
| 118 |
+
payload.conversation_id is not None,
|
| 119 |
+
payload.conversation_state is not None,
|
| 120 |
+
payload.clarification_choice is not None,
|
| 121 |
+
payload.user_location is not None,
|
| 122 |
+
)
|
| 123 |
+
)
|
| 124 |
+
if use_semantic_chat:
|
| 125 |
+
state = (
|
| 126 |
+
payload.conversation_state.to_domain()
|
| 127 |
+
if payload.conversation_state
|
| 128 |
+
else ConversationState()
|
| 129 |
+
)
|
| 130 |
+
if (
|
| 131 |
+
state.taxonomy_version is not None
|
| 132 |
+
and state.taxonomy_version != settings.places_chat_taxonomy_version
|
| 133 |
+
):
|
| 134 |
+
raise HTTPException(
|
| 135 |
+
status_code=409,
|
| 136 |
+
detail="Conversation taxonomy version is incompatible",
|
| 137 |
+
)
|
| 138 |
+
request_filters = payload.to_domain_filters()
|
| 139 |
+
hard_filters = dict(state.hard_filters)
|
| 140 |
+
for key, value in {
|
| 141 |
+
"city": request_filters.city,
|
| 142 |
+
"state": request_filters.state,
|
| 143 |
+
"price_range": request_filters.price_range,
|
| 144 |
+
"occasion": request_filters.occasion,
|
| 145 |
+
}.items():
|
| 146 |
+
if value is not None:
|
| 147 |
+
hard_filters[key] = value
|
| 148 |
+
state = replace(
|
| 149 |
+
state,
|
| 150 |
+
target_category=state.target_category or request_filters.category,
|
| 151 |
+
hard_filters=hard_filters,
|
| 152 |
+
city=state.city or request_filters.city,
|
| 153 |
+
state=state.state or request_filters.state,
|
| 154 |
+
)
|
| 155 |
+
location = payload.user_location
|
| 156 |
+
candidate_limit = (
|
| 157 |
+
payload.candidate_limit or settings.places_chat_candidate_limit
|
| 158 |
+
)
|
| 159 |
+
if candidate_limit > settings.places_chat_candidate_limit:
|
| 160 |
+
raise HTTPException(
|
| 161 |
+
status_code=422,
|
| 162 |
+
detail=(
|
| 163 |
+
"candidate_limit exceeds the configured service maximum of "
|
| 164 |
+
f"{settings.places_chat_candidate_limit}"
|
| 165 |
+
),
|
| 166 |
+
)
|
| 167 |
+
try:
|
| 168 |
+
async with asyncio.timeout(settings.request_timeout_seconds):
|
| 169 |
+
semantic_result = await semantic_use_case.execute(
|
| 170 |
+
message=payload.message,
|
| 171 |
+
state=state,
|
| 172 |
+
user_latitude=location.lat if location else None,
|
| 173 |
+
user_longitude=location.lng if location else None,
|
| 174 |
+
candidate_limit=candidate_limit,
|
| 175 |
+
result_limit=payload.limit,
|
| 176 |
+
clarification_choice=(
|
| 177 |
+
payload.clarification_choice.to_domain()
|
| 178 |
+
if payload.clarification_choice
|
| 179 |
+
else None
|
| 180 |
+
),
|
| 181 |
+
)
|
| 182 |
+
except ClarificationStateMismatchError as exc:
|
| 183 |
+
raise HTTPException(
|
| 184 |
+
status_code=409,
|
| 185 |
+
detail="Clarification choice does not match the current state",
|
| 186 |
+
) from exc
|
| 187 |
+
except TimeoutError as exc:
|
| 188 |
+
raise HTTPException(
|
| 189 |
+
status_code=503,
|
| 190 |
+
detail="Places chat timed out",
|
| 191 |
+
) from exc
|
| 192 |
+
structured = internal_chat_result_to_schema(semantic_result)
|
| 193 |
+
decision = (
|
| 194 |
+
"review"
|
| 195 |
+
if semantic_result.action == "recommendations"
|
| 196 |
+
and semantic_result.unresolved
|
| 197 |
+
else {
|
| 198 |
+
"recommendations": "auto",
|
| 199 |
+
"clarification": "clarify",
|
| 200 |
+
"no_match": "abstain",
|
| 201 |
+
}[semantic_result.action]
|
| 202 |
+
)
|
| 203 |
+
reason = (
|
| 204 |
+
semantic_result.unresolved[0]
|
| 205 |
+
if semantic_result.unresolved
|
| 206 |
+
else (
|
| 207 |
+
"sufficient_evidence"
|
| 208 |
+
if semantic_result.action == "recommendations"
|
| 209 |
+
else "catalog_exhausted"
|
| 210 |
+
)
|
| 211 |
+
)
|
| 212 |
+
return PlaceChatResponse(
|
| 213 |
+
response_id=new_response_id(),
|
| 214 |
+
nlp_trace_id=semantic_result.trace_id,
|
| 215 |
+
action=semantic_result.action,
|
| 216 |
+
message=semantic_result.message,
|
| 217 |
+
places=[
|
| 218 |
+
PlaceResultSchema(
|
| 219 |
+
id=candidate.place_id,
|
| 220 |
+
name=candidate.name,
|
| 221 |
+
score=round(candidate.content_score, 4),
|
| 222 |
+
category=candidate.category,
|
| 223 |
+
city=candidate.metadata.get("city"),
|
| 224 |
+
state=candidate.metadata.get("state"),
|
| 225 |
+
metadata={
|
| 226 |
+
**candidate.metadata,
|
| 227 |
+
"semantic_score": candidate.semantic_score,
|
| 228 |
+
"lexical_score": candidate.lexical_score,
|
| 229 |
+
"match_level": candidate.match_level,
|
| 230 |
+
"matched_reasons": list(candidate.matched_reasons),
|
| 231 |
+
},
|
| 232 |
+
)
|
| 233 |
+
for candidate in semantic_result.candidates[: payload.limit]
|
| 234 |
+
],
|
| 235 |
+
state_patch=semantic_result.state_patch,
|
| 236 |
+
location_directive=structured.location_directive,
|
| 237 |
+
clarification=structured.clarification,
|
| 238 |
+
unresolved=list(semantic_result.unresolved),
|
| 239 |
+
intent_confidence=semantic_result.intent_confidence,
|
| 240 |
+
ranking_version=semantic_result.ranking_version,
|
| 241 |
+
taxonomy_version=semantic_result.taxonomy_version,
|
| 242 |
+
uncertainty={
|
| 243 |
+
**structured.uncertainty,
|
| 244 |
+
"decision": decision,
|
| 245 |
+
"reason": reason,
|
| 246 |
+
},
|
| 247 |
+
metadata={
|
| 248 |
+
**structured.metadata,
|
| 249 |
+
"pipeline": "places-chat-semantic-v2",
|
| 250 |
+
"conversation_id": (
|
| 251 |
+
str(payload.conversation_id)
|
| 252 |
+
if payload.conversation_id is not None
|
| 253 |
+
else None
|
| 254 |
+
),
|
| 255 |
+
"turn": payload.turn,
|
| 256 |
+
},
|
| 257 |
+
)
|
| 258 |
+
|
| 259 |
+
result = await legacy_use_case.execute(
|
| 260 |
message=payload.message,
|
| 261 |
filters=payload.to_domain_filters(),
|
| 262 |
limit=payload.limit,
|
app/modules/places/api/schemas.py
CHANGED
|
@@ -1,10 +1,18 @@
|
|
| 1 |
-
from typing import Any
|
|
|
|
| 2 |
|
| 3 |
from pydantic import BaseModel, ConfigDict, Field, model_validator
|
| 4 |
|
| 5 |
from app.modules.places.application.use_cases.evaluate_place_search import (
|
| 6 |
EvaluatePlaceSearchResult,
|
| 7 |
)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 8 |
from app.modules.places.domain.models import PlaceCandidate, PlaceFilters
|
| 9 |
from app.modules.places.domain.search_metrics import (
|
| 10 |
SearchEngineMetrics,
|
|
@@ -64,6 +72,12 @@ class PlaceChatRequest(BaseModel):
|
|
| 64 |
state: str | None = Field(default=None, max_length=80)
|
| 65 |
filters: PlaceFiltersSchema = Field(default_factory=PlaceFiltersSchema)
|
| 66 |
limit: int = Field(default=5, ge=1, le=8)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 67 |
|
| 68 |
def to_domain_filters(self) -> PlaceFilters:
|
| 69 |
return PlaceFilters(
|
|
@@ -168,6 +182,15 @@ class PlaceChatResponse(BaseModel):
|
|
| 168 |
message: str
|
| 169 |
places: list[PlaceResultSchema]
|
| 170 |
metadata: dict[str, Any]
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 171 |
|
| 172 |
|
| 173 |
def place_to_schema(place: PlaceCandidate) -> PlaceResultSchema:
|
|
|
|
| 1 |
+
from typing import Any, Literal
|
| 2 |
+
from uuid import UUID
|
| 3 |
|
| 4 |
from pydantic import BaseModel, ConfigDict, Field, model_validator
|
| 5 |
|
| 6 |
from app.modules.places.application.use_cases.evaluate_place_search import (
|
| 7 |
EvaluatePlaceSearchResult,
|
| 8 |
)
|
| 9 |
+
from app.modules.places.api.internal_chat_schemas import (
|
| 10 |
+
ClarificationChoiceSchema,
|
| 11 |
+
ClarificationSchema,
|
| 12 |
+
ConversationStateSchema,
|
| 13 |
+
PlaceChatLocationDirectiveSchema,
|
| 14 |
+
UserLocationSchema,
|
| 15 |
+
)
|
| 16 |
from app.modules.places.domain.models import PlaceCandidate, PlaceFilters
|
| 17 |
from app.modules.places.domain.search_metrics import (
|
| 18 |
SearchEngineMetrics,
|
|
|
|
| 72 |
state: str | None = Field(default=None, max_length=80)
|
| 73 |
filters: PlaceFiltersSchema = Field(default_factory=PlaceFiltersSchema)
|
| 74 |
limit: int = Field(default=5, ge=1, le=8)
|
| 75 |
+
conversation_id: UUID | None = None
|
| 76 |
+
turn: int = Field(default=1, ge=1)
|
| 77 |
+
conversation_state: ConversationStateSchema | None = None
|
| 78 |
+
clarification_choice: ClarificationChoiceSchema | None = None
|
| 79 |
+
user_location: UserLocationSchema | None = None
|
| 80 |
+
candidate_limit: int | None = Field(default=None, ge=1, le=40)
|
| 81 |
|
| 82 |
def to_domain_filters(self) -> PlaceFilters:
|
| 83 |
return PlaceFilters(
|
|
|
|
| 182 |
message: str
|
| 183 |
places: list[PlaceResultSchema]
|
| 184 |
metadata: dict[str, Any]
|
| 185 |
+
action: Literal["recommendations", "clarification", "no_match"] | None = None
|
| 186 |
+
state_patch: dict[str, Any] | None = None
|
| 187 |
+
location_directive: PlaceChatLocationDirectiveSchema | None = None
|
| 188 |
+
clarification: ClarificationSchema | None = None
|
| 189 |
+
unresolved: list[str] | None = None
|
| 190 |
+
intent_confidence: float | None = Field(default=None, ge=0, le=1)
|
| 191 |
+
ranking_version: str | None = None
|
| 192 |
+
taxonomy_version: str | None = None
|
| 193 |
+
uncertainty: dict[str, Any] | None = None
|
| 194 |
|
| 195 |
|
| 196 |
def place_to_schema(place: PlaceCandidate) -> PlaceResultSchema:
|
app/modules/places/application/use_cases/chat_place_recommendations.py
CHANGED
|
@@ -1,3 +1,4 @@
|
|
|
|
|
| 1 |
from dataclasses import dataclass, replace
|
| 2 |
import logging
|
| 3 |
import math
|
|
@@ -7,6 +8,9 @@ from app.modules.places.application.ports.chat_retriever import (
|
|
| 7 |
HybridPlaceChatRetriever,
|
| 8 |
)
|
| 9 |
from app.modules.places.application.ports.intent_parser import PlaceChatIntentParser
|
|
|
|
|
|
|
|
|
|
| 10 |
from app.modules.places.application.ports.place_anchor_resolver import (
|
| 11 |
PlaceAnchorResolver,
|
| 12 |
)
|
|
@@ -36,6 +40,13 @@ from app.shared.tracing import new_trace_id
|
|
| 36 |
|
| 37 |
logger = logging.getLogger(__name__)
|
| 38 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 39 |
|
| 40 |
@dataclass(frozen=True)
|
| 41 |
class ChatPlaceRecommendationsResult:
|
|
@@ -53,6 +64,9 @@ class ChatPlaceRecommendationsResult:
|
|
| 53 |
category_source: str = "unresolved"
|
| 54 |
used_llm: bool = False
|
| 55 |
guard_reason: str | None = None
|
|
|
|
|
|
|
|
|
|
| 56 |
|
| 57 |
|
| 58 |
class ChatPlaceRecommendationsUseCase:
|
|
@@ -68,7 +82,21 @@ class ChatPlaceRecommendationsUseCase:
|
|
| 68 |
llm_enabled: bool = True,
|
| 69 |
anchor_ambiguity_delta: float = 0.15,
|
| 70 |
minimum_intent_confidence: float = 0.70,
|
|
|
|
|
|
|
|
|
|
|
|
|
| 71 |
) -> None:
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 72 |
self._intent_parser = intent_parser
|
| 73 |
self._anchor_resolver = anchor_resolver
|
| 74 |
self._retriever = retriever
|
|
@@ -79,44 +107,37 @@ class ChatPlaceRecommendationsUseCase:
|
|
| 79 |
self._llm_enabled = llm_enabled
|
| 80 |
self._anchor_ambiguity_delta = anchor_ambiguity_delta
|
| 81 |
self._minimum_intent_confidence = minimum_intent_confidence
|
|
|
|
|
|
|
|
|
|
|
|
|
| 82 |
|
| 83 |
async def execute(
|
| 84 |
self,
|
| 85 |
message: str,
|
| 86 |
state: ConversationState,
|
| 87 |
-
user_latitude: float,
|
| 88 |
-
user_longitude: float,
|
| 89 |
candidate_limit: int,
|
| 90 |
result_limit: int,
|
| 91 |
clarification_choice: ClarificationChoice | None = None,
|
| 92 |
) -> ChatPlaceRecommendationsResult:
|
| 93 |
-
|
|
|
|
|
|
|
| 94 |
trace_id = new_trace_id()
|
| 95 |
-
|
| 96 |
-
|
| 97 |
-
|
| 98 |
-
|
| 99 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
| 100 |
)
|
| 101 |
if intent.action == "clarification":
|
| 102 |
return self._clarification_result(intent, trace_id)
|
| 103 |
-
if intent.confidence < self._minimum_intent_confidence:
|
| 104 |
-
categories = tuple(
|
| 105 |
-
dict.fromkeys(
|
| 106 |
-
(
|
| 107 |
-
intent.target_category or "restaurant",
|
| 108 |
-
"restaurant",
|
| 109 |
-
"cafe",
|
| 110 |
-
"park",
|
| 111 |
-
)
|
| 112 |
-
)
|
| 113 |
-
)
|
| 114 |
-
clarified = self._with_pending_clarification(
|
| 115 |
-
intent,
|
| 116 |
-
new_category_clarification(categories, kind="intent_category"),
|
| 117 |
-
unresolved=("intent_confidence",),
|
| 118 |
-
)
|
| 119 |
-
return self._clarification_result(clarified, trace_id)
|
| 120 |
|
| 121 |
resolved = await self._resolve_entities(intent, state)
|
| 122 |
if isinstance(resolved, ChatPlaceRecommendationsResult):
|
|
@@ -127,13 +148,129 @@ class ChatPlaceRecommendationsUseCase:
|
|
| 127 |
|
| 128 |
directive = self._location_directive(intent)
|
| 129 |
if directive.source == "unresolved":
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 130 |
return self._result(
|
| 131 |
action="no_match",
|
| 132 |
-
message=
|
| 133 |
intent=intent,
|
| 134 |
directive=directive,
|
| 135 |
candidates=(),
|
| 136 |
-
unresolved=(),
|
| 137 |
trace_id=trace_id,
|
| 138 |
)
|
| 139 |
|
|
@@ -159,13 +296,18 @@ class ChatPlaceRecommendationsUseCase:
|
|
| 159 |
candidates=candidates[:result_limit],
|
| 160 |
state=state,
|
| 161 |
)
|
|
|
|
|
|
|
|
|
|
| 162 |
return self._result(
|
| 163 |
action="recommendations",
|
| 164 |
message=final_message,
|
| 165 |
intent=intent,
|
| 166 |
directive=directive,
|
| 167 |
candidates=candidates,
|
| 168 |
-
unresolved=(
|
|
|
|
|
|
|
| 169 |
trace_id=trace_id,
|
| 170 |
used_llm=used_llm,
|
| 171 |
guard_reason=guard_reason,
|
|
@@ -222,7 +364,12 @@ class ChatPlaceRecommendationsUseCase:
|
|
| 222 |
)
|
| 223 |
return self._clarification_result(clarified, "")
|
| 224 |
anchor = anchors[0]
|
| 225 |
-
resolved_location = replace(
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 226 |
explicit = ExplicitTargetLocation(
|
| 227 |
anchor_text=location.anchor_text,
|
| 228 |
place_id=anchor.place_id,
|
|
@@ -238,6 +385,36 @@ class ChatPlaceRecommendationsUseCase:
|
|
| 238 |
explicit_target_location=explicit,
|
| 239 |
),
|
| 240 |
)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 241 |
|
| 242 |
reference = resolved_intent.reference
|
| 243 |
if reference and reference.entity and not reference.place_id:
|
|
@@ -415,13 +592,138 @@ class ChatPlaceRecommendationsUseCase:
|
|
| 415 |
strict_radius=location.strict_radius,
|
| 416 |
)
|
| 417 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 418 |
async def _compose_message(
|
| 419 |
self,
|
| 420 |
intent: ParsedPlaceChatIntent,
|
| 421 |
candidates: Sequence[PlaceChatCandidate],
|
| 422 |
state: ConversationState,
|
| 423 |
) -> tuple[str, bool, str | None]:
|
| 424 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 425 |
if not self._llm_enabled:
|
| 426 |
return fallback, False, "llm_disabled"
|
| 427 |
try:
|
|
@@ -432,7 +734,7 @@ class ChatPlaceRecommendationsUseCase:
|
|
| 432 |
),
|
| 433 |
region=state.city or state.state,
|
| 434 |
places=[self._candidate_context(candidate) for candidate in candidates],
|
| 435 |
-
response_mode=
|
| 436 |
)
|
| 437 |
if any(
|
| 438 |
candidate.name.casefold() in result.message.casefold()
|
|
@@ -443,6 +745,7 @@ class ChatPlaceRecommendationsUseCase:
|
|
| 443 |
guarded = self._output_guard.validate(
|
| 444 |
message=result.message,
|
| 445 |
allowed_place_names=[candidate.name for candidate in candidates],
|
|
|
|
| 446 |
)
|
| 447 |
if guarded.used_fallback:
|
| 448 |
return fallback, False, guarded.reason
|
|
@@ -454,7 +757,14 @@ class ChatPlaceRecommendationsUseCase:
|
|
| 454 |
def _template_message(
|
| 455 |
intent: ParsedPlaceChatIntent,
|
| 456 |
candidates: Sequence[PlaceChatCandidate],
|
|
|
|
| 457 |
) -> str:
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 458 |
exact = sum(candidate.match_level == "exact" for candidate in candidates)
|
| 459 |
family = sum(candidate.match_level == "family" for candidate in candidates)
|
| 460 |
if exact:
|
|
@@ -464,6 +774,13 @@ class ChatPlaceRecommendationsUseCase:
|
|
| 464 |
category = intent.target_category or "lugar"
|
| 465 |
return f"Encontre opciones de {category} que pueden encajar con tu solicitud."
|
| 466 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 467 |
@staticmethod
|
| 468 |
def _candidate_context(candidate: PlaceChatCandidate) -> dict[str, Any]:
|
| 469 |
return {
|
|
@@ -501,17 +818,34 @@ class ChatPlaceRecommendationsUseCase:
|
|
| 501 |
unresolved: tuple[str, ...],
|
| 502 |
) -> ParsedPlaceChatIntent:
|
| 503 |
clarification = to_public_clarification(pending)
|
|
|
|
| 504 |
return replace(
|
| 505 |
intent,
|
| 506 |
action="clarification",
|
| 507 |
-
semantic_query="",
|
| 508 |
confidence=min(intent.confidence, 0.75),
|
| 509 |
state_patch=replace(
|
| 510 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 511 |
pending_clarification=pending,
|
| 512 |
),
|
| 513 |
clarification=clarification,
|
| 514 |
-
alternatives=(),
|
| 515 |
unresolved=unresolved,
|
| 516 |
clarification_message=clarification.prompt,
|
| 517 |
)
|
|
@@ -545,6 +879,17 @@ class ChatPlaceRecommendationsUseCase:
|
|
| 545 |
category_source=intent.category_source,
|
| 546 |
used_llm=used_llm,
|
| 547 |
guard_reason=guard_reason,
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 548 |
)
|
| 549 |
if trace_id:
|
| 550 |
self._log_result(result, intent)
|
|
|
|
| 1 |
+
import asyncio
|
| 2 |
from dataclasses import dataclass, replace
|
| 3 |
import logging
|
| 4 |
import math
|
|
|
|
| 8 |
HybridPlaceChatRetriever,
|
| 9 |
)
|
| 10 |
from app.modules.places.application.ports.intent_parser import PlaceChatIntentParser
|
| 11 |
+
from app.modules.places.application.ports.nearby_place_provider import (
|
| 12 |
+
NearbyPlaceProvider,
|
| 13 |
+
)
|
| 14 |
from app.modules.places.application.ports.place_anchor_resolver import (
|
| 15 |
PlaceAnchorResolver,
|
| 16 |
)
|
|
|
|
| 40 |
|
| 41 |
logger = logging.getLogger(__name__)
|
| 42 |
|
| 43 |
+
_NON_CATEGORY_ALTERNATIVE_KEYS = {
|
| 44 |
+
"location_scope",
|
| 45 |
+
"reference_entity",
|
| 46 |
+
"target_results",
|
| 47 |
+
"user_current_location",
|
| 48 |
+
}
|
| 49 |
+
|
| 50 |
|
| 51 |
@dataclass(frozen=True)
|
| 52 |
class ChatPlaceRecommendationsResult:
|
|
|
|
| 64 |
category_source: str = "unresolved"
|
| 65 |
used_llm: bool = False
|
| 66 |
guard_reason: str | None = None
|
| 67 |
+
category_hypotheses: tuple[dict[str, Any], ...] = ()
|
| 68 |
+
raw_category_phrase: str | None = None
|
| 69 |
+
intent_model_version: str = "deterministic-open-v2"
|
| 70 |
|
| 71 |
|
| 72 |
class ChatPlaceRecommendationsUseCase:
|
|
|
|
| 82 |
llm_enabled: bool = True,
|
| 83 |
anchor_ambiguity_delta: float = 0.15,
|
| 84 |
minimum_intent_confidence: float = 0.70,
|
| 85 |
+
minimum_hypothesis_confidence: float = 0.60,
|
| 86 |
+
maximum_hypothesis_gap: float = 0.15,
|
| 87 |
+
nearby_place_provider: NearbyPlaceProvider | None = None,
|
| 88 |
+
default_radius_meters: int = 5_000,
|
| 89 |
) -> None:
|
| 90 |
+
if not 0.0 <= minimum_intent_confidence <= 1.0:
|
| 91 |
+
raise ValueError("minimum_intent_confidence must be between zero and one")
|
| 92 |
+
if not 0.0 <= minimum_hypothesis_confidence <= 1.0:
|
| 93 |
+
raise ValueError(
|
| 94 |
+
"minimum_hypothesis_confidence must be between zero and one"
|
| 95 |
+
)
|
| 96 |
+
if not 0.0 <= maximum_hypothesis_gap <= 1.0:
|
| 97 |
+
raise ValueError("maximum_hypothesis_gap must be between zero and one")
|
| 98 |
+
if not 1 <= default_radius_meters <= 50_000:
|
| 99 |
+
raise ValueError("default_radius_meters must be between 1 and 50000")
|
| 100 |
self._intent_parser = intent_parser
|
| 101 |
self._anchor_resolver = anchor_resolver
|
| 102 |
self._retriever = retriever
|
|
|
|
| 107 |
self._llm_enabled = llm_enabled
|
| 108 |
self._anchor_ambiguity_delta = anchor_ambiguity_delta
|
| 109 |
self._minimum_intent_confidence = minimum_intent_confidence
|
| 110 |
+
self._minimum_hypothesis_confidence = minimum_hypothesis_confidence
|
| 111 |
+
self._maximum_hypothesis_gap = maximum_hypothesis_gap
|
| 112 |
+
self._nearby_place_provider = nearby_place_provider
|
| 113 |
+
self._default_radius_meters = default_radius_meters
|
| 114 |
|
| 115 |
async def execute(
|
| 116 |
self,
|
| 117 |
message: str,
|
| 118 |
state: ConversationState,
|
| 119 |
+
user_latitude: float | None,
|
| 120 |
+
user_longitude: float | None,
|
| 121 |
candidate_limit: int,
|
| 122 |
result_limit: int,
|
| 123 |
clarification_choice: ClarificationChoice | None = None,
|
| 124 |
) -> ChatPlaceRecommendationsResult:
|
| 125 |
+
if (user_latitude is None) != (user_longitude is None):
|
| 126 |
+
raise ValueError("user_latitude and user_longitude must be provided together")
|
| 127 |
+
has_user_location = user_latitude is not None
|
| 128 |
trace_id = new_trace_id()
|
| 129 |
+
# Both the open-vocabulary classifier and the optional BERT extractor
|
| 130 |
+
# are CPU-bound and load lazily. Keep their first inference off the
|
| 131 |
+
# async request loop.
|
| 132 |
+
intent = await asyncio.to_thread(
|
| 133 |
+
self._intent_parser.parse,
|
| 134 |
+
message,
|
| 135 |
+
state,
|
| 136 |
+
has_user_location,
|
| 137 |
+
clarification_choice,
|
| 138 |
)
|
| 139 |
if intent.action == "clarification":
|
| 140 |
return self._clarification_result(intent, trace_id)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 141 |
|
| 142 |
resolved = await self._resolve_entities(intent, state)
|
| 143 |
if isinstance(resolved, ChatPlaceRecommendationsResult):
|
|
|
|
| 148 |
|
| 149 |
directive = self._location_directive(intent)
|
| 150 |
if directive.source == "unresolved":
|
| 151 |
+
region = state.city or state.state
|
| 152 |
+
if not region:
|
| 153 |
+
return self._result(
|
| 154 |
+
action="no_match",
|
| 155 |
+
message="No pude identificar una ubicacion util para esta busqueda.",
|
| 156 |
+
intent=intent,
|
| 157 |
+
directive=directive,
|
| 158 |
+
candidates=(),
|
| 159 |
+
unresolved=("location",),
|
| 160 |
+
trace_id=trace_id,
|
| 161 |
+
)
|
| 162 |
+
directive = PlaceChatLocationDirective(
|
| 163 |
+
source="state_anchor",
|
| 164 |
+
scope="target_results",
|
| 165 |
+
anchor_text=region,
|
| 166 |
+
)
|
| 167 |
+
|
| 168 |
+
geographic_latitude = (
|
| 169 |
+
user_latitude
|
| 170 |
+
if directive.source == "user_current"
|
| 171 |
+
else intent.location.latitude
|
| 172 |
+
)
|
| 173 |
+
geographic_longitude = (
|
| 174 |
+
user_longitude
|
| 175 |
+
if directive.source == "user_current"
|
| 176 |
+
else intent.location.longitude
|
| 177 |
+
)
|
| 178 |
+
if (
|
| 179 |
+
directive.source in {"user_current", "explicit_anchor", "state_anchor"}
|
| 180 |
+
and self._nearby_place_provider is not None
|
| 181 |
+
and geographic_latitude is not None
|
| 182 |
+
and geographic_longitude is not None
|
| 183 |
+
):
|
| 184 |
+
nearby_ids = await self._nearby_place_provider.get_nearby_place_ids(
|
| 185 |
+
latitude=geographic_latitude,
|
| 186 |
+
longitude=geographic_longitude,
|
| 187 |
+
radius_meters=(
|
| 188 |
+
directive.radius_meters or self._default_radius_meters
|
| 189 |
+
),
|
| 190 |
+
)
|
| 191 |
+
if not nearby_ids:
|
| 192 |
+
return self._result(
|
| 193 |
+
action="no_match",
|
| 194 |
+
message="No encontre lugares cercanos dentro del radio solicitado.",
|
| 195 |
+
intent=intent,
|
| 196 |
+
directive=directive,
|
| 197 |
+
candidates=(),
|
| 198 |
+
unresolved=(),
|
| 199 |
+
trace_id=trace_id,
|
| 200 |
+
)
|
| 201 |
+
intent = replace(
|
| 202 |
+
intent,
|
| 203 |
+
hard_filters={
|
| 204 |
+
**intent.hard_filters,
|
| 205 |
+
"place_ids": tuple(sorted(nearby_ids)),
|
| 206 |
+
},
|
| 207 |
+
)
|
| 208 |
+
|
| 209 |
+
if intent.confidence < self._minimum_intent_confidence:
|
| 210 |
+
evidence_candidates = tuple(
|
| 211 |
+
await self._retriever.retrieve(
|
| 212 |
+
intent=intent,
|
| 213 |
+
limit=min(candidate_limit, 12),
|
| 214 |
+
)
|
| 215 |
+
)
|
| 216 |
+
pending = (
|
| 217 |
+
self._category_clarification_from_hypotheses(intent)
|
| 218 |
+
or self._category_clarification_from_candidates(evidence_candidates)
|
| 219 |
+
)
|
| 220 |
+
if pending is not None:
|
| 221 |
+
clarified = self._with_pending_clarification(
|
| 222 |
+
intent,
|
| 223 |
+
pending,
|
| 224 |
+
unresolved=("intent_confidence",),
|
| 225 |
+
)
|
| 226 |
+
return self._clarification_result(clarified, trace_id)
|
| 227 |
+
|
| 228 |
+
sufficient_candidates = tuple(
|
| 229 |
+
candidate
|
| 230 |
+
for candidate in evidence_candidates
|
| 231 |
+
if candidate.metadata.get("retrieval_diagnostics", {}).get(
|
| 232 |
+
"meets_minimum_content_score"
|
| 233 |
+
)
|
| 234 |
+
)
|
| 235 |
+
if evidence_candidates:
|
| 236 |
+
review_candidates = (
|
| 237 |
+
sufficient_candidates or evidence_candidates
|
| 238 |
+
)
|
| 239 |
+
final_message, used_llm, guard_reason = await self._compose_message(
|
| 240 |
+
intent=intent,
|
| 241 |
+
candidates=review_candidates[:result_limit],
|
| 242 |
+
state=state,
|
| 243 |
+
)
|
| 244 |
+
return self._result(
|
| 245 |
+
action="recommendations",
|
| 246 |
+
message=final_message,
|
| 247 |
+
intent=intent,
|
| 248 |
+
directive=directive,
|
| 249 |
+
candidates=review_candidates,
|
| 250 |
+
unresolved=tuple(
|
| 251 |
+
dict.fromkeys(
|
| 252 |
+
(
|
| 253 |
+
*intent.unresolved,
|
| 254 |
+
"intent_confidence",
|
| 255 |
+
*(
|
| 256 |
+
()
|
| 257 |
+
if sufficient_candidates
|
| 258 |
+
else ("retrieval_evidence",)
|
| 259 |
+
),
|
| 260 |
+
)
|
| 261 |
+
)
|
| 262 |
+
),
|
| 263 |
+
trace_id=trace_id,
|
| 264 |
+
used_llm=used_llm,
|
| 265 |
+
guard_reason=guard_reason,
|
| 266 |
+
)
|
| 267 |
return self._result(
|
| 268 |
action="no_match",
|
| 269 |
+
message=self._uncertain_intent_message(intent, evidence_candidates),
|
| 270 |
intent=intent,
|
| 271 |
directive=directive,
|
| 272 |
candidates=(),
|
| 273 |
+
unresolved=("intent_confidence",),
|
| 274 |
trace_id=trace_id,
|
| 275 |
)
|
| 276 |
|
|
|
|
| 296 |
candidates=candidates[:result_limit],
|
| 297 |
state=state,
|
| 298 |
)
|
| 299 |
+
has_sufficient_evidence = any(
|
| 300 |
+
self._meets_content_threshold(candidate) for candidate in candidates
|
| 301 |
+
)
|
| 302 |
return self._result(
|
| 303 |
action="recommendations",
|
| 304 |
message=final_message,
|
| 305 |
intent=intent,
|
| 306 |
directive=directive,
|
| 307 |
candidates=candidates,
|
| 308 |
+
unresolved=(
|
| 309 |
+
() if has_sufficient_evidence else ("retrieval_evidence",)
|
| 310 |
+
),
|
| 311 |
trace_id=trace_id,
|
| 312 |
used_llm=used_llm,
|
| 313 |
guard_reason=guard_reason,
|
|
|
|
| 364 |
)
|
| 365 |
return self._clarification_result(clarified, "")
|
| 366 |
anchor = anchors[0]
|
| 367 |
+
resolved_location = replace(
|
| 368 |
+
location,
|
| 369 |
+
resolved_place_id=anchor.place_id,
|
| 370 |
+
latitude=anchor.latitude,
|
| 371 |
+
longitude=anchor.longitude,
|
| 372 |
+
)
|
| 373 |
explicit = ExplicitTargetLocation(
|
| 374 |
anchor_text=location.anchor_text,
|
| 375 |
place_id=anchor.place_id,
|
|
|
|
| 385 |
explicit_target_location=explicit,
|
| 386 |
),
|
| 387 |
)
|
| 388 |
+
elif (
|
| 389 |
+
location.anchor_text
|
| 390 |
+
and location.resolved_place_id
|
| 391 |
+
and (location.latitude is None or location.longitude is None)
|
| 392 |
+
):
|
| 393 |
+
anchors = list(
|
| 394 |
+
await self._anchor_resolver.resolve(
|
| 395 |
+
text=location.anchor_text,
|
| 396 |
+
city=state.city,
|
| 397 |
+
state=state.state,
|
| 398 |
+
limit=5,
|
| 399 |
+
)
|
| 400 |
+
)
|
| 401 |
+
selected = next(
|
| 402 |
+
(
|
| 403 |
+
anchor
|
| 404 |
+
for anchor in anchors
|
| 405 |
+
if anchor.place_id == location.resolved_place_id
|
| 406 |
+
),
|
| 407 |
+
None,
|
| 408 |
+
)
|
| 409 |
+
if selected is not None:
|
| 410 |
+
resolved_intent = replace(
|
| 411 |
+
resolved_intent,
|
| 412 |
+
location=replace(
|
| 413 |
+
location,
|
| 414 |
+
latitude=selected.latitude,
|
| 415 |
+
longitude=selected.longitude,
|
| 416 |
+
),
|
| 417 |
+
)
|
| 418 |
|
| 419 |
reference = resolved_intent.reference
|
| 420 |
if reference and reference.entity and not reference.place_id:
|
|
|
|
| 592 |
strict_radius=location.strict_radius,
|
| 593 |
)
|
| 594 |
|
| 595 |
+
def _category_clarification_from_hypotheses(
|
| 596 |
+
self,
|
| 597 |
+
intent: ParsedPlaceChatIntent,
|
| 598 |
+
) -> PendingClarification | None:
|
| 599 |
+
scores: dict[str, float] = {}
|
| 600 |
+
labels: dict[str, str] = {}
|
| 601 |
+
if intent.target_category:
|
| 602 |
+
scores[intent.target_category] = intent.confidence
|
| 603 |
+
|
| 604 |
+
for alternative in intent.alternatives:
|
| 605 |
+
key = alternative.key.strip()
|
| 606 |
+
if not key or key in _NON_CATEGORY_ALTERNATIVE_KEYS:
|
| 607 |
+
continue
|
| 608 |
+
scores[key] = max(scores.get(key, 0.0), alternative.confidence)
|
| 609 |
+
if alternative.description.strip():
|
| 610 |
+
labels[key] = alternative.description.strip()
|
| 611 |
+
|
| 612 |
+
ranked = sorted(scores.items(), key=lambda item: (-item[1], item[0]))
|
| 613 |
+
if not ranked or ranked[0][1] < self._minimum_hypothesis_confidence:
|
| 614 |
+
return None
|
| 615 |
+
top_score = ranked[0][1]
|
| 616 |
+
ordered = tuple(
|
| 617 |
+
category
|
| 618 |
+
for category, score in ranked
|
| 619 |
+
if score >= self._minimum_hypothesis_confidence
|
| 620 |
+
and top_score - score <= self._maximum_hypothesis_gap
|
| 621 |
+
)[:5]
|
| 622 |
+
if len(ordered) < 2:
|
| 623 |
+
return None
|
| 624 |
+
return new_category_clarification(
|
| 625 |
+
ordered,
|
| 626 |
+
kind="intent_category",
|
| 627 |
+
labels=labels,
|
| 628 |
+
)
|
| 629 |
+
|
| 630 |
+
@staticmethod
|
| 631 |
+
def _category_clarification_from_candidates(
|
| 632 |
+
candidates: Sequence[PlaceChatCandidate],
|
| 633 |
+
) -> PendingClarification | None:
|
| 634 |
+
counts: dict[str, int] = {}
|
| 635 |
+
best_scores: dict[str, float] = {}
|
| 636 |
+
for candidate in candidates:
|
| 637 |
+
diagnostics = candidate.metadata.get("retrieval_diagnostics", {})
|
| 638 |
+
if diagnostics.get("meets_minimum_content_score") is False:
|
| 639 |
+
continue
|
| 640 |
+
category = (candidate.category or "").strip()
|
| 641 |
+
if not category:
|
| 642 |
+
continue
|
| 643 |
+
counts[category] = counts.get(category, 0) + 1
|
| 644 |
+
best_scores[category] = max(
|
| 645 |
+
best_scores.get(category, 0.0),
|
| 646 |
+
candidate.content_score,
|
| 647 |
+
)
|
| 648 |
+
ordered = tuple(
|
| 649 |
+
category
|
| 650 |
+
for category, _ in sorted(
|
| 651 |
+
best_scores.items(),
|
| 652 |
+
key=lambda item: (-item[1], -counts[item[0]], item[0]),
|
| 653 |
+
)[:3]
|
| 654 |
+
)
|
| 655 |
+
if len(ordered) < 2:
|
| 656 |
+
return None
|
| 657 |
+
labels = {
|
| 658 |
+
category: (
|
| 659 |
+
f"{category.replace('_', ' ').title()} "
|
| 660 |
+
f"({counts[category]} opciones encontradas)"
|
| 661 |
+
)
|
| 662 |
+
for category in ordered
|
| 663 |
+
}
|
| 664 |
+
return new_category_clarification(
|
| 665 |
+
ordered,
|
| 666 |
+
kind="intent_category",
|
| 667 |
+
labels=labels,
|
| 668 |
+
)
|
| 669 |
+
|
| 670 |
+
@staticmethod
|
| 671 |
+
def _uncertain_intent_message(
|
| 672 |
+
intent: ParsedPlaceChatIntent,
|
| 673 |
+
candidates: Sequence[PlaceChatCandidate],
|
| 674 |
+
) -> str:
|
| 675 |
+
categories = tuple(
|
| 676 |
+
dict.fromkeys(
|
| 677 |
+
candidate.category.replace("_", " ").strip().title()
|
| 678 |
+
for candidate in candidates
|
| 679 |
+
if candidate.category and candidate.category.strip()
|
| 680 |
+
)
|
| 681 |
+
)[:3]
|
| 682 |
+
if len(categories) > 1:
|
| 683 |
+
evidence = ", ".join(categories[:-1]) + f" y {categories[-1]}"
|
| 684 |
+
return (
|
| 685 |
+
f"Encontre senales relacionadas con {evidence}, pero no pude "
|
| 686 |
+
"determinar con suficiente confianza cual describe tu plan. "
|
| 687 |
+
"Cuentame que actividad quieres hacer."
|
| 688 |
+
)
|
| 689 |
+
if categories:
|
| 690 |
+
return (
|
| 691 |
+
f"La busqueda apunta a {categories[0]}, pero la intencion sigue "
|
| 692 |
+
"siendo ambigua. Cuentame que actividad quieres hacer para afinarla."
|
| 693 |
+
)
|
| 694 |
+
if intent.soft_preferences:
|
| 695 |
+
preferences = ", ".join(
|
| 696 |
+
preference.replace("_", " ")
|
| 697 |
+
for preference in intent.soft_preferences[:3]
|
| 698 |
+
)
|
| 699 |
+
return (
|
| 700 |
+
f"Entendi que buscas algo {preferences}, pero no pude determinar "
|
| 701 |
+
"con suficiente confianza el tipo de lugar. Describe la actividad "
|
| 702 |
+
"que tienes en mente."
|
| 703 |
+
)
|
| 704 |
+
return (
|
| 705 |
+
"No pude determinar con suficiente confianza el tipo de lugar. "
|
| 706 |
+
"Describe la actividad o el plan que tienes en mente."
|
| 707 |
+
)
|
| 708 |
+
|
| 709 |
async def _compose_message(
|
| 710 |
self,
|
| 711 |
intent: ParsedPlaceChatIntent,
|
| 712 |
candidates: Sequence[PlaceChatCandidate],
|
| 713 |
state: ConversationState,
|
| 714 |
) -> tuple[str, bool, str | None]:
|
| 715 |
+
has_sufficient_evidence = any(
|
| 716 |
+
self._meets_content_threshold(candidate) for candidate in candidates
|
| 717 |
+
)
|
| 718 |
+
response_mode = (
|
| 719 |
+
"low_confidence"
|
| 720 |
+
if (
|
| 721 |
+
intent.confidence < self._minimum_intent_confidence
|
| 722 |
+
or not has_sufficient_evidence
|
| 723 |
+
)
|
| 724 |
+
else "confident"
|
| 725 |
+
)
|
| 726 |
+
fallback = self._template_message(intent, candidates, response_mode)
|
| 727 |
if not self._llm_enabled:
|
| 728 |
return fallback, False, "llm_disabled"
|
| 729 |
try:
|
|
|
|
| 734 |
),
|
| 735 |
region=state.city or state.state,
|
| 736 |
places=[self._candidate_context(candidate) for candidate in candidates],
|
| 737 |
+
response_mode=response_mode,
|
| 738 |
)
|
| 739 |
if any(
|
| 740 |
candidate.name.casefold() in result.message.casefold()
|
|
|
|
| 745 |
guarded = self._output_guard.validate(
|
| 746 |
message=result.message,
|
| 747 |
allowed_place_names=[candidate.name for candidate in candidates],
|
| 748 |
+
response_mode=response_mode,
|
| 749 |
)
|
| 750 |
if guarded.used_fallback:
|
| 751 |
return fallback, False, guarded.reason
|
|
|
|
| 757 |
def _template_message(
|
| 758 |
intent: ParsedPlaceChatIntent,
|
| 759 |
candidates: Sequence[PlaceChatCandidate],
|
| 760 |
+
response_mode: str = "confident",
|
| 761 |
) -> str:
|
| 762 |
+
if response_mode == "low_confidence":
|
| 763 |
+
return (
|
| 764 |
+
"Encontre opciones semanticamente relacionadas, aunque la evidencia "
|
| 765 |
+
"todavia es debil. Revisalas como sugerencias y ajusta tu busqueda "
|
| 766 |
+
"si no representan el plan que tienes en mente."
|
| 767 |
+
)
|
| 768 |
exact = sum(candidate.match_level == "exact" for candidate in candidates)
|
| 769 |
family = sum(candidate.match_level == "family" for candidate in candidates)
|
| 770 |
if exact:
|
|
|
|
| 774 |
category = intent.target_category or "lugar"
|
| 775 |
return f"Encontre opciones de {category} que pueden encajar con tu solicitud."
|
| 776 |
|
| 777 |
+
@staticmethod
|
| 778 |
+
def _meets_content_threshold(candidate: PlaceChatCandidate) -> bool:
|
| 779 |
+
diagnostics = candidate.metadata.get("retrieval_diagnostics", {})
|
| 780 |
+
if "meets_minimum_content_score" in diagnostics:
|
| 781 |
+
return bool(diagnostics["meets_minimum_content_score"])
|
| 782 |
+
return candidate.content_score > 0.0
|
| 783 |
+
|
| 784 |
@staticmethod
|
| 785 |
def _candidate_context(candidate: PlaceChatCandidate) -> dict[str, Any]:
|
| 786 |
return {
|
|
|
|
| 818 |
unresolved: tuple[str, ...],
|
| 819 |
) -> ParsedPlaceChatIntent:
|
| 820 |
clarification = to_public_clarification(pending)
|
| 821 |
+
patch = intent.state_patch
|
| 822 |
return replace(
|
| 823 |
intent,
|
| 824 |
action="clarification",
|
|
|
|
| 825 |
confidence=min(intent.confidence, 0.75),
|
| 826 |
state_patch=replace(
|
| 827 |
+
patch,
|
| 828 |
+
target_category=(
|
| 829 |
+
patch.target_category or intent.target_category
|
| 830 |
+
),
|
| 831 |
+
hard_filters=(
|
| 832 |
+
patch.hard_filters
|
| 833 |
+
if patch.hard_filters is not None
|
| 834 |
+
else (dict(intent.hard_filters) if intent.hard_filters else None)
|
| 835 |
+
),
|
| 836 |
+
soft_preferences=(
|
| 837 |
+
patch.soft_preferences
|
| 838 |
+
if patch.soft_preferences is not None
|
| 839 |
+
else (intent.soft_preferences or None)
|
| 840 |
+
),
|
| 841 |
+
exclusions=(
|
| 842 |
+
patch.exclusions
|
| 843 |
+
if patch.exclusions is not None
|
| 844 |
+
else (intent.exclusions or None)
|
| 845 |
+
),
|
| 846 |
pending_clarification=pending,
|
| 847 |
),
|
| 848 |
clarification=clarification,
|
|
|
|
| 849 |
unresolved=unresolved,
|
| 850 |
clarification_message=clarification.prompt,
|
| 851 |
)
|
|
|
|
| 879 |
category_source=intent.category_source,
|
| 880 |
used_llm=used_llm,
|
| 881 |
guard_reason=guard_reason,
|
| 882 |
+
category_hypotheses=tuple(
|
| 883 |
+
{
|
| 884 |
+
"id": alternative.key,
|
| 885 |
+
"label": alternative.description,
|
| 886 |
+
"probability": round(alternative.confidence, 6),
|
| 887 |
+
}
|
| 888 |
+
for alternative in intent.alternatives
|
| 889 |
+
if alternative.key not in _NON_CATEGORY_ALTERNATIVE_KEYS
|
| 890 |
+
)[:5],
|
| 891 |
+
raw_category_phrase=intent.raw_category_phrase,
|
| 892 |
+
intent_model_version=intent.intent_model_version,
|
| 893 |
)
|
| 894 |
if trace_id:
|
| 895 |
self._log_result(result, intent)
|
app/modules/places/application/use_cases/chat_places.py
CHANGED
|
@@ -42,7 +42,12 @@ class ChatPlacesUseCase:
|
|
| 42 |
filters=filters,
|
| 43 |
limit=limit,
|
| 44 |
)
|
| 45 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 46 |
context_places = [place.to_llm_context() for place in places]
|
| 47 |
|
| 48 |
llm_provider = self._llm_provider.provider_name
|
|
@@ -55,18 +60,23 @@ class ChatPlacesUseCase:
|
|
| 55 |
user_intent=search_result.normalized_query,
|
| 56 |
region=filters.city or filters.state,
|
| 57 |
places=context_places,
|
|
|
|
| 58 |
)
|
| 59 |
llm_provider = llm_result.provider
|
| 60 |
llm_model = llm_result.model
|
| 61 |
guarded = self._output_guard.validate(
|
| 62 |
message=llm_result.message,
|
| 63 |
allowed_place_names=[place.name for place in places],
|
|
|
|
| 64 |
)
|
| 65 |
used_llm = not guarded.used_fallback
|
| 66 |
guard_reason = guarded.reason
|
| 67 |
final_message = guarded.message
|
| 68 |
except Exception as exc:
|
| 69 |
-
guarded = self._output_guard.fallback(
|
|
|
|
|
|
|
|
|
|
| 70 |
final_message = guarded.message
|
| 71 |
guard_reason = guarded.reason
|
| 72 |
|
|
@@ -80,6 +90,7 @@ class ChatPlacesUseCase:
|
|
| 80 |
"llm_model": llm_model,
|
| 81 |
"used_llm": used_llm,
|
| 82 |
"guard_reason": guard_reason,
|
|
|
|
| 83 |
"places_used_as_context": [place.id for place in places],
|
| 84 |
"timestamp": datetime.now(UTC).isoformat(),
|
| 85 |
},
|
|
|
|
| 42 |
filters=filters,
|
| 43 |
limit=limit,
|
| 44 |
)
|
| 45 |
+
response_mode = search_result.metrics.match_quality
|
| 46 |
+
places = (
|
| 47 |
+
[]
|
| 48 |
+
if response_mode == "no_match"
|
| 49 |
+
else [place for place in search_result.places if place.score > 0]
|
| 50 |
+
)
|
| 51 |
context_places = [place.to_llm_context() for place in places]
|
| 52 |
|
| 53 |
llm_provider = self._llm_provider.provider_name
|
|
|
|
| 60 |
user_intent=search_result.normalized_query,
|
| 61 |
region=filters.city or filters.state,
|
| 62 |
places=context_places,
|
| 63 |
+
response_mode=response_mode,
|
| 64 |
)
|
| 65 |
llm_provider = llm_result.provider
|
| 66 |
llm_model = llm_result.model
|
| 67 |
guarded = self._output_guard.validate(
|
| 68 |
message=llm_result.message,
|
| 69 |
allowed_place_names=[place.name for place in places],
|
| 70 |
+
response_mode=response_mode,
|
| 71 |
)
|
| 72 |
used_llm = not guarded.used_fallback
|
| 73 |
guard_reason = guarded.reason
|
| 74 |
final_message = guarded.message
|
| 75 |
except Exception as exc:
|
| 76 |
+
guarded = self._output_guard.fallback(
|
| 77 |
+
reason=exc.__class__.__name__,
|
| 78 |
+
response_mode=response_mode,
|
| 79 |
+
)
|
| 80 |
final_message = guarded.message
|
| 81 |
guard_reason = guarded.reason
|
| 82 |
|
|
|
|
| 90 |
"llm_model": llm_model,
|
| 91 |
"used_llm": used_llm,
|
| 92 |
"guard_reason": guard_reason,
|
| 93 |
+
"response_mode": response_mode,
|
| 94 |
"places_used_as_context": [place.id for place in places],
|
| 95 |
"timestamp": datetime.now(UTC).isoformat(),
|
| 96 |
},
|
app/modules/places/application/use_cases/search_places.py
CHANGED
|
@@ -1,4 +1,5 @@
|
|
| 1 |
from dataclasses import dataclass, replace
|
|
|
|
| 2 |
import json
|
| 3 |
from typing import Sequence
|
| 4 |
|
|
@@ -84,7 +85,10 @@ class SearchPlacesUseCase:
|
|
| 84 |
place_ids=tuple(sorted(nearby_ids)),
|
| 85 |
)
|
| 86 |
|
| 87 |
-
query_embedding =
|
|
|
|
|
|
|
|
|
|
| 88 |
candidates = await self._place_repository.search(
|
| 89 |
embedding=query_embedding,
|
| 90 |
filters=effective_filters,
|
|
|
|
| 1 |
from dataclasses import dataclass, replace
|
| 2 |
+
import asyncio
|
| 3 |
import json
|
| 4 |
from typing import Sequence
|
| 5 |
|
|
|
|
| 85 |
place_ids=tuple(sorted(nearby_ids)),
|
| 86 |
)
|
| 87 |
|
| 88 |
+
query_embedding = await asyncio.to_thread(
|
| 89 |
+
self._embedding_provider.embed_text,
|
| 90 |
+
normalized_query,
|
| 91 |
+
)
|
| 92 |
candidates = await self._place_repository.search(
|
| 93 |
embedding=query_embedding,
|
| 94 |
filters=effective_filters,
|
app/modules/places/domain/chat_intent.py
CHANGED
|
@@ -113,6 +113,8 @@ class LocationIntent:
|
|
| 113 |
resolved_place_id: str | None = None
|
| 114 |
radius_meters: int | None = None
|
| 115 |
strict_radius: bool = False
|
|
|
|
|
|
|
| 116 |
|
| 117 |
|
| 118 |
@dataclass(frozen=True)
|
|
@@ -127,6 +129,8 @@ class PlaceCategoryInference:
|
|
| 127 |
category: str
|
| 128 |
confidence: float
|
| 129 |
source: Literal["lexical_activity", "semantic_activity"]
|
|
|
|
|
|
|
| 130 |
|
| 131 |
|
| 132 |
@dataclass(frozen=True)
|
|
@@ -220,6 +224,8 @@ class ParsedPlaceChatIntent:
|
|
| 220 |
alternatives: tuple[IntentAlternative, ...] = ()
|
| 221 |
unresolved: tuple[str, ...] = ()
|
| 222 |
clarification_message: str | None = None
|
|
|
|
|
|
|
| 223 |
|
| 224 |
|
| 225 |
@dataclass(frozen=True)
|
|
|
|
| 113 |
resolved_place_id: str | None = None
|
| 114 |
radius_meters: int | None = None
|
| 115 |
strict_radius: bool = False
|
| 116 |
+
latitude: float | None = None
|
| 117 |
+
longitude: float | None = None
|
| 118 |
|
| 119 |
|
| 120 |
@dataclass(frozen=True)
|
|
|
|
| 129 |
category: str
|
| 130 |
confidence: float
|
| 131 |
source: Literal["lexical_activity", "semantic_activity"]
|
| 132 |
+
category_values: tuple[str, ...] = ()
|
| 133 |
+
label: str | None = None
|
| 134 |
|
| 135 |
|
| 136 |
@dataclass(frozen=True)
|
|
|
|
| 224 |
alternatives: tuple[IntentAlternative, ...] = ()
|
| 225 |
unresolved: tuple[str, ...] = ()
|
| 226 |
clarification_message: str | None = None
|
| 227 |
+
raw_category_phrase: str | None = None
|
| 228 |
+
intent_model_version: str = "deterministic-open-v2"
|
| 229 |
|
| 230 |
|
| 231 |
@dataclass(frozen=True)
|
app/modules/places/domain/clarifications.py
CHANGED
|
@@ -1,4 +1,6 @@
|
|
| 1 |
-
from collections.abc import Sequence
|
|
|
|
|
|
|
| 2 |
from uuid import uuid4
|
| 3 |
|
| 4 |
from app.modules.places.domain.chat_intent import (
|
|
@@ -11,36 +13,33 @@ from app.modules.places.domain.chat_intent import (
|
|
| 11 |
)
|
| 12 |
|
| 13 |
|
| 14 |
-
_CATEGORY_LABELS = {
|
| 15 |
-
"restaurant": "Restaurantes",
|
| 16 |
-
"cafe": "Cafeterias",
|
| 17 |
-
"park": "Parques",
|
| 18 |
-
"nightlife": "Fiesta y vida nocturna",
|
| 19 |
-
"sports": "Ejercicio y deporte",
|
| 20 |
-
"cinema": "Cines",
|
| 21 |
-
"shopping": "Compras",
|
| 22 |
-
"lodging": "Hospedaje",
|
| 23 |
-
}
|
| 24 |
-
|
| 25 |
-
|
| 26 |
def new_category_clarification(
|
| 27 |
categories: Sequence[str],
|
| 28 |
*,
|
| 29 |
kind: ClarificationKind = "target_category",
|
|
|
|
| 30 |
) -> PendingClarification:
|
| 31 |
unique = tuple(dict.fromkeys(category for category in categories if category))[:5]
|
| 32 |
if len(unique) < 2:
|
| 33 |
raise ValueError("a category clarification requires at least two options")
|
|
|
|
|
|
|
| 34 |
return PendingClarification(
|
| 35 |
clarification_id=str(uuid4()),
|
| 36 |
kind=kind,
|
| 37 |
options=tuple(
|
| 38 |
PendingClarificationOption(
|
| 39 |
-
option_id=
|
| 40 |
value=category,
|
| 41 |
-
label=
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 42 |
)
|
| 43 |
-
for category in unique
|
| 44 |
),
|
| 45 |
)
|
| 46 |
|
|
@@ -156,3 +155,27 @@ def _bounded_text(value: str, maximum: int) -> str:
|
|
| 156 |
if len(value) <= maximum:
|
| 157 |
return value
|
| 158 |
return value[: maximum - 1].rstrip() + "…"
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
from collections.abc import Mapping, Sequence
|
| 2 |
+
import re
|
| 3 |
+
import unicodedata
|
| 4 |
from uuid import uuid4
|
| 5 |
|
| 6 |
from app.modules.places.domain.chat_intent import (
|
|
|
|
| 13 |
)
|
| 14 |
|
| 15 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 16 |
def new_category_clarification(
|
| 17 |
categories: Sequence[str],
|
| 18 |
*,
|
| 19 |
kind: ClarificationKind = "target_category",
|
| 20 |
+
labels: Mapping[str, str] | None = None,
|
| 21 |
) -> PendingClarification:
|
| 22 |
unique = tuple(dict.fromkeys(category for category in categories if category))[:5]
|
| 23 |
if len(unique) < 2:
|
| 24 |
raise ValueError("a category clarification requires at least two options")
|
| 25 |
+
option_labels = labels or {}
|
| 26 |
+
option_ids = _category_option_ids(unique)
|
| 27 |
return PendingClarification(
|
| 28 |
clarification_id=str(uuid4()),
|
| 29 |
kind=kind,
|
| 30 |
options=tuple(
|
| 31 |
PendingClarificationOption(
|
| 32 |
+
option_id=option_id,
|
| 33 |
value=category,
|
| 34 |
+
label=_bounded_text(
|
| 35 |
+
option_labels.get(
|
| 36 |
+
category,
|
| 37 |
+
category.replace("_", " ").title(),
|
| 38 |
+
),
|
| 39 |
+
160,
|
| 40 |
+
),
|
| 41 |
)
|
| 42 |
+
for category, option_id in zip(unique, option_ids)
|
| 43 |
),
|
| 44 |
)
|
| 45 |
|
|
|
|
| 155 |
if len(value) <= maximum:
|
| 156 |
return value
|
| 157 |
return value[: maximum - 1].rstrip() + "…"
|
| 158 |
+
|
| 159 |
+
|
| 160 |
+
def _category_option_ids(values: Sequence[str]) -> tuple[str, ...]:
|
| 161 |
+
seen: set[str] = set()
|
| 162 |
+
option_ids: list[str] = []
|
| 163 |
+
for index, value in enumerate(values, start=1):
|
| 164 |
+
option_id = _category_option_id(value, index)
|
| 165 |
+
if option_id in seen:
|
| 166 |
+
suffix = f"_{index}"
|
| 167 |
+
option_id = option_id[: 64 - len(suffix)].rstrip("_-") + suffix
|
| 168 |
+
seen.add(option_id)
|
| 169 |
+
option_ids.append(option_id)
|
| 170 |
+
return tuple(option_ids)
|
| 171 |
+
|
| 172 |
+
|
| 173 |
+
def _category_option_id(value: str, index: int) -> str:
|
| 174 |
+
if re.fullmatch(r"[a-zA-Z0-9_-]{1,64}", value):
|
| 175 |
+
return value
|
| 176 |
+
ascii_value = unicodedata.normalize("NFKD", value).encode(
|
| 177 |
+
"ascii", "ignore"
|
| 178 |
+
).decode("ascii")
|
| 179 |
+
slug = re.sub(r"[^a-zA-Z0-9_-]+", "_", ascii_value).strip("_-").lower()
|
| 180 |
+
slug = slug[:54].rstrip("_-") or "category"
|
| 181 |
+
return f"{slug}_{index}"
|
app/modules/places/infrastructure/aws_pgvector_place_repository.py
CHANGED
|
@@ -9,8 +9,15 @@ from app.shared.vector_store.models import VectorMatch
|
|
| 9 |
class AwsPgvectorPlaceRepository(PlaceVectorRepository):
|
| 10 |
source_name = "pgvector"
|
| 11 |
|
| 12 |
-
def __init__(
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 13 |
self._vector_client = vector_client
|
|
|
|
|
|
|
| 14 |
|
| 15 |
async def search(
|
| 16 |
self,
|
|
@@ -24,12 +31,39 @@ class AwsPgvectorPlaceRepository(PlaceVectorRepository):
|
|
| 24 |
embedding=embedding,
|
| 25 |
filters=metadata_filter,
|
| 26 |
limit=limit,
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 27 |
)
|
| 28 |
return [_match_to_candidate(match) for match in matches]
|
| 29 |
|
| 30 |
|
| 31 |
def _match_to_candidate(match: VectorMatch) -> PlaceCandidate:
|
| 32 |
metadata = dict(match.metadata)
|
|
|
|
|
|
|
|
|
|
|
|
|
| 33 |
return PlaceCandidate(
|
| 34 |
id=match.id,
|
| 35 |
name=str(metadata.get("name") or match.id),
|
|
|
|
| 9 |
class AwsPgvectorPlaceRepository(PlaceVectorRepository):
|
| 10 |
source_name = "pgvector"
|
| 11 |
|
| 12 |
+
def __init__(
|
| 13 |
+
self,
|
| 14 |
+
vector_client: AwsPgvectorClient,
|
| 15 |
+
match_function: str = "match_places",
|
| 16 |
+
hybrid_function: str | None = None,
|
| 17 |
+
) -> None:
|
| 18 |
self._vector_client = vector_client
|
| 19 |
+
self._match_function = match_function
|
| 20 |
+
self._hybrid_function = hybrid_function
|
| 21 |
|
| 22 |
async def search(
|
| 23 |
self,
|
|
|
|
| 31 |
embedding=embedding,
|
| 32 |
filters=metadata_filter,
|
| 33 |
limit=limit,
|
| 34 |
+
function_name=self._match_function,
|
| 35 |
+
)
|
| 36 |
+
return [_match_to_candidate(match) for match in matches]
|
| 37 |
+
|
| 38 |
+
async def search_hybrid(
|
| 39 |
+
self,
|
| 40 |
+
query_text: str,
|
| 41 |
+
embedding: list[float],
|
| 42 |
+
filters: PlaceFilters,
|
| 43 |
+
limit: int,
|
| 44 |
+
) -> Sequence[PlaceCandidate]:
|
| 45 |
+
"""Use the versioned SQL hybrid contract when it is configured."""
|
| 46 |
+
|
| 47 |
+
if not self._hybrid_function:
|
| 48 |
+
return await self.search(embedding=embedding, filters=filters, limit=limit)
|
| 49 |
+
metadata_filter = filters.as_metadata_filter()
|
| 50 |
+
metadata_filter["is_active"] = True
|
| 51 |
+
matches = await self._vector_client.search_places_hybrid(
|
| 52 |
+
query_text=query_text,
|
| 53 |
+
embedding=embedding,
|
| 54 |
+
filters=metadata_filter,
|
| 55 |
+
limit=limit,
|
| 56 |
+
function_name=self._hybrid_function,
|
| 57 |
)
|
| 58 |
return [_match_to_candidate(match) for match in matches]
|
| 59 |
|
| 60 |
|
| 61 |
def _match_to_candidate(match: VectorMatch) -> PlaceCandidate:
|
| 62 |
metadata = dict(match.metadata)
|
| 63 |
+
if match.semantic_score is not None:
|
| 64 |
+
metadata["semantic_score"] = match.semantic_score
|
| 65 |
+
if match.lexical_score is not None:
|
| 66 |
+
metadata["lexical_score"] = match.lexical_score
|
| 67 |
return PlaceCandidate(
|
| 68 |
id=match.id,
|
| 69 |
name=str(metadata.get("name") or match.id),
|
app/modules/places/infrastructure/bert_intent_extractor.py
ADDED
|
@@ -0,0 +1,595 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""Lazy BERT token-classification adapter for open place-chat slots.
|
| 2 |
+
|
| 3 |
+
The extractor deliberately returns raw concepts instead of canonical place
|
| 4 |
+
categories. Category alignment belongs to the semantic catalog/retrieval
|
| 5 |
+
stage; doing it here would recreate a closed taxonomy in the model adapter.
|
| 6 |
+
|
| 7 |
+
The default loader imports ``transformers`` only on the first non-empty call.
|
| 8 |
+
Tests and alternative serving runtimes can inject a callable classifier or a
|
| 9 |
+
loader, so importing this module never requires the optional ML dependency.
|
| 10 |
+
"""
|
| 11 |
+
|
| 12 |
+
from __future__ import annotations
|
| 13 |
+
|
| 14 |
+
import math
|
| 15 |
+
import threading
|
| 16 |
+
from collections.abc import Callable, Mapping, Sequence
|
| 17 |
+
from dataclasses import dataclass
|
| 18 |
+
from typing import Any, Literal, Protocol, TypeAlias, cast
|
| 19 |
+
|
| 20 |
+
|
| 21 |
+
SlotType = Literal[
|
| 22 |
+
"CATEGORY",
|
| 23 |
+
"PREFERENCE",
|
| 24 |
+
"EXCLUSION",
|
| 25 |
+
"LOCATION",
|
| 26 |
+
"REFERENCE",
|
| 27 |
+
"RADIUS",
|
| 28 |
+
]
|
| 29 |
+
SpanPolarity = Literal["positive", "negative", "neutral"]
|
| 30 |
+
IOBPrefix = Literal["B", "I"]
|
| 31 |
+
|
| 32 |
+
_SLOT_TYPES: frozenset[str] = frozenset(
|
| 33 |
+
{
|
| 34 |
+
"CATEGORY",
|
| 35 |
+
"PREFERENCE",
|
| 36 |
+
"EXCLUSION",
|
| 37 |
+
"LOCATION",
|
| 38 |
+
"REFERENCE",
|
| 39 |
+
"RADIUS",
|
| 40 |
+
}
|
| 41 |
+
)
|
| 42 |
+
_POLARITIES: frozenset[str] = frozenset({"positive", "negative", "neutral"})
|
| 43 |
+
|
| 44 |
+
|
| 45 |
+
class BertIntentExtractionError(RuntimeError):
|
| 46 |
+
"""Base error raised by the contextual intent extractor."""
|
| 47 |
+
|
| 48 |
+
|
| 49 |
+
class BertIntentModelLoadError(BertIntentExtractionError):
|
| 50 |
+
"""The configured token-classification model could not be loaded."""
|
| 51 |
+
|
| 52 |
+
|
| 53 |
+
class BertIntentInferenceError(BertIntentExtractionError):
|
| 54 |
+
"""The loaded model failed while processing a message."""
|
| 55 |
+
|
| 56 |
+
|
| 57 |
+
class BertIntentOutputError(BertIntentExtractionError):
|
| 58 |
+
"""The model returned an invalid token-classification payload."""
|
| 59 |
+
|
| 60 |
+
|
| 61 |
+
@dataclass(frozen=True)
|
| 62 |
+
class SlotLabelDefinition:
|
| 63 |
+
"""Maps one model label to a domain slot and its semantic polarity.
|
| 64 |
+
|
| 65 |
+
Mapping is intentionally configuration-driven. For example, a model can
|
| 66 |
+
expose ``B-AMENITY`` and map ``AMENITY`` to a positive ``PREFERENCE``, or
|
| 67 |
+
expose ``B-NEGATIVE_AMENITY`` and map it to a negative ``PREFERENCE``.
|
| 68 |
+
"""
|
| 69 |
+
|
| 70 |
+
slot_type: SlotType
|
| 71 |
+
polarity: SpanPolarity = "neutral"
|
| 72 |
+
|
| 73 |
+
def __post_init__(self) -> None:
|
| 74 |
+
slot_type = str(self.slot_type).strip().upper()
|
| 75 |
+
polarity = str(self.polarity).strip().lower()
|
| 76 |
+
if slot_type not in _SLOT_TYPES:
|
| 77 |
+
raise ValueError(
|
| 78 |
+
"slot_type must be one of " + ", ".join(sorted(_SLOT_TYPES))
|
| 79 |
+
)
|
| 80 |
+
if polarity not in _POLARITIES:
|
| 81 |
+
raise ValueError(
|
| 82 |
+
"polarity must be one of " + ", ".join(sorted(_POLARITIES))
|
| 83 |
+
)
|
| 84 |
+
object.__setattr__(self, "slot_type", cast(SlotType, slot_type))
|
| 85 |
+
object.__setattr__(self, "polarity", cast(SpanPolarity, polarity))
|
| 86 |
+
|
| 87 |
+
|
| 88 |
+
@dataclass(frozen=True)
|
| 89 |
+
class IntentSpan:
|
| 90 |
+
"""One contextual slot using character offsets into the original message."""
|
| 91 |
+
|
| 92 |
+
slot_type: SlotType
|
| 93 |
+
text: str
|
| 94 |
+
start: int
|
| 95 |
+
end: int
|
| 96 |
+
polarity: SpanPolarity
|
| 97 |
+
confidence: float
|
| 98 |
+
token_count: int = 1
|
| 99 |
+
|
| 100 |
+
def __post_init__(self) -> None:
|
| 101 |
+
if self.slot_type not in _SLOT_TYPES:
|
| 102 |
+
raise ValueError(f"unsupported slot_type: {self.slot_type!r}")
|
| 103 |
+
if self.polarity not in _POLARITIES:
|
| 104 |
+
raise ValueError(f"unsupported polarity: {self.polarity!r}")
|
| 105 |
+
if self.start < 0 or self.end <= self.start:
|
| 106 |
+
raise ValueError("span offsets must satisfy 0 <= start < end")
|
| 107 |
+
if not isinstance(self.text, str) or not self.text:
|
| 108 |
+
raise ValueError("span text must not be empty")
|
| 109 |
+
if not math.isfinite(self.confidence) or not 0.0 <= self.confidence <= 1.0:
|
| 110 |
+
raise ValueError("span confidence must be finite and between 0 and 1")
|
| 111 |
+
if self.token_count <= 0:
|
| 112 |
+
raise ValueError("span token_count must be positive")
|
| 113 |
+
|
| 114 |
+
|
| 115 |
+
@dataclass(frozen=True)
|
| 116 |
+
class IntentFrame:
|
| 117 |
+
"""Open contextual interpretation of one unmodified user message."""
|
| 118 |
+
|
| 119 |
+
raw_text: str
|
| 120 |
+
spans: tuple[IntentSpan, ...]
|
| 121 |
+
confidence: float
|
| 122 |
+
model_name: str
|
| 123 |
+
model_version: str
|
| 124 |
+
|
| 125 |
+
def __post_init__(self) -> None:
|
| 126 |
+
if not isinstance(self.raw_text, str):
|
| 127 |
+
raise TypeError("raw_text must be str")
|
| 128 |
+
if not math.isfinite(self.confidence) or not 0.0 <= self.confidence <= 1.0:
|
| 129 |
+
raise ValueError("frame confidence must be finite and between 0 and 1")
|
| 130 |
+
for span in self.spans:
|
| 131 |
+
if span.end > len(self.raw_text):
|
| 132 |
+
raise ValueError("span extends beyond raw_text")
|
| 133 |
+
if self.raw_text[span.start : span.end] != span.text:
|
| 134 |
+
raise ValueError("span text must match raw_text character offsets")
|
| 135 |
+
|
| 136 |
+
def by_type(self, slot_type: SlotType) -> tuple[IntentSpan, ...]:
|
| 137 |
+
"""Return all spans of a type without normalizing their raw values."""
|
| 138 |
+
|
| 139 |
+
return tuple(span for span in self.spans if span.slot_type == slot_type)
|
| 140 |
+
|
| 141 |
+
@property
|
| 142 |
+
def categories(self) -> tuple[IntentSpan, ...]:
|
| 143 |
+
return self.by_type("CATEGORY")
|
| 144 |
+
|
| 145 |
+
@property
|
| 146 |
+
def preferences(self) -> tuple[IntentSpan, ...]:
|
| 147 |
+
return tuple(
|
| 148 |
+
span
|
| 149 |
+
for span in self.spans
|
| 150 |
+
if span.slot_type == "PREFERENCE" and span.polarity == "positive"
|
| 151 |
+
)
|
| 152 |
+
|
| 153 |
+
@property
|
| 154 |
+
def exclusions(self) -> tuple[IntentSpan, ...]:
|
| 155 |
+
return tuple(
|
| 156 |
+
span
|
| 157 |
+
for span in self.spans
|
| 158 |
+
if span.slot_type == "EXCLUSION" or span.polarity == "negative"
|
| 159 |
+
)
|
| 160 |
+
|
| 161 |
+
|
| 162 |
+
class TokenClassifier(Protocol):
|
| 163 |
+
"""Minimal shape shared by a Hugging Face pipeline and test doubles."""
|
| 164 |
+
|
| 165 |
+
def __call__(self, text: str) -> Sequence[Mapping[str, Any]]:
|
| 166 |
+
"""Return token rows containing an IOB label, score, start and end."""
|
| 167 |
+
|
| 168 |
+
|
| 169 |
+
TokenClassifierLoader: TypeAlias = Callable[
|
| 170 |
+
[str, str | None, int | str | None], TokenClassifier
|
| 171 |
+
]
|
| 172 |
+
LabelDefinitionInput: TypeAlias = (
|
| 173 |
+
SlotLabelDefinition | SlotType | tuple[SlotType, SpanPolarity]
|
| 174 |
+
)
|
| 175 |
+
|
| 176 |
+
|
| 177 |
+
DEFAULT_LABEL_DEFINITIONS: Mapping[str, SlotLabelDefinition] = {
|
| 178 |
+
"CATEGORY": SlotLabelDefinition("CATEGORY", "positive"),
|
| 179 |
+
"PREFERENCE": SlotLabelDefinition("PREFERENCE", "positive"),
|
| 180 |
+
"EXCLUSION": SlotLabelDefinition("EXCLUSION", "negative"),
|
| 181 |
+
"LOCATION": SlotLabelDefinition("LOCATION", "neutral"),
|
| 182 |
+
"REFERENCE": SlotLabelDefinition("REFERENCE", "neutral"),
|
| 183 |
+
"RADIUS": SlotLabelDefinition("RADIUS", "neutral"),
|
| 184 |
+
}
|
| 185 |
+
|
| 186 |
+
|
| 187 |
+
@dataclass(frozen=True)
|
| 188 |
+
class _TokenPrediction:
|
| 189 |
+
prefix: IOBPrefix
|
| 190 |
+
definition: SlotLabelDefinition
|
| 191 |
+
start: int
|
| 192 |
+
end: int
|
| 193 |
+
score: float
|
| 194 |
+
order: int
|
| 195 |
+
|
| 196 |
+
|
| 197 |
+
@dataclass
|
| 198 |
+
class _SpanBuilder:
|
| 199 |
+
definition: SlotLabelDefinition
|
| 200 |
+
start: int
|
| 201 |
+
end: int
|
| 202 |
+
weighted_score: float
|
| 203 |
+
score_weight: int
|
| 204 |
+
token_count: int
|
| 205 |
+
last_order: int
|
| 206 |
+
|
| 207 |
+
@classmethod
|
| 208 |
+
def from_token(cls, token: _TokenPrediction) -> _SpanBuilder:
|
| 209 |
+
weight = max(1, token.end - token.start)
|
| 210 |
+
return cls(
|
| 211 |
+
definition=token.definition,
|
| 212 |
+
start=token.start,
|
| 213 |
+
end=token.end,
|
| 214 |
+
weighted_score=token.score * weight,
|
| 215 |
+
score_weight=weight,
|
| 216 |
+
token_count=1,
|
| 217 |
+
last_order=token.order,
|
| 218 |
+
)
|
| 219 |
+
|
| 220 |
+
def append(self, token: _TokenPrediction) -> None:
|
| 221 |
+
weight = max(1, token.end - token.start)
|
| 222 |
+
self.end = max(self.end, token.end)
|
| 223 |
+
self.weighted_score += token.score * weight
|
| 224 |
+
self.score_weight += weight
|
| 225 |
+
self.token_count += 1
|
| 226 |
+
self.last_order = token.order
|
| 227 |
+
|
| 228 |
+
def build(self, raw_text: str) -> IntentSpan:
|
| 229 |
+
return IntentSpan(
|
| 230 |
+
slot_type=self.definition.slot_type,
|
| 231 |
+
text=raw_text[self.start : self.end],
|
| 232 |
+
start=self.start,
|
| 233 |
+
end=self.end,
|
| 234 |
+
polarity=self.definition.polarity,
|
| 235 |
+
confidence=self.weighted_score / self.score_weight,
|
| 236 |
+
token_count=self.token_count,
|
| 237 |
+
)
|
| 238 |
+
|
| 239 |
+
|
| 240 |
+
class BertPlaceIntentExtractor:
|
| 241 |
+
"""Extract contextual, open-value slots with a fine-tuned BERT model.
|
| 242 |
+
|
| 243 |
+
``label_definitions`` is merged over the standard six slot labels. Keys
|
| 244 |
+
refer to model labels *without* the IOB prefix. Unknown labels are ignored,
|
| 245 |
+
which lets one model expose additional tasks without coupling Places to
|
| 246 |
+
them. Values can be ``SlotLabelDefinition``, a canonical slot string, or a
|
| 247 |
+
``(slot, polarity)`` tuple.
|
| 248 |
+
"""
|
| 249 |
+
|
| 250 |
+
def __init__(
|
| 251 |
+
self,
|
| 252 |
+
model_name_or_path: str,
|
| 253 |
+
*,
|
| 254 |
+
model_version: str | None = None,
|
| 255 |
+
device: int | str | None = None,
|
| 256 |
+
label_definitions: Mapping[str, LabelDefinitionInput] | None = None,
|
| 257 |
+
minimum_token_confidence: float = 0.0,
|
| 258 |
+
classifier: TokenClassifier | None = None,
|
| 259 |
+
model_loader: TokenClassifierLoader | None = None,
|
| 260 |
+
) -> None:
|
| 261 |
+
model_name = _required_text(model_name_or_path, "model_name_or_path")
|
| 262 |
+
if not math.isfinite(minimum_token_confidence) or not (
|
| 263 |
+
0.0 <= minimum_token_confidence <= 1.0
|
| 264 |
+
):
|
| 265 |
+
raise ValueError(
|
| 266 |
+
"minimum_token_confidence must be finite and between 0 and 1"
|
| 267 |
+
)
|
| 268 |
+
if classifier is not None and model_loader is not None:
|
| 269 |
+
raise ValueError("provide classifier or model_loader, not both")
|
| 270 |
+
if classifier is not None and not callable(classifier):
|
| 271 |
+
raise TypeError("classifier must be callable")
|
| 272 |
+
|
| 273 |
+
definitions = dict(DEFAULT_LABEL_DEFINITIONS)
|
| 274 |
+
for raw_label, raw_definition in (label_definitions or {}).items():
|
| 275 |
+
label = _normalized_model_label(raw_label)
|
| 276 |
+
definitions[label] = _coerce_label_definition(raw_definition)
|
| 277 |
+
|
| 278 |
+
self._model_name = model_name
|
| 279 |
+
self._model_version = (
|
| 280 |
+
_required_text(model_version, "model_version")
|
| 281 |
+
if model_version is not None
|
| 282 |
+
else "unspecified"
|
| 283 |
+
)
|
| 284 |
+
self._revision = model_version
|
| 285 |
+
self._device = device
|
| 286 |
+
self._definitions = definitions
|
| 287 |
+
self._minimum_token_confidence = minimum_token_confidence
|
| 288 |
+
self._classifier = classifier
|
| 289 |
+
self._model_loader = model_loader or _load_transformers_classifier
|
| 290 |
+
self._load_lock = threading.Lock()
|
| 291 |
+
self._inference_lock = threading.Lock()
|
| 292 |
+
|
| 293 |
+
@property
|
| 294 |
+
def is_loaded(self) -> bool:
|
| 295 |
+
return self._classifier is not None
|
| 296 |
+
|
| 297 |
+
@property
|
| 298 |
+
def model_name(self) -> str:
|
| 299 |
+
return self._model_name
|
| 300 |
+
|
| 301 |
+
@property
|
| 302 |
+
def model_version(self) -> str:
|
| 303 |
+
return self._model_version
|
| 304 |
+
|
| 305 |
+
def extract(self, text: str) -> IntentFrame:
|
| 306 |
+
"""Extract an immutable frame while preserving ``text`` byte-for-byte."""
|
| 307 |
+
|
| 308 |
+
if not isinstance(text, str):
|
| 309 |
+
raise TypeError(f"text must be str, got {type(text).__name__}")
|
| 310 |
+
if not text.strip():
|
| 311 |
+
return IntentFrame(
|
| 312 |
+
raw_text=text,
|
| 313 |
+
spans=(),
|
| 314 |
+
confidence=0.0,
|
| 315 |
+
model_name=self._model_name,
|
| 316 |
+
model_version=self._model_version,
|
| 317 |
+
)
|
| 318 |
+
|
| 319 |
+
classifier = self._ensure_classifier()
|
| 320 |
+
try:
|
| 321 |
+
# Hugging Face pipelines and GPU modules are not guaranteed to be
|
| 322 |
+
# safe under concurrent calls. Parsing already runs in a worker
|
| 323 |
+
# thread, so serializing one model does not block the event loop.
|
| 324 |
+
with self._inference_lock:
|
| 325 |
+
raw_predictions = classifier(text)
|
| 326 |
+
except Exception as exc:
|
| 327 |
+
raise BertIntentInferenceError(
|
| 328 |
+
f"token classification failed for {self._model_name!r}: {exc}"
|
| 329 |
+
) from exc
|
| 330 |
+
|
| 331 |
+
predictions = self._prepare_predictions(raw_predictions, text)
|
| 332 |
+
spans = _decode_iob(predictions, text)
|
| 333 |
+
return IntentFrame(
|
| 334 |
+
raw_text=text,
|
| 335 |
+
spans=spans,
|
| 336 |
+
confidence=_frame_confidence(spans),
|
| 337 |
+
model_name=self._model_name,
|
| 338 |
+
model_version=self._model_version,
|
| 339 |
+
)
|
| 340 |
+
|
| 341 |
+
def _ensure_classifier(self) -> TokenClassifier:
|
| 342 |
+
classifier = self._classifier
|
| 343 |
+
if classifier is not None:
|
| 344 |
+
return classifier
|
| 345 |
+
|
| 346 |
+
with self._load_lock:
|
| 347 |
+
classifier = self._classifier
|
| 348 |
+
if classifier is not None:
|
| 349 |
+
return classifier
|
| 350 |
+
try:
|
| 351 |
+
classifier = self._model_loader(
|
| 352 |
+
self._model_name,
|
| 353 |
+
self._revision,
|
| 354 |
+
self._device,
|
| 355 |
+
)
|
| 356 |
+
except BertIntentModelLoadError:
|
| 357 |
+
raise
|
| 358 |
+
except Exception as exc:
|
| 359 |
+
raise BertIntentModelLoadError(
|
| 360 |
+
f"could not load token-classification model "
|
| 361 |
+
f"{self._model_name!r}: {exc}"
|
| 362 |
+
) from exc
|
| 363 |
+
if not callable(classifier):
|
| 364 |
+
raise BertIntentModelLoadError(
|
| 365 |
+
f"loader for {self._model_name!r} did not return a callable"
|
| 366 |
+
)
|
| 367 |
+
self._classifier = classifier
|
| 368 |
+
return classifier
|
| 369 |
+
|
| 370 |
+
def _prepare_predictions(
|
| 371 |
+
self,
|
| 372 |
+
raw_predictions: Sequence[Mapping[str, Any]],
|
| 373 |
+
text: str,
|
| 374 |
+
) -> tuple[_TokenPrediction, ...]:
|
| 375 |
+
if isinstance(raw_predictions, (str, bytes, Mapping)) or not isinstance(
|
| 376 |
+
raw_predictions, Sequence
|
| 377 |
+
):
|
| 378 |
+
raise BertIntentOutputError(
|
| 379 |
+
"token classifier output must be a sequence of mappings"
|
| 380 |
+
)
|
| 381 |
+
|
| 382 |
+
predictions: list[_TokenPrediction] = []
|
| 383 |
+
for order, row in enumerate(raw_predictions):
|
| 384 |
+
if not isinstance(row, Mapping):
|
| 385 |
+
raise BertIntentOutputError(
|
| 386 |
+
f"token classifier row {order} must be a mapping"
|
| 387 |
+
)
|
| 388 |
+
label_value = row.get("entity", row.get("entity_group", row.get("label")))
|
| 389 |
+
if not isinstance(label_value, str):
|
| 390 |
+
raise BertIntentOutputError(
|
| 391 |
+
f"token classifier row {order} has no string entity label"
|
| 392 |
+
)
|
| 393 |
+
prefix, base_label = _split_iob_label(label_value)
|
| 394 |
+
if base_label is None:
|
| 395 |
+
continue
|
| 396 |
+
definition = self._definitions.get(base_label)
|
| 397 |
+
if definition is None:
|
| 398 |
+
continue
|
| 399 |
+
|
| 400 |
+
score = _finite_score(row.get("score"), order)
|
| 401 |
+
if score < self._minimum_token_confidence:
|
| 402 |
+
continue
|
| 403 |
+
start = _integer_offset(row.get("start"), "start", order)
|
| 404 |
+
end = _integer_offset(row.get("end"), "end", order)
|
| 405 |
+
if start < 0 or end <= start or end > len(text):
|
| 406 |
+
raise BertIntentOutputError(
|
| 407 |
+
f"token classifier row {order} has invalid offsets "
|
| 408 |
+
f"start={start}, end={end}, text_length={len(text)}"
|
| 409 |
+
)
|
| 410 |
+
token_order = _token_order(row.get("index"), order)
|
| 411 |
+
predictions.append(
|
| 412 |
+
_TokenPrediction(
|
| 413 |
+
prefix=prefix,
|
| 414 |
+
definition=definition,
|
| 415 |
+
start=start,
|
| 416 |
+
end=end,
|
| 417 |
+
score=score,
|
| 418 |
+
order=token_order,
|
| 419 |
+
)
|
| 420 |
+
)
|
| 421 |
+
return tuple(
|
| 422 |
+
sorted(
|
| 423 |
+
predictions,
|
| 424 |
+
key=lambda token: (token.start, token.end, token.order),
|
| 425 |
+
)
|
| 426 |
+
)
|
| 427 |
+
|
| 428 |
+
|
| 429 |
+
def _decode_iob(
|
| 430 |
+
predictions: Sequence[_TokenPrediction],
|
| 431 |
+
raw_text: str,
|
| 432 |
+
) -> tuple[IntentSpan, ...]:
|
| 433 |
+
spans: list[IntentSpan] = []
|
| 434 |
+
current: _SpanBuilder | None = None
|
| 435 |
+
|
| 436 |
+
for token in predictions:
|
| 437 |
+
continues_current = bool(
|
| 438 |
+
token.prefix == "I"
|
| 439 |
+
and current is not None
|
| 440 |
+
and current.definition == token.definition
|
| 441 |
+
and token.start >= current.end
|
| 442 |
+
and token.order == current.last_order + 1
|
| 443 |
+
)
|
| 444 |
+
if continues_current:
|
| 445 |
+
assert current is not None
|
| 446 |
+
current.append(token)
|
| 447 |
+
continue
|
| 448 |
+
|
| 449 |
+
if current is not None:
|
| 450 |
+
spans.append(current.build(raw_text))
|
| 451 |
+
# A stray I-label begins a recoverable new span instead of losing the
|
| 452 |
+
# user's concept because of one imperfect model transition.
|
| 453 |
+
current = _SpanBuilder.from_token(token)
|
| 454 |
+
|
| 455 |
+
if current is not None:
|
| 456 |
+
spans.append(current.build(raw_text))
|
| 457 |
+
return tuple(spans)
|
| 458 |
+
|
| 459 |
+
|
| 460 |
+
def _frame_confidence(spans: Sequence[IntentSpan]) -> float:
|
| 461 |
+
if not spans:
|
| 462 |
+
return 0.0
|
| 463 |
+
weights = [max(1, span.end - span.start) for span in spans]
|
| 464 |
+
return sum(
|
| 465 |
+
span.confidence * weight for span, weight in zip(spans, weights)
|
| 466 |
+
) / sum(weights)
|
| 467 |
+
|
| 468 |
+
|
| 469 |
+
def _split_iob_label(raw_label: str) -> tuple[IOBPrefix, str | None]:
|
| 470 |
+
label = raw_label.strip().upper()
|
| 471 |
+
if not label:
|
| 472 |
+
raise BertIntentOutputError("token classifier returned an empty label")
|
| 473 |
+
if label == "O":
|
| 474 |
+
return "B", None
|
| 475 |
+
if len(label) > 2 and label[0] in {"B", "I"} and label[1] in {"-", "_"}:
|
| 476 |
+
return cast(IOBPrefix, label[0]), _normalized_model_label(label[2:])
|
| 477 |
+
# Aggregated Hugging Face pipelines may return CATEGORY instead of
|
| 478 |
+
# B-CATEGORY. Treat an unprefixed label as one complete span.
|
| 479 |
+
return "B", _normalized_model_label(label)
|
| 480 |
+
|
| 481 |
+
|
| 482 |
+
def _coerce_label_definition(value: LabelDefinitionInput) -> SlotLabelDefinition:
|
| 483 |
+
if isinstance(value, SlotLabelDefinition):
|
| 484 |
+
return value
|
| 485 |
+
if isinstance(value, str):
|
| 486 |
+
slot_type = value.strip().upper()
|
| 487 |
+
default_definition = DEFAULT_LABEL_DEFINITIONS.get(slot_type)
|
| 488 |
+
if default_definition is not None:
|
| 489 |
+
return default_definition
|
| 490 |
+
return SlotLabelDefinition(cast(SlotType, slot_type))
|
| 491 |
+
if isinstance(value, tuple) and len(value) == 2:
|
| 492 |
+
return SlotLabelDefinition(
|
| 493 |
+
cast(SlotType, value[0]),
|
| 494 |
+
cast(SpanPolarity, value[1]),
|
| 495 |
+
)
|
| 496 |
+
raise TypeError(
|
| 497 |
+
"label definition must be SlotLabelDefinition, a slot string, or a "
|
| 498 |
+
"(slot, polarity) tuple"
|
| 499 |
+
)
|
| 500 |
+
|
| 501 |
+
|
| 502 |
+
def _normalized_model_label(value: Any) -> str:
|
| 503 |
+
if not isinstance(value, str):
|
| 504 |
+
raise TypeError(f"model label must be str, got {type(value).__name__}")
|
| 505 |
+
label = value.strip().upper()
|
| 506 |
+
if not label:
|
| 507 |
+
raise ValueError("model label must not be empty")
|
| 508 |
+
return label
|
| 509 |
+
|
| 510 |
+
|
| 511 |
+
def _required_text(value: Any, field_name: str) -> str:
|
| 512 |
+
if not isinstance(value, str):
|
| 513 |
+
raise TypeError(f"{field_name} must be str, got {type(value).__name__}")
|
| 514 |
+
cleaned = value.strip()
|
| 515 |
+
if not cleaned:
|
| 516 |
+
raise ValueError(f"{field_name} must not be empty")
|
| 517 |
+
return cleaned
|
| 518 |
+
|
| 519 |
+
|
| 520 |
+
def _finite_score(value: Any, row: int) -> float:
|
| 521 |
+
if isinstance(value, bool):
|
| 522 |
+
raise BertIntentOutputError(f"token classifier row {row} has invalid score")
|
| 523 |
+
try:
|
| 524 |
+
score = float(value)
|
| 525 |
+
except (TypeError, ValueError) as exc:
|
| 526 |
+
raise BertIntentOutputError(
|
| 527 |
+
f"token classifier row {row} has no numeric score"
|
| 528 |
+
) from exc
|
| 529 |
+
if not math.isfinite(score) or not 0.0 <= score <= 1.0:
|
| 530 |
+
raise BertIntentOutputError(
|
| 531 |
+
f"token classifier row {row} score must be finite and between 0 and 1"
|
| 532 |
+
)
|
| 533 |
+
return score
|
| 534 |
+
|
| 535 |
+
|
| 536 |
+
def _integer_offset(value: Any, name: str, row: int) -> int:
|
| 537 |
+
if isinstance(value, bool) or not isinstance(value, int):
|
| 538 |
+
raise BertIntentOutputError(
|
| 539 |
+
f"token classifier row {row} has no integer {name} offset"
|
| 540 |
+
)
|
| 541 |
+
return value
|
| 542 |
+
|
| 543 |
+
|
| 544 |
+
def _token_order(value: Any, fallback: int) -> int:
|
| 545 |
+
"""Preserve Hugging Face token adjacency after ignored labels are removed."""
|
| 546 |
+
|
| 547 |
+
if value is None:
|
| 548 |
+
return fallback
|
| 549 |
+
if isinstance(value, bool) or not isinstance(value, int) or value < 0:
|
| 550 |
+
raise BertIntentOutputError(
|
| 551 |
+
"token classifier output has an invalid token index"
|
| 552 |
+
)
|
| 553 |
+
return value
|
| 554 |
+
|
| 555 |
+
|
| 556 |
+
def _load_transformers_classifier(
|
| 557 |
+
model_name_or_path: str,
|
| 558 |
+
revision: str | None,
|
| 559 |
+
device: int | str | None,
|
| 560 |
+
) -> TokenClassifier:
|
| 561 |
+
"""Load the optional Hugging Face implementation on first inference."""
|
| 562 |
+
|
| 563 |
+
try:
|
| 564 |
+
from transformers import ( # type: ignore[import-not-found]
|
| 565 |
+
AutoModelForTokenClassification,
|
| 566 |
+
AutoTokenizer,
|
| 567 |
+
pipeline,
|
| 568 |
+
)
|
| 569 |
+
except ImportError as exc:
|
| 570 |
+
raise BertIntentModelLoadError(
|
| 571 |
+
"transformers is required to load the BERT intent extractor; "
|
| 572 |
+
"install it or inject classifier/model_loader"
|
| 573 |
+
) from exc
|
| 574 |
+
|
| 575 |
+
pretrained_kwargs: dict[str, Any] = {"trust_remote_code": False}
|
| 576 |
+
if revision is not None:
|
| 577 |
+
pretrained_kwargs["revision"] = revision
|
| 578 |
+
tokenizer = AutoTokenizer.from_pretrained(
|
| 579 |
+
model_name_or_path,
|
| 580 |
+
**pretrained_kwargs,
|
| 581 |
+
)
|
| 582 |
+
model = AutoModelForTokenClassification.from_pretrained(
|
| 583 |
+
model_name_or_path,
|
| 584 |
+
**pretrained_kwargs,
|
| 585 |
+
)
|
| 586 |
+
pipeline_kwargs: dict[str, Any] = {
|
| 587 |
+
"task": "token-classification",
|
| 588 |
+
"model": model,
|
| 589 |
+
"tokenizer": tokenizer,
|
| 590 |
+
"aggregation_strategy": "none",
|
| 591 |
+
"ignore_labels": ["O"],
|
| 592 |
+
}
|
| 593 |
+
if device is not None:
|
| 594 |
+
pipeline_kwargs["device"] = device
|
| 595 |
+
return cast(TokenClassifier, pipeline(**pipeline_kwargs))
|
app/modules/places/infrastructure/deterministic_intent_parser.py
CHANGED
|
@@ -1,6 +1,7 @@
|
|
| 1 |
from dataclasses import dataclass, replace
|
| 2 |
from functools import lru_cache
|
| 3 |
import json
|
|
|
|
| 4 |
from pathlib import Path
|
| 5 |
import re
|
| 6 |
from typing import Any, Iterable
|
|
@@ -28,6 +29,12 @@ from app.modules.places.domain.chat_intent import (
|
|
| 28 |
PlaceReference,
|
| 29 |
)
|
| 30 |
from app.modules.places.domain.errors import ClarificationStateMismatchError
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 31 |
from app.shared.nlp.preprocessing.text import (
|
| 32 |
prepare_for_embedding,
|
| 33 |
tokenize_for_embeddings,
|
|
@@ -50,9 +57,15 @@ _EXCLUSION_PATTERN = re.compile(
|
|
| 50 |
r"\b(?:sin|excepto|evita(?:r)?|no\s+quiero|que\s+no\s+(?:sea|tenga))\s+"
|
| 51 |
r"(?P<value>.+?)(?="
|
| 52 |
r"\s+cerca\s+(?:de\s+la|del|de)\b|[,;]|"
|
| 53 |
-
r"\s+(?:pero|aunque)\b|\s+y\s+(?:con|que|quiero|busco)\b|$)"
|
| 54 |
)
|
| 55 |
_CURRENT_LOCATION_VALUES = {"mi", "aqui", "donde estoy", "mi ubicacion"}
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 56 |
_GENERIC_REQUEST_TOKENS = {
|
| 57 |
"dame",
|
| 58 |
"favor",
|
|
@@ -171,14 +184,29 @@ class PlaceChatTaxonomy:
|
|
| 171 |
)
|
| 172 |
|
| 173 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 174 |
class DeterministicPlaceChatIntentParser:
|
| 175 |
def __init__(
|
| 176 |
self,
|
| 177 |
taxonomy: PlaceChatTaxonomy | None = None,
|
| 178 |
activity_classifier: PlaceActivityClassifier | None = None,
|
|
|
|
| 179 |
) -> None:
|
| 180 |
self._taxonomy = taxonomy or load_place_chat_taxonomy()
|
| 181 |
self._activity_classifier = activity_classifier
|
|
|
|
| 182 |
|
| 183 |
def parse(
|
| 184 |
self,
|
|
@@ -188,19 +216,30 @@ class DeterministicPlaceChatIntentParser:
|
|
| 188 |
clarification_choice: ClarificationChoice | None = None,
|
| 189 |
) -> ParsedPlaceChatIntent:
|
| 190 |
normalized = prepare_for_embedding(message)
|
| 191 |
-
target_clause = self._target_clause(normalized)
|
| 192 |
if clarification_choice is not None:
|
| 193 |
return self._resolve_structured_clarification(
|
| 194 |
choice=clarification_choice,
|
| 195 |
state=state,
|
| 196 |
has_user_location=has_user_location,
|
| 197 |
)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 198 |
if state.pending_clarification:
|
| 199 |
pending = ensure_legacy_pending_options(state.pending_clarification)
|
| 200 |
state = replace(state, pending_clarification=pending)
|
| 201 |
-
categories = self.
|
| 202 |
inferred = (
|
| 203 |
-
self._inferred_activity_category(
|
|
|
|
|
|
|
|
|
|
| 204 |
if not categories
|
| 205 |
else None
|
| 206 |
)
|
|
@@ -220,15 +259,25 @@ class DeterministicPlaceChatIntentParser:
|
|
| 220 |
clear_pending_clarification=True,
|
| 221 |
),
|
| 222 |
)
|
| 223 |
-
return
|
| 224 |
-
|
| 225 |
-
|
| 226 |
-
|
|
|
|
|
|
|
|
|
|
| 227 |
)
|
| 228 |
|
| 229 |
-
|
| 230 |
-
|
| 231 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 232 |
excluded_texts = tuple(
|
| 233 |
match.group("value").strip(" ,.;")
|
| 234 |
for match in _EXCLUSION_PATTERN.finditer(normalized)
|
|
@@ -237,22 +286,33 @@ class DeterministicPlaceChatIntentParser:
|
|
| 237 |
# Category words inside references, exclusions, or geographic anchors
|
| 238 |
# do not describe the requested result type. In "cafeteria cerca del
|
| 239 |
# parque", only cafe is a hard category.
|
| 240 |
-
categories = self.
|
| 241 |
if len(categories) > 1:
|
| 242 |
pending = new_category_clarification(categories)
|
| 243 |
-
return
|
| 244 |
-
|
| 245 |
-
|
| 246 |
-
|
| 247 |
-
|
|
|
|
|
|
|
|
|
|
| 248 |
)
|
| 249 |
|
| 250 |
explicit_category = categories[0] if categories else None
|
| 251 |
inferred = (
|
| 252 |
-
self._inferred_activity_category(
|
|
|
|
|
|
|
|
|
|
| 253 |
if explicit_category is None
|
| 254 |
else None
|
| 255 |
)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 256 |
inferred_category = inferred.category if inferred else None
|
| 257 |
requested_category = explicit_category or inferred_category
|
| 258 |
target_category = requested_category or state.target_category
|
|
@@ -273,11 +333,36 @@ class DeterministicPlaceChatIntentParser:
|
|
| 273 |
if location_text:
|
| 274 |
location_text = _RADIUS_PATTERN.sub("", location_text).strip(" ,.;")
|
| 275 |
|
| 276 |
-
|
| 277 |
-
|
| 278 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 279 |
for excluded_text in excluded_texts
|
| 280 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 281 |
)
|
| 282 |
positive_preferences = tuple(
|
| 283 |
preference
|
|
@@ -318,25 +403,45 @@ class DeterministicPlaceChatIntentParser:
|
|
| 318 |
else inherited_reference
|
| 319 |
)
|
| 320 |
hard_filters = dict(state.hard_filters)
|
|
|
|
|
|
|
|
|
|
|
|
|
| 321 |
if "economico" in merged_preferences or _contains_any(
|
| 322 |
target_clause,
|
| 323 |
("mas barato", "economico", "barato"),
|
| 324 |
):
|
| 325 |
hard_filters["price_preference"] = "lower"
|
| 326 |
|
| 327 |
-
|
| 328 |
-
|
| 329 |
-
|
| 330 |
-
|
| 331 |
-
|
| 332 |
-
|
| 333 |
-
|
| 334 |
-
|
| 335 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 336 |
)
|
| 337 |
-
|
| 338 |
-
category = self._taxonomy.category(target_category)
|
| 339 |
-
category_values = category.storage_values if category else (target_category,)
|
| 340 |
compatible_category_values = (
|
| 341 |
category.compatible_storage_values if category else ()
|
| 342 |
)
|
|
@@ -345,24 +450,27 @@ class DeterministicPlaceChatIntentParser:
|
|
| 345 |
if (
|
| 346 |
reference_text
|
| 347 |
and location_text
|
| 348 |
-
and location_text not in _CURRENT_LOCATION_VALUES
|
| 349 |
):
|
| 350 |
-
return
|
| 351 |
-
|
| 352 |
-
|
| 353 |
-
|
| 354 |
-
|
| 355 |
-
|
| 356 |
-
|
| 357 |
-
|
| 358 |
-
|
| 359 |
-
|
| 360 |
-
|
| 361 |
-
|
| 362 |
-
|
| 363 |
-
|
| 364 |
-
|
| 365 |
-
|
|
|
|
|
|
|
|
|
|
| 366 |
)
|
| 367 |
|
| 368 |
location, explicit_location = self._location_intent(
|
|
@@ -383,14 +491,18 @@ class DeterministicPlaceChatIntentParser:
|
|
| 383 |
)
|
| 384 |
category_query_term = self._category_query_term(target_clause, category)
|
| 385 |
semantic_parts = [
|
| 386 |
-
target_category,
|
| 387 |
category_query_term,
|
| 388 |
semantic_target,
|
| 389 |
*merged_preferences,
|
| 390 |
]
|
| 391 |
if reference and reference.entity:
|
| 392 |
semantic_parts.append(reference.entity)
|
| 393 |
-
semantic_query =
|
|
|
|
|
|
|
|
|
|
|
|
|
| 394 |
|
| 395 |
patch = ConversationStatePatch(
|
| 396 |
target_category=requested_category,
|
|
@@ -408,10 +520,17 @@ class DeterministicPlaceChatIntentParser:
|
|
| 408 |
clear_reference=category_changed and reference_text is None,
|
| 409 |
taxonomy_version=self._taxonomy.version,
|
| 410 |
)
|
| 411 |
-
if explicit_category:
|
|
|
|
|
|
|
| 412 |
confidence = 0.96
|
| 413 |
elif inferred_category:
|
| 414 |
confidence = inferred.confidence if inferred else 0.88
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 415 |
else:
|
| 416 |
confidence = 0.84
|
| 417 |
if reference_text:
|
|
@@ -431,6 +550,18 @@ class DeterministicPlaceChatIntentParser:
|
|
| 431 |
compatible_category_values=compatible_category_values,
|
| 432 |
category_evidence_terms=category_evidence_terms,
|
| 433 |
category_source=category_source,
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 434 |
)
|
| 435 |
|
| 436 |
def _location_scope_clarification(
|
|
@@ -543,8 +674,10 @@ class DeterministicPlaceChatIntentParser:
|
|
| 543 |
base_state = replace(state, pending_clarification=None)
|
| 544 |
if pending.kind in ("target_category", "intent_category"):
|
| 545 |
if self._taxonomy.category(selected.value) is None:
|
| 546 |
-
|
| 547 |
-
|
|
|
|
|
|
|
| 548 |
)
|
| 549 |
resolved = self.parse(
|
| 550 |
message=self._category_message(selected.value),
|
|
@@ -780,6 +913,61 @@ class DeterministicPlaceChatIntentParser:
|
|
| 780 |
return definition.aliases[0]
|
| 781 |
return category or ""
|
| 782 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 783 |
def _location_intent(
|
| 784 |
self,
|
| 785 |
location_text: str | None,
|
|
@@ -787,7 +975,10 @@ class DeterministicPlaceChatIntentParser:
|
|
| 787 |
state: ConversationState,
|
| 788 |
has_user_location: bool,
|
| 789 |
) -> tuple[LocationIntent, ExplicitTargetLocation | None]:
|
| 790 |
-
if
|
|
|
|
|
|
|
|
|
|
| 791 |
return (
|
| 792 |
LocationIntent(
|
| 793 |
scope="user_current_location",
|
|
@@ -831,11 +1022,123 @@ class DeterministicPlaceChatIntentParser:
|
|
| 831 |
LocationIntent(
|
| 832 |
scope="user_current_location",
|
| 833 |
source="user_current",
|
|
|
|
|
|
|
| 834 |
),
|
| 835 |
None,
|
| 836 |
)
|
| 837 |
return (LocationIntent(scope="unresolved", source="none"), None)
|
| 838 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 839 |
def _matched_categories(self, normalized: str) -> tuple[str, ...]:
|
| 840 |
return _ordered_unique(
|
| 841 |
category.canonical
|
|
@@ -864,20 +1167,44 @@ class DeterministicPlaceChatIntentParser:
|
|
| 864 |
def _inferred_activity_category(
|
| 865 |
self,
|
| 866 |
normalized: str,
|
|
|
|
|
|
|
| 867 |
) -> PlaceCategoryInference | None:
|
| 868 |
-
|
| 869 |
-
|
| 870 |
-
|
| 871 |
-
|
| 872 |
-
|
| 873 |
-
|
| 874 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 875 |
if self._activity_classifier is None:
|
| 876 |
-
return
|
| 877 |
-
|
| 878 |
-
if
|
| 879 |
-
return
|
| 880 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 881 |
|
| 882 |
@staticmethod
|
| 883 |
def _target_clause(normalized: str) -> str:
|
|
@@ -1023,6 +1350,50 @@ def _contains_phrase(text: str, phrase: str) -> bool:
|
|
| 1023 |
return bool(re.search(rf"(?<!\w){re.escape(phrase)}(?!\w)", text))
|
| 1024 |
|
| 1025 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1026 |
def _contains_any(text: str, values: tuple[str, ...]) -> bool:
|
| 1027 |
return any(
|
| 1028 |
_contains_phrase(text, prepare_for_embedding(value))
|
|
|
|
| 1 |
from dataclasses import dataclass, replace
|
| 2 |
from functools import lru_cache
|
| 3 |
import json
|
| 4 |
+
import logging
|
| 5 |
from pathlib import Path
|
| 6 |
import re
|
| 7 |
from typing import Any, Iterable
|
|
|
|
| 29 |
PlaceReference,
|
| 30 |
)
|
| 31 |
from app.modules.places.domain.errors import ClarificationStateMismatchError
|
| 32 |
+
from app.modules.places.infrastructure.bert_intent_extractor import (
|
| 33 |
+
BertIntentExtractionError,
|
| 34 |
+
BertPlaceIntentExtractor,
|
| 35 |
+
IntentFrame,
|
| 36 |
+
IntentSpan,
|
| 37 |
+
)
|
| 38 |
from app.shared.nlp.preprocessing.text import (
|
| 39 |
prepare_for_embedding,
|
| 40 |
tokenize_for_embeddings,
|
|
|
|
| 57 |
r"\b(?:sin|excepto|evita(?:r)?|no\s+quiero|que\s+no\s+(?:sea|tenga))\s+"
|
| 58 |
r"(?P<value>.+?)(?="
|
| 59 |
r"\s+cerca\s+(?:de\s+la|del|de)\b|[,;]|"
|
| 60 |
+
r"\s+(?:pero|aunque|con)\b|\s+y\s+(?:con|que|quiero|busco)\b|$)"
|
| 61 |
)
|
| 62 |
_CURRENT_LOCATION_VALUES = {"mi", "aqui", "donde estoy", "mi ubicacion"}
|
| 63 |
+
_DETERMINISTIC_INTENT_VERSION = "deterministic-open-v2"
|
| 64 |
+
_RADIUS_VALUE_PATTERN = re.compile(
|
| 65 |
+
r"(?P<value>\d+(?:[.,]\d+)?)\s*"
|
| 66 |
+
r"(?P<unit>km|kilometros?|m|metros?)\b"
|
| 67 |
+
)
|
| 68 |
+
_LOGGER = logging.getLogger(__name__)
|
| 69 |
_GENERIC_REQUEST_TOKENS = {
|
| 70 |
"dame",
|
| 71 |
"favor",
|
|
|
|
| 184 |
)
|
| 185 |
|
| 186 |
|
| 187 |
+
@dataclass(frozen=True)
|
| 188 |
+
class _ContextualSignals:
|
| 189 |
+
frame: IntentFrame | None
|
| 190 |
+
category_phrases: tuple[str, ...] = ()
|
| 191 |
+
preferences: tuple[str, ...] = ()
|
| 192 |
+
exclusions: tuple[str, ...] = ()
|
| 193 |
+
location: str | None = None
|
| 194 |
+
reference: str | None = None
|
| 195 |
+
radius_meters: int | None = None
|
| 196 |
+
category_confidence: float | None = None
|
| 197 |
+
intent_model_version: str = _DETERMINISTIC_INTENT_VERSION
|
| 198 |
+
|
| 199 |
+
|
| 200 |
class DeterministicPlaceChatIntentParser:
|
| 201 |
def __init__(
|
| 202 |
self,
|
| 203 |
taxonomy: PlaceChatTaxonomy | None = None,
|
| 204 |
activity_classifier: PlaceActivityClassifier | None = None,
|
| 205 |
+
contextual_extractor: BertPlaceIntentExtractor | None = None,
|
| 206 |
) -> None:
|
| 207 |
self._taxonomy = taxonomy or load_place_chat_taxonomy()
|
| 208 |
self._activity_classifier = activity_classifier
|
| 209 |
+
self._contextual_extractor = contextual_extractor
|
| 210 |
|
| 211 |
def parse(
|
| 212 |
self,
|
|
|
|
| 216 |
clarification_choice: ClarificationChoice | None = None,
|
| 217 |
) -> ParsedPlaceChatIntent:
|
| 218 |
normalized = prepare_for_embedding(message)
|
|
|
|
| 219 |
if clarification_choice is not None:
|
| 220 |
return self._resolve_structured_clarification(
|
| 221 |
choice=clarification_choice,
|
| 222 |
state=state,
|
| 223 |
has_user_location=has_user_location,
|
| 224 |
)
|
| 225 |
+
contextual = self._contextual_signals(message)
|
| 226 |
+
# Token classification is multi-label and a valid frame can still be
|
| 227 |
+
# incomplete. Strip both model-detected context spans and any remaining
|
| 228 |
+
# high-precision legacy context before building the retrieval query. The
|
| 229 |
+
# category itself stays model/open-vocabulary first; taxonomy aliases are
|
| 230 |
+
# deliberately not restored for a successful BERT frame.
|
| 231 |
+
target_clause = self._target_clause(
|
| 232 |
+
_remove_contextual_spans(normalized, contextual.frame)
|
| 233 |
+
)
|
| 234 |
if state.pending_clarification:
|
| 235 |
pending = ensure_legacy_pending_options(state.pending_clarification)
|
| 236 |
state = replace(state, pending_clarification=pending)
|
| 237 |
+
categories = self._contextual_categories(contextual, target_clause)
|
| 238 |
inferred = (
|
| 239 |
+
self._inferred_activity_category(
|
| 240 |
+
target_clause,
|
| 241 |
+
allow_lexical_defaults=contextual.frame is None,
|
| 242 |
+
)
|
| 243 |
if not categories
|
| 244 |
else None
|
| 245 |
)
|
|
|
|
| 259 |
clear_pending_clarification=True,
|
| 260 |
),
|
| 261 |
)
|
| 262 |
+
return replace(
|
| 263 |
+
self._resolve_pending_clarification(
|
| 264 |
+
normalized=normalized,
|
| 265 |
+
state=state,
|
| 266 |
+
has_user_location=has_user_location,
|
| 267 |
+
),
|
| 268 |
+
intent_model_version=contextual.intent_model_version,
|
| 269 |
)
|
| 270 |
|
| 271 |
+
# Fuse slots independently. A CATEGORY-only frame must not erase a
|
| 272 |
+
# LOCATION, RADIUS, REFERENCE, preference, or exclusion that the model
|
| 273 |
+
# omitted. These narrow extractors are rollout fallbacks, not gates.
|
| 274 |
+
reference_text = contextual.reference or self._extract_reference(normalized)
|
| 275 |
+
location_text = contextual.location or self._extract_location(normalized)
|
| 276 |
+
radius_meters = (
|
| 277 |
+
contextual.radius_meters
|
| 278 |
+
if contextual.radius_meters is not None
|
| 279 |
+
else self._extract_radius_meters(normalized)
|
| 280 |
+
)
|
| 281 |
excluded_texts = tuple(
|
| 282 |
match.group("value").strip(" ,.;")
|
| 283 |
for match in _EXCLUSION_PATTERN.finditer(normalized)
|
|
|
|
| 286 |
# Category words inside references, exclusions, or geographic anchors
|
| 287 |
# do not describe the requested result type. In "cafeteria cerca del
|
| 288 |
# parque", only cafe is a hard category.
|
| 289 |
+
categories = self._contextual_categories(contextual, target_clause)
|
| 290 |
if len(categories) > 1:
|
| 291 |
pending = new_category_clarification(categories)
|
| 292 |
+
return replace(
|
| 293 |
+
self._clarification(
|
| 294 |
+
pending=pending,
|
| 295 |
+
unresolved=("target_category",),
|
| 296 |
+
state=state,
|
| 297 |
+
has_user_location=has_user_location,
|
| 298 |
+
),
|
| 299 |
+
intent_model_version=contextual.intent_model_version,
|
| 300 |
)
|
| 301 |
|
| 302 |
explicit_category = categories[0] if categories else None
|
| 303 |
inferred = (
|
| 304 |
+
self._inferred_activity_category(
|
| 305 |
+
target_clause,
|
| 306 |
+
allow_lexical_defaults=contextual.frame is None,
|
| 307 |
+
)
|
| 308 |
if explicit_category is None
|
| 309 |
else None
|
| 310 |
)
|
| 311 |
+
category_alternatives = (
|
| 312 |
+
self._ranked_activity_alternatives(target_clause)
|
| 313 |
+
if explicit_category is None
|
| 314 |
+
else ()
|
| 315 |
+
)
|
| 316 |
inferred_category = inferred.category if inferred else None
|
| 317 |
requested_category = explicit_category or inferred_category
|
| 318 |
target_category = requested_category or state.target_category
|
|
|
|
| 333 |
if location_text:
|
| 334 |
location_text = _RADIUS_PATTERN.sub("", location_text).strip(" ,.;")
|
| 335 |
|
| 336 |
+
legacy_preference_text = normalized if not contextual.preferences else ""
|
| 337 |
+
detected_preferences = _ordered_unique(
|
| 338 |
+
(
|
| 339 |
+
*contextual.preferences,
|
| 340 |
+
*self._matched_preferences(legacy_preference_text),
|
| 341 |
+
)
|
| 342 |
+
)
|
| 343 |
+
legacy_excluded_texts = tuple(
|
| 344 |
+
excluded_text
|
| 345 |
for excluded_text in excluded_texts
|
| 346 |
+
if not any(
|
| 347 |
+
_contains_phrase(
|
| 348 |
+
prepare_for_embedding(excluded_text),
|
| 349 |
+
prepare_for_embedding(contextual_exclusion),
|
| 350 |
+
)
|
| 351 |
+
for contextual_exclusion in contextual.exclusions
|
| 352 |
+
)
|
| 353 |
+
)
|
| 354 |
+
explicit_exclusions = _ordered_unique(
|
| 355 |
+
(
|
| 356 |
+
*contextual.exclusions,
|
| 357 |
+
*(
|
| 358 |
+
exclusion
|
| 359 |
+
for excluded_text in legacy_excluded_texts
|
| 360 |
+
for exclusion in (
|
| 361 |
+
self._matched_preferences(excluded_text)
|
| 362 |
+
or (prepare_for_embedding(excluded_text),)
|
| 363 |
+
)
|
| 364 |
+
),
|
| 365 |
+
)
|
| 366 |
)
|
| 367 |
positive_preferences = tuple(
|
| 368 |
preference
|
|
|
|
| 403 |
else inherited_reference
|
| 404 |
)
|
| 405 |
hard_filters = dict(state.hard_filters)
|
| 406 |
+
if state.city and "city" not in hard_filters:
|
| 407 |
+
hard_filters["city"] = state.city
|
| 408 |
+
if state.state and "state" not in hard_filters:
|
| 409 |
+
hard_filters["state"] = state.state
|
| 410 |
if "economico" in merged_preferences or _contains_any(
|
| 411 |
target_clause,
|
| 412 |
("mas barato", "economico", "barato"),
|
| 413 |
):
|
| 414 |
hard_filters["price_preference"] = "lower"
|
| 415 |
|
| 416 |
+
# An unresolved category is not a reason to stop retrieval. Preserve the
|
| 417 |
+
# user's open-ended concept in ``semantic_query`` and let dense/lexical
|
| 418 |
+
# retrieval provide evidence. A downstream decision policy may still ask
|
| 419 |
+
# a clarification, but its options must come from model hypotheses or real
|
| 420 |
+
# candidates rather than a fixed menu.
|
| 421 |
+
category = (
|
| 422 |
+
self._taxonomy.category(target_category)
|
| 423 |
+
if target_category is not None
|
| 424 |
+
else None
|
| 425 |
+
)
|
| 426 |
+
inferred_category_values = (
|
| 427 |
+
tuple(inferred.category_values)
|
| 428 |
+
if inferred is not None
|
| 429 |
+
and inferred.category == target_category
|
| 430 |
+
and inferred.category_values
|
| 431 |
+
else ()
|
| 432 |
+
)
|
| 433 |
+
rehydrated_category_values = self._activity_category_values(
|
| 434 |
+
target_category
|
| 435 |
+
)
|
| 436 |
+
category_values = (
|
| 437 |
+
category.storage_values
|
| 438 |
+
if category is not None
|
| 439 |
+
else (
|
| 440 |
+
inferred_category_values
|
| 441 |
+
or rehydrated_category_values
|
| 442 |
+
or ((target_category,) if target_category else ())
|
| 443 |
)
|
| 444 |
+
)
|
|
|
|
|
|
|
| 445 |
compatible_category_values = (
|
| 446 |
category.compatible_storage_values if category else ()
|
| 447 |
)
|
|
|
|
| 450 |
if (
|
| 451 |
reference_text
|
| 452 |
and location_text
|
| 453 |
+
and prepare_for_embedding(location_text) not in _CURRENT_LOCATION_VALUES
|
| 454 |
):
|
| 455 |
+
return replace(
|
| 456 |
+
self._location_scope_clarification(
|
| 457 |
+
target_category=target_category,
|
| 458 |
+
category_values=category_values,
|
| 459 |
+
compatible_category_values=compatible_category_values,
|
| 460 |
+
category_evidence_terms=category_evidence_terms,
|
| 461 |
+
explicit_category=requested_category,
|
| 462 |
+
category_source=category_source,
|
| 463 |
+
hard_filters=hard_filters,
|
| 464 |
+
preferences=merged_preferences,
|
| 465 |
+
exclusions=merged_exclusions,
|
| 466 |
+
reference=reference,
|
| 467 |
+
reference_text=reference_text,
|
| 468 |
+
location_text=location_text,
|
| 469 |
+
radius_meters=radius_meters,
|
| 470 |
+
state=state,
|
| 471 |
+
has_user_location=has_user_location,
|
| 472 |
+
),
|
| 473 |
+
intent_model_version=contextual.intent_model_version,
|
| 474 |
)
|
| 475 |
|
| 476 |
location, explicit_location = self._location_intent(
|
|
|
|
| 491 |
)
|
| 492 |
category_query_term = self._category_query_term(target_clause, category)
|
| 493 |
semantic_parts = [
|
| 494 |
+
target_category or "",
|
| 495 |
category_query_term,
|
| 496 |
semantic_target,
|
| 497 |
*merged_preferences,
|
| 498 |
]
|
| 499 |
if reference and reference.entity:
|
| 500 |
semantic_parts.append(reference.entity)
|
| 501 |
+
semantic_query = (
|
| 502 |
+
" ".join(_ordered_unique(semantic_parts)).strip()
|
| 503 |
+
or target_clause
|
| 504 |
+
or normalized
|
| 505 |
+
)
|
| 506 |
|
| 507 |
patch = ConversationStatePatch(
|
| 508 |
target_category=requested_category,
|
|
|
|
| 520 |
clear_reference=category_changed and reference_text is None,
|
| 521 |
taxonomy_version=self._taxonomy.version,
|
| 522 |
)
|
| 523 |
+
if explicit_category and contextual.category_phrases:
|
| 524 |
+
confidence = contextual.category_confidence or contextual.frame.confidence
|
| 525 |
+
elif explicit_category:
|
| 526 |
confidence = 0.96
|
| 527 |
elif inferred_category:
|
| 528 |
confidence = inferred.confidence if inferred else 0.88
|
| 529 |
+
elif target_category is None:
|
| 530 |
+
# This confidence describes category resolution, not whether the
|
| 531 |
+
# query is searchable. Keeping it below the automatic-decision band
|
| 532 |
+
# allows candidate-derived clarification without suppressing recall.
|
| 533 |
+
confidence = 0.55
|
| 534 |
else:
|
| 535 |
confidence = 0.84
|
| 536 |
if reference_text:
|
|
|
|
| 550 |
compatible_category_values=compatible_category_values,
|
| 551 |
category_evidence_terms=category_evidence_terms,
|
| 552 |
category_source=category_source,
|
| 553 |
+
alternatives=category_alternatives,
|
| 554 |
+
unresolved=("target_category",) if target_category is None else (),
|
| 555 |
+
raw_category_phrase=(
|
| 556 |
+
contextual.category_phrases[0]
|
| 557 |
+
if contextual.category_phrases
|
| 558 |
+
else (
|
| 559 |
+
semantic_target or target_clause
|
| 560 |
+
if explicit_category is None
|
| 561 |
+
else explicit_category
|
| 562 |
+
)
|
| 563 |
+
),
|
| 564 |
+
intent_model_version=contextual.intent_model_version,
|
| 565 |
)
|
| 566 |
|
| 567 |
def _location_scope_clarification(
|
|
|
|
| 674 |
base_state = replace(state, pending_clarification=None)
|
| 675 |
if pending.kind in ("target_category", "intent_category"):
|
| 676 |
if self._taxonomy.category(selected.value) is None:
|
| 677 |
+
return self._resolve_open_category_choice(
|
| 678 |
+
value=selected.value,
|
| 679 |
+
state=base_state,
|
| 680 |
+
has_user_location=has_user_location,
|
| 681 |
)
|
| 682 |
resolved = self.parse(
|
| 683 |
message=self._category_message(selected.value),
|
|
|
|
| 913 |
return definition.aliases[0]
|
| 914 |
return category or ""
|
| 915 |
|
| 916 |
+
def _resolve_open_category_choice(
|
| 917 |
+
self,
|
| 918 |
+
value: str,
|
| 919 |
+
state: ConversationState,
|
| 920 |
+
has_user_location: bool,
|
| 921 |
+
) -> ParsedPlaceChatIntent:
|
| 922 |
+
category = " ".join(value.split()).strip()
|
| 923 |
+
if not category:
|
| 924 |
+
raise ClarificationStateMismatchError(
|
| 925 |
+
"clarification category must not be empty"
|
| 926 |
+
)
|
| 927 |
+
hard_filters = dict(state.hard_filters)
|
| 928 |
+
if state.city and "city" not in hard_filters:
|
| 929 |
+
hard_filters["city"] = state.city
|
| 930 |
+
if state.state and "state" not in hard_filters:
|
| 931 |
+
hard_filters["state"] = state.state
|
| 932 |
+
location, _ = self._location_intent(
|
| 933 |
+
location_text=None,
|
| 934 |
+
radius_meters=None,
|
| 935 |
+
state=state,
|
| 936 |
+
has_user_location=has_user_location,
|
| 937 |
+
)
|
| 938 |
+
semantic_parts = [category, *state.soft_preferences]
|
| 939 |
+
if state.reference and state.reference.entity:
|
| 940 |
+
semantic_parts.append(state.reference.entity)
|
| 941 |
+
category_values = _ordered_unique(
|
| 942 |
+
(category, *self._activity_category_values(category))
|
| 943 |
+
)
|
| 944 |
+
return ParsedPlaceChatIntent(
|
| 945 |
+
action="recommendations",
|
| 946 |
+
target_category=category,
|
| 947 |
+
category_values=category_values,
|
| 948 |
+
hard_filters=hard_filters,
|
| 949 |
+
soft_preferences=state.soft_preferences,
|
| 950 |
+
exclusions=state.exclusions,
|
| 951 |
+
reference=state.reference,
|
| 952 |
+
location=location,
|
| 953 |
+
semantic_query=" ".join(_ordered_unique(semantic_parts)).strip(),
|
| 954 |
+
confidence=1.0,
|
| 955 |
+
state_patch=ConversationStatePatch(
|
| 956 |
+
target_category=category,
|
| 957 |
+
hard_filters=(
|
| 958 |
+
hard_filters if hard_filters != state.hard_filters else None
|
| 959 |
+
),
|
| 960 |
+
clear_pending_clarification=True,
|
| 961 |
+
taxonomy_version=self._taxonomy.version,
|
| 962 |
+
),
|
| 963 |
+
category_evidence_terms=(category,),
|
| 964 |
+
category_source="explicit",
|
| 965 |
+
raw_category_phrase=category,
|
| 966 |
+
intent_model_version=(
|
| 967 |
+
f"{_DETERMINISTIC_INTENT_VERSION}+dynamic-clarification-v1"
|
| 968 |
+
),
|
| 969 |
+
)
|
| 970 |
+
|
| 971 |
def _location_intent(
|
| 972 |
self,
|
| 973 |
location_text: str | None,
|
|
|
|
| 975 |
state: ConversationState,
|
| 976 |
has_user_location: bool,
|
| 977 |
) -> tuple[LocationIntent, ExplicitTargetLocation | None]:
|
| 978 |
+
if (
|
| 979 |
+
location_text
|
| 980 |
+
and prepare_for_embedding(location_text) in _CURRENT_LOCATION_VALUES
|
| 981 |
+
):
|
| 982 |
return (
|
| 983 |
LocationIntent(
|
| 984 |
scope="user_current_location",
|
|
|
|
| 1022 |
LocationIntent(
|
| 1023 |
scope="user_current_location",
|
| 1024 |
source="user_current",
|
| 1025 |
+
radius_meters=radius_meters,
|
| 1026 |
+
strict_radius=radius_meters is not None,
|
| 1027 |
),
|
| 1028 |
None,
|
| 1029 |
)
|
| 1030 |
return (LocationIntent(scope="unresolved", source="none"), None)
|
| 1031 |
|
| 1032 |
+
def _contextual_signals(self, message: str) -> _ContextualSignals:
|
| 1033 |
+
extractor = self._contextual_extractor
|
| 1034 |
+
if extractor is None:
|
| 1035 |
+
return _ContextualSignals(frame=None)
|
| 1036 |
+
|
| 1037 |
+
try:
|
| 1038 |
+
frame = extractor.extract(message)
|
| 1039 |
+
except BertIntentExtractionError as exc:
|
| 1040 |
+
configured_version = getattr(
|
| 1041 |
+
extractor,
|
| 1042 |
+
"model_version",
|
| 1043 |
+
"unspecified",
|
| 1044 |
+
)
|
| 1045 |
+
_LOGGER.warning(
|
| 1046 |
+
"BERT place-intent extraction failed; using deterministic "
|
| 1047 |
+
"fallback (model_version=%s): %s",
|
| 1048 |
+
configured_version,
|
| 1049 |
+
exc,
|
| 1050 |
+
)
|
| 1051 |
+
return _ContextualSignals(
|
| 1052 |
+
frame=None,
|
| 1053 |
+
intent_model_version=(
|
| 1054 |
+
f"{_DETERMINISTIC_INTENT_VERSION}+"
|
| 1055 |
+
f"bert-fallback:{configured_version}"
|
| 1056 |
+
),
|
| 1057 |
+
)
|
| 1058 |
+
|
| 1059 |
+
category_spans = tuple(
|
| 1060 |
+
span
|
| 1061 |
+
for span in frame.by_type("CATEGORY")
|
| 1062 |
+
if span.polarity != "negative"
|
| 1063 |
+
)
|
| 1064 |
+
positive_preferences = tuple(
|
| 1065 |
+
span
|
| 1066 |
+
for span in frame.by_type("PREFERENCE")
|
| 1067 |
+
if span.polarity == "positive"
|
| 1068 |
+
)
|
| 1069 |
+
negative_spans = tuple(
|
| 1070 |
+
span
|
| 1071 |
+
for span in frame.spans
|
| 1072 |
+
if span.slot_type == "EXCLUSION" or span.polarity == "negative"
|
| 1073 |
+
)
|
| 1074 |
+
locations = frame.by_type("LOCATION")
|
| 1075 |
+
references = frame.by_type("REFERENCE")
|
| 1076 |
+
radii = frame.by_type("RADIUS")
|
| 1077 |
+
version = (
|
| 1078 |
+
f"bert-token:{frame.model_name}@{frame.model_version}+"
|
| 1079 |
+
f"{_DETERMINISTIC_INTENT_VERSION}"
|
| 1080 |
+
)
|
| 1081 |
+
return _ContextualSignals(
|
| 1082 |
+
frame=frame,
|
| 1083 |
+
category_phrases=_span_values(category_spans),
|
| 1084 |
+
preferences=_span_values(positive_preferences),
|
| 1085 |
+
exclusions=_span_values(negative_spans),
|
| 1086 |
+
location=_first_span_value(locations),
|
| 1087 |
+
reference=_first_span_value(references),
|
| 1088 |
+
radius_meters=(
|
| 1089 |
+
_radius_meters_from_text(radii[0].text) if radii else None
|
| 1090 |
+
),
|
| 1091 |
+
category_confidence=(
|
| 1092 |
+
max(span.confidence for span in category_spans)
|
| 1093 |
+
if category_spans
|
| 1094 |
+
else None
|
| 1095 |
+
),
|
| 1096 |
+
intent_model_version=version,
|
| 1097 |
+
)
|
| 1098 |
+
|
| 1099 |
+
def _contextual_categories(
|
| 1100 |
+
self,
|
| 1101 |
+
contextual: _ContextualSignals,
|
| 1102 |
+
target_clause: str,
|
| 1103 |
+
) -> tuple[str, ...]:
|
| 1104 |
+
if not contextual.category_phrases:
|
| 1105 |
+
return (
|
| 1106 |
+
self._matched_categories(target_clause)
|
| 1107 |
+
if contextual.frame is None
|
| 1108 |
+
else ()
|
| 1109 |
+
)
|
| 1110 |
+
|
| 1111 |
+
categories: list[str] = []
|
| 1112 |
+
for phrase in contextual.category_phrases:
|
| 1113 |
+
aligned = (
|
| 1114 |
+
self._activity_classifier.classify(phrase)
|
| 1115 |
+
if self._activity_classifier is not None
|
| 1116 |
+
else None
|
| 1117 |
+
)
|
| 1118 |
+
# Semantic alignment is optional and abstaining. Unknown values
|
| 1119 |
+
# pass through unchanged; no lexical alias table is consulted once
|
| 1120 |
+
# the contextual model has supplied a category span.
|
| 1121 |
+
categories.append(aligned.category if aligned is not None else phrase)
|
| 1122 |
+
return _ordered_unique(categories)
|
| 1123 |
+
|
| 1124 |
+
def _activity_category_values(
|
| 1125 |
+
self,
|
| 1126 |
+
target_category: str | None,
|
| 1127 |
+
) -> tuple[str, ...]:
|
| 1128 |
+
"""Rehydrate an open catalog concept on conversation continuations."""
|
| 1129 |
+
|
| 1130 |
+
if target_category is None or self._activity_classifier is None:
|
| 1131 |
+
return ()
|
| 1132 |
+
get_concept = getattr(self._activity_classifier, "get_concept", None)
|
| 1133 |
+
if callable(get_concept):
|
| 1134 |
+
concept = get_concept(target_category)
|
| 1135 |
+
if concept is not None:
|
| 1136 |
+
return tuple(getattr(concept, "storage_values", ()) or ())
|
| 1137 |
+
for concept in getattr(self._activity_classifier, "concepts", ()):
|
| 1138 |
+
if getattr(concept, "id", None) == target_category:
|
| 1139 |
+
return tuple(getattr(concept, "storage_values", ()) or ())
|
| 1140 |
+
return ()
|
| 1141 |
+
|
| 1142 |
def _matched_categories(self, normalized: str) -> tuple[str, ...]:
|
| 1143 |
return _ordered_unique(
|
| 1144 |
category.canonical
|
|
|
|
| 1167 |
def _inferred_activity_category(
|
| 1168 |
self,
|
| 1169 |
normalized: str,
|
| 1170 |
+
*,
|
| 1171 |
+
allow_lexical_defaults: bool = True,
|
| 1172 |
) -> PlaceCategoryInference | None:
|
| 1173 |
+
# Prefer the injected semantic/open-vocabulary model. Lexical activity
|
| 1174 |
+
# rules remain a compatibility fallback during rollout, never a gate.
|
| 1175 |
+
if self._activity_classifier is not None:
|
| 1176 |
+
inferred = self._activity_classifier.classify(normalized)
|
| 1177 |
+
if inferred is not None:
|
| 1178 |
+
return inferred
|
| 1179 |
+
if allow_lexical_defaults:
|
| 1180 |
+
for category, patterns in _ACTIVITY_CATEGORY_DEFAULTS:
|
| 1181 |
+
if _contains_any(normalized, patterns):
|
| 1182 |
+
return PlaceCategoryInference(
|
| 1183 |
+
category=category,
|
| 1184 |
+
confidence=0.88,
|
| 1185 |
+
source="lexical_activity",
|
| 1186 |
+
)
|
| 1187 |
+
return None
|
| 1188 |
+
|
| 1189 |
+
def _ranked_activity_alternatives(
|
| 1190 |
+
self,
|
| 1191 |
+
normalized: str,
|
| 1192 |
+
) -> tuple[IntentAlternative, ...]:
|
| 1193 |
if self._activity_classifier is None:
|
| 1194 |
+
return ()
|
| 1195 |
+
rank = getattr(self._activity_classifier, "rank", None)
|
| 1196 |
+
if not callable(rank):
|
| 1197 |
+
return ()
|
| 1198 |
+
matches = rank(normalized, limit=5)
|
| 1199 |
+
return tuple(
|
| 1200 |
+
IntentAlternative(
|
| 1201 |
+
key=str(match.concept_id),
|
| 1202 |
+
description=str(match.label),
|
| 1203 |
+
confidence=max(0.0, min(1.0, (float(match.score) + 1.0) / 2.0)),
|
| 1204 |
+
)
|
| 1205 |
+
for match in matches
|
| 1206 |
+
if getattr(match, "concept_id", None)
|
| 1207 |
+
)
|
| 1208 |
|
| 1209 |
@staticmethod
|
| 1210 |
def _target_clause(normalized: str) -> str:
|
|
|
|
| 1350 |
return bool(re.search(rf"(?<!\w){re.escape(phrase)}(?!\w)", text))
|
| 1351 |
|
| 1352 |
|
| 1353 |
+
def _remove_contextual_spans(
|
| 1354 |
+
target_clause: str,
|
| 1355 |
+
frame: IntentFrame | None,
|
| 1356 |
+
) -> str:
|
| 1357 |
+
if frame is None:
|
| 1358 |
+
return target_clause
|
| 1359 |
+
cleaned = target_clause
|
| 1360 |
+
for span in frame.spans:
|
| 1361 |
+
if (
|
| 1362 |
+
span.slot_type not in {"EXCLUSION", "LOCATION", "REFERENCE", "RADIUS"}
|
| 1363 |
+
and span.polarity != "negative"
|
| 1364 |
+
):
|
| 1365 |
+
continue
|
| 1366 |
+
normalized_value = prepare_for_embedding(span.text)
|
| 1367 |
+
if normalized_value:
|
| 1368 |
+
cleaned = re.sub(
|
| 1369 |
+
rf"(?<!\w){re.escape(normalized_value)}(?!\w)",
|
| 1370 |
+
" ",
|
| 1371 |
+
cleaned,
|
| 1372 |
+
)
|
| 1373 |
+
return " ".join(cleaned.split())
|
| 1374 |
+
|
| 1375 |
+
|
| 1376 |
+
def _span_values(spans: Iterable[IntentSpan]) -> tuple[str, ...]:
|
| 1377 |
+
return _ordered_unique(
|
| 1378 |
+
" ".join(span.text.split()).strip(" ,.;") for span in spans
|
| 1379 |
+
)
|
| 1380 |
+
|
| 1381 |
+
|
| 1382 |
+
def _first_span_value(spans: tuple[IntentSpan, ...]) -> str | None:
|
| 1383 |
+
values = _span_values(spans[:1])
|
| 1384 |
+
return values[0] if values else None
|
| 1385 |
+
|
| 1386 |
+
|
| 1387 |
+
def _radius_meters_from_text(value: str) -> int | None:
|
| 1388 |
+
match = _RADIUS_VALUE_PATTERN.search(prepare_for_embedding(value))
|
| 1389 |
+
if match is None:
|
| 1390 |
+
return None
|
| 1391 |
+
distance = float(match.group("value").replace(",", "."))
|
| 1392 |
+
if match.group("unit").startswith(("km", "kilometro")):
|
| 1393 |
+
distance *= 1000
|
| 1394 |
+
return max(1, min(50_000, int(round(distance))))
|
| 1395 |
+
|
| 1396 |
+
|
| 1397 |
def _contains_any(text: str, values: tuple[str, ...]) -> bool:
|
| 1398 |
return any(
|
| 1399 |
_contains_phrase(text, prepare_for_embedding(value))
|
app/modules/places/infrastructure/hybrid_chat_retriever.py
CHANGED
|
@@ -1,5 +1,7 @@
|
|
|
|
|
| 1 |
import json
|
| 2 |
import math
|
|
|
|
| 3 |
from typing import Sequence
|
| 4 |
|
| 5 |
from app.modules.places.application.ports.place_repository import PlaceVectorRepository
|
|
@@ -23,7 +25,12 @@ _EXACT_ENTITY_PREFERENCES = {"hello_kitty"}
|
|
| 23 |
|
| 24 |
|
| 25 |
class HybridContentPlaceChatRetriever:
|
| 26 |
-
"""Fuse semantic, lexical and facet signals over a bounded vector candidate pool.
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 27 |
|
| 28 |
def __init__(
|
| 29 |
self,
|
|
@@ -50,7 +57,7 @@ class HybridContentPlaceChatRetriever:
|
|
| 50 |
intent: ParsedPlaceChatIntent,
|
| 51 |
limit: int,
|
| 52 |
) -> Sequence[PlaceChatCandidate]:
|
| 53 |
-
if intent.action != "recommendations"
|
| 54 |
return []
|
| 55 |
if limit < 1:
|
| 56 |
return []
|
|
@@ -62,36 +69,40 @@ class HybridContentPlaceChatRetriever:
|
|
| 62 |
if cached is not None:
|
| 63 |
return cached
|
| 64 |
|
| 65 |
-
embedding =
|
| 66 |
-
|
| 67 |
-
|
| 68 |
-
(
|
| 69 |
-
*intent.category_values,
|
| 70 |
-
*intent.compatible_category_values,
|
| 71 |
-
)
|
| 72 |
-
)
|
| 73 |
)
|
| 74 |
filters = PlaceFilters(
|
| 75 |
-
city=
|
| 76 |
-
state=
|
| 77 |
-
|
|
|
|
|
|
|
|
|
|
| 78 |
price_range=_optional_string(intent.hard_filters.get("price_range")),
|
| 79 |
occasion=_optional_string(intent.hard_filters.get("occasion")),
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 80 |
is_active=True,
|
| 81 |
)
|
|
|
|
| 82 |
raw_candidates = list(
|
| 83 |
-
await self.
|
|
|
|
| 84 |
embedding=embedding,
|
| 85 |
filters=filters,
|
| 86 |
-
limit=
|
| 87 |
)
|
| 88 |
)
|
| 89 |
-
candidates =
|
| 90 |
-
candidate
|
| 91 |
-
for candidate in raw_candidates
|
| 92 |
-
if self._matches_hard_category(candidate, intent)
|
| 93 |
-
and not self._matches_exclusions(candidate, intent.exclusions)
|
| 94 |
-
]
|
| 95 |
if not candidates:
|
| 96 |
return []
|
| 97 |
|
|
@@ -111,26 +122,68 @@ class HybridContentPlaceChatRetriever:
|
|
| 111 |
b=self._b,
|
| 112 |
)
|
| 113 |
lexical = normalize_bm25(raw_lexical)
|
| 114 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 115 |
theme_score, match_level, reasons = self._facet_match(
|
| 116 |
candidate,
|
| 117 |
intent,
|
| 118 |
document_tokens,
|
|
|
|
|
|
|
| 119 |
)
|
| 120 |
-
|
| 121 |
semantic_score=semantic,
|
| 122 |
lexical_score=lexical,
|
| 123 |
theme_or_reference_score=theme_score,
|
| 124 |
)
|
| 125 |
-
|
| 126 |
-
|
| 127 |
-
|
| 128 |
)
|
| 129 |
-
|
| 130 |
-
|
| 131 |
-
|
| 132 |
-
|
| 133 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 134 |
ranked.append(
|
| 135 |
PlaceChatCandidate(
|
| 136 |
place_id=candidate.id,
|
|
@@ -141,16 +194,16 @@ class HybridContentPlaceChatRetriever:
|
|
| 141 |
lexical_score=lexical,
|
| 142 |
match_level=match_level,
|
| 143 |
matched_reasons=reasons,
|
| 144 |
-
metadata=
|
| 145 |
)
|
| 146 |
)
|
| 147 |
|
| 148 |
ranked.sort(
|
| 149 |
key=lambda item: (
|
| 150 |
-
_match_level_priority(item.match_level),
|
| 151 |
-item.content_score,
|
| 152 |
-item.semantic_score,
|
| 153 |
-item.lexical_score,
|
|
|
|
| 154 |
item.place_id,
|
| 155 |
)
|
| 156 |
)
|
|
@@ -168,6 +221,34 @@ class HybridContentPlaceChatRetriever:
|
|
| 168 |
self._cache.set(cache_key, result)
|
| 169 |
return result
|
| 170 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 171 |
@staticmethod
|
| 172 |
def _cache_key(
|
| 173 |
intent: ParsedPlaceChatIntent,
|
|
@@ -176,6 +257,7 @@ class HybridContentPlaceChatRetriever:
|
|
| 176 |
) -> str:
|
| 177 |
payload = {
|
| 178 |
"query": semantic_query,
|
|
|
|
| 179 |
"categories": intent.category_values,
|
| 180 |
"compatible_categories": intent.compatible_category_values,
|
| 181 |
"category_evidence_terms": intent.category_evidence_terms,
|
|
@@ -191,59 +273,88 @@ class HybridContentPlaceChatRetriever:
|
|
| 191 |
)
|
| 192 |
|
| 193 |
@staticmethod
|
| 194 |
-
def
|
| 195 |
candidate: PlaceCandidate,
|
| 196 |
intent: ParsedPlaceChatIntent,
|
| 197 |
-
) ->
|
| 198 |
-
|
| 199 |
-
return False
|
| 200 |
-
actual = prepare_for_embedding(candidate.category).replace(" ", "_")
|
| 201 |
exact = {
|
| 202 |
-
|
| 203 |
-
for value in intent.category_values
|
|
|
|
| 204 |
}
|
| 205 |
-
if actual in exact:
|
| 206 |
-
return True
|
| 207 |
compatible = {
|
| 208 |
-
|
| 209 |
for value in intent.compatible_category_values
|
|
|
|
| 210 |
}
|
| 211 |
-
|
| 212 |
-
|
| 213 |
-
|
| 214 |
-
|
| 215 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 216 |
for term in intent.category_evidence_terms
|
| 217 |
-
if (
|
| 218 |
)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 219 |
|
| 220 |
@staticmethod
|
| 221 |
-
def
|
| 222 |
candidate: PlaceCandidate,
|
| 223 |
exclusions: tuple[str, ...],
|
| 224 |
-
) ->
|
| 225 |
if not exclusions:
|
| 226 |
-
return
|
| 227 |
-
|
| 228 |
-
|
| 229 |
-
|
| 230 |
-
|
| 231 |
-
|
| 232 |
-
|
| 233 |
-
|
| 234 |
-
|
|
|
|
| 235 |
)
|
|
|
|
| 236 |
)
|
| 237 |
)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 238 |
|
| 239 |
@staticmethod
|
| 240 |
def _facet_match(
|
| 241 |
candidate: PlaceCandidate,
|
| 242 |
intent: ParsedPlaceChatIntent,
|
| 243 |
document_tokens: list[str],
|
|
|
|
|
|
|
| 244 |
) -> tuple[float, str, tuple[str, ...]]:
|
| 245 |
token_set = set(document_tokens)
|
| 246 |
-
reasons: list[str] = [
|
|
|
|
|
|
|
| 247 |
matched_preferences = 0
|
| 248 |
exact_preference = False
|
| 249 |
for preference in intent.soft_preferences:
|
|
@@ -268,11 +379,16 @@ class HybridContentPlaceChatRetriever:
|
|
| 268 |
if reference_exact:
|
| 269 |
reasons.append(intent.reference.entity)
|
| 270 |
|
| 271 |
-
|
| 272 |
-
|
| 273 |
-
|
| 274 |
-
|
| 275 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 276 |
|
| 277 |
|
| 278 |
def _match_level_priority(level: str) -> int:
|
|
@@ -286,6 +402,12 @@ def _optional_string(value: object) -> str | None:
|
|
| 286 |
return normalized or None
|
| 287 |
|
| 288 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 289 |
def _category_evidence_tokens(candidate: PlaceCandidate) -> set[str]:
|
| 290 |
# Do not inspect candidate.document here: indexed documents intentionally
|
| 291 |
# contain broad category profiles (for example, all `entertainment` records
|
|
@@ -310,6 +432,22 @@ def _as_evidence_text(value: object) -> str:
|
|
| 310 |
return str(value)
|
| 311 |
|
| 312 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 313 |
def _unit_score(value: object) -> float:
|
| 314 |
try:
|
| 315 |
score = float(value)
|
|
@@ -318,3 +456,13 @@ def _unit_score(value: object) -> float:
|
|
| 318 |
if not math.isfinite(score):
|
| 319 |
return 0.0
|
| 320 |
return max(0.0, min(1.0, score))
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
import asyncio
|
| 2 |
import json
|
| 3 |
import math
|
| 4 |
+
import re
|
| 5 |
from typing import Sequence
|
| 6 |
|
| 7 |
from app.modules.places.application.ports.place_repository import PlaceVectorRepository
|
|
|
|
| 25 |
|
| 26 |
|
| 27 |
class HybridContentPlaceChatRetriever:
|
| 28 |
+
"""Fuse semantic, lexical and facet signals over a bounded vector candidate pool.
|
| 29 |
+
|
| 30 |
+
Categories are deliberately treated as ranking evidence, not repository
|
| 31 |
+
filters. This keeps open-vocabulary queries useful even when the parser
|
| 32 |
+
cannot map the user's wording to a canonical category.
|
| 33 |
+
"""
|
| 34 |
|
| 35 |
def __init__(
|
| 36 |
self,
|
|
|
|
| 57 |
intent: ParsedPlaceChatIntent,
|
| 58 |
limit: int,
|
| 59 |
) -> Sequence[PlaceChatCandidate]:
|
| 60 |
+
if intent.action != "recommendations":
|
| 61 |
return []
|
| 62 |
if limit < 1:
|
| 63 |
return []
|
|
|
|
| 69 |
if cached is not None:
|
| 70 |
return cached
|
| 71 |
|
| 72 |
+
embedding = await asyncio.to_thread(
|
| 73 |
+
self._embedding_provider.embed_text,
|
| 74 |
+
semantic_query,
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 75 |
)
|
| 76 |
filters = PlaceFilters(
|
| 77 |
+
city=_optional_string(intent.hard_filters.get("city")),
|
| 78 |
+
state=_optional_string(intent.hard_filters.get("state")),
|
| 79 |
+
# A canonical category is a hypothesis, not a hard constraint. A
|
| 80 |
+
# category filter here would prevent semantic retrieval from ever
|
| 81 |
+
# seeing useful places whose source taxonomy differs from ours.
|
| 82 |
+
categories=None,
|
| 83 |
price_range=_optional_string(intent.hard_filters.get("price_range")),
|
| 84 |
occasion=_optional_string(intent.hard_filters.get("occasion")),
|
| 85 |
+
place_ids=(
|
| 86 |
+
tuple(
|
| 87 |
+
str(place_id)
|
| 88 |
+
for place_id in intent.hard_filters.get("place_ids", ())
|
| 89 |
+
if str(place_id).strip()
|
| 90 |
+
)
|
| 91 |
+
if intent.hard_filters.get("place_ids") is not None
|
| 92 |
+
else None
|
| 93 |
+
),
|
| 94 |
is_active=True,
|
| 95 |
)
|
| 96 |
+
candidate_pool_limit = min(max(limit * 8, 40), 120)
|
| 97 |
raw_candidates = list(
|
| 98 |
+
await self._search_repository(
|
| 99 |
+
query_text=intent.semantic_query.strip(),
|
| 100 |
embedding=embedding,
|
| 101 |
filters=filters,
|
| 102 |
+
limit=candidate_pool_limit,
|
| 103 |
)
|
| 104 |
)
|
| 105 |
+
candidates = raw_candidates
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 106 |
if not candidates:
|
| 107 |
return []
|
| 108 |
|
|
|
|
| 122 |
b=self._b,
|
| 123 |
)
|
| 124 |
lexical = normalize_bm25(raw_lexical)
|
| 125 |
+
repository_semantic = _optional_finite_score(
|
| 126 |
+
candidate.metadata.get("semantic_score")
|
| 127 |
+
)
|
| 128 |
+
repository_lexical = _optional_finite_score(
|
| 129 |
+
candidate.metadata.get("lexical_score")
|
| 130 |
+
)
|
| 131 |
+
has_repository_channel_scores = (
|
| 132 |
+
repository_semantic is not None or repository_lexical is not None
|
| 133 |
+
)
|
| 134 |
+
semantic = _unit_score(
|
| 135 |
+
repository_semantic
|
| 136 |
+
if has_repository_channel_scores
|
| 137 |
+
else candidate.score
|
| 138 |
+
)
|
| 139 |
+
if repository_lexical is not None:
|
| 140 |
+
lexical = max(lexical, normalize_bm25(repository_lexical))
|
| 141 |
+
category_score, category_match, category_reason = (
|
| 142 |
+
self._category_affinity(candidate, intent)
|
| 143 |
+
)
|
| 144 |
theme_score, match_level, reasons = self._facet_match(
|
| 145 |
candidate,
|
| 146 |
intent,
|
| 147 |
document_tokens,
|
| 148 |
+
category_score=category_score,
|
| 149 |
+
category_reason=category_reason,
|
| 150 |
)
|
| 151 |
+
base_content_score = self._weights.score(
|
| 152 |
semantic_score=semantic,
|
| 153 |
lexical_score=lexical,
|
| 154 |
theme_or_reference_score=theme_score,
|
| 155 |
)
|
| 156 |
+
exclusion_affinity, exclusion_matches = self._exclusion_affinity(
|
| 157 |
+
candidate,
|
| 158 |
+
intent.exclusions,
|
| 159 |
)
|
| 160 |
+
# Exclusions are negative ranking evidence, not a boolean gate. A
|
| 161 |
+
# source description may mention an excluded concept in a negated
|
| 162 |
+
# form ("sin ruido"), and removing that row would invert intent.
|
| 163 |
+
content_score = max(
|
| 164 |
+
0.0,
|
| 165 |
+
base_content_score - 0.35 * exclusion_affinity,
|
| 166 |
+
)
|
| 167 |
+
# Keep weak candidates so short or novel queries do not collapse to
|
| 168 |
+
# zero results. The old minimum remains a quality diagnostic for
|
| 169 |
+
# downstream confidence/clarification policy, rather than a gate.
|
| 170 |
+
metadata = dict(candidate.metadata)
|
| 171 |
+
metadata["retrieval_diagnostics"] = {
|
| 172 |
+
"category_affinity": round(category_score, 6),
|
| 173 |
+
"category_match": category_match,
|
| 174 |
+
"exclusion_affinity": round(exclusion_affinity, 6),
|
| 175 |
+
"exclusion_matches": list(exclusion_matches),
|
| 176 |
+
"content_quality": (
|
| 177 |
+
"sufficient"
|
| 178 |
+
if content_score >= self._minimum_content_score
|
| 179 |
+
else "weak"
|
| 180 |
+
),
|
| 181 |
+
"meets_minimum_content_score": (
|
| 182 |
+
content_score >= self._minimum_content_score
|
| 183 |
+
),
|
| 184 |
+
"minimum_content_score": self._minimum_content_score,
|
| 185 |
+
"query_token_count": len(query_tokens),
|
| 186 |
+
}
|
| 187 |
ranked.append(
|
| 188 |
PlaceChatCandidate(
|
| 189 |
place_id=candidate.id,
|
|
|
|
| 194 |
lexical_score=lexical,
|
| 195 |
match_level=match_level,
|
| 196 |
matched_reasons=reasons,
|
| 197 |
+
metadata=metadata,
|
| 198 |
)
|
| 199 |
)
|
| 200 |
|
| 201 |
ranked.sort(
|
| 202 |
key=lambda item: (
|
|
|
|
| 203 |
-item.content_score,
|
| 204 |
-item.semantic_score,
|
| 205 |
-item.lexical_score,
|
| 206 |
+
_match_level_priority(item.match_level),
|
| 207 |
item.place_id,
|
| 208 |
)
|
| 209 |
)
|
|
|
|
| 221 |
self._cache.set(cache_key, result)
|
| 222 |
return result
|
| 223 |
|
| 224 |
+
async def _search_repository(
|
| 225 |
+
self,
|
| 226 |
+
query_text: str,
|
| 227 |
+
embedding: list[float],
|
| 228 |
+
filters: PlaceFilters,
|
| 229 |
+
limit: int,
|
| 230 |
+
) -> Sequence[PlaceCandidate]:
|
| 231 |
+
"""Prefer an optional independent dense+lexical repository search.
|
| 232 |
+
|
| 233 |
+
``PlaceVectorRepository`` intentionally keeps its existing contract.
|
| 234 |
+
Repositories can opt into hybrid candidate generation duck-typically,
|
| 235 |
+
while mocks and current adapters continue through ``search``.
|
| 236 |
+
"""
|
| 237 |
+
|
| 238 |
+
hybrid_search = getattr(self._place_repository, "search_hybrid", None)
|
| 239 |
+
if callable(hybrid_search):
|
| 240 |
+
return await hybrid_search(
|
| 241 |
+
query_text=query_text,
|
| 242 |
+
embedding=embedding,
|
| 243 |
+
filters=filters,
|
| 244 |
+
limit=limit,
|
| 245 |
+
)
|
| 246 |
+
return await self._place_repository.search(
|
| 247 |
+
embedding=embedding,
|
| 248 |
+
filters=filters,
|
| 249 |
+
limit=limit,
|
| 250 |
+
)
|
| 251 |
+
|
| 252 |
@staticmethod
|
| 253 |
def _cache_key(
|
| 254 |
intent: ParsedPlaceChatIntent,
|
|
|
|
| 257 |
) -> str:
|
| 258 |
payload = {
|
| 259 |
"query": semantic_query,
|
| 260 |
+
"target_category": intent.target_category,
|
| 261 |
"categories": intent.category_values,
|
| 262 |
"compatible_categories": intent.compatible_category_values,
|
| 263 |
"category_evidence_terms": intent.category_evidence_terms,
|
|
|
|
| 273 |
)
|
| 274 |
|
| 275 |
@staticmethod
|
| 276 |
+
def _category_affinity(
|
| 277 |
candidate: PlaceCandidate,
|
| 278 |
intent: ParsedPlaceChatIntent,
|
| 279 |
+
) -> tuple[float, str, str | None]:
|
| 280 |
+
actual = _normalized_category(candidate.category)
|
|
|
|
|
|
|
| 281 |
exact = {
|
| 282 |
+
normalized
|
| 283 |
+
for value in (intent.target_category, *intent.category_values)
|
| 284 |
+
if (normalized := _normalized_category(value))
|
| 285 |
}
|
|
|
|
|
|
|
| 286 |
compatible = {
|
| 287 |
+
normalized
|
| 288 |
for value in intent.compatible_category_values
|
| 289 |
+
if (normalized := _normalized_category(value))
|
| 290 |
}
|
| 291 |
+
has_category_hypothesis = bool(exact or compatible)
|
| 292 |
+
if not has_category_hypothesis:
|
| 293 |
+
return 0.0, "not_requested", None
|
| 294 |
+
|
| 295 |
+
reason = intent.target_category or next(iter(intent.category_values), None)
|
| 296 |
+
if actual and actual in exact:
|
| 297 |
+
return 1.0, "exact", reason or candidate.category
|
| 298 |
+
|
| 299 |
+
candidate_evidence_tokens = _category_evidence_tokens(candidate)
|
| 300 |
+
has_specific_evidence = any(
|
| 301 |
+
term_tokens and term_tokens <= candidate_evidence_tokens
|
| 302 |
for term in intent.category_evidence_terms
|
| 303 |
+
if (term_tokens := set(tokenize(term)))
|
| 304 |
)
|
| 305 |
+
if actual and actual in compatible:
|
| 306 |
+
if has_specific_evidence:
|
| 307 |
+
return 0.80, "compatible_with_evidence", reason
|
| 308 |
+
return 0.35, "compatible", reason
|
| 309 |
+
if has_specific_evidence:
|
| 310 |
+
return 0.65, "textual_evidence", reason
|
| 311 |
+
return 0.0, "none", None
|
| 312 |
|
| 313 |
@staticmethod
|
| 314 |
+
def _exclusion_affinity(
|
| 315 |
candidate: PlaceCandidate,
|
| 316 |
exclusions: tuple[str, ...],
|
| 317 |
+
) -> tuple[float, tuple[str, ...]]:
|
| 318 |
if not exclusions:
|
| 319 |
+
return 0.0, ()
|
| 320 |
+
evidence = prepare_for_embedding(
|
| 321 |
+
" ".join(
|
| 322 |
+
value
|
| 323 |
+
for value in (
|
| 324 |
+
candidate.name,
|
| 325 |
+
candidate.category or "",
|
| 326 |
+
candidate.document or "",
|
| 327 |
+
_as_evidence_text(candidate.metadata.get("tags")),
|
| 328 |
+
_as_evidence_text(candidate.metadata.get("short_description")),
|
| 329 |
)
|
| 330 |
+
if value
|
| 331 |
)
|
| 332 |
)
|
| 333 |
+
document_tokens = set(tokenize(evidence))
|
| 334 |
+
matches: list[str] = []
|
| 335 |
+
for exclusion in exclusions:
|
| 336 |
+
normalized = prepare_for_embedding(exclusion.replace("_", " "))
|
| 337 |
+
exclusion_tokens = set(tokenize(normalized))
|
| 338 |
+
if not exclusion_tokens or not exclusion_tokens <= document_tokens:
|
| 339 |
+
continue
|
| 340 |
+
if _is_negated_in_evidence(evidence, normalized):
|
| 341 |
+
continue
|
| 342 |
+
matches.append(exclusion)
|
| 343 |
+
unique_matches = tuple(dict.fromkeys(matches))
|
| 344 |
+
return len(unique_matches) / len(exclusions), unique_matches
|
| 345 |
|
| 346 |
@staticmethod
|
| 347 |
def _facet_match(
|
| 348 |
candidate: PlaceCandidate,
|
| 349 |
intent: ParsedPlaceChatIntent,
|
| 350 |
document_tokens: list[str],
|
| 351 |
+
category_score: float = 0.0,
|
| 352 |
+
category_reason: str | None = None,
|
| 353 |
) -> tuple[float, str, tuple[str, ...]]:
|
| 354 |
token_set = set(document_tokens)
|
| 355 |
+
reasons: list[str] = []
|
| 356 |
+
if category_score > 0 and category_reason:
|
| 357 |
+
reasons.append(category_reason)
|
| 358 |
matched_preferences = 0
|
| 359 |
exact_preference = False
|
| 360 |
for preference in intent.soft_preferences:
|
|
|
|
| 379 |
if reference_exact:
|
| 380 |
reasons.append(intent.reference.entity)
|
| 381 |
|
| 382 |
+
facet_score = max(
|
| 383 |
+
category_score,
|
| 384 |
+
preference_score,
|
| 385 |
+
1.0 if reference_exact else 0.0,
|
| 386 |
+
)
|
| 387 |
+
if reference_exact or exact_preference or category_score >= 1.0:
|
| 388 |
+
return facet_score, "exact", tuple(dict.fromkeys(reasons))
|
| 389 |
+
if matched_preferences or category_score >= 0.50:
|
| 390 |
+
return facet_score, "family", tuple(dict.fromkeys(reasons))
|
| 391 |
+
return facet_score, "broad", tuple(dict.fromkeys(reasons))
|
| 392 |
|
| 393 |
|
| 394 |
def _match_level_priority(level: str) -> int:
|
|
|
|
| 402 |
return normalized or None
|
| 403 |
|
| 404 |
|
| 405 |
+
def _normalized_category(value: object) -> str:
|
| 406 |
+
if value is None:
|
| 407 |
+
return ""
|
| 408 |
+
return prepare_for_embedding(str(value)).replace(" ", "_")
|
| 409 |
+
|
| 410 |
+
|
| 411 |
def _category_evidence_tokens(candidate: PlaceCandidate) -> set[str]:
|
| 412 |
# Do not inspect candidate.document here: indexed documents intentionally
|
| 413 |
# contain broad category profiles (for example, all `entertainment` records
|
|
|
|
| 432 |
return str(value)
|
| 433 |
|
| 434 |
|
| 435 |
+
def _is_negated_in_evidence(evidence: str, concept: str) -> bool:
|
| 436 |
+
"""Detect common source-side negation without business-category rules."""
|
| 437 |
+
|
| 438 |
+
escaped = re.escape(concept).replace(r"\ ", r"\s+")
|
| 439 |
+
negation = (
|
| 440 |
+
r"(?:sin|libre\s+de|no\s+(?:hay|tiene|ofrece)|"
|
| 441 |
+
r"evita(?:r)?|prohibid[oa]s?)"
|
| 442 |
+
)
|
| 443 |
+
return bool(
|
| 444 |
+
re.search(
|
| 445 |
+
rf"\b{negation}\s+(?:\w+\s+){{0,2}}{escaped}\b",
|
| 446 |
+
evidence,
|
| 447 |
+
)
|
| 448 |
+
)
|
| 449 |
+
|
| 450 |
+
|
| 451 |
def _unit_score(value: object) -> float:
|
| 452 |
try:
|
| 453 |
score = float(value)
|
|
|
|
| 456 |
if not math.isfinite(score):
|
| 457 |
return 0.0
|
| 458 |
return max(0.0, min(1.0, score))
|
| 459 |
+
|
| 460 |
+
|
| 461 |
+
def _optional_finite_score(value: object) -> float | None:
|
| 462 |
+
if value is None:
|
| 463 |
+
return None
|
| 464 |
+
try:
|
| 465 |
+
score = float(value)
|
| 466 |
+
except (TypeError, ValueError):
|
| 467 |
+
return None
|
| 468 |
+
return score if math.isfinite(score) else None
|
app/modules/places/infrastructure/open_vocabulary_category_classifier.py
ADDED
|
@@ -0,0 +1,368 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""Open-vocabulary place category alignment over a dynamic concept catalog."""
|
| 2 |
+
|
| 3 |
+
from __future__ import annotations
|
| 4 |
+
|
| 5 |
+
import math
|
| 6 |
+
import threading
|
| 7 |
+
from collections.abc import Mapping, Sequence
|
| 8 |
+
from dataclasses import dataclass
|
| 9 |
+
from typing import Any
|
| 10 |
+
|
| 11 |
+
from app.modules.places.domain.chat_intent import PlaceCategoryInference
|
| 12 |
+
from app.shared.nlp.embeddings.base import EmbeddingProvider
|
| 13 |
+
|
| 14 |
+
|
| 15 |
+
@dataclass(frozen=True)
|
| 16 |
+
class PlaceCategoryConcept:
|
| 17 |
+
"""A category concept supplied by configuration, a database, or an API."""
|
| 18 |
+
|
| 19 |
+
id: str
|
| 20 |
+
label: str
|
| 21 |
+
description: str
|
| 22 |
+
examples: tuple[str, ...] = ()
|
| 23 |
+
storage_values: tuple[str, ...] = ()
|
| 24 |
+
|
| 25 |
+
def __post_init__(self) -> None:
|
| 26 |
+
object.__setattr__(self, "id", _required_text(self.id, "id"))
|
| 27 |
+
object.__setattr__(self, "label", _required_text(self.label, "label"))
|
| 28 |
+
object.__setattr__(
|
| 29 |
+
self,
|
| 30 |
+
"description",
|
| 31 |
+
_required_text(self.description, "description"),
|
| 32 |
+
)
|
| 33 |
+
object.__setattr__(
|
| 34 |
+
self,
|
| 35 |
+
"examples",
|
| 36 |
+
_clean_text_values(self.examples, "examples"),
|
| 37 |
+
)
|
| 38 |
+
object.__setattr__(
|
| 39 |
+
self,
|
| 40 |
+
"storage_values",
|
| 41 |
+
_clean_text_values(self.storage_values, "storage_values"),
|
| 42 |
+
)
|
| 43 |
+
|
| 44 |
+
|
| 45 |
+
@dataclass(frozen=True)
|
| 46 |
+
class PlaceCategoryMatch:
|
| 47 |
+
"""One semantic catalog match, including separation from the next match."""
|
| 48 |
+
|
| 49 |
+
concept_id: str
|
| 50 |
+
label: str
|
| 51 |
+
description: str
|
| 52 |
+
storage_values: tuple[str, ...]
|
| 53 |
+
score: float
|
| 54 |
+
margin: float
|
| 55 |
+
|
| 56 |
+
|
| 57 |
+
@dataclass(frozen=True)
|
| 58 |
+
class _IndexedConcept:
|
| 59 |
+
concept: PlaceCategoryConcept
|
| 60 |
+
vectors: tuple[tuple[float, ...], ...]
|
| 61 |
+
|
| 62 |
+
|
| 63 |
+
ConceptInput = PlaceCategoryConcept | Mapping[str, Any]
|
| 64 |
+
|
| 65 |
+
|
| 66 |
+
class OpenVocabularyPlaceCategoryClassifier:
|
| 67 |
+
"""Ranks injected concepts without a closed classifier label head.
|
| 68 |
+
|
| 69 |
+
Concept vectors are built on first use, so constructing this component does
|
| 70 |
+
not force a transformer model to load during application startup. ``rank``
|
| 71 |
+
always exposes evidence; ``classify`` applies confidence thresholds and is
|
| 72 |
+
compatible with ``PlaceActivityClassifier``.
|
| 73 |
+
"""
|
| 74 |
+
|
| 75 |
+
def __init__(
|
| 76 |
+
self,
|
| 77 |
+
concepts: Sequence[ConceptInput],
|
| 78 |
+
embedding_provider: EmbeddingProvider,
|
| 79 |
+
*,
|
| 80 |
+
concept_embedding_provider: EmbeddingProvider | None = None,
|
| 81 |
+
minimum_similarity: float = 0.44,
|
| 82 |
+
minimum_margin: float = 0.04,
|
| 83 |
+
) -> None:
|
| 84 |
+
if not -1.0 <= minimum_similarity <= 1.0:
|
| 85 |
+
raise ValueError("minimum_similarity must be between -1 and 1")
|
| 86 |
+
if not 0.0 <= minimum_margin <= 2.0:
|
| 87 |
+
raise ValueError("minimum_margin must be between 0 and 2")
|
| 88 |
+
|
| 89 |
+
prepared_concepts = tuple(_coerce_concept(concept) for concept in concepts)
|
| 90 |
+
duplicate_ids = _duplicate_concept_ids(prepared_concepts)
|
| 91 |
+
if duplicate_ids:
|
| 92 |
+
raise ValueError(
|
| 93 |
+
"concept ids must be unique (case-insensitive): "
|
| 94 |
+
+ ", ".join(duplicate_ids)
|
| 95 |
+
)
|
| 96 |
+
|
| 97 |
+
self._concepts = prepared_concepts
|
| 98 |
+
self._concepts_by_id = {
|
| 99 |
+
concept.id.casefold(): concept for concept in prepared_concepts
|
| 100 |
+
}
|
| 101 |
+
self._embedding_provider = embedding_provider
|
| 102 |
+
self._concept_embedding_provider = (
|
| 103 |
+
concept_embedding_provider or embedding_provider
|
| 104 |
+
)
|
| 105 |
+
self._minimum_similarity = minimum_similarity
|
| 106 |
+
self._minimum_margin = minimum_margin
|
| 107 |
+
self._index: tuple[_IndexedConcept, ...] | None = None
|
| 108 |
+
self._embedding_dimension: int | None = None
|
| 109 |
+
self._index_lock = threading.Lock()
|
| 110 |
+
|
| 111 |
+
@property
|
| 112 |
+
def concepts(self) -> tuple[PlaceCategoryConcept, ...]:
|
| 113 |
+
return self._concepts
|
| 114 |
+
|
| 115 |
+
@property
|
| 116 |
+
def is_indexed(self) -> bool:
|
| 117 |
+
return self._index is not None
|
| 118 |
+
|
| 119 |
+
def get_concept(self, concept_id: str) -> PlaceCategoryConcept | None:
|
| 120 |
+
"""Resolve persisted concept IDs without rerunning classification."""
|
| 121 |
+
|
| 122 |
+
if not isinstance(concept_id, str):
|
| 123 |
+
return None
|
| 124 |
+
return self._concepts_by_id.get(concept_id.strip().casefold())
|
| 125 |
+
|
| 126 |
+
def rank(self, text: str, limit: int = 3) -> tuple[PlaceCategoryMatch, ...]:
|
| 127 |
+
"""Return semantic top-k matches without suppressing low-confidence rows."""
|
| 128 |
+
|
| 129 |
+
if isinstance(limit, bool) or not isinstance(limit, int) or limit <= 0:
|
| 130 |
+
raise ValueError("limit must be a positive integer")
|
| 131 |
+
if not isinstance(text, str):
|
| 132 |
+
raise TypeError(f"text must be str, got {type(text).__name__}")
|
| 133 |
+
query_text = text.strip()
|
| 134 |
+
if not query_text or not self._concepts:
|
| 135 |
+
return ()
|
| 136 |
+
|
| 137 |
+
query_vector = _validated_vector(
|
| 138 |
+
self._embedding_provider.embed_text(query_text),
|
| 139 |
+
context="query",
|
| 140 |
+
)
|
| 141 |
+
if query_vector is None:
|
| 142 |
+
return ()
|
| 143 |
+
|
| 144 |
+
index = self._ensure_index()
|
| 145 |
+
if not index:
|
| 146 |
+
return ()
|
| 147 |
+
if self._embedding_dimension != len(query_vector):
|
| 148 |
+
raise ValueError(
|
| 149 |
+
"query and concept embedding dimensions differ: "
|
| 150 |
+
f"query={len(query_vector)}, concepts={self._embedding_dimension}"
|
| 151 |
+
)
|
| 152 |
+
|
| 153 |
+
scores = sorted(
|
| 154 |
+
(
|
| 155 |
+
(
|
| 156 |
+
entry,
|
| 157 |
+
max(
|
| 158 |
+
_cosine_similarity(query_vector, prototype)
|
| 159 |
+
for prototype in entry.vectors
|
| 160 |
+
),
|
| 161 |
+
)
|
| 162 |
+
for entry in index
|
| 163 |
+
if entry.vectors
|
| 164 |
+
),
|
| 165 |
+
key=lambda item: (-item[1], item[0].concept.id.casefold()),
|
| 166 |
+
)
|
| 167 |
+
matches: list[PlaceCategoryMatch] = []
|
| 168 |
+
for position, (entry, score) in enumerate(scores[:limit]):
|
| 169 |
+
next_score = scores[position + 1][1] if position + 1 < len(scores) else -1.0
|
| 170 |
+
concept = entry.concept
|
| 171 |
+
matches.append(
|
| 172 |
+
PlaceCategoryMatch(
|
| 173 |
+
concept_id=concept.id,
|
| 174 |
+
label=concept.label,
|
| 175 |
+
description=concept.description,
|
| 176 |
+
storage_values=concept.storage_values,
|
| 177 |
+
score=score,
|
| 178 |
+
margin=max(0.0, score - next_score),
|
| 179 |
+
)
|
| 180 |
+
)
|
| 181 |
+
return tuple(matches)
|
| 182 |
+
|
| 183 |
+
def classify(self, text: str) -> PlaceCategoryInference | None:
|
| 184 |
+
"""Return the best concept only when its score and margin are sufficient."""
|
| 185 |
+
|
| 186 |
+
matches = self.rank(text, limit=2)
|
| 187 |
+
if not matches:
|
| 188 |
+
return None
|
| 189 |
+
best = matches[0]
|
| 190 |
+
if (
|
| 191 |
+
best.score < self._minimum_similarity
|
| 192 |
+
or best.margin < self._minimum_margin
|
| 193 |
+
):
|
| 194 |
+
return None
|
| 195 |
+
|
| 196 |
+
semantic_confidence = (best.score + 1.0) / 2.0
|
| 197 |
+
separation_confidence = min(1.0, best.margin / 0.5)
|
| 198 |
+
confidence = max(
|
| 199 |
+
0.0,
|
| 200 |
+
min(0.99, semantic_confidence * 0.85 + separation_confidence * 0.15),
|
| 201 |
+
)
|
| 202 |
+
return PlaceCategoryInference(
|
| 203 |
+
category=best.concept_id,
|
| 204 |
+
confidence=confidence,
|
| 205 |
+
source="semantic_activity",
|
| 206 |
+
category_values=best.storage_values,
|
| 207 |
+
label=best.label,
|
| 208 |
+
)
|
| 209 |
+
|
| 210 |
+
def _ensure_index(self) -> tuple[_IndexedConcept, ...]:
|
| 211 |
+
index = self._index
|
| 212 |
+
if index is not None:
|
| 213 |
+
return index
|
| 214 |
+
|
| 215 |
+
with self._index_lock:
|
| 216 |
+
index = self._index
|
| 217 |
+
if index is not None:
|
| 218 |
+
return index
|
| 219 |
+
|
| 220 |
+
texts: list[str] = []
|
| 221 |
+
owners: list[int] = []
|
| 222 |
+
for concept_index, concept in enumerate(self._concepts):
|
| 223 |
+
for semantic_text in _semantic_texts(concept):
|
| 224 |
+
texts.append(semantic_text)
|
| 225 |
+
owners.append(concept_index)
|
| 226 |
+
|
| 227 |
+
raw_vectors = self._concept_embedding_provider.embed_batch(texts)
|
| 228 |
+
if len(raw_vectors) != len(texts):
|
| 229 |
+
raise ValueError(
|
| 230 |
+
"embedding provider returned an unexpected number of concept "
|
| 231 |
+
f"vectors: returned={len(raw_vectors)}, expected={len(texts)}"
|
| 232 |
+
)
|
| 233 |
+
|
| 234 |
+
vectors_by_concept: list[list[tuple[float, ...]]] = [
|
| 235 |
+
[] for _ in self._concepts
|
| 236 |
+
]
|
| 237 |
+
embedding_dimension: int | None = None
|
| 238 |
+
for text_index, (owner, raw_vector) in enumerate(
|
| 239 |
+
zip(owners, raw_vectors)
|
| 240 |
+
):
|
| 241 |
+
vector = _validated_vector(
|
| 242 |
+
raw_vector,
|
| 243 |
+
context=f"concept text {text_index}",
|
| 244 |
+
)
|
| 245 |
+
if vector is None:
|
| 246 |
+
continue
|
| 247 |
+
if embedding_dimension is None:
|
| 248 |
+
embedding_dimension = len(vector)
|
| 249 |
+
elif len(vector) != embedding_dimension:
|
| 250 |
+
raise ValueError(
|
| 251 |
+
"concept embedding dimensions differ: "
|
| 252 |
+
f"expected={embedding_dimension}, returned={len(vector)}, "
|
| 253 |
+
f"text_index={text_index}"
|
| 254 |
+
)
|
| 255 |
+
vectors_by_concept[owner].append(vector)
|
| 256 |
+
|
| 257 |
+
index = tuple(
|
| 258 |
+
_IndexedConcept(concept=concept, vectors=tuple(vectors))
|
| 259 |
+
for concept, vectors in zip(self._concepts, vectors_by_concept)
|
| 260 |
+
if vectors
|
| 261 |
+
)
|
| 262 |
+
self._embedding_dimension = embedding_dimension
|
| 263 |
+
self._index = index
|
| 264 |
+
return index
|
| 265 |
+
|
| 266 |
+
|
| 267 |
+
def _coerce_concept(value: ConceptInput) -> PlaceCategoryConcept:
|
| 268 |
+
if isinstance(value, PlaceCategoryConcept):
|
| 269 |
+
return value
|
| 270 |
+
if not isinstance(value, Mapping):
|
| 271 |
+
raise TypeError(
|
| 272 |
+
"concept must be PlaceCategoryConcept or a mapping, got "
|
| 273 |
+
f"{type(value).__name__}"
|
| 274 |
+
)
|
| 275 |
+
missing = [key for key in ("id", "label", "description") if key not in value]
|
| 276 |
+
if missing:
|
| 277 |
+
raise ValueError("concept is missing required fields: " + ", ".join(missing))
|
| 278 |
+
return PlaceCategoryConcept(
|
| 279 |
+
id=value["id"],
|
| 280 |
+
label=value["label"],
|
| 281 |
+
description=value["description"],
|
| 282 |
+
examples=_value_sequence(value.get("examples", ()), "examples"),
|
| 283 |
+
storage_values=_value_sequence(
|
| 284 |
+
value.get("storage_values", ()), "storage_values"
|
| 285 |
+
),
|
| 286 |
+
)
|
| 287 |
+
|
| 288 |
+
|
| 289 |
+
def _value_sequence(value: Any, field_name: str) -> tuple[str, ...]:
|
| 290 |
+
if value is None:
|
| 291 |
+
return ()
|
| 292 |
+
if isinstance(value, str):
|
| 293 |
+
return (value,)
|
| 294 |
+
if not isinstance(value, Sequence):
|
| 295 |
+
raise TypeError(f"{field_name} must be a string or sequence of strings")
|
| 296 |
+
return tuple(value)
|
| 297 |
+
|
| 298 |
+
|
| 299 |
+
def _required_text(value: Any, field_name: str) -> str:
|
| 300 |
+
if not isinstance(value, str):
|
| 301 |
+
raise TypeError(f"{field_name} must be str, got {type(value).__name__}")
|
| 302 |
+
cleaned = " ".join(value.split())
|
| 303 |
+
if not cleaned:
|
| 304 |
+
raise ValueError(f"{field_name} must not be empty")
|
| 305 |
+
return cleaned
|
| 306 |
+
|
| 307 |
+
|
| 308 |
+
def _clean_text_values(values: Sequence[str], field_name: str) -> tuple[str, ...]:
|
| 309 |
+
cleaned: list[str] = []
|
| 310 |
+
seen: set[str] = set()
|
| 311 |
+
for index, value in enumerate(values):
|
| 312 |
+
item = _required_text(value, f"{field_name}[{index}]")
|
| 313 |
+
key = item.casefold()
|
| 314 |
+
if key not in seen:
|
| 315 |
+
seen.add(key)
|
| 316 |
+
cleaned.append(item)
|
| 317 |
+
return tuple(cleaned)
|
| 318 |
+
|
| 319 |
+
|
| 320 |
+
def _duplicate_concept_ids(
|
| 321 |
+
concepts: Sequence[PlaceCategoryConcept],
|
| 322 |
+
) -> tuple[str, ...]:
|
| 323 |
+
seen: set[str] = set()
|
| 324 |
+
duplicates: list[str] = []
|
| 325 |
+
for concept in concepts:
|
| 326 |
+
key = concept.id.casefold()
|
| 327 |
+
if key in seen:
|
| 328 |
+
duplicates.append(concept.id)
|
| 329 |
+
seen.add(key)
|
| 330 |
+
return tuple(duplicates)
|
| 331 |
+
|
| 332 |
+
|
| 333 |
+
def _semantic_texts(concept: PlaceCategoryConcept) -> tuple[str, ...]:
|
| 334 |
+
candidates = (
|
| 335 |
+
f"{concept.label}. {concept.description}",
|
| 336 |
+
concept.label,
|
| 337 |
+
concept.description,
|
| 338 |
+
*concept.examples,
|
| 339 |
+
)
|
| 340 |
+
return _clean_text_values(candidates, "semantic_texts")
|
| 341 |
+
|
| 342 |
+
|
| 343 |
+
def _validated_vector(
|
| 344 |
+
raw_vector: Sequence[float],
|
| 345 |
+
*,
|
| 346 |
+
context: str,
|
| 347 |
+
) -> tuple[float, ...] | None:
|
| 348 |
+
try:
|
| 349 |
+
vector = tuple(float(value) for value in raw_vector)
|
| 350 |
+
except (TypeError, ValueError) as exc:
|
| 351 |
+
raise ValueError(f"embedding for {context} is not numeric") from exc
|
| 352 |
+
if not vector or not any(value != 0.0 for value in vector):
|
| 353 |
+
return None
|
| 354 |
+
if not all(math.isfinite(value) for value in vector):
|
| 355 |
+
raise ValueError(f"embedding for {context} contains NaN or infinity")
|
| 356 |
+
return vector
|
| 357 |
+
|
| 358 |
+
|
| 359 |
+
def _cosine_similarity(
|
| 360 |
+
left: Sequence[float],
|
| 361 |
+
right: Sequence[float],
|
| 362 |
+
) -> float:
|
| 363 |
+
dot = sum(a * b for a, b in zip(left, right))
|
| 364 |
+
left_norm = math.sqrt(sum(value * value for value in left))
|
| 365 |
+
right_norm = math.sqrt(sum(value * value for value in right))
|
| 366 |
+
if left_norm == 0.0 or right_norm == 0.0:
|
| 367 |
+
return -1.0
|
| 368 |
+
return max(-1.0, min(1.0, dot / (left_norm * right_norm)))
|
app/modules/places/infrastructure/place_category_catalog.py
ADDED
|
@@ -0,0 +1,87 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""Load an open-vocabulary place concept catalog from data, not code rules."""
|
| 2 |
+
|
| 3 |
+
from __future__ import annotations
|
| 4 |
+
|
| 5 |
+
import json
|
| 6 |
+
from collections.abc import Iterable, Mapping
|
| 7 |
+
from pathlib import Path
|
| 8 |
+
from typing import Any
|
| 9 |
+
|
| 10 |
+
from app.modules.places.infrastructure.open_vocabulary_category_classifier import (
|
| 11 |
+
PlaceCategoryConcept,
|
| 12 |
+
)
|
| 13 |
+
from app.modules.places.infrastructure.place_semantic_document import PlaceTag
|
| 14 |
+
from app.shared.nlp.preprocessing.text import prepare_for_embedding
|
| 15 |
+
|
| 16 |
+
|
| 17 |
+
def load_place_category_concepts(
|
| 18 |
+
catalog_path: str | None,
|
| 19 |
+
*,
|
| 20 |
+
fallback_tags: Iterable[PlaceTag] = (),
|
| 21 |
+
) -> tuple[PlaceCategoryConcept, ...]:
|
| 22 |
+
"""Load concepts exported by the source system, or derive them from tag data.
|
| 23 |
+
|
| 24 |
+
The external catalog format is ``{"concepts": [...]}`` (a bare list is also
|
| 25 |
+
accepted). It can be refreshed independently of an application deployment.
|
| 26 |
+
The bundled tag inventory is only a backwards-compatible data fallback; no
|
| 27 |
+
category names, aliases or linguistic rules live in this module.
|
| 28 |
+
"""
|
| 29 |
+
|
| 30 |
+
if catalog_path:
|
| 31 |
+
path = Path(catalog_path).expanduser()
|
| 32 |
+
if not path.is_file():
|
| 33 |
+
raise FileNotFoundError(f"Places category catalog not found at {path}")
|
| 34 |
+
payload = json.loads(path.read_text(encoding="utf-8"))
|
| 35 |
+
records = payload.get("concepts") if isinstance(payload, Mapping) else payload
|
| 36 |
+
if not isinstance(records, list):
|
| 37 |
+
raise ValueError("Places category catalog must contain a concepts list")
|
| 38 |
+
return tuple(_concept_from_record(record) for record in records)
|
| 39 |
+
|
| 40 |
+
return _concepts_from_tags(fallback_tags)
|
| 41 |
+
|
| 42 |
+
|
| 43 |
+
def _concepts_from_tags(tags: Iterable[PlaceTag]) -> tuple[PlaceCategoryConcept, ...]:
|
| 44 |
+
concepts: list[PlaceCategoryConcept] = []
|
| 45 |
+
seen: set[str] = set()
|
| 46 |
+
for tag in tags:
|
| 47 |
+
if prepare_for_embedding(tag.category).replace(" ", "_") != "place_category":
|
| 48 |
+
continue
|
| 49 |
+
normalized = prepare_for_embedding(tag.name)
|
| 50 |
+
concept_id = normalized.replace(" ", "_")
|
| 51 |
+
if not concept_id or concept_id in seen:
|
| 52 |
+
continue
|
| 53 |
+
seen.add(concept_id)
|
| 54 |
+
concepts.append(
|
| 55 |
+
PlaceCategoryConcept(
|
| 56 |
+
id=concept_id,
|
| 57 |
+
label=tag.name,
|
| 58 |
+
description=f"Tipo de lugar registrado: {tag.name}",
|
| 59 |
+
storage_values=tuple(dict.fromkeys((tag.name, normalized, concept_id))),
|
| 60 |
+
)
|
| 61 |
+
)
|
| 62 |
+
return tuple(concepts)
|
| 63 |
+
|
| 64 |
+
|
| 65 |
+
def _concept_from_record(record: Any) -> PlaceCategoryConcept:
|
| 66 |
+
if not isinstance(record, Mapping):
|
| 67 |
+
raise ValueError("Each Places category concept must be an object")
|
| 68 |
+
try:
|
| 69 |
+
return PlaceCategoryConcept(
|
| 70 |
+
id=str(record["id"]),
|
| 71 |
+
label=str(record["label"]),
|
| 72 |
+
description=str(record["description"]),
|
| 73 |
+
examples=_text_tuple(record.get("examples")),
|
| 74 |
+
storage_values=_text_tuple(record.get("storage_values")),
|
| 75 |
+
)
|
| 76 |
+
except KeyError as exc:
|
| 77 |
+
raise ValueError(f"Missing Places category concept field: {exc.args[0]}") from exc
|
| 78 |
+
|
| 79 |
+
|
| 80 |
+
def _text_tuple(value: Any) -> tuple[str, ...]:
|
| 81 |
+
if value is None:
|
| 82 |
+
return ()
|
| 83 |
+
if isinstance(value, str):
|
| 84 |
+
return (value,)
|
| 85 |
+
if not isinstance(value, list):
|
| 86 |
+
raise ValueError("Concept examples and storage_values must be lists")
|
| 87 |
+
return tuple(str(item) for item in value)
|
app/modules/places/infrastructure/place_semantic_document.py
CHANGED
|
@@ -4,42 +4,16 @@ import json
|
|
| 4 |
from pathlib import Path
|
| 5 |
from typing import Any
|
| 6 |
|
| 7 |
-
from app.shared.nlp.embeddings.weighted_document import build_weighted_document
|
| 8 |
from app.shared.nlp.preprocessing.text import clean_text
|
| 9 |
|
| 10 |
|
| 11 |
PLACE_SEMANTIC_FIELD_WEIGHTS = {
|
| 12 |
-
"tags":
|
| 13 |
-
"category":
|
| 14 |
-
"description":
|
| 15 |
"name": 1,
|
| 16 |
}
|
| 17 |
-
PLACE_SEMANTIC_DOCUMENT_VERSION = "
|
| 18 |
-
|
| 19 |
-
|
| 20 |
-
# Broad categories from the main API are expanded into Spanish intent terms so
|
| 21 |
-
# sparse OSM records still have a useful semantic anchor.
|
| 22 |
-
CATEGORY_SEMANTIC_PROFILES = {
|
| 23 |
-
"restaurant": "restaurant restaurante comida gastronomia comer cena almuerzo desayuno",
|
| 24 |
-
"cafe": "cafe cafeteria bebidas desayuno postres conversar",
|
| 25 |
-
"bar": "bar bebidas cocteles cerveza amigos musica noche",
|
| 26 |
-
"nightlife": "nightlife vida nocturna noche baile musica bar fiesta",
|
| 27 |
-
"shopping": "shopping compras tiendas ropa calzado productos mercado centro comercial",
|
| 28 |
-
"lodging": "lodging alojamiento hotel hospedaje hostal dormir turismo viaje",
|
| 29 |
-
"park": "park parque naturaleza caminar paseo aire libre mascotas ejercicio",
|
| 30 |
-
"culture": "culture cultura museo arte historia biblioteca exposicion lectura",
|
| 31 |
-
"tourism": "tourism turismo atraccion visitar explorar paseo historia",
|
| 32 |
-
"sports": "sports deporte ejercicio entrenamiento gimnasio actividad fisica",
|
| 33 |
-
"community": "community comunidad convivencia reuniones centro comunitario actividades",
|
| 34 |
-
"family": "family familia ninos juegos convivencia actividades familiares",
|
| 35 |
-
"entertainment": "entertainment entretenimiento diversion juegos cine actividades",
|
| 36 |
-
"cinema": "cinema cine pelicula estreno sala de cine entretenimiento",
|
| 37 |
-
"library": "library biblioteca libros lectura estudio cultura",
|
| 38 |
-
"bakery": "bakery panaderia pan pasteleria reposteria",
|
| 39 |
-
"ice_cream": "ice cream heladeria helado postres dessert",
|
| 40 |
-
"market": "market mercado tianguis compras productos locales",
|
| 41 |
-
"outdoors": "outdoors aire libre mirador sendero naturaleza aventura",
|
| 42 |
-
}
|
| 43 |
|
| 44 |
|
| 45 |
@dataclass(frozen=True)
|
|
@@ -63,26 +37,28 @@ def build_place_semantic_document(
|
|
| 63 |
description: str,
|
| 64 |
resolved_tags: ResolvedPlaceTags,
|
| 65 |
) -> str:
|
| 66 |
-
|
| 67 |
-
|
| 68 |
-
|
| 69 |
-
|
| 70 |
-
|
| 71 |
-
|
| 72 |
-
|
| 73 |
-
|
| 74 |
-
|
| 75 |
-
(tags_text, PLACE_SEMANTIC_FIELD_WEIGHTS["tags"]),
|
| 76 |
-
]
|
| 77 |
)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 78 |
|
| 79 |
|
| 80 |
def semantic_category_text(category: Any) -> str:
|
| 81 |
raw_category = clean_text(str(category or "")).casefold().replace("_", " ")
|
| 82 |
if not raw_category:
|
| 83 |
return ""
|
| 84 |
-
|
| 85 |
-
return CATEGORY_SEMANTIC_PROFILES.get(profile_key, raw_category)
|
| 86 |
|
| 87 |
|
| 88 |
def resolve_place_tags(value: Any) -> ResolvedPlaceTags:
|
|
|
|
| 4 |
from pathlib import Path
|
| 5 |
from typing import Any
|
| 6 |
|
|
|
|
| 7 |
from app.shared.nlp.preprocessing.text import clean_text
|
| 8 |
|
| 9 |
|
| 10 |
PLACE_SEMANTIC_FIELD_WEIGHTS = {
|
| 11 |
+
"tags": 1,
|
| 12 |
+
"category": 1,
|
| 13 |
+
"description": 1,
|
| 14 |
"name": 1,
|
| 15 |
}
|
| 16 |
+
PLACE_SEMANTIC_DOCUMENT_VERSION = "structured-place-v3"
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 17 |
|
| 18 |
|
| 19 |
@dataclass(frozen=True)
|
|
|
|
| 37 |
description: str,
|
| 38 |
resolved_tags: ResolvedPlaceTags,
|
| 39 |
) -> str:
|
| 40 |
+
# Transformer encoders use context and sentence structure; repeating tokens
|
| 41 |
+
# to simulate weights (the old FastText strategy) distorts that context.
|
| 42 |
+
# Keep every source value once and expose its role explicitly instead.
|
| 43 |
+
fields = (
|
| 44 |
+
("Nombre", clean_text(name)),
|
| 45 |
+
("Tipo registrado", semantic_category_text(category)),
|
| 46 |
+
("Descripcion", clean_text(description)),
|
| 47 |
+
("Etiquetas", " ".join(resolved_tags.names)),
|
| 48 |
+
("Familias de etiquetas", " ".join(resolved_tags.categories)),
|
|
|
|
|
|
|
| 49 |
)
|
| 50 |
+
return " ".join(
|
| 51 |
+
f"{label}: {value}."
|
| 52 |
+
for label, value in fields
|
| 53 |
+
if value
|
| 54 |
+
).strip()
|
| 55 |
|
| 56 |
|
| 57 |
def semantic_category_text(category: Any) -> str:
|
| 58 |
raw_category = clean_text(str(category or "")).casefold().replace("_", " ")
|
| 59 |
if not raw_category:
|
| 60 |
return ""
|
| 61 |
+
return raw_category
|
|
|
|
| 62 |
|
| 63 |
|
| 64 |
def resolve_place_tags(value: Any) -> ResolvedPlaceTags:
|
app/modules/places/infrastructure/semantic_activity_classifier.py
CHANGED
|
@@ -1,204 +1,56 @@
|
|
| 1 |
-
|
| 2 |
-
from typing import Sequence
|
| 3 |
|
| 4 |
-
|
| 5 |
-
|
| 6 |
-
from
|
|
|
|
| 7 |
|
|
|
|
| 8 |
|
| 9 |
-
|
| 10 |
-
|
| 11 |
-
"comer comida restaurante",
|
| 12 |
-
"hambre tacos pizza sushi",
|
| 13 |
-
"desayunar almorzar cenar",
|
| 14 |
-
"antojo platillo cocina",
|
| 15 |
-
),
|
| 16 |
-
"cafe": (
|
| 17 |
-
"cafe cafeteria capuchino espresso",
|
| 18 |
-
"tomar cafe merendar conversar",
|
| 19 |
-
"trabajar laptop cafe",
|
| 20 |
-
),
|
| 21 |
-
"park": (
|
| 22 |
-
"parque picnic caminar relajarse",
|
| 23 |
-
"juegos infantiles areas verdes",
|
| 24 |
-
"pasear familia cesped",
|
| 25 |
-
),
|
| 26 |
-
"bar": (
|
| 27 |
-
"cerveza cocteles tragos bar",
|
| 28 |
-
"beber copas cantina",
|
| 29 |
-
),
|
| 30 |
-
"nightlife": (
|
| 31 |
-
"bailar fiesta discoteca antro",
|
| 32 |
-
"fiesta musica vida nocturna",
|
| 33 |
-
),
|
| 34 |
-
"culture": (
|
| 35 |
-
"museo arte exposicion cultura",
|
| 36 |
-
"historia galeria centro cultural",
|
| 37 |
-
),
|
| 38 |
-
"shopping": (
|
| 39 |
-
"comprar tiendas centro comercial",
|
| 40 |
-
"ropa regalos compras",
|
| 41 |
-
),
|
| 42 |
-
"sports": (
|
| 43 |
-
"ejercicio entrenar gimnasio deporte",
|
| 44 |
-
"futbol cancha nadar fitness",
|
| 45 |
-
),
|
| 46 |
-
"bakery": (
|
| 47 |
-
"pan pasteles panaderia reposteria",
|
| 48 |
-
"comprar pan pastel",
|
| 49 |
-
),
|
| 50 |
-
"ice_cream": (
|
| 51 |
-
"helado postre heladeria dulce",
|
| 52 |
-
"comer helado nieve",
|
| 53 |
-
),
|
| 54 |
-
"cinema": (
|
| 55 |
-
"pelicula cine estreno",
|
| 56 |
-
"ver pelicula pantalla",
|
| 57 |
-
),
|
| 58 |
-
"library": (
|
| 59 |
-
"leer estudiar libros biblioteca",
|
| 60 |
-
"lectura investigacion biblioteca",
|
| 61 |
-
),
|
| 62 |
-
"market": (
|
| 63 |
-
"mercado tianguis productos locales",
|
| 64 |
-
"puestos comprar alimentos mercado",
|
| 65 |
-
),
|
| 66 |
-
"outdoors": (
|
| 67 |
-
"senderismo naturaleza montana mirador",
|
| 68 |
-
"aventura aire libre paisaje",
|
| 69 |
-
),
|
| 70 |
-
"lodging": (
|
| 71 |
-
"dormir hotel hospedaje alojamiento",
|
| 72 |
-
"pasar noche hostal",
|
| 73 |
-
),
|
| 74 |
-
}
|
| 75 |
|
| 76 |
-
|
| 77 |
-
|
| 78 |
-
|
| 79 |
-
|
| 80 |
-
|
| 81 |
-
|
|
|
|
| 82 |
|
| 83 |
|
| 84 |
class SemanticPlaceActivityClassifier:
|
| 85 |
-
"""
|
| 86 |
|
| 87 |
def __init__(
|
| 88 |
self,
|
| 89 |
embedding_provider: EmbeddingProvider,
|
|
|
|
|
|
|
|
|
|
| 90 |
minimum_similarity: float = 0.44,
|
| 91 |
minimum_margin: float = 0.04,
|
| 92 |
-
window_size: int =
|
| 93 |
) -> None:
|
| 94 |
-
if not
|
| 95 |
-
raise ValueError("minimum_similarity must be between -1 and 1")
|
| 96 |
-
if not 0.0 <= minimum_margin <= 2.0:
|
| 97 |
-
raise ValueError("minimum_margin must be between 0 and 2")
|
| 98 |
-
if window_size < 2:
|
| 99 |
raise ValueError("window_size must be at least 2")
|
| 100 |
-
|
| 101 |
-
|
| 102 |
-
|
| 103 |
-
|
| 104 |
-
|
| 105 |
-
|
| 106 |
-
category: tuple(
|
| 107 |
-
vector
|
| 108 |
-
for vector in embedding_provider.embed_batch(list(prototypes))
|
| 109 |
-
if _has_magnitude(vector)
|
| 110 |
-
)
|
| 111 |
-
for category, prototypes in _CATEGORY_PROTOTYPES.items()
|
| 112 |
-
}
|
| 113 |
-
|
| 114 |
-
def classify(self, text: str) -> PlaceCategoryInference | None:
|
| 115 |
-
tokens = tokenize_for_embeddings(text)
|
| 116 |
-
if not tokens:
|
| 117 |
-
return None
|
| 118 |
-
if len(tokens) == 1 and tokens[0] in _GENERIC_SINGLE_TOKEN_INTENTS:
|
| 119 |
-
return None
|
| 120 |
-
|
| 121 |
-
query_vectors = tuple(
|
| 122 |
-
vector
|
| 123 |
-
for vector in self._embedding_provider.embed_batch(
|
| 124 |
-
self._query_segments(tokens)
|
| 125 |
-
)
|
| 126 |
-
if _has_magnitude(vector)
|
| 127 |
)
|
| 128 |
-
if not query_vectors:
|
| 129 |
-
return None
|
| 130 |
|
| 131 |
-
|
| 132 |
-
|
| 133 |
-
|
| 134 |
-
category,
|
| 135 |
-
max(
|
| 136 |
-
_cosine_similarity(query, prototype)
|
| 137 |
-
for query in query_vectors
|
| 138 |
-
for prototype in prototypes
|
| 139 |
-
),
|
| 140 |
-
)
|
| 141 |
-
for category, prototypes in self._prototype_vectors.items()
|
| 142 |
-
if prototypes
|
| 143 |
-
),
|
| 144 |
-
key=lambda item: (-item[1], item[0]),
|
| 145 |
-
)
|
| 146 |
-
if not scores:
|
| 147 |
-
return None
|
| 148 |
|
| 149 |
-
|
| 150 |
-
|
| 151 |
-
margin = best_score - second_score
|
| 152 |
-
if (
|
| 153 |
-
best_score < self._minimum_similarity
|
| 154 |
-
or margin < self._minimum_margin
|
| 155 |
-
):
|
| 156 |
-
return None
|
| 157 |
-
|
| 158 |
-
confidence = min(
|
| 159 |
-
0.92,
|
| 160 |
-
max(
|
| 161 |
-
0.74,
|
| 162 |
-
0.74
|
| 163 |
-
+ (best_score - self._minimum_similarity) * 0.30
|
| 164 |
-
+ min(margin, 0.20) * 0.35,
|
| 165 |
-
),
|
| 166 |
-
)
|
| 167 |
-
return PlaceCategoryInference(
|
| 168 |
-
category=category,
|
| 169 |
-
confidence=confidence,
|
| 170 |
-
source="semantic_activity",
|
| 171 |
-
)
|
| 172 |
-
|
| 173 |
-
def _query_segments(self, tokens: list[str]) -> list[str]:
|
| 174 |
-
full_text = " ".join(tokens)
|
| 175 |
-
if len(tokens) <= self._window_size:
|
| 176 |
-
return [full_text]
|
| 177 |
-
|
| 178 |
-
segments = [full_text]
|
| 179 |
-
stride = max(2, self._window_size // 2)
|
| 180 |
-
for start in range(0, len(tokens), stride):
|
| 181 |
-
window = tokens[start : start + self._window_size]
|
| 182 |
-
if len(window) >= 2:
|
| 183 |
-
segments.append(" ".join(window))
|
| 184 |
-
if start + self._window_size >= len(tokens):
|
| 185 |
-
break
|
| 186 |
-
return list(dict.fromkeys(segments))
|
| 187 |
-
|
| 188 |
-
|
| 189 |
-
def _has_magnitude(vector: Sequence[float]) -> bool:
|
| 190 |
-
return any(float(value) != 0.0 for value in vector)
|
| 191 |
|
|
|
|
|
|
|
| 192 |
|
| 193 |
-
def
|
| 194 |
-
|
| 195 |
-
right: Sequence[float],
|
| 196 |
-
) -> float:
|
| 197 |
-
if len(left) != len(right) or not left:
|
| 198 |
-
return -1.0
|
| 199 |
-
dot = sum(float(a) * float(b) for a, b in zip(left, right))
|
| 200 |
-
left_norm = math.sqrt(sum(float(value) ** 2 for value in left))
|
| 201 |
-
right_norm = math.sqrt(sum(float(value) ** 2 for value in right))
|
| 202 |
-
if left_norm == 0.0 or right_norm == 0.0:
|
| 203 |
-
return -1.0
|
| 204 |
-
return max(-1.0, min(1.0, dot / (left_norm * right_norm)))
|
|
|
|
| 1 |
+
"""Backward-compatible facade for the data-driven category aligner.
|
|
|
|
| 2 |
|
| 3 |
+
The previous implementation embedded a fixed category/prototype dictionary in
|
| 4 |
+
Python. This facade keeps the import path for callers while requiring concepts
|
| 5 |
+
to come from configuration, the source API, or a catalog export.
|
| 6 |
+
"""
|
| 7 |
|
| 8 |
+
from __future__ import annotations
|
| 9 |
|
| 10 |
+
from collections.abc import Mapping, Sequence
|
| 11 |
+
from typing import Any
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 12 |
|
| 13 |
+
from app.modules.places.domain.chat_intent import PlaceCategoryInference
|
| 14 |
+
from app.modules.places.infrastructure.open_vocabulary_category_classifier import (
|
| 15 |
+
OpenVocabularyPlaceCategoryClassifier,
|
| 16 |
+
PlaceCategoryConcept,
|
| 17 |
+
PlaceCategoryMatch,
|
| 18 |
+
)
|
| 19 |
+
from app.shared.nlp.embeddings.base import EmbeddingProvider
|
| 20 |
|
| 21 |
|
| 22 |
class SemanticPlaceActivityClassifier:
|
| 23 |
+
"""Compatibility adapter with no built-in categories or lexical rules."""
|
| 24 |
|
| 25 |
def __init__(
|
| 26 |
self,
|
| 27 |
embedding_provider: EmbeddingProvider,
|
| 28 |
+
concepts: Sequence[PlaceCategoryConcept | Mapping[str, Any]],
|
| 29 |
+
*,
|
| 30 |
+
concept_embedding_provider: EmbeddingProvider | None = None,
|
| 31 |
minimum_similarity: float = 0.44,
|
| 32 |
minimum_margin: float = 0.04,
|
| 33 |
+
window_size: int | None = None,
|
| 34 |
) -> None:
|
| 35 |
+
if window_size is not None and window_size < 2:
|
|
|
|
|
|
|
|
|
|
|
|
|
| 36 |
raise ValueError("window_size must be at least 2")
|
| 37 |
+
self._delegate = OpenVocabularyPlaceCategoryClassifier(
|
| 38 |
+
concepts=concepts,
|
| 39 |
+
embedding_provider=embedding_provider,
|
| 40 |
+
concept_embedding_provider=concept_embedding_provider,
|
| 41 |
+
minimum_similarity=minimum_similarity,
|
| 42 |
+
minimum_margin=minimum_margin,
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 43 |
)
|
|
|
|
|
|
|
| 44 |
|
| 45 |
+
@property
|
| 46 |
+
def concepts(self) -> tuple[PlaceCategoryConcept, ...]:
|
| 47 |
+
return self._delegate.concepts
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 48 |
|
| 49 |
+
def get_concept(self, concept_id: str) -> PlaceCategoryConcept | None:
|
| 50 |
+
return self._delegate.get_concept(concept_id)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 51 |
|
| 52 |
+
def classify(self, text: str) -> PlaceCategoryInference | None:
|
| 53 |
+
return self._delegate.classify(text)
|
| 54 |
|
| 55 |
+
def rank(self, text: str, limit: int = 3) -> tuple[PlaceCategoryMatch, ...]:
|
| 56 |
+
return self._delegate.rank(text, limit=limit)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
app/modules/places/infrastructure/semantic_place_ranker.py
CHANGED
|
@@ -10,11 +10,16 @@ from app.modules.places.infrastructure.place_semantic_document import (
|
|
| 10 |
class SemanticPlaceRanker(PlaceRanker):
|
| 11 |
"""Preserve the cosine-similarity order returned by PGVector."""
|
| 12 |
|
| 13 |
-
engine_name = "
|
| 14 |
score_metric = "cosine_similarity"
|
| 15 |
field_weights = PLACE_SEMANTIC_FIELD_WEIGHTS
|
| 16 |
|
| 17 |
-
def __init__(
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 18 |
self.ranking_parameters = {"dimension": float(dimension)}
|
| 19 |
|
| 20 |
def rank(
|
|
|
|
| 10 |
class SemanticPlaceRanker(PlaceRanker):
|
| 11 |
"""Preserve the cosine-similarity order returned by PGVector."""
|
| 12 |
|
| 13 |
+
engine_name = "semantic_embeddings"
|
| 14 |
score_metric = "cosine_similarity"
|
| 15 |
field_weights = PLACE_SEMANTIC_FIELD_WEIGHTS
|
| 16 |
|
| 17 |
+
def __init__(
|
| 18 |
+
self,
|
| 19 |
+
dimension: int = 300,
|
| 20 |
+
model_name: str = "semantic_embeddings",
|
| 21 |
+
) -> None:
|
| 22 |
+
self.engine_name = model_name
|
| 23 |
self.ranking_parameters = {"dimension": float(dimension)}
|
| 24 |
|
| 25 |
def rank(
|
app/shared/config/settings.py
CHANGED
|
@@ -167,6 +167,82 @@ class Settings(BaseSettings):
|
|
| 167 |
alias="FASTTEXT_AUTO_DOWNLOAD",
|
| 168 |
)
|
| 169 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 170 |
bm25_k1: float = Field(default=1.5, gt=0, alias="BM25_K1")
|
| 171 |
bm25_b: float = Field(default=0.75, ge=0, le=1, alias="BM25_B")
|
| 172 |
bm25_relevance_threshold: float = Field(
|
|
@@ -218,6 +294,24 @@ class Settings(BaseSettings):
|
|
| 218 |
le=1.0,
|
| 219 |
alias="PLACES_CHAT_AMBIGUITY_DELTA",
|
| 220 |
)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 221 |
places_chat_ranking_version: str = Field(
|
| 222 |
default="places-chat-v2",
|
| 223 |
min_length=1,
|
|
@@ -230,6 +324,31 @@ class Settings(BaseSettings):
|
|
| 230 |
max_length=64,
|
| 231 |
alias="PLACES_CHAT_TAXONOMY_VERSION",
|
| 232 |
)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 233 |
|
| 234 |
log_level: str = Field(default="INFO", alias="LOG_LEVEL")
|
| 235 |
request_timeout_seconds: int = Field(
|
|
@@ -291,6 +410,61 @@ class Settings(BaseSettings):
|
|
| 291 |
@model_validator(mode="after")
|
| 292 |
def validate_post_feed_security(self) -> "Settings":
|
| 293 |
self.vector_store_provider = self.vector_store_provider.strip().lower()
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 294 |
allowed_resources = {"places", "posts", "users", "clubs", "groups", "events"}
|
| 295 |
for resource_type, thresholds in self.global_search_resource_thresholds.items():
|
| 296 |
if resource_type not in allowed_resources:
|
|
|
|
| 167 |
alias="FASTTEXT_AUTO_DOWNLOAD",
|
| 168 |
)
|
| 169 |
|
| 170 |
+
# Places can migrate independently from posts, global search and feed
|
| 171 |
+
# embeddings. Defaults preserve the current FastText contract; enabling a
|
| 172 |
+
# Sentence-Transformer is an explicit, reversible deployment choice.
|
| 173 |
+
places_embedding_provider: str = Field(
|
| 174 |
+
default="fasttext",
|
| 175 |
+
alias="PLACES_EMBEDDING_PROVIDER",
|
| 176 |
+
)
|
| 177 |
+
places_embedding_dimension: int = Field(
|
| 178 |
+
default=300,
|
| 179 |
+
gt=0,
|
| 180 |
+
alias="PLACES_EMBEDDING_DIMENSION",
|
| 181 |
+
)
|
| 182 |
+
places_embedding_model: str = Field(
|
| 183 |
+
default="facebook/fasttext-es-vectors",
|
| 184 |
+
min_length=1,
|
| 185 |
+
alias="PLACES_EMBEDDING_MODEL",
|
| 186 |
+
)
|
| 187 |
+
places_embedding_version: str = Field(
|
| 188 |
+
default="common-crawl-300-v1",
|
| 189 |
+
min_length=1,
|
| 190 |
+
alias="PLACES_EMBEDDING_VERSION",
|
| 191 |
+
)
|
| 192 |
+
places_embedding_query_prefix: str = Field(
|
| 193 |
+
default="",
|
| 194 |
+
alias="PLACES_EMBEDDING_QUERY_PREFIX",
|
| 195 |
+
)
|
| 196 |
+
places_embedding_passage_prefix: str = Field(
|
| 197 |
+
default="",
|
| 198 |
+
alias="PLACES_EMBEDDING_PASSAGE_PREFIX",
|
| 199 |
+
)
|
| 200 |
+
places_embedding_batch_size: int = Field(
|
| 201 |
+
default=32,
|
| 202 |
+
ge=1,
|
| 203 |
+
le=512,
|
| 204 |
+
alias="PLACES_EMBEDDING_BATCH_SIZE",
|
| 205 |
+
)
|
| 206 |
+
places_embedding_device: str | None = Field(
|
| 207 |
+
default=None,
|
| 208 |
+
alias="PLACES_EMBEDDING_DEVICE",
|
| 209 |
+
)
|
| 210 |
+
places_category_catalog_path: str | None = Field(
|
| 211 |
+
default=None,
|
| 212 |
+
alias="PLACES_CATEGORY_CATALOG_PATH",
|
| 213 |
+
)
|
| 214 |
+
places_category_min_similarity: float = Field(
|
| 215 |
+
default=0.44,
|
| 216 |
+
ge=-1.0,
|
| 217 |
+
le=1.0,
|
| 218 |
+
alias="PLACES_CATEGORY_MIN_SIMILARITY",
|
| 219 |
+
)
|
| 220 |
+
places_category_min_margin: float = Field(
|
| 221 |
+
default=0.04,
|
| 222 |
+
ge=0.0,
|
| 223 |
+
le=2.0,
|
| 224 |
+
alias="PLACES_CATEGORY_MIN_MARGIN",
|
| 225 |
+
)
|
| 226 |
+
places_pgvector_match_function: str = Field(
|
| 227 |
+
default="match_places",
|
| 228 |
+
min_length=1,
|
| 229 |
+
alias="PLACES_PGVECTOR_MATCH_FUNCTION",
|
| 230 |
+
)
|
| 231 |
+
places_pgvector_hybrid_function: str | None = Field(
|
| 232 |
+
default=None,
|
| 233 |
+
alias="PLACES_PGVECTOR_HYBRID_FUNCTION",
|
| 234 |
+
)
|
| 235 |
+
places_pgvector_upsert_function: str = Field(
|
| 236 |
+
default="upsert_place_embedding",
|
| 237 |
+
min_length=1,
|
| 238 |
+
alias="PLACES_PGVECTOR_UPSERT_FUNCTION",
|
| 239 |
+
)
|
| 240 |
+
places_pgvector_hash_function: str = Field(
|
| 241 |
+
default="get_place_content_hashes",
|
| 242 |
+
min_length=1,
|
| 243 |
+
alias="PLACES_PGVECTOR_HASH_FUNCTION",
|
| 244 |
+
)
|
| 245 |
+
|
| 246 |
bm25_k1: float = Field(default=1.5, gt=0, alias="BM25_K1")
|
| 247 |
bm25_b: float = Field(default=0.75, ge=0, le=1, alias="BM25_B")
|
| 248 |
bm25_relevance_threshold: float = Field(
|
|
|
|
| 294 |
le=1.0,
|
| 295 |
alias="PLACES_CHAT_AMBIGUITY_DELTA",
|
| 296 |
)
|
| 297 |
+
places_chat_hypothesis_min_confidence: float = Field(
|
| 298 |
+
default=0.60,
|
| 299 |
+
ge=0.0,
|
| 300 |
+
le=1.0,
|
| 301 |
+
alias="PLACES_CHAT_HYPOTHESIS_MIN_CONFIDENCE",
|
| 302 |
+
)
|
| 303 |
+
places_chat_hypothesis_max_gap: float = Field(
|
| 304 |
+
default=0.15,
|
| 305 |
+
ge=0.0,
|
| 306 |
+
le=1.0,
|
| 307 |
+
alias="PLACES_CHAT_HYPOTHESIS_MAX_GAP",
|
| 308 |
+
)
|
| 309 |
+
places_chat_default_radius_meters: int = Field(
|
| 310 |
+
default=5_000,
|
| 311 |
+
ge=1,
|
| 312 |
+
le=50_000,
|
| 313 |
+
alias="PLACES_CHAT_DEFAULT_RADIUS_METERS",
|
| 314 |
+
)
|
| 315 |
places_chat_ranking_version: str = Field(
|
| 316 |
default="places-chat-v2",
|
| 317 |
min_length=1,
|
|
|
|
| 324 |
max_length=64,
|
| 325 |
alias="PLACES_CHAT_TAXONOMY_VERSION",
|
| 326 |
)
|
| 327 |
+
# Contextual intent extraction is opt-in. ``disabled`` is accepted as an
|
| 328 |
+
# operational alias for the deterministic-only path so deployments can
|
| 329 |
+
# explicitly turn the optional model off without changing code.
|
| 330 |
+
places_chat_intent_provider: str = Field(
|
| 331 |
+
default="deterministic",
|
| 332 |
+
alias="PLACES_CHAT_INTENT_PROVIDER",
|
| 333 |
+
)
|
| 334 |
+
places_chat_bert_model_path: str | None = Field(
|
| 335 |
+
default=None,
|
| 336 |
+
alias="PLACES_CHAT_BERT_MODEL_PATH",
|
| 337 |
+
)
|
| 338 |
+
places_chat_bert_model_version: str | None = Field(
|
| 339 |
+
default=None,
|
| 340 |
+
alias="PLACES_CHAT_BERT_MODEL_VERSION",
|
| 341 |
+
)
|
| 342 |
+
places_chat_bert_device: str | None = Field(
|
| 343 |
+
default=None,
|
| 344 |
+
alias="PLACES_CHAT_BERT_DEVICE",
|
| 345 |
+
)
|
| 346 |
+
places_chat_bert_min_token_confidence: float = Field(
|
| 347 |
+
default=0.60,
|
| 348 |
+
ge=0.0,
|
| 349 |
+
le=1.0,
|
| 350 |
+
alias="PLACES_CHAT_BERT_MIN_TOKEN_CONFIDENCE",
|
| 351 |
+
)
|
| 352 |
|
| 353 |
log_level: str = Field(default="INFO", alias="LOG_LEVEL")
|
| 354 |
request_timeout_seconds: int = Field(
|
|
|
|
| 410 |
@model_validator(mode="after")
|
| 411 |
def validate_post_feed_security(self) -> "Settings":
|
| 412 |
self.vector_store_provider = self.vector_store_provider.strip().lower()
|
| 413 |
+
self.places_embedding_provider = self.places_embedding_provider.strip().lower()
|
| 414 |
+
if self.places_embedding_device is not None:
|
| 415 |
+
self.places_embedding_device = (
|
| 416 |
+
self.places_embedding_device.strip() or None
|
| 417 |
+
)
|
| 418 |
+
if self.places_category_catalog_path is not None:
|
| 419 |
+
self.places_category_catalog_path = (
|
| 420 |
+
self.places_category_catalog_path.strip() or None
|
| 421 |
+
)
|
| 422 |
+
if self.places_pgvector_hybrid_function is not None:
|
| 423 |
+
self.places_pgvector_hybrid_function = (
|
| 424 |
+
self.places_pgvector_hybrid_function.strip() or None
|
| 425 |
+
)
|
| 426 |
+
if self.places_embedding_provider not in {
|
| 427 |
+
"fasttext",
|
| 428 |
+
"mock",
|
| 429 |
+
"sentence_transformer",
|
| 430 |
+
"bert",
|
| 431 |
+
}:
|
| 432 |
+
raise ValueError(
|
| 433 |
+
"PLACES_EMBEDDING_PROVIDER debe ser fasttext, mock, "
|
| 434 |
+
"sentence_transformer o bert"
|
| 435 |
+
)
|
| 436 |
+
self.places_chat_intent_provider = (
|
| 437 |
+
self.places_chat_intent_provider.strip().lower()
|
| 438 |
+
)
|
| 439 |
+
if self.places_chat_intent_provider not in {
|
| 440 |
+
"disabled",
|
| 441 |
+
"deterministic",
|
| 442 |
+
"bert",
|
| 443 |
+
}:
|
| 444 |
+
raise ValueError(
|
| 445 |
+
"PLACES_CHAT_INTENT_PROVIDER debe ser disabled, "
|
| 446 |
+
"deterministic o bert"
|
| 447 |
+
)
|
| 448 |
+
if self.places_chat_bert_model_path is not None:
|
| 449 |
+
self.places_chat_bert_model_path = (
|
| 450 |
+
self.places_chat_bert_model_path.strip() or None
|
| 451 |
+
)
|
| 452 |
+
if self.places_chat_bert_model_version is not None:
|
| 453 |
+
self.places_chat_bert_model_version = (
|
| 454 |
+
self.places_chat_bert_model_version.strip() or None
|
| 455 |
+
)
|
| 456 |
+
if self.places_chat_bert_device is not None:
|
| 457 |
+
self.places_chat_bert_device = (
|
| 458 |
+
self.places_chat_bert_device.strip() or None
|
| 459 |
+
)
|
| 460 |
+
if (
|
| 461 |
+
self.places_chat_intent_provider == "bert"
|
| 462 |
+
and self.places_chat_bert_model_path is None
|
| 463 |
+
):
|
| 464 |
+
raise ValueError(
|
| 465 |
+
"PLACES_CHAT_BERT_MODEL_PATH es obligatorio cuando "
|
| 466 |
+
"PLACES_CHAT_INTENT_PROVIDER=bert"
|
| 467 |
+
)
|
| 468 |
allowed_resources = {"places", "posts", "users", "clubs", "groups", "events"}
|
| 469 |
for resource_type, thresholds in self.global_search_resource_thresholds.items():
|
| 470 |
if resource_type not in allowed_resources:
|
app/shared/dependencies.py
CHANGED
|
@@ -4,7 +4,10 @@ from app.shared.cache.memory import SimpleTTLCache
|
|
| 4 |
from app.shared.config.settings import get_settings
|
| 5 |
from app.shared.nlp.embeddings.base import EmbeddingProvider
|
| 6 |
from app.shared.nlp.embeddings.cached import CachedEmbeddingProvider
|
| 7 |
-
from app.shared.nlp.embeddings.factory import
|
|
|
|
|
|
|
|
|
|
| 8 |
from app.shared.nlp.llm.base import LLMProvider
|
| 9 |
from app.shared.nlp.llm.groq_llama import GroqLlamaProvider
|
| 10 |
from app.shared.nlp.llm.mock import MockLLMProvider
|
|
@@ -19,6 +22,32 @@ def get_embedding_provider() -> EmbeddingProvider:
|
|
| 19 |
)
|
| 20 |
|
| 21 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 22 |
@lru_cache
|
| 23 |
def get_llm_provider() -> LLMProvider:
|
| 24 |
settings = get_settings()
|
|
|
|
| 4 |
from app.shared.config.settings import get_settings
|
| 5 |
from app.shared.nlp.embeddings.base import EmbeddingProvider
|
| 6 |
from app.shared.nlp.embeddings.cached import CachedEmbeddingProvider
|
| 7 |
+
from app.shared.nlp.embeddings.factory import (
|
| 8 |
+
create_embedding_provider,
|
| 9 |
+
create_place_embedding_provider,
|
| 10 |
+
)
|
| 11 |
from app.shared.nlp.llm.base import LLMProvider
|
| 12 |
from app.shared.nlp.llm.groq_llama import GroqLlamaProvider
|
| 13 |
from app.shared.nlp.llm.mock import MockLLMProvider
|
|
|
|
| 22 |
)
|
| 23 |
|
| 24 |
|
| 25 |
+
@lru_cache
|
| 26 |
+
def get_place_embedding_provider() -> EmbeddingProvider:
|
| 27 |
+
"""Return the query encoder dedicated to Places.
|
| 28 |
+
|
| 29 |
+
It intentionally has a separate cache and configuration so a 768-dimensional
|
| 30 |
+
BERT rollout cannot invalidate or break the global 300-dimensional indexes.
|
| 31 |
+
"""
|
| 32 |
+
|
| 33 |
+
settings = get_settings()
|
| 34 |
+
return CachedEmbeddingProvider(
|
| 35 |
+
provider=create_place_embedding_provider(settings, text_role="query"),
|
| 36 |
+
cache=SimpleTTLCache(default_ttl_seconds=settings.embedding_cache_ttl_seconds),
|
| 37 |
+
)
|
| 38 |
+
|
| 39 |
+
|
| 40 |
+
@lru_cache
|
| 41 |
+
def get_place_passage_embedding_provider() -> EmbeddingProvider:
|
| 42 |
+
"""Return the Places passage encoder (for E5-style asymmetric models)."""
|
| 43 |
+
|
| 44 |
+
settings = get_settings()
|
| 45 |
+
return CachedEmbeddingProvider(
|
| 46 |
+
provider=create_place_embedding_provider(settings, text_role="passage"),
|
| 47 |
+
cache=SimpleTTLCache(default_ttl_seconds=settings.embedding_cache_ttl_seconds),
|
| 48 |
+
)
|
| 49 |
+
|
| 50 |
+
|
| 51 |
@lru_cache
|
| 52 |
def get_llm_provider() -> LLMProvider:
|
| 53 |
settings = get_settings()
|
app/shared/nlp/embeddings/cached.py
CHANGED
|
@@ -22,4 +22,32 @@ class CachedEmbeddingProvider(EmbeddingProvider):
|
|
| 22 |
return embedding
|
| 23 |
|
| 24 |
def embed_batch(self, texts: list[str]) -> list[list[float]]:
|
| 25 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 22 |
return embedding
|
| 23 |
|
| 24 |
def embed_batch(self, texts: list[str]) -> list[list[float]]:
|
| 25 |
+
if not texts:
|
| 26 |
+
return []
|
| 27 |
+
|
| 28 |
+
results: list[list[float] | None] = [None] * len(texts)
|
| 29 |
+
missing_positions: dict[str, list[int]] = {}
|
| 30 |
+
for index, text in enumerate(texts):
|
| 31 |
+
cache_key = f"embedding:{text}"
|
| 32 |
+
cached = self._cache.get(cache_key)
|
| 33 |
+
if cached is not None:
|
| 34 |
+
results[index] = cached
|
| 35 |
+
else:
|
| 36 |
+
missing_positions.setdefault(text, []).append(index)
|
| 37 |
+
|
| 38 |
+
missing_texts = list(missing_positions)
|
| 39 |
+
if missing_texts:
|
| 40 |
+
generated = self._provider.embed_batch(missing_texts)
|
| 41 |
+
if len(generated) != len(missing_texts):
|
| 42 |
+
raise ValueError(
|
| 43 |
+
"Embedding provider returned an unexpected batch size: "
|
| 44 |
+
f"returned={len(generated)}, expected={len(missing_texts)}"
|
| 45 |
+
)
|
| 46 |
+
for text, embedding in zip(missing_texts, generated):
|
| 47 |
+
self._cache.set(f"embedding:{text}", embedding)
|
| 48 |
+
for index in missing_positions[text]:
|
| 49 |
+
results[index] = embedding
|
| 50 |
+
|
| 51 |
+
if any(result is None for result in results):
|
| 52 |
+
raise RuntimeError("Embedding batch cache left unresolved positions")
|
| 53 |
+
return [result for result in results if result is not None]
|
app/shared/nlp/embeddings/factory.py
CHANGED
|
@@ -2,6 +2,9 @@ from app.shared.config.settings import Settings
|
|
| 2 |
from app.shared.nlp.embeddings.base import EmbeddingProvider
|
| 3 |
from app.shared.nlp.embeddings.fasttext import FastTextEmbeddingProvider
|
| 4 |
from app.shared.nlp.embeddings.mock import MockEmbeddingProvider
|
|
|
|
|
|
|
|
|
|
| 5 |
|
| 6 |
|
| 7 |
def create_embedding_provider(settings: Settings) -> EmbeddingProvider:
|
|
@@ -17,3 +20,50 @@ def create_embedding_provider(settings: Settings) -> EmbeddingProvider:
|
|
| 17 |
if provider == "mock":
|
| 18 |
return MockEmbeddingProvider(dimension=settings.embedding_dimension)
|
| 19 |
raise ValueError(f"Unsupported EMBEDDING_PROVIDER: {settings.embedding_provider}")
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 2 |
from app.shared.nlp.embeddings.base import EmbeddingProvider
|
| 3 |
from app.shared.nlp.embeddings.fasttext import FastTextEmbeddingProvider
|
| 4 |
from app.shared.nlp.embeddings.mock import MockEmbeddingProvider
|
| 5 |
+
from app.shared.nlp.embeddings.sentence_transformer import (
|
| 6 |
+
SentenceTransformerEmbeddingProvider,
|
| 7 |
+
)
|
| 8 |
|
| 9 |
|
| 10 |
def create_embedding_provider(settings: Settings) -> EmbeddingProvider:
|
|
|
|
| 20 |
if provider == "mock":
|
| 21 |
return MockEmbeddingProvider(dimension=settings.embedding_dimension)
|
| 22 |
raise ValueError(f"Unsupported EMBEDDING_PROVIDER: {settings.embedding_provider}")
|
| 23 |
+
|
| 24 |
+
|
| 25 |
+
def create_place_embedding_provider(
|
| 26 |
+
settings: Settings,
|
| 27 |
+
*,
|
| 28 |
+
text_role: str = "query",
|
| 29 |
+
) -> EmbeddingProvider:
|
| 30 |
+
"""Create the Places-only embedding provider.
|
| 31 |
+
|
| 32 |
+
Keeping this separate from ``create_embedding_provider`` prevents a Places
|
| 33 |
+
migration from changing post, feed or global-search vector dimensions.
|
| 34 |
+
``text_role`` selects the query/passage prefix required by retrieval models
|
| 35 |
+
such as E5 while sharing the same model weights.
|
| 36 |
+
"""
|
| 37 |
+
|
| 38 |
+
if text_role not in {"query", "passage"}:
|
| 39 |
+
raise ValueError("text_role must be 'query' or 'passage'")
|
| 40 |
+
|
| 41 |
+
provider = settings.places_embedding_provider.casefold()
|
| 42 |
+
if provider == "fasttext":
|
| 43 |
+
return FastTextEmbeddingProvider(
|
| 44 |
+
model_path=settings.fasttext_model_path,
|
| 45 |
+
expected_dimension=settings.places_embedding_dimension,
|
| 46 |
+
repo_id=settings.fasttext_model_repo_id,
|
| 47 |
+
filename=settings.fasttext_model_filename,
|
| 48 |
+
auto_download=settings.fasttext_auto_download,
|
| 49 |
+
)
|
| 50 |
+
if provider == "mock":
|
| 51 |
+
return MockEmbeddingProvider(dimension=settings.places_embedding_dimension)
|
| 52 |
+
if provider in {"sentence_transformer", "bert"}:
|
| 53 |
+
prefix = (
|
| 54 |
+
settings.places_embedding_query_prefix
|
| 55 |
+
if text_role == "query"
|
| 56 |
+
else settings.places_embedding_passage_prefix
|
| 57 |
+
)
|
| 58 |
+
return SentenceTransformerEmbeddingProvider(
|
| 59 |
+
model_name_or_path=settings.places_embedding_model,
|
| 60 |
+
expected_dimension=settings.places_embedding_dimension,
|
| 61 |
+
batch_size=settings.places_embedding_batch_size,
|
| 62 |
+
device=settings.places_embedding_device,
|
| 63 |
+
text_prefix=prefix,
|
| 64 |
+
normalize_embeddings=True,
|
| 65 |
+
)
|
| 66 |
+
raise ValueError(
|
| 67 |
+
"Unsupported PLACES_EMBEDDING_PROVIDER: "
|
| 68 |
+
f"{settings.places_embedding_provider}"
|
| 69 |
+
)
|
app/shared/nlp/embeddings/sentence_transformer.py
ADDED
|
@@ -0,0 +1,288 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""Lazy Sentence-Transformer embeddings with strict output validation.
|
| 2 |
+
|
| 3 |
+
The optional ``sentence-transformers`` dependency is intentionally imported only
|
| 4 |
+
when the first non-empty text is embedded. This keeps application startup and
|
| 5 |
+
test discovery independent from heavyweight model loading.
|
| 6 |
+
"""
|
| 7 |
+
|
| 8 |
+
from __future__ import annotations
|
| 9 |
+
|
| 10 |
+
import math
|
| 11 |
+
import threading
|
| 12 |
+
from collections.abc import Callable, Sequence
|
| 13 |
+
from typing import Any, Protocol
|
| 14 |
+
|
| 15 |
+
from app.shared.nlp.embeddings.base import EmbeddingProvider
|
| 16 |
+
|
| 17 |
+
|
| 18 |
+
class SentenceTransformerModel(Protocol):
|
| 19 |
+
"""Small portion of the SentenceTransformer API used by this provider."""
|
| 20 |
+
|
| 21 |
+
def encode(self, sentences: Sequence[str], **kwargs: Any) -> Any: ...
|
| 22 |
+
|
| 23 |
+
def get_sentence_embedding_dimension(self) -> int | None: ...
|
| 24 |
+
|
| 25 |
+
|
| 26 |
+
ModelLoader = Callable[[str, str | None], SentenceTransformerModel]
|
| 27 |
+
|
| 28 |
+
|
| 29 |
+
_SHARED_MODELS: dict[tuple[str, str | None], SentenceTransformerModel] = {}
|
| 30 |
+
_SHARED_MODELS_LOCK = threading.Lock()
|
| 31 |
+
_MODEL_INFERENCE_LOCKS: dict[int, threading.Lock] = {}
|
| 32 |
+
|
| 33 |
+
|
| 34 |
+
class SentenceTransformerEmbeddingError(RuntimeError):
|
| 35 |
+
"""Base error raised when a sentence embedding cannot be produced safely."""
|
| 36 |
+
|
| 37 |
+
|
| 38 |
+
class SentenceTransformerModelLoadError(SentenceTransformerEmbeddingError):
|
| 39 |
+
"""Raised when the configured model or its runtime cannot be loaded."""
|
| 40 |
+
|
| 41 |
+
|
| 42 |
+
class SentenceTransformerInferenceError(SentenceTransformerEmbeddingError):
|
| 43 |
+
"""Raised when model inference fails."""
|
| 44 |
+
|
| 45 |
+
|
| 46 |
+
class SentenceTransformerDimensionError(ValueError):
|
| 47 |
+
"""Raised when a model returns a vector with an unexpected dimension."""
|
| 48 |
+
|
| 49 |
+
|
| 50 |
+
class SentenceTransformerEmbeddingProvider(EmbeddingProvider):
|
| 51 |
+
"""Batch Sentence-Transformer provider with lazy, thread-safe model loading.
|
| 52 |
+
|
| 53 |
+
``text_prefix`` supports retrieval models such as E5, whose query and
|
| 54 |
+
passage encoders use the same weights but require different input prefixes.
|
| 55 |
+
Configure separate provider instances for queries and documents when those
|
| 56 |
+
prefixes differ.
|
| 57 |
+
"""
|
| 58 |
+
|
| 59 |
+
def __init__(
|
| 60 |
+
self,
|
| 61 |
+
model_name_or_path: str,
|
| 62 |
+
expected_dimension: int,
|
| 63 |
+
*,
|
| 64 |
+
batch_size: int = 32,
|
| 65 |
+
device: str | None = None,
|
| 66 |
+
text_prefix: str = "",
|
| 67 |
+
normalize_embeddings: bool = True,
|
| 68 |
+
model_loader: ModelLoader | None = None,
|
| 69 |
+
) -> None:
|
| 70 |
+
if not model_name_or_path.strip():
|
| 71 |
+
raise ValueError("model_name_or_path must not be empty")
|
| 72 |
+
if expected_dimension <= 0:
|
| 73 |
+
raise ValueError("expected_dimension must be greater than zero")
|
| 74 |
+
if batch_size <= 0:
|
| 75 |
+
raise ValueError("batch_size must be greater than zero")
|
| 76 |
+
|
| 77 |
+
self.model_name_or_path = model_name_or_path
|
| 78 |
+
self.dimension = int(expected_dimension)
|
| 79 |
+
self.batch_size = int(batch_size)
|
| 80 |
+
self.device = device
|
| 81 |
+
self.text_prefix = text_prefix
|
| 82 |
+
self.normalize_embeddings = normalize_embeddings
|
| 83 |
+
|
| 84 |
+
self._model_loader = model_loader or _load_sentence_transformer
|
| 85 |
+
self._model: SentenceTransformerModel | None = None
|
| 86 |
+
self._model_lock = threading.Lock()
|
| 87 |
+
|
| 88 |
+
@property
|
| 89 |
+
def is_loaded(self) -> bool:
|
| 90 |
+
"""Whether the heavyweight model has already been initialized."""
|
| 91 |
+
|
| 92 |
+
return self._model is not None
|
| 93 |
+
|
| 94 |
+
def embed_text(self, text: str) -> list[float]:
|
| 95 |
+
return self.embed_batch([text])[0]
|
| 96 |
+
|
| 97 |
+
def embed_batch(self, texts: list[str]) -> list[list[float]]:
|
| 98 |
+
if not texts:
|
| 99 |
+
return []
|
| 100 |
+
|
| 101 |
+
prepared: list[str] = []
|
| 102 |
+
positions: list[int] = []
|
| 103 |
+
for index, text in enumerate(texts):
|
| 104 |
+
if not isinstance(text, str):
|
| 105 |
+
raise TypeError(
|
| 106 |
+
f"texts[{index}] must be str, got {type(text).__name__}"
|
| 107 |
+
)
|
| 108 |
+
stripped = text.strip()
|
| 109 |
+
if stripped:
|
| 110 |
+
positions.append(index)
|
| 111 |
+
prepared.append(f"{self.text_prefix}{stripped}")
|
| 112 |
+
|
| 113 |
+
embeddings = [[0.0] * self.dimension for _ in texts]
|
| 114 |
+
if not prepared:
|
| 115 |
+
return embeddings
|
| 116 |
+
|
| 117 |
+
model = self._get_model()
|
| 118 |
+
try:
|
| 119 |
+
with _inference_lock_for(model):
|
| 120 |
+
raw_embeddings = model.encode(
|
| 121 |
+
prepared,
|
| 122 |
+
batch_size=self.batch_size,
|
| 123 |
+
convert_to_numpy=True,
|
| 124 |
+
normalize_embeddings=False,
|
| 125 |
+
show_progress_bar=False,
|
| 126 |
+
)
|
| 127 |
+
except Exception as exc:
|
| 128 |
+
raise SentenceTransformerInferenceError(
|
| 129 |
+
"Sentence-Transformer inference failed for "
|
| 130 |
+
f"model {self.model_name_or_path!r}: {exc}"
|
| 131 |
+
) from exc
|
| 132 |
+
|
| 133 |
+
vectors = self._validate_and_convert(raw_embeddings, len(prepared))
|
| 134 |
+
for position, vector in zip(positions, vectors):
|
| 135 |
+
embeddings[position] = vector
|
| 136 |
+
return embeddings
|
| 137 |
+
|
| 138 |
+
def _get_model(self) -> SentenceTransformerModel:
|
| 139 |
+
model = self._model
|
| 140 |
+
if model is not None:
|
| 141 |
+
return model
|
| 142 |
+
|
| 143 |
+
with self._model_lock:
|
| 144 |
+
model = self._model
|
| 145 |
+
if model is not None:
|
| 146 |
+
return model
|
| 147 |
+
try:
|
| 148 |
+
model = self._model_loader(self.model_name_or_path, self.device)
|
| 149 |
+
except SentenceTransformerEmbeddingError:
|
| 150 |
+
raise
|
| 151 |
+
except Exception as exc:
|
| 152 |
+
raise SentenceTransformerModelLoadError(
|
| 153 |
+
"Could not load Sentence-Transformer model "
|
| 154 |
+
f"{self.model_name_or_path!r}: {exc}"
|
| 155 |
+
) from exc
|
| 156 |
+
|
| 157 |
+
self._validate_model_dimension(model)
|
| 158 |
+
self._model = model
|
| 159 |
+
return model
|
| 160 |
+
|
| 161 |
+
def _validate_model_dimension(self, model: SentenceTransformerModel) -> None:
|
| 162 |
+
dimension_getter = getattr(
|
| 163 |
+
model, "get_sentence_embedding_dimension", None
|
| 164 |
+
)
|
| 165 |
+
if not callable(dimension_getter):
|
| 166 |
+
return
|
| 167 |
+
try:
|
| 168 |
+
reported_dimension = dimension_getter()
|
| 169 |
+
except Exception as exc:
|
| 170 |
+
raise SentenceTransformerModelLoadError(
|
| 171 |
+
"Could not inspect the embedding dimension for "
|
| 172 |
+
f"model {self.model_name_or_path!r}: {exc}"
|
| 173 |
+
) from exc
|
| 174 |
+
if reported_dimension is None:
|
| 175 |
+
return
|
| 176 |
+
if int(reported_dimension) != self.dimension:
|
| 177 |
+
raise SentenceTransformerDimensionError(
|
| 178 |
+
"Sentence-Transformer model dimension does not match the "
|
| 179 |
+
"configured embedding dimension: "
|
| 180 |
+
f"model={reported_dimension}, configured={self.dimension}, "
|
| 181 |
+
f"name={self.model_name_or_path!r}"
|
| 182 |
+
)
|
| 183 |
+
|
| 184 |
+
def _validate_and_convert(
|
| 185 |
+
self,
|
| 186 |
+
raw_embeddings: Any,
|
| 187 |
+
expected_count: int,
|
| 188 |
+
) -> list[list[float]]:
|
| 189 |
+
converted = (
|
| 190 |
+
raw_embeddings.tolist()
|
| 191 |
+
if hasattr(raw_embeddings, "tolist")
|
| 192 |
+
else raw_embeddings
|
| 193 |
+
)
|
| 194 |
+
try:
|
| 195 |
+
rows = list(converted)
|
| 196 |
+
except (TypeError, ValueError) as exc:
|
| 197 |
+
raise SentenceTransformerInferenceError(
|
| 198 |
+
"Sentence-Transformer returned a non-iterable embedding result"
|
| 199 |
+
) from exc
|
| 200 |
+
|
| 201 |
+
# Some compatible runtimes collapse a one-item batch to one dimension.
|
| 202 |
+
if expected_count == 1 and rows and _is_scalar(rows[0]):
|
| 203 |
+
rows = [rows]
|
| 204 |
+
if len(rows) != expected_count:
|
| 205 |
+
raise SentenceTransformerInferenceError(
|
| 206 |
+
"Sentence-Transformer returned an unexpected number of vectors: "
|
| 207 |
+
f"returned={len(rows)}, expected={expected_count}"
|
| 208 |
+
)
|
| 209 |
+
|
| 210 |
+
vectors: list[list[float]] = []
|
| 211 |
+
for row_index, row in enumerate(rows):
|
| 212 |
+
values_source = row.tolist() if hasattr(row, "tolist") else row
|
| 213 |
+
try:
|
| 214 |
+
values = [float(value) for value in values_source]
|
| 215 |
+
except (TypeError, ValueError) as exc:
|
| 216 |
+
raise SentenceTransformerInferenceError(
|
| 217 |
+
"Sentence-Transformer returned a non-numeric vector at "
|
| 218 |
+
f"batch index {row_index}"
|
| 219 |
+
) from exc
|
| 220 |
+
if len(values) != self.dimension:
|
| 221 |
+
raise SentenceTransformerDimensionError(
|
| 222 |
+
"Sentence-Transformer returned an unexpected vector "
|
| 223 |
+
f"dimension at batch index {row_index}: "
|
| 224 |
+
f"returned={len(values)}, expected={self.dimension}"
|
| 225 |
+
)
|
| 226 |
+
if not all(math.isfinite(value) for value in values):
|
| 227 |
+
raise SentenceTransformerInferenceError(
|
| 228 |
+
"Sentence-Transformer returned NaN or infinity at "
|
| 229 |
+
f"batch index {row_index}"
|
| 230 |
+
)
|
| 231 |
+
if self.normalize_embeddings:
|
| 232 |
+
values = _l2_normalize_nonzero(values, row_index)
|
| 233 |
+
vectors.append(values)
|
| 234 |
+
return vectors
|
| 235 |
+
|
| 236 |
+
|
| 237 |
+
def _load_sentence_transformer(
|
| 238 |
+
model_name_or_path: str,
|
| 239 |
+
device: str | None,
|
| 240 |
+
) -> SentenceTransformerModel:
|
| 241 |
+
cache_key = (model_name_or_path, device)
|
| 242 |
+
cached = _SHARED_MODELS.get(cache_key)
|
| 243 |
+
if cached is not None:
|
| 244 |
+
return cached
|
| 245 |
+
with _SHARED_MODELS_LOCK:
|
| 246 |
+
cached = _SHARED_MODELS.get(cache_key)
|
| 247 |
+
if cached is not None:
|
| 248 |
+
return cached
|
| 249 |
+
try:
|
| 250 |
+
from sentence_transformers import SentenceTransformer
|
| 251 |
+
except ImportError as exc:
|
| 252 |
+
raise SentenceTransformerModelLoadError(
|
| 253 |
+
"sentence-transformers is required for BERT embeddings. "
|
| 254 |
+
"Install the project's optional Sentence-Transformer dependencies."
|
| 255 |
+
) from exc
|
| 256 |
+
|
| 257 |
+
try:
|
| 258 |
+
model = SentenceTransformer(model_name_or_path, device=device)
|
| 259 |
+
except Exception as exc:
|
| 260 |
+
raise SentenceTransformerModelLoadError(
|
| 261 |
+
"Could not initialize Sentence-Transformer model "
|
| 262 |
+
f"{model_name_or_path!r}: {exc}"
|
| 263 |
+
) from exc
|
| 264 |
+
_SHARED_MODELS[cache_key] = model
|
| 265 |
+
return model
|
| 266 |
+
|
| 267 |
+
|
| 268 |
+
def _is_scalar(value: Any) -> bool:
|
| 269 |
+
return isinstance(value, (int, float, complex))
|
| 270 |
+
|
| 271 |
+
|
| 272 |
+
def _inference_lock_for(model: SentenceTransformerModel) -> threading.Lock:
|
| 273 |
+
model_id = id(model)
|
| 274 |
+
with _SHARED_MODELS_LOCK:
|
| 275 |
+
return _MODEL_INFERENCE_LOCKS.setdefault(model_id, threading.Lock())
|
| 276 |
+
|
| 277 |
+
|
| 278 |
+
def _l2_normalize_nonzero(
|
| 279 |
+
vector: list[float],
|
| 280 |
+
row_index: int,
|
| 281 |
+
) -> list[float]:
|
| 282 |
+
norm = math.sqrt(sum(value * value for value in vector))
|
| 283 |
+
if norm == 0:
|
| 284 |
+
raise SentenceTransformerInferenceError(
|
| 285 |
+
"Sentence-Transformer returned a zero-norm vector for non-empty "
|
| 286 |
+
f"text at batch index {row_index}"
|
| 287 |
+
)
|
| 288 |
+
return [value / norm for value in vector]
|
app/shared/vector_store/aws_pgvector.py
CHANGED
|
@@ -53,14 +53,48 @@ class AwsPgvectorClient:
|
|
| 53 |
embedding: list[float],
|
| 54 |
filters: dict[str, Any],
|
| 55 |
limit: int,
|
|
|
|
| 56 |
) -> list[VectorMatch]:
|
| 57 |
return await self._match(
|
| 58 |
-
function_name=
|
| 59 |
embedding=embedding,
|
| 60 |
filters=filters,
|
| 61 |
limit=limit,
|
| 62 |
)
|
| 63 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 64 |
async def match_posts(
|
| 65 |
self,
|
| 66 |
embedding: list[float],
|
|
@@ -108,6 +142,7 @@ class AwsPgvectorClient:
|
|
| 108 |
|
| 109 |
async def check_read_contract(self) -> dict[str, Any]:
|
| 110 |
"""Check that read-only pgvector functions are visible and executable."""
|
|
|
|
| 111 |
try:
|
| 112 |
async with self.connection() as connection:
|
| 113 |
vector_row = await connection.fetchrow(
|
|
@@ -120,10 +155,10 @@ class AwsPgvectorClient:
|
|
| 120 |
"exists": False,
|
| 121 |
"executable": False,
|
| 122 |
}
|
| 123 |
-
for function_name, signature in
|
| 124 |
}
|
| 125 |
if vector_available:
|
| 126 |
-
for function_name, signature in
|
| 127 |
row = await connection.fetchrow(
|
| 128 |
"""
|
| 129 |
SELECT
|
|
@@ -169,9 +204,13 @@ class AwsPgvectorClient:
|
|
| 169 |
"functions": functions,
|
| 170 |
}
|
| 171 |
|
| 172 |
-
async def fetch_place_content_hashes(
|
|
|
|
|
|
|
|
|
|
|
|
|
| 173 |
return await self._fetch_content_hashes(
|
| 174 |
-
function_name=
|
| 175 |
ids=ids,
|
| 176 |
)
|
| 177 |
|
|
@@ -205,10 +244,15 @@ class AwsPgvectorClient:
|
|
| 205 |
async def upsert_place_embeddings(
|
| 206 |
self,
|
| 207 |
records: list[VectorUpsertRecord],
|
|
|
|
|
|
|
|
|
|
| 208 |
) -> None:
|
| 209 |
await self._upsert_records(
|
| 210 |
-
function_name=
|
| 211 |
records=records,
|
|
|
|
|
|
|
| 212 |
)
|
| 213 |
|
| 214 |
async def upsert_post_embeddings(
|
|
@@ -323,6 +367,8 @@ class AwsPgvectorClient:
|
|
| 323 |
self,
|
| 324 |
function_name: str,
|
| 325 |
records: list[VectorUpsertRecord],
|
|
|
|
|
|
|
| 326 |
) -> None:
|
| 327 |
if not records:
|
| 328 |
return
|
|
@@ -346,8 +392,8 @@ class AwsPgvectorClient:
|
|
| 346 |
json.dumps(record.metadata, ensure_ascii=False),
|
| 347 |
vector_literal(record.embedding),
|
| 348 |
record.content_hash,
|
| 349 |
-
self._settings.embedding_model,
|
| 350 |
-
self._settings.embedding_version,
|
| 351 |
record.is_active,
|
| 352 |
)
|
| 353 |
for record in records
|
|
@@ -425,6 +471,18 @@ def _read_contract_is_ready(
|
|
| 425 |
)
|
| 426 |
|
| 427 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 428 |
def _credentials_for_role(settings: Settings, role: str) -> tuple[str | None, str | None]:
|
| 429 |
if role == "writer":
|
| 430 |
return (
|
|
|
|
| 53 |
embedding: list[float],
|
| 54 |
filters: dict[str, Any],
|
| 55 |
limit: int,
|
| 56 |
+
function_name: str = "match_places",
|
| 57 |
) -> list[VectorMatch]:
|
| 58 |
return await self._match(
|
| 59 |
+
function_name=function_name,
|
| 60 |
embedding=embedding,
|
| 61 |
filters=filters,
|
| 62 |
limit=limit,
|
| 63 |
)
|
| 64 |
|
| 65 |
+
async def search_places_hybrid(
|
| 66 |
+
self,
|
| 67 |
+
query_text: str,
|
| 68 |
+
embedding: list[float],
|
| 69 |
+
filters: dict[str, Any],
|
| 70 |
+
limit: int,
|
| 71 |
+
function_name: str,
|
| 72 |
+
) -> list[VectorMatch]:
|
| 73 |
+
"""Search a versioned Places index using independent dense and lexical pools."""
|
| 74 |
+
|
| 75 |
+
query = (
|
| 76 |
+
f"SELECT * FROM {quote_identifier(function_name)}("
|
| 77 |
+
"$1::text, $2::vector, $3::integer, $4::jsonb)"
|
| 78 |
+
)
|
| 79 |
+
try:
|
| 80 |
+
async with self.connection() as connection:
|
| 81 |
+
rows = await connection.fetch(
|
| 82 |
+
query,
|
| 83 |
+
query_text,
|
| 84 |
+
vector_literal(embedding),
|
| 85 |
+
limit,
|
| 86 |
+
json.dumps(filters, ensure_ascii=False),
|
| 87 |
+
)
|
| 88 |
+
except asyncpg.exceptions.UndefinedFunctionError as exc:
|
| 89 |
+
raise AppError(
|
| 90 |
+
"Places hybrid SQL contract is missing. Run the versioned "
|
| 91 |
+
"Places semantic embedding migration and grant EXECUTE to the "
|
| 92 |
+
"reader role.",
|
| 93 |
+
code="pgvector_places_hybrid_contract_missing",
|
| 94 |
+
status_code=503,
|
| 95 |
+
) from exc
|
| 96 |
+
return [_row_to_vector_match(row) for row in rows]
|
| 97 |
+
|
| 98 |
async def match_posts(
|
| 99 |
self,
|
| 100 |
embedding: list[float],
|
|
|
|
| 142 |
|
| 143 |
async def check_read_contract(self) -> dict[str, Any]:
|
| 144 |
"""Check that read-only pgvector functions are visible and executable."""
|
| 145 |
+
signatures = _configured_read_contract_signatures(self._settings)
|
| 146 |
try:
|
| 147 |
async with self.connection() as connection:
|
| 148 |
vector_row = await connection.fetchrow(
|
|
|
|
| 155 |
"exists": False,
|
| 156 |
"executable": False,
|
| 157 |
}
|
| 158 |
+
for function_name, signature in signatures.items()
|
| 159 |
}
|
| 160 |
if vector_available:
|
| 161 |
+
for function_name, signature in signatures.items():
|
| 162 |
row = await connection.fetchrow(
|
| 163 |
"""
|
| 164 |
SELECT
|
|
|
|
| 204 |
"functions": functions,
|
| 205 |
}
|
| 206 |
|
| 207 |
+
async def fetch_place_content_hashes(
|
| 208 |
+
self,
|
| 209 |
+
ids: Iterable[str],
|
| 210 |
+
function_name: str = "get_place_content_hashes",
|
| 211 |
+
) -> dict[str, str]:
|
| 212 |
return await self._fetch_content_hashes(
|
| 213 |
+
function_name=function_name,
|
| 214 |
ids=ids,
|
| 215 |
)
|
| 216 |
|
|
|
|
| 244 |
async def upsert_place_embeddings(
|
| 245 |
self,
|
| 246 |
records: list[VectorUpsertRecord],
|
| 247 |
+
function_name: str = "upsert_place_embedding",
|
| 248 |
+
embedding_model: str | None = None,
|
| 249 |
+
embedding_version: str | None = None,
|
| 250 |
) -> None:
|
| 251 |
await self._upsert_records(
|
| 252 |
+
function_name=function_name,
|
| 253 |
records=records,
|
| 254 |
+
embedding_model=embedding_model,
|
| 255 |
+
embedding_version=embedding_version,
|
| 256 |
)
|
| 257 |
|
| 258 |
async def upsert_post_embeddings(
|
|
|
|
| 367 |
self,
|
| 368 |
function_name: str,
|
| 369 |
records: list[VectorUpsertRecord],
|
| 370 |
+
embedding_model: str | None = None,
|
| 371 |
+
embedding_version: str | None = None,
|
| 372 |
) -> None:
|
| 373 |
if not records:
|
| 374 |
return
|
|
|
|
| 392 |
json.dumps(record.metadata, ensure_ascii=False),
|
| 393 |
vector_literal(record.embedding),
|
| 394 |
record.content_hash,
|
| 395 |
+
embedding_model or self._settings.embedding_model,
|
| 396 |
+
embedding_version or self._settings.embedding_version,
|
| 397 |
record.is_active,
|
| 398 |
)
|
| 399 |
for record in records
|
|
|
|
| 471 |
)
|
| 472 |
|
| 473 |
|
| 474 |
+
def _configured_read_contract_signatures(
|
| 475 |
+
settings: Settings,
|
| 476 |
+
) -> dict[str, str]:
|
| 477 |
+
signatures = dict(READ_CONTRACT_SIGNATURES)
|
| 478 |
+
match_name = settings.places_pgvector_match_function
|
| 479 |
+
signatures[match_name] = f"{match_name}(vector, integer, jsonb)"
|
| 480 |
+
hybrid_name = settings.places_pgvector_hybrid_function
|
| 481 |
+
if hybrid_name:
|
| 482 |
+
signatures[hybrid_name] = f"{hybrid_name}(text, vector, integer, jsonb)"
|
| 483 |
+
return signatures
|
| 484 |
+
|
| 485 |
+
|
| 486 |
def _credentials_for_role(settings: Settings, role: str) -> tuple[str | None, str | None]:
|
| 487 |
if role == "writer":
|
| 488 |
return (
|
requirements-training.txt
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
-r requirements.txt
|
| 2 |
+
accelerate>=0.33,<2.0
|
| 3 |
+
torch>=2.1,<3.0
|
requirements.txt
CHANGED
|
@@ -10,5 +10,7 @@ huggingface-hub>=0.34,<2.0
|
|
| 10 |
numpy>=1.26,<2.0
|
| 11 |
scipy>=1.10,<1.15
|
| 12 |
scikit-learn>=1.5,<1.7
|
|
|
|
|
|
|
| 13 |
pytest>=8.0,<9.0
|
| 14 |
pytest-asyncio>=0.23,<1.0
|
|
|
|
| 10 |
numpy>=1.26,<2.0
|
| 11 |
scipy>=1.10,<1.15
|
| 12 |
scikit-learn>=1.5,<1.7
|
| 13 |
+
sentence-transformers>=3.0,<6.0
|
| 14 |
+
transformers>=4.44,<5.0
|
| 15 |
pytest>=8.0,<9.0
|
| 16 |
pytest-asyncio>=0.23,<1.0
|
scripts/train_place_intent_bert.py
ADDED
|
@@ -0,0 +1,501 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""Fine-tune a Hugging Face token classifier for open place-chat slots.
|
| 2 |
+
|
| 3 |
+
The input files are JSONL. Each non-empty line has this shape::
|
| 4 |
+
|
| 5 |
+
{
|
| 6 |
+
"text": "Quiero donas artesanales sin ruido",
|
| 7 |
+
"spans": [
|
| 8 |
+
{"start": 7, "end": 25, "slot": "CATEGORY"},
|
| 9 |
+
{"start": 30, "end": 35, "slot": "EXCLUSION"}
|
| 10 |
+
]
|
| 11 |
+
}
|
| 12 |
+
|
| 13 |
+
Offsets use Python's half-open character convention: ``text[start:end]``.
|
| 14 |
+
Only domain-independent slot types are accepted. Category values remain raw
|
| 15 |
+
text, so this training path does not recreate a closed business taxonomy.
|
| 16 |
+
|
| 17 |
+
Imports for Transformers and PyTorch are deliberately lazy. Importing this
|
| 18 |
+
module, validating data, and running ``--help`` do not require the optional ML
|
| 19 |
+
runtime.
|
| 20 |
+
"""
|
| 21 |
+
|
| 22 |
+
from __future__ import annotations
|
| 23 |
+
|
| 24 |
+
import argparse
|
| 25 |
+
import inspect
|
| 26 |
+
import json
|
| 27 |
+
import math
|
| 28 |
+
from dataclasses import dataclass
|
| 29 |
+
from datetime import datetime, timezone
|
| 30 |
+
from pathlib import Path
|
| 31 |
+
from typing import Any, Mapping, Sequence
|
| 32 |
+
|
| 33 |
+
|
| 34 |
+
SLOT_TYPES: tuple[str, ...] = (
|
| 35 |
+
"CATEGORY",
|
| 36 |
+
"PREFERENCE",
|
| 37 |
+
"EXCLUSION",
|
| 38 |
+
"LOCATION",
|
| 39 |
+
"REFERENCE",
|
| 40 |
+
"RADIUS",
|
| 41 |
+
)
|
| 42 |
+
LABELS: tuple[str, ...] = (
|
| 43 |
+
"O",
|
| 44 |
+
*(label for slot in SLOT_TYPES for label in (f"B-{slot}", f"I-{slot}")),
|
| 45 |
+
)
|
| 46 |
+
LABEL_TO_ID: Mapping[str, int] = {label: index for index, label in enumerate(LABELS)}
|
| 47 |
+
ID_TO_LABEL: Mapping[int, str] = {index: label for label, index in LABEL_TO_ID.items()}
|
| 48 |
+
IGNORED_LABEL_ID = -100
|
| 49 |
+
|
| 50 |
+
|
| 51 |
+
@dataclass(frozen=True)
|
| 52 |
+
class LabeledSpan:
|
| 53 |
+
"""One validated, open-value slot annotation."""
|
| 54 |
+
|
| 55 |
+
start: int
|
| 56 |
+
end: int
|
| 57 |
+
slot: str
|
| 58 |
+
|
| 59 |
+
|
| 60 |
+
@dataclass(frozen=True)
|
| 61 |
+
class IntentTrainingExample:
|
| 62 |
+
"""One validated token-classification example."""
|
| 63 |
+
|
| 64 |
+
text: str
|
| 65 |
+
spans: tuple[LabeledSpan, ...]
|
| 66 |
+
|
| 67 |
+
|
| 68 |
+
def validate_training_example(
|
| 69 |
+
payload: Any,
|
| 70 |
+
*,
|
| 71 |
+
context: str = "example",
|
| 72 |
+
) -> IntentTrainingExample:
|
| 73 |
+
"""Validate one JSON-compatible example and normalize slot casing.
|
| 74 |
+
|
| 75 |
+
Spans must be disjoint. This is stricter than silently selecting one label
|
| 76 |
+
for an overlapping token and makes annotation errors fail before training.
|
| 77 |
+
"""
|
| 78 |
+
|
| 79 |
+
if not isinstance(payload, Mapping):
|
| 80 |
+
raise ValueError(f"{context}: expected a JSON object")
|
| 81 |
+
|
| 82 |
+
text = payload.get("text")
|
| 83 |
+
if not isinstance(text, str) or not text.strip():
|
| 84 |
+
raise ValueError(f"{context}: text must be a non-empty string")
|
| 85 |
+
|
| 86 |
+
raw_spans = payload.get("spans")
|
| 87 |
+
if not isinstance(raw_spans, list):
|
| 88 |
+
raise ValueError(f"{context}: spans must be a list")
|
| 89 |
+
|
| 90 |
+
spans: list[LabeledSpan] = []
|
| 91 |
+
for index, raw_span in enumerate(raw_spans):
|
| 92 |
+
span_context = f"{context}, span {index}"
|
| 93 |
+
if not isinstance(raw_span, Mapping):
|
| 94 |
+
raise ValueError(f"{span_context}: expected a JSON object")
|
| 95 |
+
|
| 96 |
+
start = _integer_offset(raw_span.get("start"), "start", span_context)
|
| 97 |
+
end = _integer_offset(raw_span.get("end"), "end", span_context)
|
| 98 |
+
if start < 0 or end <= start or end > len(text):
|
| 99 |
+
raise ValueError(
|
| 100 |
+
f"{span_context}: offsets must satisfy "
|
| 101 |
+
f"0 <= start < end <= {len(text)}; got start={start}, end={end}"
|
| 102 |
+
)
|
| 103 |
+
if not text[start:end].strip():
|
| 104 |
+
raise ValueError(f"{span_context}: annotated text cannot be blank")
|
| 105 |
+
|
| 106 |
+
raw_slot = raw_span.get("slot", raw_span.get("label"))
|
| 107 |
+
if not isinstance(raw_slot, str) or not raw_slot.strip():
|
| 108 |
+
raise ValueError(f"{span_context}: slot must be a non-empty string")
|
| 109 |
+
slot = raw_slot.strip().upper()
|
| 110 |
+
if slot not in SLOT_TYPES:
|
| 111 |
+
raise ValueError(
|
| 112 |
+
f"{span_context}: unsupported slot {raw_slot!r}; expected one of "
|
| 113 |
+
+ ", ".join(SLOT_TYPES)
|
| 114 |
+
)
|
| 115 |
+
spans.append(LabeledSpan(start=start, end=end, slot=slot))
|
| 116 |
+
|
| 117 |
+
spans.sort(key=lambda span: (span.start, span.end, span.slot))
|
| 118 |
+
for previous, current in zip(spans, spans[1:]):
|
| 119 |
+
if current.start < previous.end:
|
| 120 |
+
raise ValueError(
|
| 121 |
+
f"{context}: overlapping spans "
|
| 122 |
+
f"[{previous.start}, {previous.end}) and "
|
| 123 |
+
f"[{current.start}, {current.end})"
|
| 124 |
+
)
|
| 125 |
+
|
| 126 |
+
return IntentTrainingExample(text=text, spans=tuple(spans))
|
| 127 |
+
|
| 128 |
+
|
| 129 |
+
def read_jsonl(path: Path) -> list[IntentTrainingExample]:
|
| 130 |
+
"""Read and validate a UTF-8 JSONL dataset with contextual errors."""
|
| 131 |
+
|
| 132 |
+
if not path.is_file():
|
| 133 |
+
raise FileNotFoundError(f"Dataset not found: {path}")
|
| 134 |
+
|
| 135 |
+
examples: list[IntentTrainingExample] = []
|
| 136 |
+
with path.open("r", encoding="utf-8") as source:
|
| 137 |
+
for line_number, raw_line in enumerate(source, start=1):
|
| 138 |
+
if not raw_line.strip():
|
| 139 |
+
continue
|
| 140 |
+
try:
|
| 141 |
+
payload = json.loads(raw_line)
|
| 142 |
+
except json.JSONDecodeError as exc:
|
| 143 |
+
raise ValueError(
|
| 144 |
+
f"{path}, line {line_number}: invalid JSON: {exc.msg}"
|
| 145 |
+
) from exc
|
| 146 |
+
examples.append(
|
| 147 |
+
validate_training_example(
|
| 148 |
+
payload,
|
| 149 |
+
context=f"{path}, line {line_number}",
|
| 150 |
+
)
|
| 151 |
+
)
|
| 152 |
+
|
| 153 |
+
if not examples:
|
| 154 |
+
raise ValueError(f"Dataset has no examples: {path}")
|
| 155 |
+
return examples
|
| 156 |
+
|
| 157 |
+
|
| 158 |
+
def align_spans_to_token_offsets(
|
| 159 |
+
text: str,
|
| 160 |
+
spans: Sequence[LabeledSpan],
|
| 161 |
+
token_offsets: Sequence[Sequence[int]],
|
| 162 |
+
*,
|
| 163 |
+
label_to_id: Mapping[str, int] = LABEL_TO_ID,
|
| 164 |
+
) -> list[int]:
|
| 165 |
+
"""Align character spans to tokenizer offsets using IOB labels.
|
| 166 |
+
|
| 167 |
+
Any token with a non-empty intersection with a span receives that slot.
|
| 168 |
+
This handles subword tokenizers while preserving character-level source
|
| 169 |
+
annotations. Special/padding tokens, represented by ``(0, 0)``, receive
|
| 170 |
+
the standard ``-100`` ignore label.
|
| 171 |
+
|
| 172 |
+
Every annotated span must be covered by at least one token. Consequently,
|
| 173 |
+
truncation cannot silently turn a positive span into ``O``.
|
| 174 |
+
"""
|
| 175 |
+
|
| 176 |
+
if not isinstance(text, str):
|
| 177 |
+
raise TypeError("text must be str")
|
| 178 |
+
|
| 179 |
+
_validate_label_mapping(label_to_id)
|
| 180 |
+
normalized_spans = tuple(spans)
|
| 181 |
+
_validate_span_objects(text, normalized_spans)
|
| 182 |
+
|
| 183 |
+
seen_span_indexes: set[int] = set()
|
| 184 |
+
labels: list[int] = []
|
| 185 |
+
previous_token_start = -1
|
| 186 |
+
for token_index, raw_offset in enumerate(token_offsets):
|
| 187 |
+
token_start, token_end = _token_offset(raw_offset, token_index, len(text))
|
| 188 |
+
if token_start == token_end == 0:
|
| 189 |
+
labels.append(IGNORED_LABEL_ID)
|
| 190 |
+
continue
|
| 191 |
+
if token_start < previous_token_start:
|
| 192 |
+
raise ValueError("token offsets must be ordered by start position")
|
| 193 |
+
previous_token_start = token_start
|
| 194 |
+
|
| 195 |
+
matching = [
|
| 196 |
+
span_index
|
| 197 |
+
for span_index, span in enumerate(normalized_spans)
|
| 198 |
+
if token_start < span.end and span.start < token_end
|
| 199 |
+
]
|
| 200 |
+
if len(matching) > 1:
|
| 201 |
+
raise ValueError(
|
| 202 |
+
f"token {token_index} [{token_start}, {token_end}) intersects "
|
| 203 |
+
"multiple annotated spans"
|
| 204 |
+
)
|
| 205 |
+
if not matching:
|
| 206 |
+
labels.append(label_to_id["O"])
|
| 207 |
+
continue
|
| 208 |
+
|
| 209 |
+
span_index = matching[0]
|
| 210 |
+
span = normalized_spans[span_index]
|
| 211 |
+
prefix = "I" if span_index in seen_span_indexes else "B"
|
| 212 |
+
labels.append(label_to_id[f"{prefix}-{span.slot}"])
|
| 213 |
+
seen_span_indexes.add(span_index)
|
| 214 |
+
|
| 215 |
+
missing = [
|
| 216 |
+
span
|
| 217 |
+
for span_index, span in enumerate(normalized_spans)
|
| 218 |
+
if span_index not in seen_span_indexes
|
| 219 |
+
]
|
| 220 |
+
if missing:
|
| 221 |
+
details = ", ".join(
|
| 222 |
+
f"{span.slot}[{span.start}, {span.end})" for span in missing
|
| 223 |
+
)
|
| 224 |
+
raise ValueError(
|
| 225 |
+
"annotated spans were not covered by tokenizer offsets "
|
| 226 |
+
f"(possibly truncated): {details}"
|
| 227 |
+
)
|
| 228 |
+
return labels
|
| 229 |
+
|
| 230 |
+
|
| 231 |
+
def encode_examples(
|
| 232 |
+
examples: Sequence[IntentTrainingExample],
|
| 233 |
+
tokenizer: Any,
|
| 234 |
+
*,
|
| 235 |
+
max_length: int,
|
| 236 |
+
) -> list[dict[str, Any]]:
|
| 237 |
+
"""Tokenize and align examples without depending on a dataset library."""
|
| 238 |
+
|
| 239 |
+
if isinstance(max_length, bool) or not isinstance(max_length, int) or max_length <= 0:
|
| 240 |
+
raise ValueError("max_length must be a positive integer")
|
| 241 |
+
|
| 242 |
+
encoded_examples: list[dict[str, Any]] = []
|
| 243 |
+
for example_index, example in enumerate(examples):
|
| 244 |
+
encoding = tokenizer(
|
| 245 |
+
example.text,
|
| 246 |
+
truncation=True,
|
| 247 |
+
max_length=max_length,
|
| 248 |
+
return_offsets_mapping=True,
|
| 249 |
+
)
|
| 250 |
+
if not isinstance(encoding, Mapping):
|
| 251 |
+
raise TypeError(
|
| 252 |
+
f"tokenizer output for example {example_index} must be a mapping"
|
| 253 |
+
)
|
| 254 |
+
if "offset_mapping" not in encoding:
|
| 255 |
+
raise ValueError(
|
| 256 |
+
"tokenizer did not return offset_mapping; a fast tokenizer is required"
|
| 257 |
+
)
|
| 258 |
+
|
| 259 |
+
offsets = encoding["offset_mapping"]
|
| 260 |
+
if not isinstance(offsets, Sequence) or isinstance(offsets, (str, bytes)):
|
| 261 |
+
raise ValueError(
|
| 262 |
+
f"tokenizer offset_mapping for example {example_index} "
|
| 263 |
+
"must be a sequence"
|
| 264 |
+
)
|
| 265 |
+
feature = {
|
| 266 |
+
key: value for key, value in encoding.items() if key != "offset_mapping"
|
| 267 |
+
}
|
| 268 |
+
feature["labels"] = align_spans_to_token_offsets(
|
| 269 |
+
example.text,
|
| 270 |
+
example.spans,
|
| 271 |
+
offsets,
|
| 272 |
+
)
|
| 273 |
+
input_ids = feature.get("input_ids")
|
| 274 |
+
if not isinstance(input_ids, Sequence) or isinstance(input_ids, (str, bytes)):
|
| 275 |
+
raise ValueError(
|
| 276 |
+
f"tokenizer output for example {example_index} has no input_ids sequence"
|
| 277 |
+
)
|
| 278 |
+
if len(input_ids) != len(feature["labels"]):
|
| 279 |
+
raise ValueError(
|
| 280 |
+
f"tokenizer output for example {example_index} has mismatched "
|
| 281 |
+
"input_ids and offset_mapping lengths"
|
| 282 |
+
)
|
| 283 |
+
encoded_examples.append(feature)
|
| 284 |
+
return encoded_examples
|
| 285 |
+
|
| 286 |
+
|
| 287 |
+
def main(argv: Sequence[str] | None = None) -> None:
|
| 288 |
+
args = build_parser().parse_args(argv)
|
| 289 |
+
_run_training(args)
|
| 290 |
+
|
| 291 |
+
|
| 292 |
+
def build_parser() -> argparse.ArgumentParser:
|
| 293 |
+
parser = argparse.ArgumentParser(description=__doc__)
|
| 294 |
+
parser.add_argument("--train-file", required=True, help="Training JSONL path")
|
| 295 |
+
parser.add_argument(
|
| 296 |
+
"--validation-file",
|
| 297 |
+
help="Optional validation JSONL path",
|
| 298 |
+
)
|
| 299 |
+
parser.add_argument("--output-dir", required=True)
|
| 300 |
+
parser.add_argument(
|
| 301 |
+
"--base-model",
|
| 302 |
+
default="dccuchile/bert-base-spanish-wwm-cased",
|
| 303 |
+
help="Hugging Face model id or local model directory",
|
| 304 |
+
)
|
| 305 |
+
parser.add_argument("--epochs", type=_positive_float, default=3.0)
|
| 306 |
+
parser.add_argument("--batch-size", type=_positive_int, default=16)
|
| 307 |
+
parser.add_argument("--learning-rate", type=_positive_float, default=2e-5)
|
| 308 |
+
parser.add_argument("--max-length", type=_positive_int, default=256)
|
| 309 |
+
parser.add_argument("--seed", type=int, default=42)
|
| 310 |
+
return parser
|
| 311 |
+
|
| 312 |
+
|
| 313 |
+
def _run_training(args: argparse.Namespace) -> None:
|
| 314 |
+
# Optional heavyweight imports stay behind argument parsing so ``--help``
|
| 315 |
+
# remains available in API and CI environments without the ML toolchain.
|
| 316 |
+
try:
|
| 317 |
+
import transformers
|
| 318 |
+
from transformers import (
|
| 319 |
+
AutoModelForTokenClassification,
|
| 320 |
+
AutoTokenizer,
|
| 321 |
+
DataCollatorForTokenClassification,
|
| 322 |
+
Trainer,
|
| 323 |
+
TrainingArguments,
|
| 324 |
+
set_seed,
|
| 325 |
+
)
|
| 326 |
+
except ImportError as exc:
|
| 327 |
+
raise RuntimeError(
|
| 328 |
+
"Training requires transformers with its PyTorch backend. "
|
| 329 |
+
"Install the optional ML dependencies before running this script."
|
| 330 |
+
) from exc
|
| 331 |
+
|
| 332 |
+
train_path = Path(args.train_file)
|
| 333 |
+
validation_path = Path(args.validation_file) if args.validation_file else None
|
| 334 |
+
train_examples = read_jsonl(train_path)
|
| 335 |
+
validation_examples = read_jsonl(validation_path) if validation_path else []
|
| 336 |
+
|
| 337 |
+
set_seed(args.seed)
|
| 338 |
+
tokenizer = AutoTokenizer.from_pretrained(args.base_model, use_fast=True)
|
| 339 |
+
if not getattr(tokenizer, "is_fast", False):
|
| 340 |
+
raise RuntimeError(
|
| 341 |
+
"The selected model does not provide a fast tokenizer with offsets"
|
| 342 |
+
)
|
| 343 |
+
|
| 344 |
+
encoded_train = encode_examples(
|
| 345 |
+
train_examples,
|
| 346 |
+
tokenizer,
|
| 347 |
+
max_length=args.max_length,
|
| 348 |
+
)
|
| 349 |
+
encoded_validation = encode_examples(
|
| 350 |
+
validation_examples,
|
| 351 |
+
tokenizer,
|
| 352 |
+
max_length=args.max_length,
|
| 353 |
+
)
|
| 354 |
+
|
| 355 |
+
model = AutoModelForTokenClassification.from_pretrained(
|
| 356 |
+
args.base_model,
|
| 357 |
+
num_labels=len(LABELS),
|
| 358 |
+
label2id=dict(LABEL_TO_ID),
|
| 359 |
+
id2label=dict(ID_TO_LABEL),
|
| 360 |
+
ignore_mismatched_sizes=True,
|
| 361 |
+
)
|
| 362 |
+
|
| 363 |
+
output_dir = Path(args.output_dir)
|
| 364 |
+
output_dir.mkdir(parents=True, exist_ok=True)
|
| 365 |
+
has_validation = bool(encoded_validation)
|
| 366 |
+
strategy = "epoch" if has_validation else "no"
|
| 367 |
+
training_kwargs: dict[str, Any] = {
|
| 368 |
+
"output_dir": str(output_dir),
|
| 369 |
+
"num_train_epochs": args.epochs,
|
| 370 |
+
"per_device_train_batch_size": args.batch_size,
|
| 371 |
+
"per_device_eval_batch_size": args.batch_size,
|
| 372 |
+
"learning_rate": args.learning_rate,
|
| 373 |
+
"seed": args.seed,
|
| 374 |
+
"data_seed": args.seed,
|
| 375 |
+
"save_strategy": "epoch",
|
| 376 |
+
"logging_strategy": "steps",
|
| 377 |
+
"logging_steps": 25,
|
| 378 |
+
"report_to": [],
|
| 379 |
+
"load_best_model_at_end": has_validation,
|
| 380 |
+
}
|
| 381 |
+
parameter_names = inspect.signature(TrainingArguments.__init__).parameters
|
| 382 |
+
strategy_parameter = (
|
| 383 |
+
"eval_strategy" if "eval_strategy" in parameter_names else "evaluation_strategy"
|
| 384 |
+
)
|
| 385 |
+
training_kwargs[strategy_parameter] = strategy
|
| 386 |
+
|
| 387 |
+
trainer = Trainer(
|
| 388 |
+
model=model,
|
| 389 |
+
args=TrainingArguments(**training_kwargs),
|
| 390 |
+
train_dataset=encoded_train,
|
| 391 |
+
eval_dataset=encoded_validation if has_validation else None,
|
| 392 |
+
data_collator=DataCollatorForTokenClassification(tokenizer=tokenizer),
|
| 393 |
+
tokenizer=tokenizer,
|
| 394 |
+
)
|
| 395 |
+
train_result = trainer.train()
|
| 396 |
+
trainer.save_model(str(output_dir))
|
| 397 |
+
tokenizer.save_pretrained(str(output_dir))
|
| 398 |
+
|
| 399 |
+
manifest = {
|
| 400 |
+
"schema_version": 1,
|
| 401 |
+
"task": "token-classification",
|
| 402 |
+
"architecture": "bert-open-place-intent",
|
| 403 |
+
"base_model": args.base_model,
|
| 404 |
+
"slots": list(SLOT_TYPES),
|
| 405 |
+
"labels": list(LABELS),
|
| 406 |
+
"label_to_id": dict(LABEL_TO_ID),
|
| 407 |
+
"training_examples": len(train_examples),
|
| 408 |
+
"validation_examples": len(validation_examples),
|
| 409 |
+
"epochs": args.epochs,
|
| 410 |
+
"batch_size": args.batch_size,
|
| 411 |
+
"learning_rate": args.learning_rate,
|
| 412 |
+
"max_length": args.max_length,
|
| 413 |
+
"seed": args.seed,
|
| 414 |
+
"train_loss": _finite_or_none(getattr(train_result, "training_loss", None)),
|
| 415 |
+
"transformers_version": getattr(transformers, "__version__", "unknown"),
|
| 416 |
+
"trained_at": datetime.now(timezone.utc).isoformat(),
|
| 417 |
+
}
|
| 418 |
+
(output_dir / "place_intent_training_manifest.json").write_text(
|
| 419 |
+
json.dumps(manifest, indent=2, ensure_ascii=False) + "\n",
|
| 420 |
+
encoding="utf-8",
|
| 421 |
+
)
|
| 422 |
+
|
| 423 |
+
|
| 424 |
+
def _integer_offset(value: Any, field: str, context: str) -> int:
|
| 425 |
+
if isinstance(value, bool) or not isinstance(value, int):
|
| 426 |
+
raise ValueError(f"{context}: {field} must be an integer")
|
| 427 |
+
return value
|
| 428 |
+
|
| 429 |
+
|
| 430 |
+
def _validate_span_objects(text: str, spans: Sequence[LabeledSpan]) -> None:
|
| 431 |
+
previous: LabeledSpan | None = None
|
| 432 |
+
for index, span in enumerate(spans):
|
| 433 |
+
if not isinstance(span, LabeledSpan):
|
| 434 |
+
raise TypeError(f"span {index} must be LabeledSpan")
|
| 435 |
+
if span.slot not in SLOT_TYPES:
|
| 436 |
+
raise ValueError(f"span {index} has unsupported slot {span.slot!r}")
|
| 437 |
+
if span.start < 0 or span.end <= span.start or span.end > len(text):
|
| 438 |
+
raise ValueError(f"span {index} has invalid offsets")
|
| 439 |
+
if previous is not None and span.start < previous.end:
|
| 440 |
+
raise ValueError("spans must be sorted and non-overlapping")
|
| 441 |
+
previous = span
|
| 442 |
+
|
| 443 |
+
|
| 444 |
+
def _validate_label_mapping(label_to_id: Mapping[str, int]) -> None:
|
| 445 |
+
missing = [label for label in LABELS if label not in label_to_id]
|
| 446 |
+
if missing:
|
| 447 |
+
raise ValueError("label_to_id is missing labels: " + ", ".join(missing))
|
| 448 |
+
|
| 449 |
+
|
| 450 |
+
def _token_offset(
|
| 451 |
+
raw_offset: Sequence[int],
|
| 452 |
+
token_index: int,
|
| 453 |
+
text_length: int,
|
| 454 |
+
) -> tuple[int, int]:
|
| 455 |
+
if (
|
| 456 |
+
isinstance(raw_offset, (str, bytes))
|
| 457 |
+
or not isinstance(raw_offset, Sequence)
|
| 458 |
+
or len(raw_offset) != 2
|
| 459 |
+
):
|
| 460 |
+
raise ValueError(f"token offset {token_index} must contain start and end")
|
| 461 |
+
start, end = raw_offset
|
| 462 |
+
if (
|
| 463 |
+
isinstance(start, bool)
|
| 464 |
+
or isinstance(end, bool)
|
| 465 |
+
or not isinstance(start, int)
|
| 466 |
+
or not isinstance(end, int)
|
| 467 |
+
):
|
| 468 |
+
raise ValueError(f"token offset {token_index} must contain integers")
|
| 469 |
+
if start == end == 0:
|
| 470 |
+
return 0, 0
|
| 471 |
+
if start < 0 or end <= start or end > text_length:
|
| 472 |
+
raise ValueError(
|
| 473 |
+
f"token offset {token_index} is invalid for text length {text_length}"
|
| 474 |
+
)
|
| 475 |
+
return start, end
|
| 476 |
+
|
| 477 |
+
|
| 478 |
+
def _positive_int(value: str) -> int:
|
| 479 |
+
parsed = int(value)
|
| 480 |
+
if parsed <= 0:
|
| 481 |
+
raise argparse.ArgumentTypeError("must be greater than zero")
|
| 482 |
+
return parsed
|
| 483 |
+
|
| 484 |
+
|
| 485 |
+
def _positive_float(value: str) -> float:
|
| 486 |
+
parsed = float(value)
|
| 487 |
+
if not math.isfinite(parsed) or parsed <= 0:
|
| 488 |
+
raise argparse.ArgumentTypeError("must be a finite value greater than zero")
|
| 489 |
+
return parsed
|
| 490 |
+
|
| 491 |
+
|
| 492 |
+
def _finite_or_none(value: Any) -> float | None:
|
| 493 |
+
try:
|
| 494 |
+
parsed = float(value)
|
| 495 |
+
except (TypeError, ValueError):
|
| 496 |
+
return None
|
| 497 |
+
return parsed if math.isfinite(parsed) else None
|
| 498 |
+
|
| 499 |
+
|
| 500 |
+
if __name__ == "__main__":
|
| 501 |
+
main()
|
scripts/train_place_retriever.py
ADDED
|
@@ -0,0 +1,147 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""Fine-tune a bi-encoder for open-vocabulary place retrieval.
|
| 2 |
+
|
| 3 |
+
Input is JSONL with at least ``query`` and ``positive``. ``hard_negatives`` may
|
| 4 |
+
contain semantically close but incorrect place documents. The script keeps all
|
| 5 |
+
runtime imports lazy so the API can still run with FastText during rollout.
|
| 6 |
+
"""
|
| 7 |
+
|
| 8 |
+
from __future__ import annotations
|
| 9 |
+
|
| 10 |
+
import argparse
|
| 11 |
+
import json
|
| 12 |
+
import random
|
| 13 |
+
from pathlib import Path
|
| 14 |
+
from typing import Any
|
| 15 |
+
|
| 16 |
+
|
| 17 |
+
def main() -> None:
|
| 18 |
+
args = _parse_args()
|
| 19 |
+
random.seed(args.seed)
|
| 20 |
+
|
| 21 |
+
try:
|
| 22 |
+
from sentence_transformers import InputExample, SentenceTransformer, losses
|
| 23 |
+
from torch.utils.data import DataLoader
|
| 24 |
+
except ImportError as exc:
|
| 25 |
+
raise RuntimeError(
|
| 26 |
+
"Install sentence-transformers and its PyTorch runtime before training"
|
| 27 |
+
) from exc
|
| 28 |
+
|
| 29 |
+
rows = _read_training_rows(Path(args.train_file))
|
| 30 |
+
examples = [
|
| 31 |
+
InputExample(
|
| 32 |
+
texts=[
|
| 33 |
+
f"query: {row['query']}",
|
| 34 |
+
f"passage: {row['positive']}",
|
| 35 |
+
*(
|
| 36 |
+
f"passage: {negative}"
|
| 37 |
+
for negative in row["hard_negatives"][: args.max_hard_negatives]
|
| 38 |
+
),
|
| 39 |
+
]
|
| 40 |
+
)
|
| 41 |
+
for row in rows
|
| 42 |
+
]
|
| 43 |
+
random.shuffle(examples)
|
| 44 |
+
|
| 45 |
+
model = SentenceTransformer(args.base_model, device=args.device)
|
| 46 |
+
data_loader = DataLoader(
|
| 47 |
+
examples,
|
| 48 |
+
shuffle=True,
|
| 49 |
+
batch_size=args.batch_size,
|
| 50 |
+
drop_last=len(examples) >= args.batch_size,
|
| 51 |
+
)
|
| 52 |
+
cached_loss = getattr(losses, "CachedMultipleNegativesRankingLoss", None)
|
| 53 |
+
if cached_loss is not None:
|
| 54 |
+
train_loss = cached_loss(model, mini_batch_size=args.mini_batch_size)
|
| 55 |
+
else:
|
| 56 |
+
train_loss = losses.MultipleNegativesRankingLoss(model)
|
| 57 |
+
|
| 58 |
+
output = Path(args.output_dir)
|
| 59 |
+
output.mkdir(parents=True, exist_ok=True)
|
| 60 |
+
warmup_steps = max(1, int(len(data_loader) * args.epochs * args.warmup_ratio))
|
| 61 |
+
model.fit(
|
| 62 |
+
train_objectives=[(data_loader, train_loss)],
|
| 63 |
+
epochs=args.epochs,
|
| 64 |
+
warmup_steps=warmup_steps,
|
| 65 |
+
optimizer_params={"lr": args.learning_rate},
|
| 66 |
+
output_path=str(output),
|
| 67 |
+
show_progress_bar=True,
|
| 68 |
+
)
|
| 69 |
+
(output / "places_training_manifest.json").write_text(
|
| 70 |
+
json.dumps(
|
| 71 |
+
{
|
| 72 |
+
"base_model": args.base_model,
|
| 73 |
+
"training_examples": len(examples),
|
| 74 |
+
"epochs": args.epochs,
|
| 75 |
+
"batch_size": args.batch_size,
|
| 76 |
+
"learning_rate": args.learning_rate,
|
| 77 |
+
"query_prefix": "query: ",
|
| 78 |
+
"passage_prefix": "passage: ",
|
| 79 |
+
"objective": type(train_loss).__name__,
|
| 80 |
+
"seed": args.seed,
|
| 81 |
+
},
|
| 82 |
+
indent=2,
|
| 83 |
+
ensure_ascii=False,
|
| 84 |
+
),
|
| 85 |
+
encoding="utf-8",
|
| 86 |
+
)
|
| 87 |
+
|
| 88 |
+
|
| 89 |
+
def _read_training_rows(path: Path) -> list[dict[str, Any]]:
|
| 90 |
+
if not path.is_file():
|
| 91 |
+
raise FileNotFoundError(f"Training dataset not found: {path}")
|
| 92 |
+
rows: list[dict[str, Any]] = []
|
| 93 |
+
for line_number, raw_line in enumerate(
|
| 94 |
+
path.read_text(encoding="utf-8").splitlines(),
|
| 95 |
+
start=1,
|
| 96 |
+
):
|
| 97 |
+
if not raw_line.strip():
|
| 98 |
+
continue
|
| 99 |
+
payload = json.loads(raw_line)
|
| 100 |
+
query = _required_text(payload.get("query"), line_number, "query")
|
| 101 |
+
positive = _required_text(payload.get("positive"), line_number, "positive")
|
| 102 |
+
raw_negatives = payload.get("hard_negatives", [])
|
| 103 |
+
if isinstance(raw_negatives, str):
|
| 104 |
+
raw_negatives = [raw_negatives]
|
| 105 |
+
if not isinstance(raw_negatives, list):
|
| 106 |
+
raise ValueError(
|
| 107 |
+
f"Line {line_number}: hard_negatives must be a list of strings"
|
| 108 |
+
)
|
| 109 |
+
negatives = [
|
| 110 |
+
_required_text(value, line_number, "hard_negatives")
|
| 111 |
+
for value in raw_negatives
|
| 112 |
+
]
|
| 113 |
+
rows.append(
|
| 114 |
+
{"query": query, "positive": positive, "hard_negatives": negatives}
|
| 115 |
+
)
|
| 116 |
+
if len(rows) < 2:
|
| 117 |
+
raise ValueError("At least two training examples are required")
|
| 118 |
+
return rows
|
| 119 |
+
|
| 120 |
+
|
| 121 |
+
def _required_text(value: Any, line_number: int, field: str) -> str:
|
| 122 |
+
if not isinstance(value, str) or not value.strip():
|
| 123 |
+
raise ValueError(f"Line {line_number}: {field} must be a non-empty string")
|
| 124 |
+
return " ".join(value.split())
|
| 125 |
+
|
| 126 |
+
|
| 127 |
+
def _parse_args() -> argparse.Namespace:
|
| 128 |
+
parser = argparse.ArgumentParser(description=__doc__)
|
| 129 |
+
parser.add_argument("--train-file", required=True)
|
| 130 |
+
parser.add_argument("--output-dir", required=True)
|
| 131 |
+
parser.add_argument(
|
| 132 |
+
"--base-model",
|
| 133 |
+
default="intfloat/multilingual-e5-base",
|
| 134 |
+
)
|
| 135 |
+
parser.add_argument("--device", default=None)
|
| 136 |
+
parser.add_argument("--epochs", type=int, default=2)
|
| 137 |
+
parser.add_argument("--batch-size", type=int, default=16)
|
| 138 |
+
parser.add_argument("--mini-batch-size", type=int, default=8)
|
| 139 |
+
parser.add_argument("--max-hard-negatives", type=int, default=3)
|
| 140 |
+
parser.add_argument("--learning-rate", type=float, default=2e-5)
|
| 141 |
+
parser.add_argument("--warmup-ratio", type=float, default=0.1)
|
| 142 |
+
parser.add_argument("--seed", type=int, default=42)
|
| 143 |
+
return parser.parse_args()
|
| 144 |
+
|
| 145 |
+
|
| 146 |
+
if __name__ == "__main__":
|
| 147 |
+
main()
|
sql/migrations/20260716_02_places_semantic_v1.sql
ADDED
|
@@ -0,0 +1,255 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
-- Additive Places-only Sentence-Transformer index (multilingual-e5-base: 768d).
|
| 2 |
+
-- This migration never alters or drops the existing FastText VECTOR(300) table.
|
| 3 |
+
|
| 4 |
+
BEGIN;
|
| 5 |
+
|
| 6 |
+
CREATE EXTENSION IF NOT EXISTS vector;
|
| 7 |
+
|
| 8 |
+
CREATE TABLE IF NOT EXISTS public.place_embeddings_semantic_v1 (
|
| 9 |
+
external_id TEXT PRIMARY KEY,
|
| 10 |
+
document TEXT NOT NULL,
|
| 11 |
+
metadata JSONB NOT NULL DEFAULT '{}'::jsonb,
|
| 12 |
+
embedding VECTOR(768) NOT NULL,
|
| 13 |
+
content_hash TEXT NOT NULL,
|
| 14 |
+
embedding_model TEXT NOT NULL,
|
| 15 |
+
embedding_version TEXT NOT NULL,
|
| 16 |
+
is_active BOOLEAN NOT NULL DEFAULT true,
|
| 17 |
+
textsearch TSVECTOR GENERATED ALWAYS AS (
|
| 18 |
+
to_tsvector('simple', document)
|
| 19 |
+
) STORED,
|
| 20 |
+
updated_at TIMESTAMPTZ NOT NULL DEFAULT now()
|
| 21 |
+
);
|
| 22 |
+
|
| 23 |
+
CREATE INDEX IF NOT EXISTS place_embeddings_semantic_v1_embedding_hnsw_idx
|
| 24 |
+
ON public.place_embeddings_semantic_v1
|
| 25 |
+
USING hnsw (embedding vector_cosine_ops);
|
| 26 |
+
|
| 27 |
+
CREATE INDEX IF NOT EXISTS place_embeddings_semantic_v1_textsearch_gin_idx
|
| 28 |
+
ON public.place_embeddings_semantic_v1 USING gin (textsearch);
|
| 29 |
+
|
| 30 |
+
CREATE INDEX IF NOT EXISTS place_embeddings_semantic_v1_metadata_gin_idx
|
| 31 |
+
ON public.place_embeddings_semantic_v1 USING gin (metadata);
|
| 32 |
+
|
| 33 |
+
CREATE OR REPLACE FUNCTION public.match_places_semantic_v1(
|
| 34 |
+
query_embedding VECTOR(768),
|
| 35 |
+
match_count INTEGER,
|
| 36 |
+
filters JSONB DEFAULT '{}'::jsonb
|
| 37 |
+
)
|
| 38 |
+
RETURNS TABLE (
|
| 39 |
+
external_id TEXT,
|
| 40 |
+
document TEXT,
|
| 41 |
+
metadata JSONB,
|
| 42 |
+
score DOUBLE PRECISION
|
| 43 |
+
)
|
| 44 |
+
LANGUAGE sql
|
| 45 |
+
STABLE
|
| 46 |
+
SECURITY DEFINER
|
| 47 |
+
SET search_path = public
|
| 48 |
+
AS $$
|
| 49 |
+
SELECT
|
| 50 |
+
place.external_id,
|
| 51 |
+
place.document,
|
| 52 |
+
place.metadata,
|
| 53 |
+
1 - (place.embedding <=> query_embedding) AS score
|
| 54 |
+
FROM public.place_embeddings_semantic_v1 AS place
|
| 55 |
+
WHERE place.is_active = true
|
| 56 |
+
AND COALESCE((filters->>'is_active')::boolean, true) = true
|
| 57 |
+
AND ((filters ? 'city') IS FALSE OR lower(place.metadata->>'city') = lower(filters->>'city'))
|
| 58 |
+
AND ((filters ? 'state') IS FALSE OR lower(place.metadata->>'state') = lower(filters->>'state'))
|
| 59 |
+
AND ((filters ? 'category') IS FALSE OR lower(place.metadata->>'category') = lower(filters->>'category'))
|
| 60 |
+
AND ((filters ? 'price_range') IS FALSE OR place.metadata->>'price_range' = filters->>'price_range')
|
| 61 |
+
AND ((filters ? 'occasion') IS FALSE OR place.metadata->>'occasion' ILIKE ('%' || (filters->>'occasion') || '%'))
|
| 62 |
+
AND (
|
| 63 |
+
(filters ? 'place_ids') IS FALSE
|
| 64 |
+
OR place.external_id IN (SELECT jsonb_array_elements_text(filters->'place_ids'))
|
| 65 |
+
)
|
| 66 |
+
ORDER BY place.embedding <=> query_embedding, place.external_id
|
| 67 |
+
LIMIT GREATEST(match_count, 0);
|
| 68 |
+
$$;
|
| 69 |
+
|
| 70 |
+
CREATE OR REPLACE FUNCTION public.search_places_semantic_v1(
|
| 71 |
+
query_text TEXT,
|
| 72 |
+
query_embedding VECTOR(768),
|
| 73 |
+
match_count INTEGER,
|
| 74 |
+
filters JSONB DEFAULT '{}'::jsonb
|
| 75 |
+
)
|
| 76 |
+
RETURNS TABLE (
|
| 77 |
+
external_id TEXT,
|
| 78 |
+
document TEXT,
|
| 79 |
+
metadata JSONB,
|
| 80 |
+
score DOUBLE PRECISION,
|
| 81 |
+
semantic_score DOUBLE PRECISION,
|
| 82 |
+
lexical_score DOUBLE PRECISION
|
| 83 |
+
)
|
| 84 |
+
LANGUAGE sql
|
| 85 |
+
STABLE
|
| 86 |
+
SECURITY DEFINER
|
| 87 |
+
SET search_path = public
|
| 88 |
+
AS $$
|
| 89 |
+
WITH dense AS (
|
| 90 |
+
SELECT
|
| 91 |
+
place.external_id,
|
| 92 |
+
1 - (place.embedding <=> query_embedding) AS semantic_score,
|
| 93 |
+
row_number() OVER (
|
| 94 |
+
ORDER BY place.embedding <=> query_embedding, place.external_id
|
| 95 |
+
) AS dense_rank
|
| 96 |
+
FROM public.place_embeddings_semantic_v1 AS place
|
| 97 |
+
WHERE place.is_active = true
|
| 98 |
+
AND COALESCE((filters->>'is_active')::boolean, true) = true
|
| 99 |
+
AND ((filters ? 'city') IS FALSE OR lower(place.metadata->>'city') = lower(filters->>'city'))
|
| 100 |
+
AND ((filters ? 'state') IS FALSE OR lower(place.metadata->>'state') = lower(filters->>'state'))
|
| 101 |
+
AND ((filters ? 'category') IS FALSE OR lower(place.metadata->>'category') = lower(filters->>'category'))
|
| 102 |
+
AND ((filters ? 'price_range') IS FALSE OR place.metadata->>'price_range' = filters->>'price_range')
|
| 103 |
+
AND ((filters ? 'occasion') IS FALSE OR place.metadata->>'occasion' ILIKE ('%' || (filters->>'occasion') || '%'))
|
| 104 |
+
AND (
|
| 105 |
+
(filters ? 'place_ids') IS FALSE
|
| 106 |
+
OR place.external_id IN (SELECT jsonb_array_elements_text(filters->'place_ids'))
|
| 107 |
+
)
|
| 108 |
+
ORDER BY place.embedding <=> query_embedding, place.external_id
|
| 109 |
+
LIMIT GREATEST(match_count * 4, 100)
|
| 110 |
+
),
|
| 111 |
+
lexical AS (
|
| 112 |
+
SELECT
|
| 113 |
+
place.external_id,
|
| 114 |
+
ts_rank_cd(
|
| 115 |
+
place.textsearch,
|
| 116 |
+
websearch_to_tsquery('simple', COALESCE(query_text, ''))
|
| 117 |
+
)::DOUBLE PRECISION AS lexical_score,
|
| 118 |
+
row_number() OVER (
|
| 119 |
+
ORDER BY
|
| 120 |
+
ts_rank_cd(
|
| 121 |
+
place.textsearch,
|
| 122 |
+
websearch_to_tsquery('simple', COALESCE(query_text, ''))
|
| 123 |
+
) DESC,
|
| 124 |
+
place.external_id
|
| 125 |
+
) AS lexical_rank
|
| 126 |
+
FROM public.place_embeddings_semantic_v1 AS place
|
| 127 |
+
WHERE place.is_active = true
|
| 128 |
+
AND COALESCE((filters->>'is_active')::boolean, true) = true
|
| 129 |
+
AND ((filters ? 'city') IS FALSE OR lower(place.metadata->>'city') = lower(filters->>'city'))
|
| 130 |
+
AND ((filters ? 'state') IS FALSE OR lower(place.metadata->>'state') = lower(filters->>'state'))
|
| 131 |
+
AND ((filters ? 'category') IS FALSE OR lower(place.metadata->>'category') = lower(filters->>'category'))
|
| 132 |
+
AND ((filters ? 'price_range') IS FALSE OR place.metadata->>'price_range' = filters->>'price_range')
|
| 133 |
+
AND ((filters ? 'occasion') IS FALSE OR place.metadata->>'occasion' ILIKE ('%' || (filters->>'occasion') || '%'))
|
| 134 |
+
AND (
|
| 135 |
+
(filters ? 'place_ids') IS FALSE
|
| 136 |
+
OR place.external_id IN (SELECT jsonb_array_elements_text(filters->'place_ids'))
|
| 137 |
+
)
|
| 138 |
+
AND query_text IS NOT NULL
|
| 139 |
+
AND btrim(query_text) <> ''
|
| 140 |
+
AND place.textsearch @@ websearch_to_tsquery('simple', query_text)
|
| 141 |
+
ORDER BY lexical_score DESC, place.external_id
|
| 142 |
+
LIMIT GREATEST(match_count * 4, 100)
|
| 143 |
+
),
|
| 144 |
+
unioned AS (
|
| 145 |
+
SELECT
|
| 146 |
+
COALESCE(dense.external_id, lexical.external_id) AS external_id,
|
| 147 |
+
dense.semantic_score,
|
| 148 |
+
lexical.lexical_score,
|
| 149 |
+
dense.dense_rank,
|
| 150 |
+
lexical.lexical_rank
|
| 151 |
+
FROM dense
|
| 152 |
+
FULL OUTER JOIN lexical USING (external_id)
|
| 153 |
+
),
|
| 154 |
+
scored AS (
|
| 155 |
+
SELECT
|
| 156 |
+
unioned.*,
|
| 157 |
+
(
|
| 158 |
+
COALESCE(1.0 / (60.0 + unioned.dense_rank), 0.0)
|
| 159 |
+
+ COALESCE(1.0 / (60.0 + unioned.lexical_rank), 0.0)
|
| 160 |
+
) AS rrf_score
|
| 161 |
+
FROM unioned
|
| 162 |
+
)
|
| 163 |
+
SELECT
|
| 164 |
+
place.external_id,
|
| 165 |
+
place.document,
|
| 166 |
+
place.metadata,
|
| 167 |
+
LEAST(
|
| 168 |
+
1.0,
|
| 169 |
+
0.75 * GREATEST(COALESCE(scored.semantic_score, 0.0), 0.0)
|
| 170 |
+
+ 0.15 * LEAST(COALESCE(scored.lexical_score, 0.0), 1.0)
|
| 171 |
+
+ 0.10 * LEAST(scored.rrf_score * 31.0, 1.0)
|
| 172 |
+
) AS score,
|
| 173 |
+
scored.semantic_score,
|
| 174 |
+
scored.lexical_score
|
| 175 |
+
FROM scored
|
| 176 |
+
JOIN public.place_embeddings_semantic_v1 AS place USING (external_id)
|
| 177 |
+
ORDER BY score DESC, place.external_id
|
| 178 |
+
LIMIT GREATEST(match_count, 0);
|
| 179 |
+
$$;
|
| 180 |
+
|
| 181 |
+
CREATE OR REPLACE FUNCTION public.upsert_place_embedding_semantic_v1(
|
| 182 |
+
p_external_id TEXT,
|
| 183 |
+
p_document TEXT,
|
| 184 |
+
p_metadata JSONB,
|
| 185 |
+
p_embedding VECTOR(768),
|
| 186 |
+
p_content_hash TEXT,
|
| 187 |
+
p_embedding_model TEXT,
|
| 188 |
+
p_embedding_version TEXT,
|
| 189 |
+
p_is_active BOOLEAN
|
| 190 |
+
)
|
| 191 |
+
RETURNS VOID
|
| 192 |
+
LANGUAGE sql
|
| 193 |
+
SECURITY DEFINER
|
| 194 |
+
SET search_path = public
|
| 195 |
+
AS $$
|
| 196 |
+
INSERT INTO public.place_embeddings_semantic_v1 (
|
| 197 |
+
external_id, document, metadata, embedding, content_hash,
|
| 198 |
+
embedding_model, embedding_version, is_active, updated_at
|
| 199 |
+
) VALUES (
|
| 200 |
+
p_external_id, p_document, p_metadata, p_embedding, p_content_hash,
|
| 201 |
+
p_embedding_model, p_embedding_version, p_is_active, now()
|
| 202 |
+
)
|
| 203 |
+
ON CONFLICT (external_id) DO UPDATE SET
|
| 204 |
+
document = EXCLUDED.document,
|
| 205 |
+
metadata = EXCLUDED.metadata,
|
| 206 |
+
embedding = EXCLUDED.embedding,
|
| 207 |
+
content_hash = EXCLUDED.content_hash,
|
| 208 |
+
embedding_model = EXCLUDED.embedding_model,
|
| 209 |
+
embedding_version = EXCLUDED.embedding_version,
|
| 210 |
+
is_active = EXCLUDED.is_active,
|
| 211 |
+
updated_at = now();
|
| 212 |
+
$$;
|
| 213 |
+
|
| 214 |
+
CREATE OR REPLACE FUNCTION public.get_place_content_hashes_semantic_v1(
|
| 215 |
+
p_external_ids TEXT[]
|
| 216 |
+
)
|
| 217 |
+
RETURNS TABLE (external_id TEXT, content_hash TEXT)
|
| 218 |
+
LANGUAGE sql
|
| 219 |
+
STABLE
|
| 220 |
+
SECURITY DEFINER
|
| 221 |
+
SET search_path = public
|
| 222 |
+
AS $$
|
| 223 |
+
SELECT place.external_id, place.content_hash
|
| 224 |
+
FROM public.place_embeddings_semantic_v1 AS place
|
| 225 |
+
WHERE place.external_id = ANY(p_external_ids);
|
| 226 |
+
$$;
|
| 227 |
+
|
| 228 |
+
REVOKE ALL ON TABLE public.place_embeddings_semantic_v1 FROM PUBLIC;
|
| 229 |
+
REVOKE ALL ON FUNCTION public.match_places_semantic_v1(VECTOR, INTEGER, JSONB) FROM PUBLIC;
|
| 230 |
+
REVOKE ALL ON FUNCTION public.search_places_semantic_v1(TEXT, VECTOR, INTEGER, JSONB) FROM PUBLIC;
|
| 231 |
+
REVOKE ALL ON FUNCTION public.upsert_place_embedding_semantic_v1(TEXT, TEXT, JSONB, VECTOR, TEXT, TEXT, TEXT, BOOLEAN) FROM PUBLIC;
|
| 232 |
+
REVOKE ALL ON FUNCTION public.get_place_content_hashes_semantic_v1(TEXT[]) FROM PUBLIC;
|
| 233 |
+
|
| 234 |
+
DO $$
|
| 235 |
+
BEGIN
|
| 236 |
+
IF EXISTS (SELECT 1 FROM pg_roles WHERE rolname = 'nlp_reader') THEN
|
| 237 |
+
EXECUTE 'GRANT USAGE ON SCHEMA public TO nlp_reader';
|
| 238 |
+
EXECUTE 'GRANT EXECUTE ON FUNCTION public.match_places_semantic_v1(VECTOR, INTEGER, JSONB) TO nlp_reader';
|
| 239 |
+
EXECUTE 'GRANT EXECUTE ON FUNCTION public.search_places_semantic_v1(TEXT, VECTOR, INTEGER, JSONB) TO nlp_reader';
|
| 240 |
+
END IF;
|
| 241 |
+
IF EXISTS (SELECT 1 FROM pg_roles WHERE rolname = 'nlp_writer') THEN
|
| 242 |
+
EXECUTE 'GRANT USAGE ON SCHEMA public TO nlp_writer';
|
| 243 |
+
EXECUTE 'GRANT EXECUTE ON FUNCTION public.upsert_place_embedding_semantic_v1(TEXT, TEXT, JSONB, VECTOR, TEXT, TEXT, TEXT, BOOLEAN) TO nlp_writer';
|
| 244 |
+
EXECUTE 'GRANT EXECUTE ON FUNCTION public.get_place_content_hashes_semantic_v1(TEXT[]) TO nlp_writer';
|
| 245 |
+
END IF;
|
| 246 |
+
END
|
| 247 |
+
$$;
|
| 248 |
+
|
| 249 |
+
-- Execute after creating the least-privilege roles used by the service:
|
| 250 |
+
-- GRANT EXECUTE ON FUNCTION public.match_places_semantic_v1(VECTOR, INTEGER, JSONB) TO nlp_reader;
|
| 251 |
+
-- GRANT EXECUTE ON FUNCTION public.search_places_semantic_v1(TEXT, VECTOR, INTEGER, JSONB) TO nlp_reader;
|
| 252 |
+
-- GRANT EXECUTE ON FUNCTION public.upsert_place_embedding_semantic_v1(TEXT, TEXT, JSONB, VECTOR, TEXT, TEXT, TEXT, BOOLEAN) TO nlp_writer;
|
| 253 |
+
-- GRANT EXECUTE ON FUNCTION public.get_place_content_hashes_semantic_v1(TEXT[]) TO nlp_writer;
|
| 254 |
+
|
| 255 |
+
COMMIT;
|
sql/verify_places_semantic_v1.sql
ADDED
|
@@ -0,0 +1,46 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
BEGIN;
|
| 2 |
+
SET TRANSACTION READ ONLY;
|
| 3 |
+
|
| 4 |
+
DO $$
|
| 5 |
+
DECLARE
|
| 6 |
+
embedding_type TEXT;
|
| 7 |
+
BEGIN
|
| 8 |
+
SELECT format_type(attribute.atttypid, attribute.atttypmod)
|
| 9 |
+
INTO embedding_type
|
| 10 |
+
FROM pg_attribute AS attribute
|
| 11 |
+
WHERE attribute.attrelid = 'public.place_embeddings_semantic_v1'::regclass
|
| 12 |
+
AND attribute.attname = 'embedding'
|
| 13 |
+
AND NOT attribute.attisdropped;
|
| 14 |
+
|
| 15 |
+
IF embedding_type IS DISTINCT FROM 'vector(768)' THEN
|
| 16 |
+
RAISE EXCEPTION 'Expected place_embeddings_semantic_v1.embedding vector(768), got %', embedding_type;
|
| 17 |
+
END IF;
|
| 18 |
+
IF to_regprocedure('public.match_places_semantic_v1(vector,integer,jsonb)') IS NULL THEN
|
| 19 |
+
RAISE EXCEPTION 'match_places_semantic_v1 is missing';
|
| 20 |
+
END IF;
|
| 21 |
+
IF to_regprocedure('public.search_places_semantic_v1(text,vector,integer,jsonb)') IS NULL THEN
|
| 22 |
+
RAISE EXCEPTION 'search_places_semantic_v1 is missing';
|
| 23 |
+
END IF;
|
| 24 |
+
IF NOT EXISTS (
|
| 25 |
+
SELECT 1
|
| 26 |
+
FROM pg_indexes
|
| 27 |
+
WHERE schemaname = 'public'
|
| 28 |
+
AND tablename = 'place_embeddings_semantic_v1'
|
| 29 |
+
AND indexdef ILIKE '%USING hnsw%'
|
| 30 |
+
) THEN
|
| 31 |
+
RAISE EXCEPTION 'Places semantic HNSW index is missing';
|
| 32 |
+
END IF;
|
| 33 |
+
END
|
| 34 |
+
$$;
|
| 35 |
+
|
| 36 |
+
-- Inspect this plan after a representative backfill. With enough rows, the
|
| 37 |
+
-- nearest-neighbor branch should use the HNSW index rather than materializing
|
| 38 |
+
-- the entire eligible corpus before ordering.
|
| 39 |
+
EXPLAIN (ANALYZE, BUFFERS, COSTS, VERBOSE)
|
| 40 |
+
SELECT place.external_id
|
| 41 |
+
FROM public.place_embeddings_semantic_v1 AS place
|
| 42 |
+
WHERE place.is_active = true
|
| 43 |
+
ORDER BY place.embedding <=> (array_fill(0::real, ARRAY[768])::vector)
|
| 44 |
+
LIMIT 20;
|
| 45 |
+
|
| 46 |
+
ROLLBACK;
|
tests/conftest.py
CHANGED
|
@@ -5,6 +5,10 @@ os.environ["EMBEDDING_PROVIDER"] = "mock"
|
|
| 5 |
os.environ["EMBEDDING_DIMENSION"] = "16"
|
| 6 |
os.environ["EMBEDDING_MODEL"] = "mock-embedding"
|
| 7 |
os.environ["EMBEDDING_VERSION"] = "test-v1"
|
|
|
|
|
|
|
|
|
|
|
|
|
| 8 |
os.environ["GROQ_API_KEY"] = ""
|
| 9 |
os.environ["SEARCH_INTERNAL_TOKEN"] = "test-search-token"
|
| 10 |
os.environ["NLP_SERVICE_TOKEN"] = "test-nlp-service-token"
|
|
|
|
| 5 |
os.environ["EMBEDDING_DIMENSION"] = "16"
|
| 6 |
os.environ["EMBEDDING_MODEL"] = "mock-embedding"
|
| 7 |
os.environ["EMBEDDING_VERSION"] = "test-v1"
|
| 8 |
+
os.environ["PLACES_EMBEDDING_PROVIDER"] = "mock"
|
| 9 |
+
os.environ["PLACES_EMBEDDING_DIMENSION"] = "16"
|
| 10 |
+
os.environ["PLACES_EMBEDDING_MODEL"] = "mock-place-embedding"
|
| 11 |
+
os.environ["PLACES_EMBEDDING_VERSION"] = "test-places-v1"
|
| 12 |
os.environ["GROQ_API_KEY"] = ""
|
| 13 |
os.environ["SEARCH_INTERNAL_TOKEN"] = "test-search-token"
|
| 14 |
os.environ["NLP_SERVICE_TOKEN"] = "test-nlp-service-token"
|
tests/test_api_endpoints.py
CHANGED
|
@@ -20,14 +20,14 @@ def test_places_search_endpoint() -> None:
|
|
| 20 |
payload = response.json()
|
| 21 |
assert payload["query"] == "lugares tranquilos para cenar"
|
| 22 |
assert payload["places"]
|
| 23 |
-
assert payload["metrics"]["engine"] == "
|
| 24 |
assert payload["metrics"]["candidate_retrieval"] == "mock_embeddings"
|
| 25 |
assert payload["metrics"]["score_metric"] == "cosine_similarity"
|
| 26 |
assert payload["metrics"]["ranking_parameters"] == {"dimension": 16.0}
|
| 27 |
assert payload["metrics"]["field_weights"] == {
|
| 28 |
-
"tags":
|
| 29 |
-
"category":
|
| 30 |
-
"description":
|
| 31 |
"name": 1,
|
| 32 |
}
|
| 33 |
assert payload["metrics"]["returned_count"] == len(payload["places"])
|
|
@@ -76,7 +76,7 @@ def test_places_chat_endpoint_returns_trace_and_structured_places() -> None:
|
|
| 76 |
response = client.post(
|
| 77 |
"/places/chat",
|
| 78 |
json={
|
| 79 |
-
"message": "
|
| 80 |
"city": "Tuxtla Gutierrez",
|
| 81 |
"filters": {"occasion": "pareja", "is_active": True},
|
| 82 |
"limit": 3,
|
|
@@ -92,6 +92,72 @@ def test_places_chat_endpoint_returns_trace_and_structured_places() -> None:
|
|
| 92 |
assert payload["metadata"]["places_used_as_context"]
|
| 93 |
|
| 94 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 95 |
def test_places_recommendations_returns_llm_message_and_semantic_metadata() -> None:
|
| 96 |
client = TestClient(create_app())
|
| 97 |
|
|
@@ -109,7 +175,7 @@ def test_places_recommendations_returns_llm_message_and_semantic_metadata() -> N
|
|
| 109 |
payload = response.json()
|
| 110 |
assert payload["message"]
|
| 111 |
assert payload["places"]
|
| 112 |
-
assert payload["metrics"]["engine"] == "
|
| 113 |
assert payload["metrics"]["score_metric"] == "cosine_similarity"
|
| 114 |
assert payload["metrics"]["returned_count"] == len(payload["places"])
|
| 115 |
assert payload["metrics"]["candidate_retrieval"] == "mock_embeddings"
|
|
@@ -118,7 +184,7 @@ def test_places_recommendations_returns_llm_message_and_semantic_metadata() -> N
|
|
| 118 |
assert payload["metrics"]["scope"] == "current_query"
|
| 119 |
assert payload["metrics"]["ground_truth_available"] is False
|
| 120 |
assert "evaluation_metrics" not in payload
|
| 121 |
-
assert payload["metadata"]["ranking"] == "
|
| 122 |
assert payload["metadata"]["response_mode"] == "confident"
|
| 123 |
assert payload["metadata"]["used_llm"] is True
|
| 124 |
|
|
|
|
| 20 |
payload = response.json()
|
| 21 |
assert payload["query"] == "lugares tranquilos para cenar"
|
| 22 |
assert payload["places"]
|
| 23 |
+
assert payload["metrics"]["engine"] == "mock-place-embedding"
|
| 24 |
assert payload["metrics"]["candidate_retrieval"] == "mock_embeddings"
|
| 25 |
assert payload["metrics"]["score_metric"] == "cosine_similarity"
|
| 26 |
assert payload["metrics"]["ranking_parameters"] == {"dimension": 16.0}
|
| 27 |
assert payload["metrics"]["field_weights"] == {
|
| 28 |
+
"tags": 1,
|
| 29 |
+
"category": 1,
|
| 30 |
+
"description": 1,
|
| 31 |
"name": 1,
|
| 32 |
}
|
| 33 |
assert payload["metrics"]["returned_count"] == len(payload["places"])
|
|
|
|
| 76 |
response = client.post(
|
| 77 |
"/places/chat",
|
| 78 |
json={
|
| 79 |
+
"message": "cafe tranquilo postres platica",
|
| 80 |
"city": "Tuxtla Gutierrez",
|
| 81 |
"filters": {"occasion": "pareja", "is_active": True},
|
| 82 |
"limit": 3,
|
|
|
|
| 92 |
assert payload["metadata"]["places_used_as_context"]
|
| 93 |
|
| 94 |
|
| 95 |
+
def test_places_chat_opt_in_uses_semantic_conversation_contract() -> None:
|
| 96 |
+
client = TestClient(create_app())
|
| 97 |
+
|
| 98 |
+
response = client.post(
|
| 99 |
+
"/places/chat",
|
| 100 |
+
json={
|
| 101 |
+
"conversation_id": "3a4723f6-260d-4c11-b9d6-089f07a4f338",
|
| 102 |
+
"turn": 1,
|
| 103 |
+
"message": "una cafeteria tranquila",
|
| 104 |
+
"conversation_state": {
|
| 105 |
+
"city": "Tuxtla Gutierrez",
|
| 106 |
+
"taxonomy_version": "places-taxonomy-v1",
|
| 107 |
+
},
|
| 108 |
+
"user_location": {"lat": 16.7531, "lng": -93.1156},
|
| 109 |
+
"filters": {"is_active": True},
|
| 110 |
+
"candidate_limit": 5,
|
| 111 |
+
"limit": 1,
|
| 112 |
+
},
|
| 113 |
+
)
|
| 114 |
+
|
| 115 |
+
assert response.status_code == 200
|
| 116 |
+
payload = response.json()
|
| 117 |
+
assert payload["action"] == "recommendations"
|
| 118 |
+
assert payload["state_patch"]["target_category"] == "cafe"
|
| 119 |
+
assert payload["location_directive"]["source"] == "user_current"
|
| 120 |
+
assert payload["uncertainty"]["decision"] == "auto"
|
| 121 |
+
assert payload["metadata"]["pipeline"] == "places-chat-semantic-v2"
|
| 122 |
+
assert len(payload["places"]) == 1
|
| 123 |
+
|
| 124 |
+
|
| 125 |
+
def test_places_chat_opt_in_rejects_incompatible_conversation_state() -> None:
|
| 126 |
+
client = TestClient(create_app())
|
| 127 |
+
|
| 128 |
+
response = client.post(
|
| 129 |
+
"/places/chat",
|
| 130 |
+
json={
|
| 131 |
+
"conversation_id": "3a4723f6-260d-4c11-b9d6-089f07a4f338",
|
| 132 |
+
"message": "donas",
|
| 133 |
+
"conversation_state": {"taxonomy_version": "obsolete-v0"},
|
| 134 |
+
"user_location": {"lat": 16.7531, "lng": -93.1156},
|
| 135 |
+
},
|
| 136 |
+
)
|
| 137 |
+
|
| 138 |
+
assert response.status_code == 409
|
| 139 |
+
|
| 140 |
+
|
| 141 |
+
def test_places_chat_opt_in_maps_stale_clarification_to_conflict() -> None:
|
| 142 |
+
client = TestClient(create_app())
|
| 143 |
+
|
| 144 |
+
response = client.post(
|
| 145 |
+
"/places/chat",
|
| 146 |
+
json={
|
| 147 |
+
"conversation_id": "3a4723f6-260d-4c11-b9d6-089f07a4f338",
|
| 148 |
+
"message": "donas",
|
| 149 |
+
"conversation_state": {"taxonomy_version": "places-taxonomy-v1"},
|
| 150 |
+
"clarification_choice": {
|
| 151 |
+
"clarification_id": "00000000-0000-4000-8000-000000000000",
|
| 152 |
+
"option_id": "bakery",
|
| 153 |
+
},
|
| 154 |
+
"user_location": {"lat": 16.7531, "lng": -93.1156},
|
| 155 |
+
},
|
| 156 |
+
)
|
| 157 |
+
|
| 158 |
+
assert response.status_code == 409
|
| 159 |
+
|
| 160 |
+
|
| 161 |
def test_places_recommendations_returns_llm_message_and_semantic_metadata() -> None:
|
| 162 |
client = TestClient(create_app())
|
| 163 |
|
|
|
|
| 175 |
payload = response.json()
|
| 176 |
assert payload["message"]
|
| 177 |
assert payload["places"]
|
| 178 |
+
assert payload["metrics"]["engine"] == "mock-place-embedding"
|
| 179 |
assert payload["metrics"]["score_metric"] == "cosine_similarity"
|
| 180 |
assert payload["metrics"]["returned_count"] == len(payload["places"])
|
| 181 |
assert payload["metrics"]["candidate_retrieval"] == "mock_embeddings"
|
|
|
|
| 184 |
assert payload["metrics"]["scope"] == "current_query"
|
| 185 |
assert payload["metrics"]["ground_truth_available"] is False
|
| 186 |
assert "evaluation_metrics" not in payload
|
| 187 |
+
assert payload["metadata"]["ranking"] == "mock-place-embedding"
|
| 188 |
assert payload["metadata"]["response_mode"] == "confident"
|
| 189 |
assert payload["metadata"]["used_llm"] is True
|
| 190 |
|
tests/test_bert_place_intent_extractor.py
ADDED
|
@@ -0,0 +1,277 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
import math
|
| 2 |
+
|
| 3 |
+
import pytest
|
| 4 |
+
|
| 5 |
+
from app.modules.places.infrastructure.bert_intent_extractor import (
|
| 6 |
+
BertIntentInferenceError,
|
| 7 |
+
BertIntentModelLoadError,
|
| 8 |
+
BertIntentOutputError,
|
| 9 |
+
BertPlaceIntentExtractor,
|
| 10 |
+
)
|
| 11 |
+
|
| 12 |
+
|
| 13 |
+
class FakeTokenClassifier:
|
| 14 |
+
def __init__(self, predictions: list[dict[str, object]]) -> None:
|
| 15 |
+
self.predictions = predictions
|
| 16 |
+
self.calls: list[str] = []
|
| 17 |
+
|
| 18 |
+
def __call__(self, text: str) -> list[dict[str, object]]:
|
| 19 |
+
self.calls.append(text)
|
| 20 |
+
return list(self.predictions)
|
| 21 |
+
|
| 22 |
+
|
| 23 |
+
def _prediction(
|
| 24 |
+
text: str,
|
| 25 |
+
value: str,
|
| 26 |
+
entity: str,
|
| 27 |
+
score: float,
|
| 28 |
+
*,
|
| 29 |
+
after: int = 0,
|
| 30 |
+
) -> dict[str, object]:
|
| 31 |
+
start = text.index(value, after)
|
| 32 |
+
return {
|
| 33 |
+
"entity": entity,
|
| 34 |
+
"score": score,
|
| 35 |
+
"start": start,
|
| 36 |
+
"end": start + len(value),
|
| 37 |
+
"word": value,
|
| 38 |
+
}
|
| 39 |
+
|
| 40 |
+
|
| 41 |
+
def test_extractor_loads_lazily_and_preserves_raw_open_value_spans() -> None:
|
| 42 |
+
text = (
|
| 43 |
+
"Quiero donas artesanales, sin ruido, cerca del Parque Central a 2 km."
|
| 44 |
+
)
|
| 45 |
+
predictions = [
|
| 46 |
+
_prediction(text, "donas", "B-CATEGORY", 0.96),
|
| 47 |
+
_prediction(text, "artesanales", "I-CATEGORY", 0.90),
|
| 48 |
+
_prediction(text, "ruido", "B-EXCLUSION", 0.94),
|
| 49 |
+
_prediction(text, "Parque", "B-LOCATION", 0.91),
|
| 50 |
+
_prediction(text, "Central", "I-LOCATION", 0.89),
|
| 51 |
+
_prediction(text, "2", "B-RADIUS", 0.88),
|
| 52 |
+
_prediction(text, "km", "I-RADIUS", 0.86),
|
| 53 |
+
]
|
| 54 |
+
classifier = FakeTokenClassifier(predictions)
|
| 55 |
+
loader_calls: list[tuple[str, str | None, int | str | None]] = []
|
| 56 |
+
|
| 57 |
+
def loader(
|
| 58 |
+
model_name: str,
|
| 59 |
+
revision: str | None,
|
| 60 |
+
device: int | str | None,
|
| 61 |
+
) -> FakeTokenClassifier:
|
| 62 |
+
loader_calls.append((model_name, revision, device))
|
| 63 |
+
return classifier
|
| 64 |
+
|
| 65 |
+
extractor = BertPlaceIntentExtractor(
|
| 66 |
+
"frimeet/places-intent-bert",
|
| 67 |
+
model_version="2026-07-16",
|
| 68 |
+
device="cpu",
|
| 69 |
+
model_loader=loader,
|
| 70 |
+
)
|
| 71 |
+
|
| 72 |
+
assert extractor.is_loaded is False
|
| 73 |
+
assert loader_calls == []
|
| 74 |
+
|
| 75 |
+
frame = extractor.extract(text)
|
| 76 |
+
|
| 77 |
+
assert extractor.is_loaded is True
|
| 78 |
+
assert loader_calls == [
|
| 79 |
+
("frimeet/places-intent-bert", "2026-07-16", "cpu")
|
| 80 |
+
]
|
| 81 |
+
assert frame.raw_text == text
|
| 82 |
+
assert frame.model_name == "frimeet/places-intent-bert"
|
| 83 |
+
assert frame.model_version == "2026-07-16"
|
| 84 |
+
assert [span.text for span in frame.spans] == [
|
| 85 |
+
"donas artesanales",
|
| 86 |
+
"ruido",
|
| 87 |
+
"Parque Central",
|
| 88 |
+
"2 km",
|
| 89 |
+
]
|
| 90 |
+
assert frame.categories[0].text == "donas artesanales"
|
| 91 |
+
assert frame.categories[0].polarity == "positive"
|
| 92 |
+
assert frame.categories[0].token_count == 2
|
| 93 |
+
assert frame.exclusions[0].text == "ruido"
|
| 94 |
+
assert frame.exclusions[0].polarity == "negative"
|
| 95 |
+
assert frame.by_type("LOCATION")[0].text == "Parque Central"
|
| 96 |
+
assert frame.by_type("RADIUS")[0].text == "2 km"
|
| 97 |
+
assert 0.86 <= frame.confidence <= 0.96
|
| 98 |
+
for span in frame.spans:
|
| 99 |
+
assert text[span.start : span.end] == span.text
|
| 100 |
+
|
| 101 |
+
extractor.extract(text)
|
| 102 |
+
assert len(loader_calls) == 1
|
| 103 |
+
assert classifier.calls == [text, text]
|
| 104 |
+
|
| 105 |
+
|
| 106 |
+
def test_custom_model_labels_map_to_slots_and_contextual_polarity() -> None:
|
| 107 |
+
text = "Busco cafecito con terraza pero evito ruido"
|
| 108 |
+
classifier = FakeTokenClassifier(
|
| 109 |
+
[
|
| 110 |
+
_prediction(text, "cafecito", "B-TARGET", 0.93),
|
| 111 |
+
_prediction(text, "terraza", "B-AMENITY", 0.91),
|
| 112 |
+
_prediction(text, "ruido", "B-NEGATIVE_AMENITY", 0.95),
|
| 113 |
+
_prediction(text, "Busco", "B-UNRELATED_HEAD", 0.99),
|
| 114 |
+
]
|
| 115 |
+
)
|
| 116 |
+
extractor = BertPlaceIntentExtractor(
|
| 117 |
+
"injected-model",
|
| 118 |
+
classifier=classifier,
|
| 119 |
+
label_definitions={
|
| 120 |
+
"TARGET": "CATEGORY",
|
| 121 |
+
"AMENITY": ("PREFERENCE", "positive"),
|
| 122 |
+
"NEGATIVE_AMENITY": ("PREFERENCE", "negative"),
|
| 123 |
+
},
|
| 124 |
+
)
|
| 125 |
+
|
| 126 |
+
frame = extractor.extract(text)
|
| 127 |
+
|
| 128 |
+
assert [span.text for span in frame.categories] == ["cafecito"]
|
| 129 |
+
assert [span.text for span in frame.preferences] == ["terraza"]
|
| 130 |
+
assert [span.text for span in frame.exclusions] == ["ruido"]
|
| 131 |
+
assert frame.exclusions[0].slot_type == "PREFERENCE"
|
| 132 |
+
assert "UNRELATED_HEAD" not in {span.slot_type for span in frame.spans}
|
| 133 |
+
|
| 134 |
+
|
| 135 |
+
def test_stray_i_label_is_recovered_but_a_skipped_token_breaks_the_span() -> None:
|
| 136 |
+
text = "terraza muy tranquila"
|
| 137 |
+
classifier = FakeTokenClassifier(
|
| 138 |
+
[
|
| 139 |
+
_prediction(text, "terraza", "I-PREFERENCE", 0.92),
|
| 140 |
+
_prediction(text, "muy", "I-PREFERENCE", 0.30),
|
| 141 |
+
_prediction(text, "tranquila", "I-PREFERENCE", 0.90),
|
| 142 |
+
]
|
| 143 |
+
)
|
| 144 |
+
extractor = BertPlaceIntentExtractor(
|
| 145 |
+
"injected-model",
|
| 146 |
+
classifier=classifier,
|
| 147 |
+
minimum_token_confidence=0.8,
|
| 148 |
+
)
|
| 149 |
+
|
| 150 |
+
frame = extractor.extract(text)
|
| 151 |
+
|
| 152 |
+
assert [span.text for span in frame.preferences] == ["terraza", "tranquila"]
|
| 153 |
+
assert all(span.token_count == 1 for span in frame.preferences)
|
| 154 |
+
|
| 155 |
+
|
| 156 |
+
def test_huggingface_token_index_breaks_span_across_ignored_o_tokens() -> None:
|
| 157 |
+
text = "cafe cerca del parque"
|
| 158 |
+
first = _prediction(text, "cafe", "B-CATEGORY", 0.94)
|
| 159 |
+
first["index"] = 1
|
| 160 |
+
second = _prediction(text, "parque", "I-CATEGORY", 0.91)
|
| 161 |
+
second["index"] = 5
|
| 162 |
+
classifier = FakeTokenClassifier([first, second])
|
| 163 |
+
|
| 164 |
+
frame = BertPlaceIntentExtractor(
|
| 165 |
+
"injected-model",
|
| 166 |
+
classifier=classifier,
|
| 167 |
+
).extract(text)
|
| 168 |
+
|
| 169 |
+
assert [span.text for span in frame.categories] == ["cafe", "parque"]
|
| 170 |
+
assert all(span.token_count == 1 for span in frame.categories)
|
| 171 |
+
|
| 172 |
+
|
| 173 |
+
def test_empty_text_returns_an_empty_frame_without_loading_the_model() -> None:
|
| 174 |
+
def loader(
|
| 175 |
+
_model_name: str,
|
| 176 |
+
_revision: str | None,
|
| 177 |
+
_device: int | str | None,
|
| 178 |
+
) -> FakeTokenClassifier:
|
| 179 |
+
raise AssertionError("empty text must not load transformers")
|
| 180 |
+
|
| 181 |
+
extractor = BertPlaceIntentExtractor("lazy-model", model_loader=loader)
|
| 182 |
+
|
| 183 |
+
frame = extractor.extract(" \t\n")
|
| 184 |
+
|
| 185 |
+
assert frame.raw_text == " \t\n"
|
| 186 |
+
assert frame.spans == ()
|
| 187 |
+
assert frame.confidence == 0.0
|
| 188 |
+
assert extractor.is_loaded is False
|
| 189 |
+
|
| 190 |
+
|
| 191 |
+
def test_injected_classifier_does_not_require_transformers_and_accepts_group_labels() -> None:
|
| 192 |
+
text = "Parque México"
|
| 193 |
+
classifier = FakeTokenClassifier(
|
| 194 |
+
[
|
| 195 |
+
{
|
| 196 |
+
"entity_group": "LOCATION",
|
| 197 |
+
"score": 0.97,
|
| 198 |
+
"start": 0,
|
| 199 |
+
"end": len(text),
|
| 200 |
+
}
|
| 201 |
+
]
|
| 202 |
+
)
|
| 203 |
+
extractor = BertPlaceIntentExtractor("no-transformers-needed", classifier=classifier)
|
| 204 |
+
|
| 205 |
+
frame = extractor.extract(text)
|
| 206 |
+
|
| 207 |
+
assert frame.by_type("LOCATION")[0].text == text
|
| 208 |
+
assert frame.by_type("LOCATION")[0].confidence == pytest.approx(0.97)
|
| 209 |
+
|
| 210 |
+
|
| 211 |
+
def test_loader_inference_and_malformed_output_errors_have_context() -> None:
|
| 212 |
+
def broken_loader(
|
| 213 |
+
_model_name: str,
|
| 214 |
+
_revision: str | None,
|
| 215 |
+
_device: int | str | None,
|
| 216 |
+
) -> FakeTokenClassifier:
|
| 217 |
+
raise OSError("model cache unavailable")
|
| 218 |
+
|
| 219 |
+
extractor = BertPlaceIntentExtractor("missing-model", model_loader=broken_loader)
|
| 220 |
+
with pytest.raises(
|
| 221 |
+
BertIntentModelLoadError,
|
| 222 |
+
match="missing-model.*model cache unavailable",
|
| 223 |
+
):
|
| 224 |
+
extractor.extract("donas")
|
| 225 |
+
|
| 226 |
+
class BrokenClassifier:
|
| 227 |
+
def __call__(self, _text: str) -> list[dict[str, object]]:
|
| 228 |
+
raise RuntimeError("backend crashed")
|
| 229 |
+
|
| 230 |
+
extractor = BertPlaceIntentExtractor("broken-model", classifier=BrokenClassifier())
|
| 231 |
+
with pytest.raises(
|
| 232 |
+
BertIntentInferenceError,
|
| 233 |
+
match="broken-model.*backend crashed",
|
| 234 |
+
):
|
| 235 |
+
extractor.extract("donas")
|
| 236 |
+
|
| 237 |
+
malformed = FakeTokenClassifier(
|
| 238 |
+
[{"entity": "B-CATEGORY", "score": 0.9, "start": 0}]
|
| 239 |
+
)
|
| 240 |
+
extractor = BertPlaceIntentExtractor("bad-output", classifier=malformed)
|
| 241 |
+
with pytest.raises(BertIntentOutputError, match="integer end offset"):
|
| 242 |
+
extractor.extract("donas")
|
| 243 |
+
|
| 244 |
+
|
| 245 |
+
def test_frame_and_span_confidence_are_length_weighted() -> None:
|
| 246 |
+
text = "cafe silencioso"
|
| 247 |
+
classifier = FakeTokenClassifier(
|
| 248 |
+
[
|
| 249 |
+
_prediction(text, "cafe", "B-CATEGORY", 1.0),
|
| 250 |
+
_prediction(text, "silencioso", "B-PREFERENCE", 0.5),
|
| 251 |
+
]
|
| 252 |
+
)
|
| 253 |
+
frame = BertPlaceIntentExtractor("model", classifier=classifier).extract(text)
|
| 254 |
+
|
| 255 |
+
expected = (4 * 1.0 + 10 * 0.5) / 14
|
| 256 |
+
assert frame.confidence == pytest.approx(expected)
|
| 257 |
+
assert math.isfinite(frame.confidence)
|
| 258 |
+
|
| 259 |
+
|
| 260 |
+
def test_constructor_rejects_invalid_configuration() -> None:
|
| 261 |
+
classifier = FakeTokenClassifier([])
|
| 262 |
+
with pytest.raises(ValueError, match="model_name_or_path"):
|
| 263 |
+
BertPlaceIntentExtractor(" ")
|
| 264 |
+
with pytest.raises(ValueError, match="minimum_token_confidence"):
|
| 265 |
+
BertPlaceIntentExtractor("model", minimum_token_confidence=1.1)
|
| 266 |
+
with pytest.raises(ValueError, match="classifier or model_loader"):
|
| 267 |
+
BertPlaceIntentExtractor(
|
| 268 |
+
"model",
|
| 269 |
+
classifier=classifier,
|
| 270 |
+
model_loader=lambda _name, _revision, _device: classifier,
|
| 271 |
+
)
|
| 272 |
+
with pytest.raises(ValueError, match="slot_type"):
|
| 273 |
+
BertPlaceIntentExtractor(
|
| 274 |
+
"model",
|
| 275 |
+
classifier=classifier,
|
| 276 |
+
label_definitions={"CUSTOM": "NOT_A_SLOT"}, # type: ignore[dict-item]
|
| 277 |
+
)
|
tests/test_embeddings.py
CHANGED
|
@@ -1,4 +1,7 @@
|
|
| 1 |
from app.shared.nlp.embeddings.mock import MockEmbeddingProvider
|
|
|
|
|
|
|
|
|
|
| 2 |
|
| 3 |
|
| 4 |
def test_mock_embedding_provider_is_deterministic() -> None:
|
|
@@ -9,3 +12,27 @@ def test_mock_embedding_provider_is_deterministic() -> None:
|
|
| 9 |
|
| 10 |
assert first == second
|
| 11 |
assert len(first) == provider.dimension
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
from app.shared.nlp.embeddings.mock import MockEmbeddingProvider
|
| 2 |
+
from app.shared.nlp.embeddings.base import EmbeddingProvider
|
| 3 |
+
from app.shared.nlp.embeddings.cached import CachedEmbeddingProvider
|
| 4 |
+
from app.shared.cache.memory import SimpleTTLCache
|
| 5 |
|
| 6 |
|
| 7 |
def test_mock_embedding_provider_is_deterministic() -> None:
|
|
|
|
| 12 |
|
| 13 |
assert first == second
|
| 14 |
assert len(first) == provider.dimension
|
| 15 |
+
|
| 16 |
+
|
| 17 |
+
class RecordingBatchProvider(EmbeddingProvider):
|
| 18 |
+
def __init__(self) -> None:
|
| 19 |
+
self.batches: list[list[str]] = []
|
| 20 |
+
|
| 21 |
+
def embed_text(self, text: str) -> list[float]:
|
| 22 |
+
raise AssertionError("batch cache should use embed_batch for misses")
|
| 23 |
+
|
| 24 |
+
def embed_batch(self, texts: list[str]) -> list[list[float]]:
|
| 25 |
+
self.batches.append(list(texts))
|
| 26 |
+
return [[float(len(text))] for text in texts]
|
| 27 |
+
|
| 28 |
+
|
| 29 |
+
def test_cached_provider_batches_unique_misses_and_preserves_order() -> None:
|
| 30 |
+
inner = RecordingBatchProvider()
|
| 31 |
+
provider = CachedEmbeddingProvider(inner, SimpleTTLCache())
|
| 32 |
+
|
| 33 |
+
first = provider.embed_batch(["donas", "cafe", "donas"])
|
| 34 |
+
second = provider.embed_batch(["cafe", "parque"])
|
| 35 |
+
|
| 36 |
+
assert first == [[5.0], [4.0], [5.0]]
|
| 37 |
+
assert second == [[4.0], [6.0]]
|
| 38 |
+
assert inner.batches == [["donas", "cafe"], ["parque"]]
|
tests/test_hybrid_chat_retriever.py
ADDED
|
@@ -0,0 +1,232 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
import pytest
|
| 2 |
+
|
| 3 |
+
from app.modules.places.domain.chat_intent import (
|
| 4 |
+
ConversationStatePatch,
|
| 5 |
+
LocationIntent,
|
| 6 |
+
ParsedPlaceChatIntent,
|
| 7 |
+
)
|
| 8 |
+
from app.modules.places.domain.models import PlaceCandidate
|
| 9 |
+
from app.modules.places.infrastructure.hybrid_chat_retriever import (
|
| 10 |
+
HybridContentPlaceChatRetriever,
|
| 11 |
+
)
|
| 12 |
+
from app.shared.nlp.embeddings.mock import MockEmbeddingProvider
|
| 13 |
+
|
| 14 |
+
|
| 15 |
+
def _intent(
|
| 16 |
+
query: str,
|
| 17 |
+
*,
|
| 18 |
+
target_category: str | None = None,
|
| 19 |
+
category_values: tuple[str, ...] = (),
|
| 20 |
+
hard_filters: dict[str, object] | None = None,
|
| 21 |
+
exclusions: tuple[str, ...] = (),
|
| 22 |
+
) -> ParsedPlaceChatIntent:
|
| 23 |
+
return ParsedPlaceChatIntent(
|
| 24 |
+
action="recommendations",
|
| 25 |
+
target_category=target_category,
|
| 26 |
+
category_values=category_values,
|
| 27 |
+
hard_filters=hard_filters or {},
|
| 28 |
+
soft_preferences=(),
|
| 29 |
+
exclusions=exclusions,
|
| 30 |
+
reference=None,
|
| 31 |
+
location=LocationIntent(
|
| 32 |
+
scope="target_results",
|
| 33 |
+
source="user_current",
|
| 34 |
+
),
|
| 35 |
+
semantic_query=query,
|
| 36 |
+
confidence=0.90,
|
| 37 |
+
state_patch=ConversationStatePatch(),
|
| 38 |
+
)
|
| 39 |
+
|
| 40 |
+
|
| 41 |
+
class RecordingRepository:
|
| 42 |
+
source_name = "test"
|
| 43 |
+
|
| 44 |
+
def __init__(self, candidates: list[PlaceCandidate]) -> None:
|
| 45 |
+
self.candidates = candidates
|
| 46 |
+
self.calls: list[dict[str, object]] = []
|
| 47 |
+
|
| 48 |
+
async def search(self, embedding, filters, limit):
|
| 49 |
+
self.calls.append(
|
| 50 |
+
{
|
| 51 |
+
"embedding": embedding,
|
| 52 |
+
"filters": filters,
|
| 53 |
+
"limit": limit,
|
| 54 |
+
}
|
| 55 |
+
)
|
| 56 |
+
return self.candidates
|
| 57 |
+
|
| 58 |
+
|
| 59 |
+
@pytest.mark.asyncio
|
| 60 |
+
async def test_open_vocabulary_recommendation_retrieves_without_category() -> None:
|
| 61 |
+
repository = RecordingRepository(
|
| 62 |
+
[
|
| 63 |
+
PlaceCandidate(
|
| 64 |
+
id="donut_shop",
|
| 65 |
+
name="Dulce Circular",
|
| 66 |
+
category="bakery",
|
| 67 |
+
score=0.0,
|
| 68 |
+
metadata={"tags": "postres artesanales"},
|
| 69 |
+
)
|
| 70 |
+
]
|
| 71 |
+
)
|
| 72 |
+
retriever = HybridContentPlaceChatRetriever(
|
| 73 |
+
embedding_provider=MockEmbeddingProvider(dimension=16),
|
| 74 |
+
place_repository=repository,
|
| 75 |
+
minimum_content_score=0.95,
|
| 76 |
+
)
|
| 77 |
+
|
| 78 |
+
candidates = await retriever.retrieve(
|
| 79 |
+
intent=_intent(
|
| 80 |
+
"donas",
|
| 81 |
+
hard_filters={"city": "Tuxtla Gutierrez", "state": "Chiapas"},
|
| 82 |
+
),
|
| 83 |
+
limit=3,
|
| 84 |
+
)
|
| 85 |
+
|
| 86 |
+
assert [candidate.place_id for candidate in candidates] == ["donut_shop"]
|
| 87 |
+
assert repository.calls[0]["filters"].categories is None
|
| 88 |
+
assert repository.calls[0]["filters"].city == "Tuxtla Gutierrez"
|
| 89 |
+
assert repository.calls[0]["filters"].state == "Chiapas"
|
| 90 |
+
diagnostics = candidates[0].metadata["retrieval_diagnostics"]
|
| 91 |
+
assert diagnostics == {
|
| 92 |
+
"category_affinity": 0.0,
|
| 93 |
+
"category_match": "not_requested",
|
| 94 |
+
"exclusion_affinity": 0.0,
|
| 95 |
+
"exclusion_matches": [],
|
| 96 |
+
"content_quality": "weak",
|
| 97 |
+
"meets_minimum_content_score": False,
|
| 98 |
+
"minimum_content_score": 0.95,
|
| 99 |
+
"query_token_count": 1,
|
| 100 |
+
}
|
| 101 |
+
|
| 102 |
+
|
| 103 |
+
@pytest.mark.asyncio
|
| 104 |
+
async def test_category_boosts_ranking_but_does_not_remove_other_categories() -> None:
|
| 105 |
+
repository = RecordingRepository(
|
| 106 |
+
[
|
| 107 |
+
PlaceCandidate(
|
| 108 |
+
id="park",
|
| 109 |
+
name="Parque Vecinal",
|
| 110 |
+
category="park",
|
| 111 |
+
score=0.2,
|
| 112 |
+
),
|
| 113 |
+
PlaceCandidate(
|
| 114 |
+
id="cafe",
|
| 115 |
+
name="Salon Urbano",
|
| 116 |
+
category="cafe",
|
| 117 |
+
score=0.2,
|
| 118 |
+
),
|
| 119 |
+
]
|
| 120 |
+
)
|
| 121 |
+
retriever = HybridContentPlaceChatRetriever(
|
| 122 |
+
embedding_provider=MockEmbeddingProvider(dimension=16),
|
| 123 |
+
place_repository=repository,
|
| 124 |
+
)
|
| 125 |
+
|
| 126 |
+
candidates = await retriever.retrieve(
|
| 127 |
+
intent=_intent(
|
| 128 |
+
"sitio agradable",
|
| 129 |
+
target_category="cafe",
|
| 130 |
+
category_values=("cafe", "cafeteria"),
|
| 131 |
+
),
|
| 132 |
+
limit=5,
|
| 133 |
+
)
|
| 134 |
+
|
| 135 |
+
assert [candidate.place_id for candidate in candidates] == ["cafe", "park"]
|
| 136 |
+
assert candidates[0].metadata["retrieval_diagnostics"]["category_match"] == (
|
| 137 |
+
"exact"
|
| 138 |
+
)
|
| 139 |
+
assert candidates[1].metadata["retrieval_diagnostics"]["category_match"] == (
|
| 140 |
+
"none"
|
| 141 |
+
)
|
| 142 |
+
|
| 143 |
+
|
| 144 |
+
class HybridRepository:
|
| 145 |
+
source_name = "test_hybrid"
|
| 146 |
+
|
| 147 |
+
def __init__(self) -> None:
|
| 148 |
+
self.hybrid_call: dict[str, object] | None = None
|
| 149 |
+
|
| 150 |
+
async def search_hybrid(self, query_text, embedding, filters, limit):
|
| 151 |
+
self.hybrid_call = {
|
| 152 |
+
"query_text": query_text,
|
| 153 |
+
"embedding": embedding,
|
| 154 |
+
"filters": filters,
|
| 155 |
+
"limit": limit,
|
| 156 |
+
}
|
| 157 |
+
return [
|
| 158 |
+
PlaceCandidate(
|
| 159 |
+
id="hybrid_result",
|
| 160 |
+
name="Resultado Hibrido",
|
| 161 |
+
category="bakery",
|
| 162 |
+
# The fused repository score is not a semantic score. This
|
| 163 |
+
# row came from the independent lexical candidate pool.
|
| 164 |
+
score=0.99,
|
| 165 |
+
metadata={"lexical_score": 1.0},
|
| 166 |
+
)
|
| 167 |
+
]
|
| 168 |
+
|
| 169 |
+
async def search(self, embedding, filters, limit):
|
| 170 |
+
del embedding, filters, limit
|
| 171 |
+
raise AssertionError("search must not run when search_hybrid is available")
|
| 172 |
+
|
| 173 |
+
|
| 174 |
+
@pytest.mark.asyncio
|
| 175 |
+
async def test_optional_repository_hybrid_search_is_preferred() -> None:
|
| 176 |
+
repository = HybridRepository()
|
| 177 |
+
retriever = HybridContentPlaceChatRetriever(
|
| 178 |
+
embedding_provider=MockEmbeddingProvider(dimension=16),
|
| 179 |
+
place_repository=repository,
|
| 180 |
+
)
|
| 181 |
+
|
| 182 |
+
candidates = await retriever.retrieve(
|
| 183 |
+
intent=_intent("Donas cerca"),
|
| 184 |
+
limit=3,
|
| 185 |
+
)
|
| 186 |
+
|
| 187 |
+
assert [candidate.place_id for candidate in candidates] == ["hybrid_result"]
|
| 188 |
+
assert repository.hybrid_call is not None
|
| 189 |
+
assert repository.hybrid_call["query_text"] == "Donas cerca"
|
| 190 |
+
assert repository.hybrid_call["filters"].categories is None
|
| 191 |
+
assert repository.hybrid_call["limit"] == 40
|
| 192 |
+
assert candidates[0].semantic_score == 0.0
|
| 193 |
+
assert candidates[0].lexical_score > 0.0
|
| 194 |
+
|
| 195 |
+
|
| 196 |
+
@pytest.mark.asyncio
|
| 197 |
+
async def test_exclusions_penalize_instead_of_dropping_and_respect_negation() -> None:
|
| 198 |
+
repository = RecordingRepository(
|
| 199 |
+
[
|
| 200 |
+
PlaceCandidate(
|
| 201 |
+
id="quiet",
|
| 202 |
+
name="Patio Sereno",
|
| 203 |
+
category="cafe",
|
| 204 |
+
score=0.5,
|
| 205 |
+
document="Un espacio tranquilo sin ruido exterior",
|
| 206 |
+
),
|
| 207 |
+
PlaceCandidate(
|
| 208 |
+
id="noisy",
|
| 209 |
+
name="Foro Central",
|
| 210 |
+
category="cafe",
|
| 211 |
+
score=0.5,
|
| 212 |
+
document="Musica y ruido durante toda la noche",
|
| 213 |
+
),
|
| 214 |
+
]
|
| 215 |
+
)
|
| 216 |
+
retriever = HybridContentPlaceChatRetriever(
|
| 217 |
+
embedding_provider=MockEmbeddingProvider(dimension=16),
|
| 218 |
+
place_repository=repository,
|
| 219 |
+
)
|
| 220 |
+
|
| 221 |
+
candidates = await retriever.retrieve(
|
| 222 |
+
intent=_intent("lugar tranquilo", exclusions=("ruido",)),
|
| 223 |
+
limit=5,
|
| 224 |
+
)
|
| 225 |
+
|
| 226 |
+
assert [candidate.place_id for candidate in candidates] == ["quiet", "noisy"]
|
| 227 |
+
quiet_diagnostics = candidates[0].metadata["retrieval_diagnostics"]
|
| 228 |
+
noisy_diagnostics = candidates[1].metadata["retrieval_diagnostics"]
|
| 229 |
+
assert quiet_diagnostics["exclusion_affinity"] == 0.0
|
| 230 |
+
assert noisy_diagnostics["exclusion_affinity"] == 1.0
|
| 231 |
+
assert noisy_diagnostics["exclusion_matches"] == ["ruido"]
|
| 232 |
+
assert candidates[0].content_score > candidates[1].content_score
|
tests/test_internal_places_chat.py
CHANGED
|
@@ -52,8 +52,9 @@ def test_internal_chat_returns_only_technical_candidates() -> None:
|
|
| 52 |
"strict_radius": False,
|
| 53 |
}
|
| 54 |
assert payload["candidates"]
|
| 55 |
-
assert
|
| 56 |
-
|
|
|
|
| 57 |
}
|
| 58 |
assert set(payload["candidates"][0]) == {
|
| 59 |
"place_id",
|
|
@@ -114,7 +115,7 @@ def test_internal_chat_recommends_for_short_explicit_categories(
|
|
| 114 |
}
|
| 115 |
|
| 116 |
|
| 117 |
-
def
|
| 118 |
client = TestClient(create_app())
|
| 119 |
request = {
|
| 120 |
**BASE_REQUEST,
|
|
@@ -130,9 +131,7 @@ def test_internal_chat_never_returns_parks_for_a_cafe_query() -> None:
|
|
| 130 |
assert response.status_code == 200
|
| 131 |
payload = response.json()
|
| 132 |
assert payload["action"] == "recommendations"
|
| 133 |
-
assert
|
| 134 |
-
"place_1"
|
| 135 |
-
}
|
| 136 |
assert payload["location_directive"]["source"] == "explicit_anchor"
|
| 137 |
assert payload["location_directive"]["anchor_place_id"] == "place_6"
|
| 138 |
|
|
@@ -296,7 +295,7 @@ def test_internal_chat_rejects_a_stale_structured_choice() -> None:
|
|
| 296 |
assert response.status_code == 409
|
| 297 |
|
| 298 |
|
| 299 |
-
def
|
| 300 |
client = TestClient(create_app())
|
| 301 |
request = {
|
| 302 |
**BASE_REQUEST,
|
|
@@ -311,8 +310,10 @@ def test_internal_chat_applies_content_exclusions_before_ranking() -> None:
|
|
| 311 |
|
| 312 |
assert response.status_code == 200
|
| 313 |
payload = response.json()
|
| 314 |
-
assert payload["action"] == "
|
| 315 |
-
|
|
|
|
|
|
|
| 316 |
assert payload["state_patch"]["exclusions"] == ["tranquilo"]
|
| 317 |
|
| 318 |
|
|
|
|
| 52 |
"strict_radius": False,
|
| 53 |
}
|
| 54 |
assert payload["candidates"]
|
| 55 |
+
assert payload["candidates"][0]["place_id"] == "place_1"
|
| 56 |
+
assert "place_1" in {
|
| 57 |
+
candidate["place_id"] for candidate in payload["candidates"]
|
| 58 |
}
|
| 59 |
assert set(payload["candidates"][0]) == {
|
| 60 |
"place_id",
|
|
|
|
| 115 |
}
|
| 116 |
|
| 117 |
|
| 118 |
+
def test_internal_chat_soft_category_ranks_cafe_first_near_park_anchor() -> None:
|
| 119 |
client = TestClient(create_app())
|
| 120 |
request = {
|
| 121 |
**BASE_REQUEST,
|
|
|
|
| 131 |
assert response.status_code == 200
|
| 132 |
payload = response.json()
|
| 133 |
assert payload["action"] == "recommendations"
|
| 134 |
+
assert payload["candidates"][0]["place_id"] == "place_1"
|
|
|
|
|
|
|
| 135 |
assert payload["location_directive"]["source"] == "explicit_anchor"
|
| 136 |
assert payload["location_directive"]["anchor_place_id"] == "place_6"
|
| 137 |
|
|
|
|
| 295 |
assert response.status_code == 409
|
| 296 |
|
| 297 |
|
| 298 |
+
def test_internal_chat_applies_content_exclusions_as_negative_ranking_evidence() -> None:
|
| 299 |
client = TestClient(create_app())
|
| 300 |
request = {
|
| 301 |
**BASE_REQUEST,
|
|
|
|
| 310 |
|
| 311 |
assert response.status_code == 200
|
| 312 |
payload = response.json()
|
| 313 |
+
assert payload["action"] == "recommendations"
|
| 314 |
+
candidate_ids = [candidate["place_id"] for candidate in payload["candidates"]]
|
| 315 |
+
assert "place_1" in candidate_ids
|
| 316 |
+
assert candidate_ids[0] != "place_1"
|
| 317 |
assert payload["state_patch"]["exclusions"] == ["tranquilo"]
|
| 318 |
|
| 319 |
|
tests/test_main_api_place_source.py
CHANGED
|
@@ -31,7 +31,7 @@ def test_place_to_source_record_maps_api_place() -> None:
|
|
| 31 |
assert len(record.content_hash) == 64
|
| 32 |
|
| 33 |
|
| 34 |
-
def
|
| 35 |
record = place_to_source_record(
|
| 36 |
{
|
| 37 |
"id": "place_weighted",
|
|
@@ -48,17 +48,17 @@ def test_place_to_source_record_resolves_and_weights_numeric_tags() -> None:
|
|
| 48 |
assert record.metadata["tags"] == "Compras,Ropa barata"
|
| 49 |
assert record.metadata["tag_ids"] == [29, 187, 9999]
|
| 50 |
assert record.metadata["unknown_tag_ids"] == [9999]
|
| 51 |
-
assert record.metadata["semantic_document_version"] == "
|
| 52 |
assert record.document.count("Nombre Ambiguo") == 1
|
| 53 |
-
assert record.document.count("Venta de prendas y accesorios
|
| 54 |
-
assert record.document.count("Compras Ropa barata") ==
|
| 55 |
-
assert
|
| 56 |
assert "osm" not in record.document
|
| 57 |
assert "Direccion que no debe influir" not in record.document
|
| 58 |
assert "9999" not in record.document
|
| 59 |
|
| 60 |
|
| 61 |
-
def
|
| 62 |
park = place_to_source_record(
|
| 63 |
{"id": "park", "name": "Area Uno", "category": "park"}
|
| 64 |
)
|
|
@@ -66,8 +66,10 @@ def test_semantic_documents_keep_canonical_and_local_category_terms() -> None:
|
|
| 66 |
{"id": "shopping", "name": "Area Dos", "category": "shopping"}
|
| 67 |
)
|
| 68 |
|
| 69 |
-
assert park is not None and "
|
| 70 |
-
assert shopping is not None and "
|
|
|
|
|
|
|
| 71 |
|
| 72 |
|
| 73 |
def test_place_tag_catalog_contains_complete_supplied_mapping() -> None:
|
|
|
|
| 31 |
assert len(record.content_hash) == 64
|
| 32 |
|
| 33 |
|
| 34 |
+
def test_place_to_source_record_resolves_tags_without_token_repetition() -> None:
|
| 35 |
record = place_to_source_record(
|
| 36 |
{
|
| 37 |
"id": "place_weighted",
|
|
|
|
| 48 |
assert record.metadata["tags"] == "Compras,Ropa barata"
|
| 49 |
assert record.metadata["tag_ids"] == [29, 187, 9999]
|
| 50 |
assert record.metadata["unknown_tag_ids"] == [9999]
|
| 51 |
+
assert record.metadata["semantic_document_version"] == "structured-place-v3"
|
| 52 |
assert record.document.count("Nombre Ambiguo") == 1
|
| 53 |
+
assert record.document.count("Venta de prendas y accesorios") == 1
|
| 54 |
+
assert record.document.count("Compras Ropa barata") == 1
|
| 55 |
+
assert "Tipo registrado: shopping." in record.document
|
| 56 |
assert "osm" not in record.document
|
| 57 |
assert "Direccion que no debe influir" not in record.document
|
| 58 |
assert "9999" not in record.document
|
| 59 |
|
| 60 |
|
| 61 |
+
def test_semantic_documents_keep_source_category_without_manual_expansion() -> None:
|
| 62 |
park = place_to_source_record(
|
| 63 |
{"id": "park", "name": "Area Uno", "category": "park"}
|
| 64 |
)
|
|
|
|
| 66 |
{"id": "shopping", "name": "Area Dos", "category": "shopping"}
|
| 67 |
)
|
| 68 |
|
| 69 |
+
assert park is not None and "Tipo registrado: park." in park.document
|
| 70 |
+
assert shopping is not None and "Tipo registrado: shopping." in shopping.document
|
| 71 |
+
assert "park parque naturaleza" not in park.document
|
| 72 |
+
assert "shopping compras tiendas" not in shopping.document
|
| 73 |
|
| 74 |
|
| 75 |
def test_place_tag_catalog_contains_complete_supplied_mapping() -> None:
|
tests/test_open_vocabulary_category_classifier.py
ADDED
|
@@ -0,0 +1,228 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
import math
|
| 2 |
+
|
| 3 |
+
import pytest
|
| 4 |
+
|
| 5 |
+
from app.modules.places.infrastructure.open_vocabulary_category_classifier import (
|
| 6 |
+
OpenVocabularyPlaceCategoryClassifier,
|
| 7 |
+
PlaceCategoryConcept,
|
| 8 |
+
)
|
| 9 |
+
from app.shared.nlp.embeddings.base import EmbeddingProvider
|
| 10 |
+
|
| 11 |
+
|
| 12 |
+
class ControlledEmbeddingProvider(EmbeddingProvider):
|
| 13 |
+
def __init__(self, vectors: dict[str, list[float]]) -> None:
|
| 14 |
+
self.vectors = vectors
|
| 15 |
+
self.text_calls: list[str] = []
|
| 16 |
+
self.batch_calls: list[list[str]] = []
|
| 17 |
+
|
| 18 |
+
def embed_text(self, text: str) -> list[float]:
|
| 19 |
+
self.text_calls.append(text)
|
| 20 |
+
return self.vectors.get(text, [0.0, 0.0, 0.0])
|
| 21 |
+
|
| 22 |
+
def embed_batch(self, texts: list[str]) -> list[list[float]]:
|
| 23 |
+
self.batch_calls.append(list(texts))
|
| 24 |
+
return [self.vectors.get(text, [0.0, 0.0, 0.0]) for text in texts]
|
| 25 |
+
|
| 26 |
+
|
| 27 |
+
def _catalog_vectors() -> dict[str, list[float]]:
|
| 28 |
+
return {
|
| 29 |
+
"Dulces horneados. Lugares con masas dulces y glaseadas": [1.0, 0.0, 0.0],
|
| 30 |
+
"Dulces horneados": [1.0, 0.0, 0.0],
|
| 31 |
+
"Lugares con masas dulces y glaseadas": [1.0, 0.0, 0.0],
|
| 32 |
+
"quiero una dona artesanal": [1.0, 0.0, 0.0],
|
| 33 |
+
"Naturaleza urbana. Espacios abiertos con vegetacion": [0.0, 1.0, 0.0],
|
| 34 |
+
"Naturaleza urbana": [0.0, 1.0, 0.0],
|
| 35 |
+
"Espacios abiertos con vegetacion": [0.0, 1.0, 0.0],
|
| 36 |
+
"quiero caminar entre arboles": [0.0, 1.0, 0.0],
|
| 37 |
+
"se me antojaron donitas": [0.9, 0.1, 0.0],
|
| 38 |
+
"quiero salir": [1.0, 1.0, 0.0],
|
| 39 |
+
"algo completamente distinto": [0.0, 0.0, 1.0],
|
| 40 |
+
}
|
| 41 |
+
|
| 42 |
+
|
| 43 |
+
def _concepts() -> list[PlaceCategoryConcept]:
|
| 44 |
+
return [
|
| 45 |
+
PlaceCategoryConcept(
|
| 46 |
+
id="sweet_baked_goods",
|
| 47 |
+
label="Dulces horneados",
|
| 48 |
+
description="Lugares con masas dulces y glaseadas",
|
| 49 |
+
examples=("quiero una dona artesanal",),
|
| 50 |
+
storage_values=("baked_goods", "pastry_vendor"),
|
| 51 |
+
),
|
| 52 |
+
PlaceCategoryConcept(
|
| 53 |
+
id="urban_nature",
|
| 54 |
+
label="Naturaleza urbana",
|
| 55 |
+
description="Espacios abiertos con vegetacion",
|
| 56 |
+
examples=("quiero caminar entre arboles",),
|
| 57 |
+
storage_values=("green_area",),
|
| 58 |
+
),
|
| 59 |
+
]
|
| 60 |
+
|
| 61 |
+
|
| 62 |
+
def test_rank_uses_injected_catalog_and_exposes_score_margin_and_storage_values() -> None:
|
| 63 |
+
embeddings = ControlledEmbeddingProvider(_catalog_vectors())
|
| 64 |
+
classifier = OpenVocabularyPlaceCategoryClassifier(
|
| 65 |
+
_concepts(),
|
| 66 |
+
embeddings,
|
| 67 |
+
)
|
| 68 |
+
|
| 69 |
+
assert classifier.is_indexed is False
|
| 70 |
+
assert embeddings.batch_calls == []
|
| 71 |
+
|
| 72 |
+
matches = classifier.rank("se me antojaron donitas", limit=2)
|
| 73 |
+
|
| 74 |
+
assert classifier.is_indexed is True
|
| 75 |
+
assert len(embeddings.batch_calls) == 1
|
| 76 |
+
assert [match.concept_id for match in matches] == [
|
| 77 |
+
"sweet_baked_goods",
|
| 78 |
+
"urban_nature",
|
| 79 |
+
]
|
| 80 |
+
assert matches[0].storage_values == ("baked_goods", "pastry_vendor")
|
| 81 |
+
assert matches[0].score == pytest.approx(0.9938837)
|
| 82 |
+
assert matches[0].margin > 0.88
|
| 83 |
+
assert matches[1].margin >= 1.0
|
| 84 |
+
assert classifier.get_concept("SWEET_BAKED_GOODS") == _concepts()[0]
|
| 85 |
+
|
| 86 |
+
|
| 87 |
+
def test_rank_limit_still_calculates_top_margin_against_runner_up() -> None:
|
| 88 |
+
classifier = OpenVocabularyPlaceCategoryClassifier(
|
| 89 |
+
_concepts(),
|
| 90 |
+
ControlledEmbeddingProvider(_catalog_vectors()),
|
| 91 |
+
)
|
| 92 |
+
|
| 93 |
+
only_match = classifier.rank("se me antojaron donitas", limit=1)
|
| 94 |
+
|
| 95 |
+
assert len(only_match) == 1
|
| 96 |
+
assert only_match[0].margin > 0.88
|
| 97 |
+
|
| 98 |
+
|
| 99 |
+
def test_query_and_concept_encoders_can_use_distinct_e5_prefix_roles() -> None:
|
| 100 |
+
vectors = _catalog_vectors()
|
| 101 |
+
query_embeddings = ControlledEmbeddingProvider(
|
| 102 |
+
{"se me antojaron donitas": vectors["se me antojaron donitas"]}
|
| 103 |
+
)
|
| 104 |
+
concept_embeddings = ControlledEmbeddingProvider(vectors)
|
| 105 |
+
classifier = OpenVocabularyPlaceCategoryClassifier(
|
| 106 |
+
_concepts(),
|
| 107 |
+
query_embeddings,
|
| 108 |
+
concept_embedding_provider=concept_embeddings,
|
| 109 |
+
)
|
| 110 |
+
|
| 111 |
+
matches = classifier.rank("se me antojaron donitas", limit=1)
|
| 112 |
+
|
| 113 |
+
assert matches[0].concept_id == "sweet_baked_goods"
|
| 114 |
+
assert query_embeddings.text_calls == ["se me antojaron donitas"]
|
| 115 |
+
assert query_embeddings.batch_calls == []
|
| 116 |
+
assert len(concept_embeddings.batch_calls) == 1
|
| 117 |
+
|
| 118 |
+
|
| 119 |
+
def test_classify_is_compatible_with_place_activity_classifier() -> None:
|
| 120 |
+
classifier = OpenVocabularyPlaceCategoryClassifier(
|
| 121 |
+
_concepts(),
|
| 122 |
+
ControlledEmbeddingProvider(_catalog_vectors()),
|
| 123 |
+
minimum_similarity=0.5,
|
| 124 |
+
minimum_margin=0.05,
|
| 125 |
+
)
|
| 126 |
+
|
| 127 |
+
result = classifier.classify("se me antojaron donitas")
|
| 128 |
+
|
| 129 |
+
assert result is not None
|
| 130 |
+
assert result.category == "sweet_baked_goods"
|
| 131 |
+
assert result.source == "semantic_activity"
|
| 132 |
+
assert result.category_values == ("baked_goods", "pastry_vendor")
|
| 133 |
+
assert result.label == "Dulces horneados"
|
| 134 |
+
assert 0.0 <= result.confidence <= 0.99
|
| 135 |
+
|
| 136 |
+
|
| 137 |
+
def test_classify_abstains_when_candidates_are_tied_or_similarity_is_low() -> None:
|
| 138 |
+
classifier = OpenVocabularyPlaceCategoryClassifier(
|
| 139 |
+
_concepts(),
|
| 140 |
+
ControlledEmbeddingProvider(_catalog_vectors()),
|
| 141 |
+
minimum_similarity=0.5,
|
| 142 |
+
minimum_margin=0.05,
|
| 143 |
+
)
|
| 144 |
+
|
| 145 |
+
assert classifier.classify("quiero salir") is None
|
| 146 |
+
assert classifier.classify("algo completamente distinto") is None
|
| 147 |
+
|
| 148 |
+
|
| 149 |
+
def test_mapping_catalog_is_supported_without_code_level_categories() -> None:
|
| 150 |
+
concept = {
|
| 151 |
+
"id": "orbital_archive",
|
| 152 |
+
"label": "Archivo orbital",
|
| 153 |
+
"description": "Colecciones documentales sobre misiones espaciales",
|
| 154 |
+
"examples": "quiero estudiar expediciones fuera de la Tierra",
|
| 155 |
+
"storage_values": ["space_records"],
|
| 156 |
+
}
|
| 157 |
+
semantic_text = (
|
| 158 |
+
"Archivo orbital. Colecciones documentales sobre misiones espaciales"
|
| 159 |
+
)
|
| 160 |
+
embeddings = ControlledEmbeddingProvider(
|
| 161 |
+
{
|
| 162 |
+
semantic_text: [1.0, 0.0, 0.0],
|
| 163 |
+
"Archivo orbital": [1.0, 0.0, 0.0],
|
| 164 |
+
"Colecciones documentales sobre misiones espaciales": [1.0, 0.0, 0.0],
|
| 165 |
+
"quiero estudiar expediciones fuera de la Tierra": [1.0, 0.0, 0.0],
|
| 166 |
+
"busco documentos de misiones espaciales": [1.0, 0.0, 0.0],
|
| 167 |
+
}
|
| 168 |
+
)
|
| 169 |
+
classifier = OpenVocabularyPlaceCategoryClassifier([concept], embeddings)
|
| 170 |
+
|
| 171 |
+
match = classifier.rank("busco documentos de misiones espaciales", limit=1)[0]
|
| 172 |
+
|
| 173 |
+
assert match.concept_id == "orbital_archive"
|
| 174 |
+
assert match.storage_values == ("space_records",)
|
| 175 |
+
|
| 176 |
+
|
| 177 |
+
def test_blank_or_zero_query_and_empty_catalog_return_no_matches() -> None:
|
| 178 |
+
embeddings = ControlledEmbeddingProvider(_catalog_vectors())
|
| 179 |
+
classifier = OpenVocabularyPlaceCategoryClassifier(_concepts(), embeddings)
|
| 180 |
+
|
| 181 |
+
assert classifier.rank(" ") == ()
|
| 182 |
+
assert classifier.rank("unknown") == ()
|
| 183 |
+
assert classifier.classify("unknown") is None
|
| 184 |
+
assert embeddings.batch_calls == []
|
| 185 |
+
|
| 186 |
+
empty = OpenVocabularyPlaceCategoryClassifier([], embeddings)
|
| 187 |
+
assert empty.rank("se me antojaron donitas") == ()
|
| 188 |
+
|
| 189 |
+
|
| 190 |
+
def test_catalog_and_rank_inputs_are_validated() -> None:
|
| 191 |
+
embeddings = ControlledEmbeddingProvider({})
|
| 192 |
+
with pytest.raises(ValueError, match="unique"):
|
| 193 |
+
OpenVocabularyPlaceCategoryClassifier(
|
| 194 |
+
[
|
| 195 |
+
{
|
| 196 |
+
"id": "Dynamic",
|
| 197 |
+
"label": "First",
|
| 198 |
+
"description": "First concept",
|
| 199 |
+
},
|
| 200 |
+
{
|
| 201 |
+
"id": "dynamic",
|
| 202 |
+
"label": "Second",
|
| 203 |
+
"description": "Second concept",
|
| 204 |
+
},
|
| 205 |
+
],
|
| 206 |
+
embeddings,
|
| 207 |
+
)
|
| 208 |
+
with pytest.raises(ValueError, match="missing required fields"):
|
| 209 |
+
OpenVocabularyPlaceCategoryClassifier(
|
| 210 |
+
[{"id": "incomplete", "label": "Incomplete"}],
|
| 211 |
+
embeddings,
|
| 212 |
+
)
|
| 213 |
+
|
| 214 |
+
classifier = OpenVocabularyPlaceCategoryClassifier([], embeddings)
|
| 215 |
+
with pytest.raises(ValueError, match="positive integer"):
|
| 216 |
+
classifier.rank("query", limit=0)
|
| 217 |
+
with pytest.raises(TypeError, match="text must be str"):
|
| 218 |
+
classifier.rank(42) # type: ignore[arg-type]
|
| 219 |
+
|
| 220 |
+
|
| 221 |
+
def test_non_finite_vectors_are_rejected() -> None:
|
| 222 |
+
embeddings = ControlledEmbeddingProvider(
|
| 223 |
+
{"query": [math.nan, 0.0, 1.0]}
|
| 224 |
+
)
|
| 225 |
+
classifier = OpenVocabularyPlaceCategoryClassifier(_concepts(), embeddings)
|
| 226 |
+
|
| 227 |
+
with pytest.raises(ValueError, match="NaN or infinity"):
|
| 228 |
+
classifier.rank("query")
|
tests/test_pgvector_readiness.py
CHANGED
|
@@ -1,4 +1,9 @@
|
|
| 1 |
-
from
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 2 |
|
| 3 |
|
| 4 |
def _functions(search_v2: bool) -> dict[str, dict[str, object]]:
|
|
@@ -20,3 +25,19 @@ def test_readiness_rejects_legacy_search_function() -> None:
|
|
| 20 |
|
| 21 |
def test_readiness_accepts_search_candidate_v2_contract() -> None:
|
| 22 |
assert _read_contract_is_ready(True, _functions(search_v2=True)) is True
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
from types import SimpleNamespace
|
| 2 |
+
|
| 3 |
+
from app.shared.vector_store.aws_pgvector import (
|
| 4 |
+
_configured_read_contract_signatures,
|
| 5 |
+
_read_contract_is_ready,
|
| 6 |
+
)
|
| 7 |
|
| 8 |
|
| 9 |
def _functions(search_v2: bool) -> dict[str, dict[str, object]]:
|
|
|
|
| 25 |
|
| 26 |
def test_readiness_accepts_search_candidate_v2_contract() -> None:
|
| 27 |
assert _read_contract_is_ready(True, _functions(search_v2=True)) is True
|
| 28 |
+
|
| 29 |
+
|
| 30 |
+
def test_readiness_includes_configured_places_semantic_contract() -> None:
|
| 31 |
+
signatures = _configured_read_contract_signatures(
|
| 32 |
+
SimpleNamespace(
|
| 33 |
+
places_pgvector_match_function="match_places_semantic_v1",
|
| 34 |
+
places_pgvector_hybrid_function="search_places_semantic_v1",
|
| 35 |
+
)
|
| 36 |
+
)
|
| 37 |
+
|
| 38 |
+
assert signatures["match_places_semantic_v1"] == (
|
| 39 |
+
"match_places_semantic_v1(vector, integer, jsonb)"
|
| 40 |
+
)
|
| 41 |
+
assert signatures["search_places_semantic_v1"] == (
|
| 42 |
+
"search_places_semantic_v1(text, vector, integer, jsonb)"
|
| 43 |
+
)
|
tests/test_place_chat_intent_parser.py
CHANGED
|
@@ -5,9 +5,13 @@ from app.modules.places.domain.chat_intent import (
|
|
| 5 |
ConversationState,
|
| 6 |
ExplicitTargetLocation,
|
| 7 |
PendingClarification,
|
|
|
|
| 8 |
PlaceCategoryInference,
|
| 9 |
PlaceReference,
|
| 10 |
)
|
|
|
|
|
|
|
|
|
|
| 11 |
from app.modules.places.domain.errors import ClarificationStateMismatchError
|
| 12 |
from app.modules.places.infrastructure.deterministic_intent_parser import (
|
| 13 |
DeterministicPlaceChatIntentParser,
|
|
@@ -122,15 +126,31 @@ def test_explicit_radius_is_strict_and_removed_from_anchor_text() -> None:
|
|
| 122 |
assert intent.location.strict_radius is True
|
| 123 |
|
| 124 |
|
| 125 |
-
def
|
| 126 |
intent = DeterministicPlaceChatIntentParser().parse(
|
| 127 |
message="quiero un lugar bonito para salir",
|
| 128 |
state=ConversationState(),
|
| 129 |
has_user_location=True,
|
| 130 |
)
|
| 131 |
|
| 132 |
-
assert intent.action == "
|
|
|
|
|
|
|
| 133 |
assert intent.unresolved == ("target_category",)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 134 |
|
| 135 |
|
| 136 |
def test_food_activity_defaults_to_restaurant_without_clarification() -> None:
|
|
@@ -353,7 +373,7 @@ def test_pending_scope_is_resolved_by_structured_choice_without_repeating() -> N
|
|
| 353 |
def test_structured_choice_rejects_a_stale_clarification_id() -> None:
|
| 354 |
parser = DeterministicPlaceChatIntentParser()
|
| 355 |
first = parser.parse(
|
| 356 |
-
message="
|
| 357 |
state=ConversationState(),
|
| 358 |
has_user_location=True,
|
| 359 |
)
|
|
@@ -410,3 +430,336 @@ def test_exclusion_is_kept_out_of_positive_preferences() -> None:
|
|
| 410 |
assert "musica" not in intent.soft_preferences
|
| 411 |
assert intent.exclusions == ("musica",)
|
| 412 |
assert intent.state_patch.as_dict()["exclusions"] == ["musica"]
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 5 |
ConversationState,
|
| 6 |
ExplicitTargetLocation,
|
| 7 |
PendingClarification,
|
| 8 |
+
PendingClarificationOption,
|
| 9 |
PlaceCategoryInference,
|
| 10 |
PlaceReference,
|
| 11 |
)
|
| 12 |
+
from app.modules.places.infrastructure.bert_intent_extractor import (
|
| 13 |
+
BertPlaceIntentExtractor,
|
| 14 |
+
)
|
| 15 |
from app.modules.places.domain.errors import ClarificationStateMismatchError
|
| 16 |
from app.modules.places.infrastructure.deterministic_intent_parser import (
|
| 17 |
DeterministicPlaceChatIntentParser,
|
|
|
|
| 126 |
assert intent.location.strict_radius is True
|
| 127 |
|
| 128 |
|
| 129 |
+
def test_missing_category_preserves_open_concept_for_retrieval() -> None:
|
| 130 |
intent = DeterministicPlaceChatIntentParser().parse(
|
| 131 |
message="quiero un lugar bonito para salir",
|
| 132 |
state=ConversationState(),
|
| 133 |
has_user_location=True,
|
| 134 |
)
|
| 135 |
|
| 136 |
+
assert intent.action == "recommendations"
|
| 137 |
+
assert intent.target_category is None
|
| 138 |
+
assert "bonito" in intent.semantic_query
|
| 139 |
assert intent.unresolved == ("target_category",)
|
| 140 |
+
assert intent.state_patch.pending_clarification is None
|
| 141 |
+
|
| 142 |
+
|
| 143 |
+
def test_unknown_category_word_is_not_dropped_before_dense_retrieval() -> None:
|
| 144 |
+
intent = DeterministicPlaceChatIntentParser().parse(
|
| 145 |
+
message="quiero donas artesanales",
|
| 146 |
+
state=ConversationState(),
|
| 147 |
+
has_user_location=True,
|
| 148 |
+
)
|
| 149 |
+
|
| 150 |
+
assert intent.action == "recommendations"
|
| 151 |
+
assert intent.target_category is None
|
| 152 |
+
assert "donas" in intent.semantic_query
|
| 153 |
+
assert intent.raw_category_phrase is not None
|
| 154 |
|
| 155 |
|
| 156 |
def test_food_activity_defaults_to_restaurant_without_clarification() -> None:
|
|
|
|
| 373 |
def test_structured_choice_rejects_a_stale_clarification_id() -> None:
|
| 374 |
parser = DeterministicPlaceChatIntentParser()
|
| 375 |
first = parser.parse(
|
| 376 |
+
message="cafeterias como la de Hello Kitty cerca del Parque Central",
|
| 377 |
state=ConversationState(),
|
| 378 |
has_user_location=True,
|
| 379 |
)
|
|
|
|
| 430 |
assert "musica" not in intent.soft_preferences
|
| 431 |
assert intent.exclusions == ("musica",)
|
| 432 |
assert intent.state_patch.as_dict()["exclusions"] == ["musica"]
|
| 433 |
+
|
| 434 |
+
|
| 435 |
+
def test_unknown_exclusion_is_preserved_and_does_not_consume_positive_context() -> None:
|
| 436 |
+
intent = DeterministicPlaceChatIntentParser().parse(
|
| 437 |
+
message="una cafeteria sin ruido con terraza",
|
| 438 |
+
state=ConversationState(),
|
| 439 |
+
has_user_location=True,
|
| 440 |
+
)
|
| 441 |
+
|
| 442 |
+
assert intent.action == "recommendations"
|
| 443 |
+
assert intent.exclusions == ("ruido",)
|
| 444 |
+
assert "terraza" in intent.semantic_query
|
| 445 |
+
assert "ruido" not in intent.semantic_query
|
| 446 |
+
|
| 447 |
+
|
| 448 |
+
def test_bert_slots_drive_open_category_exclusion_and_location_before_legacy() -> None:
|
| 449 |
+
message = "quiero donas artesanales sin ruido cerca de la plaza"
|
| 450 |
+
|
| 451 |
+
def token(value: str, entity: str, score: float) -> dict[str, object]:
|
| 452 |
+
start = message.index(value)
|
| 453 |
+
return {
|
| 454 |
+
"entity": entity,
|
| 455 |
+
"score": score,
|
| 456 |
+
"start": start,
|
| 457 |
+
"end": start + len(value),
|
| 458 |
+
}
|
| 459 |
+
|
| 460 |
+
class Classifier:
|
| 461 |
+
def __call__(self, _text: str) -> list[dict[str, object]]:
|
| 462 |
+
return [
|
| 463 |
+
token("donas", "B-CATEGORY", 0.96),
|
| 464 |
+
token("artesanales", "I-CATEGORY", 0.94),
|
| 465 |
+
token("ruido", "B-EXCLUSION", 0.95),
|
| 466 |
+
token("plaza", "B-LOCATION", 0.93),
|
| 467 |
+
]
|
| 468 |
+
|
| 469 |
+
extractor = BertPlaceIntentExtractor(
|
| 470 |
+
"places-intent-test",
|
| 471 |
+
model_version="test-v1",
|
| 472 |
+
classifier=Classifier(),
|
| 473 |
+
)
|
| 474 |
+
intent = DeterministicPlaceChatIntentParser(
|
| 475 |
+
contextual_extractor=extractor,
|
| 476 |
+
).parse(
|
| 477 |
+
message=message,
|
| 478 |
+
state=ConversationState(),
|
| 479 |
+
has_user_location=True,
|
| 480 |
+
)
|
| 481 |
+
|
| 482 |
+
assert intent.action == "recommendations"
|
| 483 |
+
assert intent.target_category == "donas artesanales"
|
| 484 |
+
assert intent.category_values == ("donas artesanales",)
|
| 485 |
+
assert intent.raw_category_phrase == "donas artesanales"
|
| 486 |
+
assert intent.exclusions == ("ruido",)
|
| 487 |
+
assert intent.location.scope == "target_results"
|
| 488 |
+
assert intent.location.anchor_text == "plaza"
|
| 489 |
+
assert "donas artesanales" in intent.semantic_query
|
| 490 |
+
assert "ruido" not in intent.semantic_query
|
| 491 |
+
assert intent.state_patch.target_category == "donas artesanales"
|
| 492 |
+
assert intent.intent_model_version == (
|
| 493 |
+
"bert-token:places-intent-test@test-v1+deterministic-open-v2"
|
| 494 |
+
)
|
| 495 |
+
|
| 496 |
+
|
| 497 |
+
def test_bert_raw_category_is_aligned_semantically_to_dynamic_storage_values() -> None:
|
| 498 |
+
message = "quiero donas artesanales"
|
| 499 |
+
start = message.index("donas")
|
| 500 |
+
|
| 501 |
+
class TokenClassifier:
|
| 502 |
+
def __call__(self, _text: str) -> list[dict[str, object]]:
|
| 503 |
+
return [
|
| 504 |
+
{
|
| 505 |
+
"entity": "B-CATEGORY",
|
| 506 |
+
"score": 0.96,
|
| 507 |
+
"start": start,
|
| 508 |
+
"end": len(message),
|
| 509 |
+
}
|
| 510 |
+
]
|
| 511 |
+
|
| 512 |
+
class Concept:
|
| 513 |
+
id = "donut_shop"
|
| 514 |
+
storage_values = ("bakery", "dessert")
|
| 515 |
+
|
| 516 |
+
class ActivityClassifier:
|
| 517 |
+
concepts = (Concept(),)
|
| 518 |
+
|
| 519 |
+
def classify(self, text: str) -> PlaceCategoryInference | None:
|
| 520 |
+
assert text == "donas artesanales"
|
| 521 |
+
return PlaceCategoryInference(
|
| 522 |
+
category="donut_shop",
|
| 523 |
+
confidence=0.91,
|
| 524 |
+
source="semantic_activity",
|
| 525 |
+
category_values=("bakery", "dessert"),
|
| 526 |
+
label="Donas",
|
| 527 |
+
)
|
| 528 |
+
|
| 529 |
+
intent = DeterministicPlaceChatIntentParser(
|
| 530 |
+
activity_classifier=ActivityClassifier(),
|
| 531 |
+
contextual_extractor=BertPlaceIntentExtractor(
|
| 532 |
+
"places-intent-test",
|
| 533 |
+
classifier=TokenClassifier(),
|
| 534 |
+
),
|
| 535 |
+
).parse(
|
| 536 |
+
message=message,
|
| 537 |
+
state=ConversationState(),
|
| 538 |
+
has_user_location=True,
|
| 539 |
+
)
|
| 540 |
+
|
| 541 |
+
assert intent.target_category == "donut_shop"
|
| 542 |
+
assert intent.category_values == ("bakery", "dessert")
|
| 543 |
+
assert intent.raw_category_phrase == "donas artesanales"
|
| 544 |
+
|
| 545 |
+
|
| 546 |
+
def test_bert_failure_falls_open_and_exposes_fallback_model_version() -> None:
|
| 547 |
+
class BrokenClassifier:
|
| 548 |
+
def __call__(self, _text: str) -> list[dict[str, object]]:
|
| 549 |
+
raise RuntimeError("inference backend unavailable")
|
| 550 |
+
|
| 551 |
+
extractor = BertPlaceIntentExtractor(
|
| 552 |
+
"broken-intent-model",
|
| 553 |
+
model_version="broken-v1",
|
| 554 |
+
classifier=BrokenClassifier(),
|
| 555 |
+
)
|
| 556 |
+
intent = DeterministicPlaceChatIntentParser(
|
| 557 |
+
contextual_extractor=extractor,
|
| 558 |
+
).parse(
|
| 559 |
+
message="recomiendame una cafeteria tranquila",
|
| 560 |
+
state=ConversationState(),
|
| 561 |
+
has_user_location=True,
|
| 562 |
+
)
|
| 563 |
+
|
| 564 |
+
assert intent.action == "recommendations"
|
| 565 |
+
assert intent.target_category == "cafe"
|
| 566 |
+
assert "tranquilo" in intent.soft_preferences
|
| 567 |
+
assert intent.intent_model_version == (
|
| 568 |
+
"deterministic-open-v2+bert-fallback:broken-v1"
|
| 569 |
+
)
|
| 570 |
+
|
| 571 |
+
|
| 572 |
+
def test_successful_bert_frame_does_not_reapply_manual_category_aliases() -> None:
|
| 573 |
+
extractor = BertPlaceIntentExtractor(
|
| 574 |
+
"places-intent-test",
|
| 575 |
+
classifier=lambda _text: [],
|
| 576 |
+
)
|
| 577 |
+
|
| 578 |
+
intent = DeterministicPlaceChatIntentParser(
|
| 579 |
+
contextual_extractor=extractor,
|
| 580 |
+
).parse(
|
| 581 |
+
message="cafeteria con una variacion no etiquetada",
|
| 582 |
+
state=ConversationState(),
|
| 583 |
+
has_user_location=True,
|
| 584 |
+
)
|
| 585 |
+
|
| 586 |
+
assert intent.target_category is None
|
| 587 |
+
assert intent.category_values == ()
|
| 588 |
+
assert "cafeteria" in intent.semantic_query
|
| 589 |
+
assert intent.category_source == "unresolved"
|
| 590 |
+
|
| 591 |
+
|
| 592 |
+
def test_partial_bert_frame_fuses_missing_context_slots_independently() -> None:
|
| 593 |
+
message = "cafeteria tranquila sin ruido cerca del centro"
|
| 594 |
+
category_start = message.index("cafeteria")
|
| 595 |
+
extractor = BertPlaceIntentExtractor(
|
| 596 |
+
"places-intent-test",
|
| 597 |
+
classifier=lambda _text: [
|
| 598 |
+
{
|
| 599 |
+
"entity": "B-CATEGORY",
|
| 600 |
+
"score": 0.97,
|
| 601 |
+
"start": category_start,
|
| 602 |
+
"end": category_start + len("cafeteria"),
|
| 603 |
+
}
|
| 604 |
+
],
|
| 605 |
+
)
|
| 606 |
+
|
| 607 |
+
intent = DeterministicPlaceChatIntentParser(
|
| 608 |
+
contextual_extractor=extractor,
|
| 609 |
+
).parse(
|
| 610 |
+
message=message,
|
| 611 |
+
state=ConversationState(),
|
| 612 |
+
has_user_location=True,
|
| 613 |
+
)
|
| 614 |
+
|
| 615 |
+
# The model-provided open category is preserved (no taxonomy remap), while
|
| 616 |
+
# omitted slots use narrow rollout fallbacks instead of disappearing.
|
| 617 |
+
assert intent.target_category == "cafeteria"
|
| 618 |
+
assert intent.location.scope == "target_results"
|
| 619 |
+
assert intent.location.source == "current_message"
|
| 620 |
+
assert intent.location.anchor_text == "centro"
|
| 621 |
+
assert intent.exclusions == ("ruido",)
|
| 622 |
+
assert "tranquilo" in intent.soft_preferences
|
| 623 |
+
assert "centro" not in intent.semantic_query
|
| 624 |
+
assert "ruido" not in intent.semantic_query
|
| 625 |
+
|
| 626 |
+
|
| 627 |
+
def test_bert_preference_reference_and_radius_remain_open_raw_signals() -> None:
|
| 628 |
+
message = "busco salones de te estilo Casa Azul con terraza maximo 2 km"
|
| 629 |
+
|
| 630 |
+
def token(value: str, entity: str) -> dict[str, object]:
|
| 631 |
+
start = message.index(value)
|
| 632 |
+
return {
|
| 633 |
+
"entity": entity,
|
| 634 |
+
"score": 0.94,
|
| 635 |
+
"start": start,
|
| 636 |
+
"end": start + len(value),
|
| 637 |
+
}
|
| 638 |
+
|
| 639 |
+
class Classifier:
|
| 640 |
+
def __call__(self, _text: str) -> list[dict[str, object]]:
|
| 641 |
+
return [
|
| 642 |
+
token("salones", "B-CATEGORY"),
|
| 643 |
+
token("de", "I-CATEGORY"),
|
| 644 |
+
token("te", "I-CATEGORY"),
|
| 645 |
+
token("Casa", "B-REFERENCE"),
|
| 646 |
+
token("Azul", "I-REFERENCE"),
|
| 647 |
+
token("terraza", "B-PREFERENCE"),
|
| 648 |
+
token("2", "B-RADIUS"),
|
| 649 |
+
token("km", "I-RADIUS"),
|
| 650 |
+
]
|
| 651 |
+
|
| 652 |
+
intent = DeterministicPlaceChatIntentParser(
|
| 653 |
+
contextual_extractor=BertPlaceIntentExtractor(
|
| 654 |
+
"places-intent-test",
|
| 655 |
+
classifier=Classifier(),
|
| 656 |
+
),
|
| 657 |
+
).parse(
|
| 658 |
+
message=message,
|
| 659 |
+
state=ConversationState(),
|
| 660 |
+
has_user_location=True,
|
| 661 |
+
)
|
| 662 |
+
|
| 663 |
+
assert intent.target_category == "salones de te"
|
| 664 |
+
assert intent.soft_preferences == ("terraza",)
|
| 665 |
+
assert intent.reference is not None
|
| 666 |
+
assert intent.reference.entity == "Casa Azul"
|
| 667 |
+
assert intent.location.source == "user_current"
|
| 668 |
+
assert intent.location.radius_meters == 2000
|
| 669 |
+
assert intent.location.strict_radius is True
|
| 670 |
+
|
| 671 |
+
|
| 672 |
+
def test_dynamic_category_clarification_selection_is_not_taxonomy_gated() -> None:
|
| 673 |
+
pending = PendingClarification(
|
| 674 |
+
clarification_id="dynamic-category-1",
|
| 675 |
+
kind="intent_category",
|
| 676 |
+
options=(
|
| 677 |
+
PendingClarificationOption(
|
| 678 |
+
option_id="donuts",
|
| 679 |
+
value="donas artesanales",
|
| 680 |
+
label="Donas artesanales",
|
| 681 |
+
),
|
| 682 |
+
PendingClarificationOption(
|
| 683 |
+
option_id="desserts",
|
| 684 |
+
value="postres frios",
|
| 685 |
+
label="Postres frios",
|
| 686 |
+
),
|
| 687 |
+
),
|
| 688 |
+
)
|
| 689 |
+
state = ConversationState(
|
| 690 |
+
soft_preferences=("tranquilo",),
|
| 691 |
+
exclusions=("ruido",),
|
| 692 |
+
pending_clarification=pending,
|
| 693 |
+
city="Puebla",
|
| 694 |
+
)
|
| 695 |
+
|
| 696 |
+
intent = DeterministicPlaceChatIntentParser().parse(
|
| 697 |
+
message="Donas artesanales",
|
| 698 |
+
state=state,
|
| 699 |
+
has_user_location=True,
|
| 700 |
+
clarification_choice=ClarificationChoice(
|
| 701 |
+
clarification_id="dynamic-category-1",
|
| 702 |
+
option_id="donuts",
|
| 703 |
+
),
|
| 704 |
+
)
|
| 705 |
+
|
| 706 |
+
assert intent.action == "recommendations"
|
| 707 |
+
assert intent.target_category == "donas artesanales"
|
| 708 |
+
assert intent.category_values == ("donas artesanales",)
|
| 709 |
+
assert intent.raw_category_phrase == "donas artesanales"
|
| 710 |
+
assert intent.soft_preferences == ("tranquilo",)
|
| 711 |
+
assert intent.exclusions == ("ruido",)
|
| 712 |
+
assert intent.hard_filters["city"] == "Puebla"
|
| 713 |
+
assert intent.state_patch.target_category == "donas artesanales"
|
| 714 |
+
assert intent.state_patch.clear_pending_clarification is True
|
| 715 |
+
assert "dynamic-clarification-v1" in intent.intent_model_version
|
| 716 |
+
|
| 717 |
+
|
| 718 |
+
def test_open_catalog_category_values_are_rehydrated_on_following_turn() -> None:
|
| 719 |
+
class Concept:
|
| 720 |
+
id = "donut_shop"
|
| 721 |
+
storage_values = ("bakery", "dessert")
|
| 722 |
+
|
| 723 |
+
class ActivityClassifier:
|
| 724 |
+
concepts = (Concept(),)
|
| 725 |
+
|
| 726 |
+
def classify(self, text: str) -> PlaceCategoryInference | None:
|
| 727 |
+
if "donas" in text:
|
| 728 |
+
return PlaceCategoryInference(
|
| 729 |
+
category="donut_shop",
|
| 730 |
+
confidence=0.91,
|
| 731 |
+
source="semantic_activity",
|
| 732 |
+
category_values=("bakery", "dessert"),
|
| 733 |
+
label="Donas",
|
| 734 |
+
)
|
| 735 |
+
return None
|
| 736 |
+
|
| 737 |
+
parser = DeterministicPlaceChatIntentParser(
|
| 738 |
+
activity_classifier=ActivityClassifier(),
|
| 739 |
+
)
|
| 740 |
+
first = parser.parse(
|
| 741 |
+
message="quiero donas glaseadas",
|
| 742 |
+
state=ConversationState(),
|
| 743 |
+
has_user_location=True,
|
| 744 |
+
)
|
| 745 |
+
continued = parser.parse(
|
| 746 |
+
message="mas barato",
|
| 747 |
+
state=ConversationState(target_category=first.target_category),
|
| 748 |
+
has_user_location=True,
|
| 749 |
+
)
|
| 750 |
+
|
| 751 |
+
assert first.target_category == "donut_shop"
|
| 752 |
+
assert first.category_values == ("bakery", "dessert")
|
| 753 |
+
assert continued.target_category == "donut_shop"
|
| 754 |
+
assert continued.category_values == ("bakery", "dessert")
|
| 755 |
+
|
| 756 |
+
|
| 757 |
+
def test_generic_short_request_never_produces_an_empty_embedding_query() -> None:
|
| 758 |
+
intent = DeterministicPlaceChatIntentParser().parse(
|
| 759 |
+
message="dame opciones",
|
| 760 |
+
state=ConversationState(),
|
| 761 |
+
has_user_location=True,
|
| 762 |
+
)
|
| 763 |
+
|
| 764 |
+
assert intent.action == "recommendations"
|
| 765 |
+
assert intent.semantic_query == "dame opciones"
|
tests/test_place_chat_recommendations_use_case.py
CHANGED
|
@@ -1,3 +1,5 @@
|
|
|
|
|
|
|
|
| 1 |
import pytest
|
| 2 |
|
| 3 |
from app.modules.places.application.use_cases.chat_place_recommendations import (
|
|
@@ -10,8 +12,11 @@ from app.modules.places.domain.clarifications import (
|
|
| 10 |
from app.modules.places.domain.chat_intent import (
|
| 11 |
ClarificationChoice,
|
| 12 |
ConversationState,
|
|
|
|
|
|
|
| 13 |
PendingClarification,
|
| 14 |
PendingClarificationOption,
|
|
|
|
| 15 |
PlaceReference,
|
| 16 |
ResolvedPlaceAnchor,
|
| 17 |
)
|
|
@@ -33,21 +38,32 @@ from app.shared.nlp.llm.mock import MockLLMProvider
|
|
| 33 |
from app.shared.nlp.llm.output_guard import PlaceChatOutputGuard
|
| 34 |
|
| 35 |
|
| 36 |
-
def build_use_case(
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 37 |
embedding = MockEmbeddingProvider(dimension=16)
|
| 38 |
return ChatPlaceRecommendationsUseCase(
|
| 39 |
-
intent_parser=DeterministicPlaceChatIntentParser(),
|
| 40 |
anchor_resolver=anchor_resolver or MockPlaceAnchorResolver(),
|
| 41 |
-
retriever=
|
| 42 |
-
|
| 43 |
-
|
| 44 |
-
|
|
|
|
|
|
|
|
|
|
| 45 |
),
|
| 46 |
llm_provider=MockLLMProvider(),
|
| 47 |
output_guard=PlaceChatOutputGuard(),
|
| 48 |
ranking_version="places-chat-v2",
|
| 49 |
taxonomy_version="places-taxonomy-v1",
|
| 50 |
llm_enabled=llm_enabled,
|
|
|
|
| 51 |
)
|
| 52 |
|
| 53 |
|
|
@@ -68,6 +84,29 @@ def test_public_anchor_buttons_are_bounded_for_the_main_api_contract() -> None:
|
|
| 68 |
assert len(clarification.options[0].message) <= 200
|
| 69 |
|
| 70 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 71 |
@pytest.mark.asyncio
|
| 72 |
async def test_llm_cannot_change_action_or_candidates() -> None:
|
| 73 |
without_llm = await build_use_case(llm_enabled=False).execute(
|
|
@@ -95,6 +134,326 @@ async def test_llm_cannot_change_action_or_candidates() -> None:
|
|
| 95 |
assert with_llm.guard_reason == "candidate_name_deferred_to_main_api"
|
| 96 |
|
| 97 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 98 |
class AmbiguousAnchorResolver:
|
| 99 |
async def resolve(self, text, city, state, limit=3):
|
| 100 |
del text, city, state, limit
|
|
@@ -308,7 +667,7 @@ class MixedCategoryRepository:
|
|
| 308 |
("shopping", "shopping_exact"),
|
| 309 |
),
|
| 310 |
)
|
| 311 |
-
async def
|
| 312 |
message: str,
|
| 313 |
expected_id: str,
|
| 314 |
) -> None:
|
|
@@ -325,11 +684,23 @@ async def test_short_category_queries_survive_sparse_scores_without_family_noise
|
|
| 325 |
|
| 326 |
candidates = await retriever.retrieve(intent=intent, limit=5)
|
| 327 |
|
| 328 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 329 |
|
| 330 |
|
| 331 |
@pytest.mark.asyncio
|
| 332 |
-
async def
|
| 333 |
embedding = MockEmbeddingProvider(dimension=16)
|
| 334 |
retriever = HybridContentPlaceChatRetriever(
|
| 335 |
embedding_provider=embedding,
|
|
@@ -344,16 +715,18 @@ async def test_hard_category_and_controlled_theme_relaxation() -> None:
|
|
| 344 |
|
| 345 |
candidates = await retriever.retrieve(intent=intent, limit=10)
|
| 346 |
|
| 347 |
-
assert
|
|
|
|
| 348 |
"exact",
|
| 349 |
"family",
|
| 350 |
"broad",
|
| 351 |
-
|
| 352 |
-
|
| 353 |
-
"
|
| 354 |
-
|
| 355 |
-
|
| 356 |
-
]
|
|
|
|
| 357 |
|
| 358 |
|
| 359 |
class ReferenceLocationResolver:
|
|
|
|
| 1 |
+
from dataclasses import replace
|
| 2 |
+
|
| 3 |
import pytest
|
| 4 |
|
| 5 |
from app.modules.places.application.use_cases.chat_place_recommendations import (
|
|
|
|
| 12 |
from app.modules.places.domain.chat_intent import (
|
| 13 |
ClarificationChoice,
|
| 14 |
ConversationState,
|
| 15 |
+
ExplicitTargetLocation,
|
| 16 |
+
IntentAlternative,
|
| 17 |
PendingClarification,
|
| 18 |
PendingClarificationOption,
|
| 19 |
+
PlaceChatCandidate,
|
| 20 |
PlaceReference,
|
| 21 |
ResolvedPlaceAnchor,
|
| 22 |
)
|
|
|
|
| 38 |
from app.shared.nlp.llm.output_guard import PlaceChatOutputGuard
|
| 39 |
|
| 40 |
|
| 41 |
+
def build_use_case(
|
| 42 |
+
*,
|
| 43 |
+
llm_enabled: bool,
|
| 44 |
+
anchor_resolver=None,
|
| 45 |
+
intent_parser=None,
|
| 46 |
+
retriever=None,
|
| 47 |
+
nearby_place_provider=None,
|
| 48 |
+
):
|
| 49 |
embedding = MockEmbeddingProvider(dimension=16)
|
| 50 |
return ChatPlaceRecommendationsUseCase(
|
| 51 |
+
intent_parser=intent_parser or DeterministicPlaceChatIntentParser(),
|
| 52 |
anchor_resolver=anchor_resolver or MockPlaceAnchorResolver(),
|
| 53 |
+
retriever=(
|
| 54 |
+
retriever
|
| 55 |
+
or HybridContentPlaceChatRetriever(
|
| 56 |
+
embedding_provider=embedding,
|
| 57 |
+
place_repository=MockPlaceVectorRepository(embedding),
|
| 58 |
+
minimum_content_score=0.20,
|
| 59 |
+
)
|
| 60 |
),
|
| 61 |
llm_provider=MockLLMProvider(),
|
| 62 |
output_guard=PlaceChatOutputGuard(),
|
| 63 |
ranking_version="places-chat-v2",
|
| 64 |
taxonomy_version="places-taxonomy-v1",
|
| 65 |
llm_enabled=llm_enabled,
|
| 66 |
+
nearby_place_provider=nearby_place_provider,
|
| 67 |
)
|
| 68 |
|
| 69 |
|
|
|
|
| 84 |
assert len(clarification.options[0].message) <= 200
|
| 85 |
|
| 86 |
|
| 87 |
+
def test_open_category_buttons_use_safe_ids_and_preserve_raw_values() -> None:
|
| 88 |
+
from app.modules.places.domain.clarifications import new_category_clarification
|
| 89 |
+
|
| 90 |
+
pending = new_category_clarification(
|
| 91 |
+
("donas artesanales", "café de especialidad"),
|
| 92 |
+
kind="intent_category",
|
| 93 |
+
)
|
| 94 |
+
clarification = to_public_clarification(pending)
|
| 95 |
+
|
| 96 |
+
assert [option.option_id for option in pending.options] == [
|
| 97 |
+
"donas_artesanales_1",
|
| 98 |
+
"cafe_de_especialidad_2",
|
| 99 |
+
]
|
| 100 |
+
assert [option.value for option in pending.options] == [
|
| 101 |
+
"donas artesanales",
|
| 102 |
+
"café de especialidad",
|
| 103 |
+
]
|
| 104 |
+
assert [option.option_id for option in clarification.options] == [
|
| 105 |
+
"donas_artesanales_1",
|
| 106 |
+
"cafe_de_especialidad_2",
|
| 107 |
+
]
|
| 108 |
+
|
| 109 |
+
|
| 110 |
@pytest.mark.asyncio
|
| 111 |
async def test_llm_cannot_change_action_or_candidates() -> None:
|
| 112 |
without_llm = await build_use_case(llm_enabled=False).execute(
|
|
|
|
| 134 |
assert with_llm.guard_reason == "candidate_name_deferred_to_main_api"
|
| 135 |
|
| 136 |
|
| 137 |
+
class LowConfidenceIntentParser:
|
| 138 |
+
def __init__(self, alternatives: tuple[IntentAlternative, ...]) -> None:
|
| 139 |
+
self._alternatives = alternatives
|
| 140 |
+
|
| 141 |
+
def parse(
|
| 142 |
+
self,
|
| 143 |
+
message,
|
| 144 |
+
state,
|
| 145 |
+
has_user_location,
|
| 146 |
+
clarification_choice=None,
|
| 147 |
+
):
|
| 148 |
+
del message, clarification_choice
|
| 149 |
+
parsed = DeterministicPlaceChatIntentParser().parse(
|
| 150 |
+
message="una panaderia tranquila",
|
| 151 |
+
state=state,
|
| 152 |
+
has_user_location=has_user_location,
|
| 153 |
+
)
|
| 154 |
+
return replace(
|
| 155 |
+
parsed,
|
| 156 |
+
semantic_query="algo dulce tranquilo",
|
| 157 |
+
confidence=0.45,
|
| 158 |
+
alternatives=self._alternatives,
|
| 159 |
+
category_source="semantic_activity",
|
| 160 |
+
)
|
| 161 |
+
|
| 162 |
+
|
| 163 |
+
class CategoryEvidenceRetriever:
|
| 164 |
+
def __init__(self) -> None:
|
| 165 |
+
self.calls = 0
|
| 166 |
+
|
| 167 |
+
async def retrieve(self, intent, limit):
|
| 168 |
+
del intent, limit
|
| 169 |
+
self.calls += 1
|
| 170 |
+
return [
|
| 171 |
+
PlaceChatCandidate(
|
| 172 |
+
place_id="bakery_1",
|
| 173 |
+
name="Panaderia Local",
|
| 174 |
+
category="bakery",
|
| 175 |
+
content_score=0.62,
|
| 176 |
+
semantic_score=0.64,
|
| 177 |
+
lexical_score=0.20,
|
| 178 |
+
match_level="broad",
|
| 179 |
+
matched_reasons=("algo dulce",),
|
| 180 |
+
)
|
| 181 |
+
]
|
| 182 |
+
|
| 183 |
+
|
| 184 |
+
@pytest.mark.asyncio
|
| 185 |
+
async def test_low_confidence_uses_dynamic_category_hypotheses() -> None:
|
| 186 |
+
parser = LowConfidenceIntentParser(
|
| 187 |
+
alternatives=(
|
| 188 |
+
IntentAlternative(
|
| 189 |
+
key="bakery",
|
| 190 |
+
description="Panaderias artesanales",
|
| 191 |
+
confidence=0.68,
|
| 192 |
+
),
|
| 193 |
+
IntentAlternative(
|
| 194 |
+
key="ice_cream",
|
| 195 |
+
description="Postres y heladerias",
|
| 196 |
+
confidence=0.64,
|
| 197 |
+
),
|
| 198 |
+
)
|
| 199 |
+
)
|
| 200 |
+
|
| 201 |
+
result = await build_use_case(
|
| 202 |
+
llm_enabled=False,
|
| 203 |
+
intent_parser=parser,
|
| 204 |
+
).execute(
|
| 205 |
+
message="quiero algo dulce y tranquilo",
|
| 206 |
+
state=ConversationState(),
|
| 207 |
+
user_latitude=16.7531,
|
| 208 |
+
user_longitude=-93.1156,
|
| 209 |
+
candidate_limit=5,
|
| 210 |
+
result_limit=3,
|
| 211 |
+
)
|
| 212 |
+
|
| 213 |
+
assert result.action == "clarification"
|
| 214 |
+
assert result.clarification is not None
|
| 215 |
+
assert [option.option_id for option in result.clarification.options] == [
|
| 216 |
+
"bakery",
|
| 217 |
+
"ice_cream",
|
| 218 |
+
]
|
| 219 |
+
assert [option.label for option in result.clarification.options] == [
|
| 220 |
+
"Panaderias artesanales",
|
| 221 |
+
"Postres y heladerias",
|
| 222 |
+
]
|
| 223 |
+
assert result.state_patch["target_category"] == "bakery"
|
| 224 |
+
assert result.state_patch["soft_preferences"] == ["tranquilo"]
|
| 225 |
+
assert [
|
| 226 |
+
option["id"]
|
| 227 |
+
for option in result.state_patch["pending_clarification"]["options"]
|
| 228 |
+
] == ["bakery", "ice_cream"]
|
| 229 |
+
|
| 230 |
+
|
| 231 |
+
@pytest.mark.asyncio
|
| 232 |
+
async def test_low_confidence_without_top_k_uses_retrieval_evidence_not_fixed_menu() -> None:
|
| 233 |
+
retriever = CategoryEvidenceRetriever()
|
| 234 |
+
result = await build_use_case(
|
| 235 |
+
llm_enabled=False,
|
| 236 |
+
intent_parser=LowConfidenceIntentParser(alternatives=()),
|
| 237 |
+
retriever=retriever,
|
| 238 |
+
).execute(
|
| 239 |
+
message="quiero algo dulce y tranquilo",
|
| 240 |
+
state=ConversationState(),
|
| 241 |
+
user_latitude=16.7531,
|
| 242 |
+
user_longitude=-93.1156,
|
| 243 |
+
candidate_limit=5,
|
| 244 |
+
result_limit=3,
|
| 245 |
+
)
|
| 246 |
+
|
| 247 |
+
assert retriever.calls == 1
|
| 248 |
+
assert result.action == "recommendations"
|
| 249 |
+
assert result.clarification is None
|
| 250 |
+
assert [candidate.place_id for candidate in result.candidates] == ["bakery_1"]
|
| 251 |
+
assert "intent_confidence" in result.unresolved
|
| 252 |
+
assert "restaurante" not in result.message.casefold()
|
| 253 |
+
assert "cafeteria" not in result.message.casefold()
|
| 254 |
+
assert "parque" not in result.message.casefold()
|
| 255 |
+
assert result.state_patch["soft_preferences"] == ["tranquilo"]
|
| 256 |
+
|
| 257 |
+
|
| 258 |
+
@pytest.mark.asyncio
|
| 259 |
+
async def test_low_quality_hypotheses_do_not_create_an_arbitrary_menu() -> None:
|
| 260 |
+
parser = LowConfidenceIntentParser(
|
| 261 |
+
alternatives=(
|
| 262 |
+
IntentAlternative("bakery", "Panaderias", 0.55),
|
| 263 |
+
IntentAlternative("outdoors", "Espacios abiertos", 0.52),
|
| 264 |
+
)
|
| 265 |
+
)
|
| 266 |
+
|
| 267 |
+
result = await build_use_case(
|
| 268 |
+
llm_enabled=False,
|
| 269 |
+
intent_parser=parser,
|
| 270 |
+
retriever=CategoryEvidenceRetriever(),
|
| 271 |
+
).execute(
|
| 272 |
+
message="algo dificil de interpretar",
|
| 273 |
+
state=ConversationState(),
|
| 274 |
+
user_latitude=16.7531,
|
| 275 |
+
user_longitude=-93.1156,
|
| 276 |
+
candidate_limit=5,
|
| 277 |
+
result_limit=3,
|
| 278 |
+
)
|
| 279 |
+
|
| 280 |
+
assert result.action == "recommendations"
|
| 281 |
+
assert result.clarification is None
|
| 282 |
+
assert "intent_confidence" in result.unresolved
|
| 283 |
+
|
| 284 |
+
|
| 285 |
+
class WeakEvidenceRetriever:
|
| 286 |
+
async def retrieve(self, intent, limit):
|
| 287 |
+
del intent, limit
|
| 288 |
+
return [
|
| 289 |
+
PlaceChatCandidate(
|
| 290 |
+
place_id="weak_1",
|
| 291 |
+
name="Coincidencia tenue",
|
| 292 |
+
category="cafe",
|
| 293 |
+
content_score=0.0,
|
| 294 |
+
semantic_score=0.0,
|
| 295 |
+
lexical_score=0.0,
|
| 296 |
+
match_level="broad",
|
| 297 |
+
matched_reasons=(),
|
| 298 |
+
metadata={
|
| 299 |
+
"retrieval_diagnostics": {
|
| 300 |
+
"meets_minimum_content_score": False,
|
| 301 |
+
}
|
| 302 |
+
},
|
| 303 |
+
)
|
| 304 |
+
]
|
| 305 |
+
|
| 306 |
+
|
| 307 |
+
@pytest.mark.asyncio
|
| 308 |
+
async def test_weak_candidates_are_returned_as_reviewable_not_confident() -> None:
|
| 309 |
+
result = await build_use_case(
|
| 310 |
+
llm_enabled=False,
|
| 311 |
+
retriever=WeakEvidenceRetriever(),
|
| 312 |
+
).execute(
|
| 313 |
+
message="una cafeteria",
|
| 314 |
+
state=ConversationState(),
|
| 315 |
+
user_latitude=16.7531,
|
| 316 |
+
user_longitude=-93.1156,
|
| 317 |
+
candidate_limit=5,
|
| 318 |
+
result_limit=3,
|
| 319 |
+
)
|
| 320 |
+
|
| 321 |
+
assert result.action == "recommendations"
|
| 322 |
+
assert result.unresolved == ("retrieval_evidence",)
|
| 323 |
+
assert [candidate.place_id for candidate in result.candidates] == ["weak_1"]
|
| 324 |
+
assert "evidencia" in result.message.casefold()
|
| 325 |
+
|
| 326 |
+
|
| 327 |
+
class RecordingNearbyProvider:
|
| 328 |
+
def __init__(self) -> None:
|
| 329 |
+
self.call = None
|
| 330 |
+
|
| 331 |
+
async def get_nearby_place_ids(self, latitude, longitude, radius_meters):
|
| 332 |
+
self.call = (latitude, longitude, radius_meters)
|
| 333 |
+
return {"near_2", "near_1"}
|
| 334 |
+
|
| 335 |
+
|
| 336 |
+
class RecordingFilteredRetriever:
|
| 337 |
+
def __init__(self) -> None:
|
| 338 |
+
self.intent = None
|
| 339 |
+
|
| 340 |
+
async def retrieve(self, intent, limit):
|
| 341 |
+
del limit
|
| 342 |
+
self.intent = intent
|
| 343 |
+
return [
|
| 344 |
+
PlaceChatCandidate(
|
| 345 |
+
place_id="near_1",
|
| 346 |
+
name="Cercano",
|
| 347 |
+
category="cafe",
|
| 348 |
+
content_score=0.8,
|
| 349 |
+
semantic_score=0.8,
|
| 350 |
+
lexical_score=0.4,
|
| 351 |
+
match_level="exact",
|
| 352 |
+
matched_reasons=("cafe",),
|
| 353 |
+
metadata={
|
| 354 |
+
"retrieval_diagnostics": {
|
| 355 |
+
"meets_minimum_content_score": True,
|
| 356 |
+
}
|
| 357 |
+
},
|
| 358 |
+
)
|
| 359 |
+
]
|
| 360 |
+
|
| 361 |
+
|
| 362 |
+
@pytest.mark.asyncio
|
| 363 |
+
async def test_current_location_ids_are_applied_before_content_retrieval() -> None:
|
| 364 |
+
nearby = RecordingNearbyProvider()
|
| 365 |
+
retriever = RecordingFilteredRetriever()
|
| 366 |
+
|
| 367 |
+
result = await build_use_case(
|
| 368 |
+
llm_enabled=False,
|
| 369 |
+
retriever=retriever,
|
| 370 |
+
nearby_place_provider=nearby,
|
| 371 |
+
).execute(
|
| 372 |
+
message="una cafeteria",
|
| 373 |
+
state=ConversationState(),
|
| 374 |
+
user_latitude=16.7531,
|
| 375 |
+
user_longitude=-93.1156,
|
| 376 |
+
candidate_limit=5,
|
| 377 |
+
result_limit=3,
|
| 378 |
+
)
|
| 379 |
+
|
| 380 |
+
assert result.action == "recommendations"
|
| 381 |
+
assert nearby.call == (16.7531, -93.1156, 5_000)
|
| 382 |
+
assert retriever.intent.hard_filters["place_ids"] == ("near_1", "near_2")
|
| 383 |
+
|
| 384 |
+
|
| 385 |
+
class CoordinateAnchorResolver:
|
| 386 |
+
async def resolve(self, text, city, state, limit=3):
|
| 387 |
+
del text, city, state, limit
|
| 388 |
+
return [
|
| 389 |
+
ResolvedPlaceAnchor(
|
| 390 |
+
place_id="anchor_centro",
|
| 391 |
+
name="Centro",
|
| 392 |
+
latitude=16.75,
|
| 393 |
+
longitude=-93.12,
|
| 394 |
+
score=0.95,
|
| 395 |
+
)
|
| 396 |
+
]
|
| 397 |
+
|
| 398 |
+
|
| 399 |
+
@pytest.mark.asyncio
|
| 400 |
+
async def test_explicit_anchor_coordinates_filter_before_retrieval() -> None:
|
| 401 |
+
nearby = RecordingNearbyProvider()
|
| 402 |
+
retriever = RecordingFilteredRetriever()
|
| 403 |
+
|
| 404 |
+
result = await build_use_case(
|
| 405 |
+
llm_enabled=False,
|
| 406 |
+
anchor_resolver=CoordinateAnchorResolver(),
|
| 407 |
+
retriever=retriever,
|
| 408 |
+
nearby_place_provider=nearby,
|
| 409 |
+
).execute(
|
| 410 |
+
message="una cafeteria cerca del centro",
|
| 411 |
+
state=ConversationState(),
|
| 412 |
+
user_latitude=16.70,
|
| 413 |
+
user_longitude=-93.10,
|
| 414 |
+
candidate_limit=5,
|
| 415 |
+
result_limit=3,
|
| 416 |
+
)
|
| 417 |
+
|
| 418 |
+
assert result.action == "recommendations"
|
| 419 |
+
assert result.location_directive.source == "explicit_anchor"
|
| 420 |
+
assert nearby.call == (16.75, -93.12, 5_000)
|
| 421 |
+
assert retriever.intent.hard_filters["place_ids"] == ("near_1", "near_2")
|
| 422 |
+
|
| 423 |
+
|
| 424 |
+
@pytest.mark.asyncio
|
| 425 |
+
async def test_persisted_resolved_anchor_keeps_nearby_filter_on_followup() -> None:
|
| 426 |
+
nearby = RecordingNearbyProvider()
|
| 427 |
+
retriever = RecordingFilteredRetriever()
|
| 428 |
+
|
| 429 |
+
result = await build_use_case(
|
| 430 |
+
llm_enabled=False,
|
| 431 |
+
anchor_resolver=CoordinateAnchorResolver(),
|
| 432 |
+
retriever=retriever,
|
| 433 |
+
nearby_place_provider=nearby,
|
| 434 |
+
).execute(
|
| 435 |
+
message="otra cafeteria tranquila",
|
| 436 |
+
state=ConversationState(
|
| 437 |
+
explicit_target_location=ExplicitTargetLocation(
|
| 438 |
+
anchor_text="Centro",
|
| 439 |
+
place_id="anchor_centro",
|
| 440 |
+
label="Centro",
|
| 441 |
+
radius_meters=3_000,
|
| 442 |
+
strict_radius=True,
|
| 443 |
+
)
|
| 444 |
+
),
|
| 445 |
+
user_latitude=None,
|
| 446 |
+
user_longitude=None,
|
| 447 |
+
candidate_limit=5,
|
| 448 |
+
result_limit=3,
|
| 449 |
+
)
|
| 450 |
+
|
| 451 |
+
assert result.action == "recommendations"
|
| 452 |
+
assert result.location_directive.source == "state_anchor"
|
| 453 |
+
assert nearby.call == (16.75, -93.12, 3_000)
|
| 454 |
+
assert retriever.intent.hard_filters["place_ids"] == ("near_1", "near_2")
|
| 455 |
+
|
| 456 |
+
|
| 457 |
class AmbiguousAnchorResolver:
|
| 458 |
async def resolve(self, text, city, state, limit=3):
|
| 459 |
del text, city, state, limit
|
|
|
|
| 667 |
("shopping", "shopping_exact"),
|
| 668 |
),
|
| 669 |
)
|
| 670 |
+
async def test_short_category_queries_survive_sparse_scores_without_being_filtered(
|
| 671 |
message: str,
|
| 672 |
expected_id: str,
|
| 673 |
) -> None:
|
|
|
|
| 684 |
|
| 685 |
candidates = await retriever.retrieve(intent=intent, limit=5)
|
| 686 |
|
| 687 |
+
candidate_ids = [candidate.place_id for candidate in candidates]
|
| 688 |
+
assert expected_id in candidate_ids
|
| 689 |
+
assert set(candidate_ids) == {
|
| 690 |
+
"park_exact",
|
| 691 |
+
"cinema_compatible",
|
| 692 |
+
"entertainment_noise",
|
| 693 |
+
"shopping_exact",
|
| 694 |
+
"family_noise",
|
| 695 |
+
}
|
| 696 |
+
assert all(
|
| 697 |
+
candidate.metadata["retrieval_diagnostics"]["content_quality"] == "weak"
|
| 698 |
+
for candidate in candidates
|
| 699 |
+
)
|
| 700 |
|
| 701 |
|
| 702 |
@pytest.mark.asyncio
|
| 703 |
+
async def test_category_is_ranking_evidence_instead_of_a_hard_filter() -> None:
|
| 704 |
embedding = MockEmbeddingProvider(dimension=16)
|
| 705 |
retriever = HybridContentPlaceChatRetriever(
|
| 706 |
embedding_provider=embedding,
|
|
|
|
| 715 |
|
| 716 |
candidates = await retriever.retrieve(intent=intent, limit=10)
|
| 717 |
|
| 718 |
+
assert {candidate.place_id for candidate in candidates} == {
|
| 719 |
+
"park",
|
| 720 |
"exact",
|
| 721 |
"family",
|
| 722 |
"broad",
|
| 723 |
+
}
|
| 724 |
+
diagnostics = {
|
| 725 |
+
candidate.place_id: candidate.metadata["retrieval_diagnostics"]
|
| 726 |
+
for candidate in candidates
|
| 727 |
+
}
|
| 728 |
+
assert diagnostics["park"]["category_match"] == "none"
|
| 729 |
+
assert diagnostics["exact"]["category_match"] == "exact"
|
| 730 |
|
| 731 |
|
| 732 |
class ReferenceLocationResolver:
|
tests/test_place_embedding_configuration.py
ADDED
|
@@ -0,0 +1,182 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
import json
|
| 2 |
+
|
| 3 |
+
import pytest
|
| 4 |
+
from pydantic import ValidationError
|
| 5 |
+
|
| 6 |
+
from app.modules.places.api import dependencies as place_dependencies
|
| 7 |
+
from app.modules.places.infrastructure.bert_intent_extractor import (
|
| 8 |
+
BertPlaceIntentExtractor,
|
| 9 |
+
)
|
| 10 |
+
from app.modules.places.infrastructure.place_category_catalog import (
|
| 11 |
+
load_place_category_concepts,
|
| 12 |
+
)
|
| 13 |
+
from app.shared.config.settings import Settings
|
| 14 |
+
from app.shared.nlp.embeddings.factory import create_place_embedding_provider
|
| 15 |
+
from app.shared.nlp.embeddings.mock import MockEmbeddingProvider
|
| 16 |
+
from app.shared.nlp.embeddings.sentence_transformer import (
|
| 17 |
+
SentenceTransformerEmbeddingProvider,
|
| 18 |
+
)
|
| 19 |
+
|
| 20 |
+
|
| 21 |
+
def test_places_embedding_provider_is_independent_from_global_dimension() -> None:
|
| 22 |
+
settings = Settings(
|
| 23 |
+
_env_file=None,
|
| 24 |
+
ENV="local",
|
| 25 |
+
EMBEDDING_PROVIDER="mock",
|
| 26 |
+
EMBEDDING_DIMENSION=16,
|
| 27 |
+
PLACES_EMBEDDING_PROVIDER="mock",
|
| 28 |
+
PLACES_EMBEDDING_DIMENSION=384,
|
| 29 |
+
PLACES_EMBEDDING_MODEL="places-test",
|
| 30 |
+
)
|
| 31 |
+
|
| 32 |
+
provider = create_place_embedding_provider(settings)
|
| 33 |
+
|
| 34 |
+
assert isinstance(provider, MockEmbeddingProvider)
|
| 35 |
+
assert provider.dimension == 384
|
| 36 |
+
assert settings.embedding_dimension == 16
|
| 37 |
+
|
| 38 |
+
|
| 39 |
+
def test_blank_optional_places_runtime_values_are_normalized_to_none() -> None:
|
| 40 |
+
settings = Settings(
|
| 41 |
+
_env_file=None,
|
| 42 |
+
PLACES_EMBEDDING_DEVICE=" ",
|
| 43 |
+
PLACES_CATEGORY_CATALOG_PATH="",
|
| 44 |
+
PLACES_PGVECTOR_HYBRID_FUNCTION=" ",
|
| 45 |
+
)
|
| 46 |
+
|
| 47 |
+
assert settings.places_embedding_device is None
|
| 48 |
+
assert settings.places_category_catalog_path is None
|
| 49 |
+
assert settings.places_pgvector_hybrid_function is None
|
| 50 |
+
|
| 51 |
+
|
| 52 |
+
def test_sentence_transformer_factory_configures_query_and_passage_prefixes() -> None:
|
| 53 |
+
settings = Settings(
|
| 54 |
+
_env_file=None,
|
| 55 |
+
ENV="local",
|
| 56 |
+
PLACES_EMBEDDING_PROVIDER="sentence_transformer",
|
| 57 |
+
PLACES_EMBEDDING_DIMENSION=768,
|
| 58 |
+
PLACES_EMBEDDING_MODEL="intfloat/multilingual-e5-base",
|
| 59 |
+
PLACES_EMBEDDING_QUERY_PREFIX="query: ",
|
| 60 |
+
PLACES_EMBEDDING_PASSAGE_PREFIX="passage: ",
|
| 61 |
+
)
|
| 62 |
+
|
| 63 |
+
query = create_place_embedding_provider(settings, text_role="query")
|
| 64 |
+
passage = create_place_embedding_provider(settings, text_role="passage")
|
| 65 |
+
|
| 66 |
+
assert isinstance(query, SentenceTransformerEmbeddingProvider)
|
| 67 |
+
assert isinstance(passage, SentenceTransformerEmbeddingProvider)
|
| 68 |
+
assert query.text_prefix == "query: "
|
| 69 |
+
assert passage.text_prefix == "passage: "
|
| 70 |
+
assert query.is_loaded is False
|
| 71 |
+
assert passage.is_loaded is False
|
| 72 |
+
|
| 73 |
+
|
| 74 |
+
def test_quoted_e5_prefixes_preserve_the_significant_space(tmp_path) -> None:
|
| 75 |
+
env_file = tmp_path / ".env"
|
| 76 |
+
env_file.write_text(
|
| 77 |
+
'PLACES_EMBEDDING_QUERY_PREFIX="query: "\n'
|
| 78 |
+
'PLACES_EMBEDDING_PASSAGE_PREFIX="passage: "\n',
|
| 79 |
+
encoding="utf-8",
|
| 80 |
+
)
|
| 81 |
+
|
| 82 |
+
settings = Settings(_env_file=env_file)
|
| 83 |
+
|
| 84 |
+
assert settings.places_embedding_query_prefix == "query: "
|
| 85 |
+
assert settings.places_embedding_passage_prefix == "passage: "
|
| 86 |
+
|
| 87 |
+
|
| 88 |
+
def test_places_embedding_provider_rejects_unknown_backend() -> None:
|
| 89 |
+
with pytest.raises(ValidationError, match="PLACES_EMBEDDING_PROVIDER"):
|
| 90 |
+
Settings(
|
| 91 |
+
_env_file=None,
|
| 92 |
+
ENV="local",
|
| 93 |
+
PLACES_EMBEDDING_PROVIDER="closed_taxonomy_magic",
|
| 94 |
+
)
|
| 95 |
+
|
| 96 |
+
|
| 97 |
+
def test_external_place_concept_catalog_is_data_driven(tmp_path) -> None:
|
| 98 |
+
path = tmp_path / "concepts.json"
|
| 99 |
+
path.write_text(
|
| 100 |
+
json.dumps(
|
| 101 |
+
{
|
| 102 |
+
"concepts": [
|
| 103 |
+
{
|
| 104 |
+
"id": "donut_shop",
|
| 105 |
+
"label": "Donas",
|
| 106 |
+
"description": "Lugar especializado en donas artesanales",
|
| 107 |
+
"examples": ["algo dulce glaseado"],
|
| 108 |
+
"storage_values": ["bakery", "dessert"],
|
| 109 |
+
}
|
| 110 |
+
]
|
| 111 |
+
}
|
| 112 |
+
),
|
| 113 |
+
encoding="utf-8",
|
| 114 |
+
)
|
| 115 |
+
|
| 116 |
+
concepts = load_place_category_concepts(str(path))
|
| 117 |
+
|
| 118 |
+
assert [concept.id for concept in concepts] == ["donut_shop"]
|
| 119 |
+
assert concepts[0].storage_values == ("bakery", "dessert")
|
| 120 |
+
|
| 121 |
+
|
| 122 |
+
def test_place_chat_intent_provider_defaults_to_deterministic() -> None:
|
| 123 |
+
settings = Settings(_env_file=None, ENV="local")
|
| 124 |
+
|
| 125 |
+
assert settings.places_chat_intent_provider == "deterministic"
|
| 126 |
+
assert settings.places_chat_bert_model_path is None
|
| 127 |
+
|
| 128 |
+
|
| 129 |
+
def test_place_chat_bert_provider_is_explicit_and_validated() -> None:
|
| 130 |
+
settings = Settings(
|
| 131 |
+
_env_file=None,
|
| 132 |
+
ENV="local",
|
| 133 |
+
PLACES_CHAT_INTENT_PROVIDER="BERT",
|
| 134 |
+
PLACES_CHAT_BERT_MODEL_PATH="models/places-intent",
|
| 135 |
+
PLACES_CHAT_BERT_MODEL_VERSION="intent-v3",
|
| 136 |
+
PLACES_CHAT_BERT_DEVICE="cpu",
|
| 137 |
+
PLACES_CHAT_BERT_MIN_TOKEN_CONFIDENCE=0.72,
|
| 138 |
+
)
|
| 139 |
+
|
| 140 |
+
assert settings.places_chat_intent_provider == "bert"
|
| 141 |
+
assert settings.places_chat_bert_model_path == "models/places-intent"
|
| 142 |
+
assert settings.places_chat_bert_model_version == "intent-v3"
|
| 143 |
+
assert settings.places_chat_bert_device == "cpu"
|
| 144 |
+
assert settings.places_chat_bert_min_token_confidence == pytest.approx(0.72)
|
| 145 |
+
|
| 146 |
+
with pytest.raises(ValidationError, match="PLACES_CHAT_BERT_MODEL_PATH"):
|
| 147 |
+
Settings(
|
| 148 |
+
_env_file=None,
|
| 149 |
+
ENV="local",
|
| 150 |
+
PLACES_CHAT_INTENT_PROVIDER="bert",
|
| 151 |
+
)
|
| 152 |
+
|
| 153 |
+
|
| 154 |
+
def test_place_chat_dependency_builds_bert_extractor_without_loading_it(
|
| 155 |
+
monkeypatch: pytest.MonkeyPatch,
|
| 156 |
+
) -> None:
|
| 157 |
+
settings = Settings(
|
| 158 |
+
_env_file=None,
|
| 159 |
+
ENV="local",
|
| 160 |
+
PLACES_EMBEDDING_PROVIDER="mock",
|
| 161 |
+
PLACES_CHAT_INTENT_PROVIDER="bert",
|
| 162 |
+
PLACES_CHAT_BERT_MODEL_PATH="models/places-intent",
|
| 163 |
+
PLACES_CHAT_BERT_MODEL_VERSION="intent-v3",
|
| 164 |
+
PLACES_CHAT_BERT_DEVICE="cpu",
|
| 165 |
+
)
|
| 166 |
+
monkeypatch.setattr(place_dependencies, "get_settings", lambda: settings)
|
| 167 |
+
monkeypatch.setattr(
|
| 168 |
+
place_dependencies,
|
| 169 |
+
"load_place_category_concepts",
|
| 170 |
+
lambda *_args, **_kwargs: (),
|
| 171 |
+
)
|
| 172 |
+
place_dependencies.get_place_chat_intent_parser.cache_clear()
|
| 173 |
+
try:
|
| 174 |
+
parser = place_dependencies.get_place_chat_intent_parser()
|
| 175 |
+
extractor = parser._contextual_extractor
|
| 176 |
+
|
| 177 |
+
assert isinstance(extractor, BertPlaceIntentExtractor)
|
| 178 |
+
assert extractor.model_name == "models/places-intent"
|
| 179 |
+
assert extractor.model_version == "intent-v3"
|
| 180 |
+
assert extractor.is_loaded is False
|
| 181 |
+
finally:
|
| 182 |
+
place_dependencies.get_place_chat_intent_parser.cache_clear()
|
tests/test_places_use_cases.py
CHANGED
|
@@ -11,7 +11,10 @@ from app.modules.places.infrastructure.mock_place_repository import MockPlaceVec
|
|
| 11 |
from app.shared.nlp.embeddings.mock import MockEmbeddingProvider
|
| 12 |
from app.shared.nlp.llm.base import LLMProvider, LLMResult, PlaceResponseMode
|
| 13 |
from app.shared.nlp.llm.mock import MockLLMProvider
|
| 14 |
-
from app.shared.nlp.llm.output_guard import
|
|
|
|
|
|
|
|
|
|
| 15 |
|
| 16 |
|
| 17 |
@pytest.mark.asyncio
|
|
@@ -243,9 +246,10 @@ async def test_chat_places_falls_back_when_llm_fails() -> None:
|
|
| 243 |
limit=3,
|
| 244 |
)
|
| 245 |
|
| 246 |
-
assert result.message ==
|
| 247 |
assert result.places
|
| 248 |
assert result.metadata["used_llm"] is False
|
|
|
|
| 249 |
|
| 250 |
|
| 251 |
class FailingLLMProvider(LLMProvider):
|
|
|
|
| 11 |
from app.shared.nlp.embeddings.mock import MockEmbeddingProvider
|
| 12 |
from app.shared.nlp.llm.base import LLMProvider, LLMResult, PlaceResponseMode
|
| 13 |
from app.shared.nlp.llm.mock import MockLLMProvider
|
| 14 |
+
from app.shared.nlp.llm.output_guard import (
|
| 15 |
+
LOW_CONFIDENCE_PLACE_CHAT_FALLBACK,
|
| 16 |
+
PlaceChatOutputGuard,
|
| 17 |
+
)
|
| 18 |
|
| 19 |
|
| 20 |
@pytest.mark.asyncio
|
|
|
|
| 246 |
limit=3,
|
| 247 |
)
|
| 248 |
|
| 249 |
+
assert result.message == LOW_CONFIDENCE_PLACE_CHAT_FALLBACK
|
| 250 |
assert result.places
|
| 251 |
assert result.metadata["used_llm"] is False
|
| 252 |
+
assert result.metadata["response_mode"] == "low_confidence"
|
| 253 |
|
| 254 |
|
| 255 |
class FailingLLMProvider(LLMProvider):
|
tests/test_semantic_activity_classifier.py
CHANGED
|
@@ -1,6 +1,9 @@
|
|
| 1 |
from app.modules.places.infrastructure.semantic_activity_classifier import (
|
| 2 |
SemanticPlaceActivityClassifier,
|
| 3 |
)
|
|
|
|
|
|
|
|
|
|
| 4 |
from app.shared.nlp.embeddings.base import EmbeddingProvider
|
| 5 |
from app.shared.nlp.preprocessing.text import prepare_for_embedding
|
| 6 |
|
|
@@ -45,9 +48,26 @@ class ControlledEmbeddingProvider(EmbeddingProvider):
|
|
| 45 |
return [self.embed_text(text) for text in texts]
|
| 46 |
|
| 47 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 48 |
def test_classifier_finds_activity_inside_a_long_message() -> None:
|
| 49 |
classifier = SemanticPlaceActivityClassifier(
|
| 50 |
embedding_provider=ControlledEmbeddingProvider(),
|
|
|
|
| 51 |
)
|
| 52 |
|
| 53 |
result = classifier.classify(
|
|
@@ -62,6 +82,7 @@ def test_classifier_finds_activity_inside_a_long_message() -> None:
|
|
| 62 |
def test_classifier_abstains_when_the_best_categories_are_tied() -> None:
|
| 63 |
classifier = SemanticPlaceActivityClassifier(
|
| 64 |
embedding_provider=ControlledEmbeddingProvider(),
|
|
|
|
| 65 |
)
|
| 66 |
|
| 67 |
result = classifier.classify("mezcla")
|
|
@@ -72,6 +93,7 @@ def test_classifier_abstains_when_the_best_categories_are_tied() -> None:
|
|
| 72 |
def test_classifier_abstains_for_a_generic_single_word_request() -> None:
|
| 73 |
classifier = SemanticPlaceActivityClassifier(
|
| 74 |
embedding_provider=ControlledEmbeddingProvider(),
|
|
|
|
| 75 |
)
|
| 76 |
|
| 77 |
assert classifier.classify("salir") is None
|
|
|
|
| 1 |
from app.modules.places.infrastructure.semantic_activity_classifier import (
|
| 2 |
SemanticPlaceActivityClassifier,
|
| 3 |
)
|
| 4 |
+
from app.modules.places.infrastructure.open_vocabulary_category_classifier import (
|
| 5 |
+
PlaceCategoryConcept,
|
| 6 |
+
)
|
| 7 |
from app.shared.nlp.embeddings.base import EmbeddingProvider
|
| 8 |
from app.shared.nlp.preprocessing.text import prepare_for_embedding
|
| 9 |
|
|
|
|
| 48 |
return [self.embed_text(text) for text in texts]
|
| 49 |
|
| 50 |
|
| 51 |
+
CONCEPTS = (
|
| 52 |
+
PlaceCategoryConcept(
|
| 53 |
+
id="restaurant",
|
| 54 |
+
label="comida restaurante",
|
| 55 |
+
description="hambre tacos pizza sushi antojo platillo cocina",
|
| 56 |
+
storage_values=("restaurant",),
|
| 57 |
+
),
|
| 58 |
+
PlaceCategoryConcept(
|
| 59 |
+
id="sports",
|
| 60 |
+
label="ejercicio gimnasio deporte",
|
| 61 |
+
description="entrenar futbol cancha nadar fitness",
|
| 62 |
+
storage_values=("sports",),
|
| 63 |
+
),
|
| 64 |
+
)
|
| 65 |
+
|
| 66 |
+
|
| 67 |
def test_classifier_finds_activity_inside_a_long_message() -> None:
|
| 68 |
classifier = SemanticPlaceActivityClassifier(
|
| 69 |
embedding_provider=ControlledEmbeddingProvider(),
|
| 70 |
+
concepts=CONCEPTS,
|
| 71 |
)
|
| 72 |
|
| 73 |
result = classifier.classify(
|
|
|
|
| 82 |
def test_classifier_abstains_when_the_best_categories_are_tied() -> None:
|
| 83 |
classifier = SemanticPlaceActivityClassifier(
|
| 84 |
embedding_provider=ControlledEmbeddingProvider(),
|
| 85 |
+
concepts=CONCEPTS,
|
| 86 |
)
|
| 87 |
|
| 88 |
result = classifier.classify("mezcla")
|
|
|
|
| 93 |
def test_classifier_abstains_for_a_generic_single_word_request() -> None:
|
| 94 |
classifier = SemanticPlaceActivityClassifier(
|
| 95 |
embedding_provider=ControlledEmbeddingProvider(),
|
| 96 |
+
concepts=CONCEPTS,
|
| 97 |
)
|
| 98 |
|
| 99 |
assert classifier.classify("salir") is None
|
tests/test_sentence_transformer_embeddings.py
ADDED
|
@@ -0,0 +1,223 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
import math
|
| 2 |
+
|
| 3 |
+
import pytest
|
| 4 |
+
|
| 5 |
+
from app.shared.nlp.embeddings.sentence_transformer import (
|
| 6 |
+
SentenceTransformerDimensionError,
|
| 7 |
+
SentenceTransformerEmbeddingProvider,
|
| 8 |
+
SentenceTransformerInferenceError,
|
| 9 |
+
SentenceTransformerModelLoadError,
|
| 10 |
+
)
|
| 11 |
+
|
| 12 |
+
|
| 13 |
+
class FakeSentenceTransformer:
|
| 14 |
+
def __init__(
|
| 15 |
+
self,
|
| 16 |
+
vectors: dict[str, list[float]],
|
| 17 |
+
dimension: int = 3,
|
| 18 |
+
) -> None:
|
| 19 |
+
self.vectors = vectors
|
| 20 |
+
self.dimension = dimension
|
| 21 |
+
self.calls: list[tuple[list[str], dict[str, object]]] = []
|
| 22 |
+
|
| 23 |
+
def get_sentence_embedding_dimension(self) -> int:
|
| 24 |
+
return self.dimension
|
| 25 |
+
|
| 26 |
+
def encode(self, sentences: list[str], **kwargs: object) -> list[list[float]]:
|
| 27 |
+
self.calls.append((list(sentences), kwargs))
|
| 28 |
+
return [self.vectors[text] for text in sentences]
|
| 29 |
+
|
| 30 |
+
|
| 31 |
+
def test_provider_loads_lazily_and_batches_non_empty_texts() -> None:
|
| 32 |
+
model = FakeSentenceTransformer(
|
| 33 |
+
{
|
| 34 |
+
"query: donas": [3.0, 4.0, 0.0],
|
| 35 |
+
"query: cafecito": [0.0, 0.0, 2.0],
|
| 36 |
+
}
|
| 37 |
+
)
|
| 38 |
+
loader_calls: list[tuple[str, str | None]] = []
|
| 39 |
+
|
| 40 |
+
def loader(name: str, device: str | None) -> FakeSentenceTransformer:
|
| 41 |
+
loader_calls.append((name, device))
|
| 42 |
+
return model
|
| 43 |
+
|
| 44 |
+
provider = SentenceTransformerEmbeddingProvider(
|
| 45 |
+
"domain-model",
|
| 46 |
+
expected_dimension=3,
|
| 47 |
+
batch_size=8,
|
| 48 |
+
device="cpu",
|
| 49 |
+
text_prefix="query: ",
|
| 50 |
+
model_loader=loader,
|
| 51 |
+
)
|
| 52 |
+
|
| 53 |
+
assert provider.is_loaded is False
|
| 54 |
+
assert provider.dimension == 3
|
| 55 |
+
assert loader_calls == []
|
| 56 |
+
|
| 57 |
+
embeddings = provider.embed_batch([" donas ", " ", "cafecito"])
|
| 58 |
+
|
| 59 |
+
assert provider.is_loaded is True
|
| 60 |
+
assert loader_calls == [("domain-model", "cpu")]
|
| 61 |
+
assert embeddings[0] == pytest.approx([0.6, 0.8, 0.0])
|
| 62 |
+
assert embeddings[1] == [0.0, 0.0, 0.0]
|
| 63 |
+
assert embeddings[2] == pytest.approx([0.0, 0.0, 1.0])
|
| 64 |
+
assert model.calls == [
|
| 65 |
+
(
|
| 66 |
+
["query: donas", "query: cafecito"],
|
| 67 |
+
{
|
| 68 |
+
"batch_size": 8,
|
| 69 |
+
"convert_to_numpy": True,
|
| 70 |
+
"normalize_embeddings": False,
|
| 71 |
+
"show_progress_bar": False,
|
| 72 |
+
},
|
| 73 |
+
)
|
| 74 |
+
]
|
| 75 |
+
|
| 76 |
+
|
| 77 |
+
def test_empty_text_returns_zero_without_loading_model() -> None:
|
| 78 |
+
def loader(
|
| 79 |
+
_name: str, _device: str | None
|
| 80 |
+
) -> FakeSentenceTransformer:
|
| 81 |
+
raise AssertionError("empty input must not load the model")
|
| 82 |
+
|
| 83 |
+
provider = SentenceTransformerEmbeddingProvider(
|
| 84 |
+
"domain-model",
|
| 85 |
+
expected_dimension=3,
|
| 86 |
+
model_loader=loader,
|
| 87 |
+
)
|
| 88 |
+
|
| 89 |
+
assert provider.embed_text("\t \n") == [0.0, 0.0, 0.0]
|
| 90 |
+
assert provider.embed_batch([]) == []
|
| 91 |
+
assert provider.is_loaded is False
|
| 92 |
+
|
| 93 |
+
|
| 94 |
+
def test_model_is_loaded_only_once_across_calls() -> None:
|
| 95 |
+
model = FakeSentenceTransformer({"first": [1.0, 0.0, 0.0], "second": [0.0, 1.0, 0.0]})
|
| 96 |
+
load_count = 0
|
| 97 |
+
|
| 98 |
+
def loader(
|
| 99 |
+
_name: str, _device: str | None
|
| 100 |
+
) -> FakeSentenceTransformer:
|
| 101 |
+
nonlocal load_count
|
| 102 |
+
load_count += 1
|
| 103 |
+
return model
|
| 104 |
+
|
| 105 |
+
provider = SentenceTransformerEmbeddingProvider(
|
| 106 |
+
"domain-model",
|
| 107 |
+
expected_dimension=3,
|
| 108 |
+
model_loader=loader,
|
| 109 |
+
)
|
| 110 |
+
|
| 111 |
+
provider.embed_text("first")
|
| 112 |
+
provider.embed_text("second")
|
| 113 |
+
|
| 114 |
+
assert load_count == 1
|
| 115 |
+
assert len(model.calls) == 2
|
| 116 |
+
|
| 117 |
+
|
| 118 |
+
def test_provider_rejects_reported_model_dimension_mismatch() -> None:
|
| 119 |
+
model = FakeSentenceTransformer({}, dimension=4)
|
| 120 |
+
provider = SentenceTransformerEmbeddingProvider(
|
| 121 |
+
"wrong-model",
|
| 122 |
+
expected_dimension=3,
|
| 123 |
+
model_loader=lambda _name, _device: model,
|
| 124 |
+
)
|
| 125 |
+
|
| 126 |
+
with pytest.raises(
|
| 127 |
+
SentenceTransformerDimensionError,
|
| 128 |
+
match="model=4, configured=3",
|
| 129 |
+
):
|
| 130 |
+
provider.embed_text("donas")
|
| 131 |
+
|
| 132 |
+
assert provider.is_loaded is False
|
| 133 |
+
|
| 134 |
+
|
| 135 |
+
def test_provider_rejects_bad_inference_output() -> None:
|
| 136 |
+
model = FakeSentenceTransformer({"donas": [1.0, 2.0]}, dimension=3)
|
| 137 |
+
provider = SentenceTransformerEmbeddingProvider(
|
| 138 |
+
"bad-output-model",
|
| 139 |
+
expected_dimension=3,
|
| 140 |
+
model_loader=lambda _name, _device: model,
|
| 141 |
+
)
|
| 142 |
+
|
| 143 |
+
with pytest.raises(
|
| 144 |
+
SentenceTransformerDimensionError,
|
| 145 |
+
match="returned=2, expected=3",
|
| 146 |
+
):
|
| 147 |
+
provider.embed_text("donas")
|
| 148 |
+
|
| 149 |
+
|
| 150 |
+
def test_provider_wraps_loader_and_inference_failures_with_context() -> None:
|
| 151 |
+
def broken_loader(_name: str, _device: str | None) -> FakeSentenceTransformer:
|
| 152 |
+
raise OSError("model cache unavailable")
|
| 153 |
+
|
| 154 |
+
provider = SentenceTransformerEmbeddingProvider(
|
| 155 |
+
"missing-model",
|
| 156 |
+
expected_dimension=3,
|
| 157 |
+
model_loader=broken_loader,
|
| 158 |
+
)
|
| 159 |
+
with pytest.raises(
|
| 160 |
+
SentenceTransformerModelLoadError,
|
| 161 |
+
match="missing-model.*model cache unavailable",
|
| 162 |
+
):
|
| 163 |
+
provider.embed_text("donas")
|
| 164 |
+
|
| 165 |
+
class BrokenModel(FakeSentenceTransformer):
|
| 166 |
+
def encode(
|
| 167 |
+
self, sentences: list[str], **kwargs: object
|
| 168 |
+
) -> list[list[float]]:
|
| 169 |
+
raise RuntimeError("backend crashed")
|
| 170 |
+
|
| 171 |
+
broken_model = BrokenModel({}, dimension=3)
|
| 172 |
+
provider = SentenceTransformerEmbeddingProvider(
|
| 173 |
+
"broken-model",
|
| 174 |
+
expected_dimension=3,
|
| 175 |
+
model_loader=lambda _name, _device: broken_model,
|
| 176 |
+
)
|
| 177 |
+
with pytest.raises(
|
| 178 |
+
SentenceTransformerInferenceError,
|
| 179 |
+
match="broken-model.*backend crashed",
|
| 180 |
+
):
|
| 181 |
+
provider.embed_text("donas")
|
| 182 |
+
|
| 183 |
+
|
| 184 |
+
def test_provider_rejects_zero_norm_and_non_finite_vectors() -> None:
|
| 185 |
+
zero_model = FakeSentenceTransformer({"zero": [0.0, 0.0, 0.0]})
|
| 186 |
+
provider = SentenceTransformerEmbeddingProvider(
|
| 187 |
+
"zero-model",
|
| 188 |
+
expected_dimension=3,
|
| 189 |
+
model_loader=lambda _name, _device: zero_model,
|
| 190 |
+
)
|
| 191 |
+
with pytest.raises(SentenceTransformerInferenceError, match="zero-norm"):
|
| 192 |
+
provider.embed_text("zero")
|
| 193 |
+
|
| 194 |
+
nan_model = FakeSentenceTransformer({"nan": [math.nan, 0.0, 1.0]})
|
| 195 |
+
provider = SentenceTransformerEmbeddingProvider(
|
| 196 |
+
"nan-model",
|
| 197 |
+
expected_dimension=3,
|
| 198 |
+
model_loader=lambda _name, _device: nan_model,
|
| 199 |
+
)
|
| 200 |
+
with pytest.raises(
|
| 201 |
+
SentenceTransformerInferenceError,
|
| 202 |
+
match="NaN or infinity",
|
| 203 |
+
):
|
| 204 |
+
provider.embed_text("nan")
|
| 205 |
+
|
| 206 |
+
|
| 207 |
+
def test_provider_validates_constructor_and_input() -> None:
|
| 208 |
+
with pytest.raises(ValueError, match="model_name_or_path"):
|
| 209 |
+
SentenceTransformerEmbeddingProvider(" ", expected_dimension=3)
|
| 210 |
+
with pytest.raises(ValueError, match="expected_dimension"):
|
| 211 |
+
SentenceTransformerEmbeddingProvider("model", expected_dimension=0)
|
| 212 |
+
with pytest.raises(ValueError, match="batch_size"):
|
| 213 |
+
SentenceTransformerEmbeddingProvider(
|
| 214 |
+
"model", expected_dimension=3, batch_size=0
|
| 215 |
+
)
|
| 216 |
+
|
| 217 |
+
provider = SentenceTransformerEmbeddingProvider(
|
| 218 |
+
"model",
|
| 219 |
+
expected_dimension=3,
|
| 220 |
+
model_loader=lambda _name, _device: FakeSentenceTransformer({}),
|
| 221 |
+
)
|
| 222 |
+
with pytest.raises(TypeError, match=r"texts\[1\].*int"):
|
| 223 |
+
provider.embed_batch(["valid", 42]) # type: ignore[list-item]
|
tests/test_sql_contract.py
CHANGED
|
@@ -103,3 +103,29 @@ def test_global_search_sql_does_not_directly_cast_untrusted_event_metadata() ->
|
|
| 103 |
sql = path.read_text(encoding="utf-8")
|
| 104 |
assert "NULLIF(e.metadata->>'start_time', '')::timestamptz" not in sql
|
| 105 |
assert "(e.metadata->>'duration_minutes')::integer" not in sql
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 103 |
sql = path.read_text(encoding="utf-8")
|
| 104 |
assert "NULLIF(e.metadata->>'start_time', '')::timestamptz" not in sql
|
| 105 |
assert "(e.metadata->>'duration_minutes')::integer" not in sql
|
| 106 |
+
|
| 107 |
+
|
| 108 |
+
def test_places_semantic_index_is_additive_and_hybrid() -> None:
|
| 109 |
+
migration = Path(
|
| 110 |
+
"sql/migrations/20260716_02_places_semantic_v1.sql"
|
| 111 |
+
).read_text(encoding="utf-8")
|
| 112 |
+
upper = migration.upper()
|
| 113 |
+
|
| 114 |
+
assert "PLACE_EMBEDDINGS_SEMANTIC_V1" in upper
|
| 115 |
+
assert "VECTOR(768)" in upper
|
| 116 |
+
assert "SEARCH_PLACES_SEMANTIC_V1" in upper
|
| 117 |
+
assert "FULL OUTER JOIN" in upper
|
| 118 |
+
assert "WEBSEARCH_TO_TSQUERY" in upper
|
| 119 |
+
assert "USING HNSW" in upper
|
| 120 |
+
assert "AS MATERIALIZED" not in upper
|
| 121 |
+
assert "FROM PUBLIC.PLACE_EMBEDDINGS_SEMANTIC_V1 AS PLACE" in upper
|
| 122 |
+
assert "ALTER TABLE PUBLIC.PLACE_EMBEDDINGS" not in upper
|
| 123 |
+
assert "DROP TABLE" not in upper
|
| 124 |
+
|
| 125 |
+
verifier = Path("sql/verify_places_semantic_v1.sql")
|
| 126 |
+
assert verifier.exists()
|
| 127 |
+
verifier_sql = verifier.read_text(encoding="utf-8").upper()
|
| 128 |
+
assert "SET TRANSACTION READ ONLY" in verifier_sql
|
| 129 |
+
assert "VECTOR(768)" in verifier_sql
|
| 130 |
+
assert "USING HNSW" in verifier_sql
|
| 131 |
+
assert "EXPLAIN (ANALYZE, BUFFERS" in verifier_sql
|