AlleksDev commited on
Commit
baaa888
·
unverified ·
1 Parent(s): 27d6159

Add: Bert

Browse files
This view is limited to 50 files because it contains too many changes.   See raw diff
Files changed (50) hide show
  1. .env.example +41 -0
  2. Dockerfile +9 -6
  3. README.md +73 -25
  4. app/jobs/sync_place_embeddings.py +19 -8
  5. app/modules/places/api/dependencies.py +70 -12
  6. app/modules/places/api/internal_chat_schemas.py +39 -2
  7. app/modules/places/api/router.py +170 -4
  8. app/modules/places/api/schemas.py +24 -1
  9. app/modules/places/application/use_cases/chat_place_recommendations.py +379 -34
  10. app/modules/places/application/use_cases/chat_places.py +13 -2
  11. app/modules/places/application/use_cases/search_places.py +5 -1
  12. app/modules/places/domain/chat_intent.py +6 -0
  13. app/modules/places/domain/clarifications.py +39 -16
  14. app/modules/places/infrastructure/aws_pgvector_place_repository.py +35 -1
  15. app/modules/places/infrastructure/bert_intent_extractor.py +595 -0
  16. app/modules/places/infrastructure/deterministic_intent_parser.py +440 -69
  17. app/modules/places/infrastructure/hybrid_chat_retriever.py +214 -66
  18. app/modules/places/infrastructure/open_vocabulary_category_classifier.py +368 -0
  19. app/modules/places/infrastructure/place_category_catalog.py +87 -0
  20. app/modules/places/infrastructure/place_semantic_document.py +19 -43
  21. app/modules/places/infrastructure/semantic_activity_classifier.py +36 -184
  22. app/modules/places/infrastructure/semantic_place_ranker.py +7 -2
  23. app/shared/config/settings.py +174 -0
  24. app/shared/dependencies.py +30 -1
  25. app/shared/nlp/embeddings/cached.py +29 -1
  26. app/shared/nlp/embeddings/factory.py +50 -0
  27. app/shared/nlp/embeddings/sentence_transformer.py +288 -0
  28. app/shared/vector_store/aws_pgvector.py +66 -8
  29. requirements-training.txt +3 -0
  30. requirements.txt +2 -0
  31. scripts/train_place_intent_bert.py +501 -0
  32. scripts/train_place_retriever.py +147 -0
  33. sql/migrations/20260716_02_places_semantic_v1.sql +255 -0
  34. sql/verify_places_semantic_v1.sql +46 -0
  35. tests/conftest.py +4 -0
  36. tests/test_api_endpoints.py +73 -7
  37. tests/test_bert_place_intent_extractor.py +277 -0
  38. tests/test_embeddings.py +27 -0
  39. tests/test_hybrid_chat_retriever.py +232 -0
  40. tests/test_internal_places_chat.py +10 -9
  41. tests/test_main_api_place_source.py +10 -8
  42. tests/test_open_vocabulary_category_classifier.py +228 -0
  43. tests/test_pgvector_readiness.py +22 -1
  44. tests/test_place_chat_intent_parser.py +356 -3
  45. tests/test_place_chat_recommendations_use_case.py +389 -16
  46. tests/test_place_embedding_configuration.py +182 -0
  47. tests/test_places_use_cases.py +6 -2
  48. tests/test_semantic_activity_classifier.py +22 -0
  49. tests/test_sentence_transformer_embeddings.py +223 -0
  50. tests/test_sql_contract.py +26 -0
.env.example CHANGED
@@ -57,6 +57,37 @@ FASTTEXT_MODEL_REPO_ID=facebook/fasttext-es-vectors
57
  FASTTEXT_MODEL_FILENAME=model.bin
58
  FASTTEXT_AUTO_DOWNLOAD=true
59
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
60
  BM25_K1=1.5
61
  BM25_B=0.75
62
  BM25_RELEVANCE_THRESHOLD=3.0
@@ -70,7 +101,17 @@ PLACES_CHAT_CANDIDATE_LIMIT=30
70
  PLACES_CHAT_MIN_CONTENT_SCORE=0.20
71
  PLACES_CHAT_INTENT_MIN_CONFIDENCE=0.70
72
  PLACES_CHAT_AMBIGUITY_DELTA=0.15
 
 
 
73
  PLACES_CHAT_RANKING_VERSION=places-chat-v2
 
 
 
 
 
 
 
74
  PLACES_CHAT_TAXONOMY_VERSION=places-taxonomy-v1
75
 
76
  LOG_LEVEL=INFO
 
57
  FASTTEXT_MODEL_FILENAME=model.bin
58
  FASTTEXT_AUTO_DOWNLOAD=true
59
 
60
+ # Embeddings exclusivos de Places. Mantener FastText hasta aplicar/backfillear
61
+ # sql/migrations/20260716_02_places_semantic_v1.sql.
62
+ PLACES_EMBEDDING_PROVIDER=fasttext
63
+ PLACES_EMBEDDING_DIMENSION=300
64
+ PLACES_EMBEDDING_MODEL=facebook/fasttext-es-vectors
65
+ PLACES_EMBEDDING_VERSION=common-crawl-300-v1
66
+ PLACES_EMBEDDING_QUERY_PREFIX=
67
+ PLACES_EMBEDDING_PASSAGE_PREFIX=
68
+ PLACES_EMBEDDING_BATCH_SIZE=32
69
+ PLACES_EMBEDDING_DEVICE=
70
+ PLACES_CATEGORY_CATALOG_PATH=
71
+ PLACES_CATEGORY_MIN_SIMILARITY=0.44
72
+ PLACES_CATEGORY_MIN_MARGIN=0.04
73
+ PLACES_PGVECTOR_MATCH_FUNCTION=match_places
74
+ PLACES_PGVECTOR_HYBRID_FUNCTION=
75
+ PLACES_PGVECTOR_UPSERT_FUNCTION=upsert_place_embedding
76
+ PLACES_PGVECTOR_HASH_FUNCTION=get_place_content_hashes
77
+
78
+ # Perfil BERT recomendado despues del backfill:
79
+ # PLACES_EMBEDDING_PROVIDER=sentence_transformer
80
+ # PLACES_EMBEDDING_DIMENSION=768
81
+ # PLACES_EMBEDDING_MODEL=intfloat/multilingual-e5-base
82
+ # PLACES_EMBEDDING_VERSION=places-e5-domain-v1
83
+ # Quoting preserves the significant trailing space used by E5.
84
+ # PLACES_EMBEDDING_QUERY_PREFIX="query: "
85
+ # PLACES_EMBEDDING_PASSAGE_PREFIX="passage: "
86
+ # PLACES_PGVECTOR_MATCH_FUNCTION=match_places_semantic_v1
87
+ # PLACES_PGVECTOR_HYBRID_FUNCTION=search_places_semantic_v1
88
+ # PLACES_PGVECTOR_UPSERT_FUNCTION=upsert_place_embedding_semantic_v1
89
+ # PLACES_PGVECTOR_HASH_FUNCTION=get_place_content_hashes_semantic_v1
90
+
91
  BM25_K1=1.5
92
  BM25_B=0.75
93
  BM25_RELEVANCE_THRESHOLD=3.0
 
101
  PLACES_CHAT_MIN_CONTENT_SCORE=0.20
102
  PLACES_CHAT_INTENT_MIN_CONFIDENCE=0.70
103
  PLACES_CHAT_AMBIGUITY_DELTA=0.15
104
+ PLACES_CHAT_HYPOTHESIS_MIN_CONFIDENCE=0.60
105
+ PLACES_CHAT_HYPOTHESIS_MAX_GAP=0.15
106
+ PLACES_CHAT_DEFAULT_RADIUS_METERS=5000
107
  PLACES_CHAT_RANKING_VERSION=places-chat-v2
108
+ PLACES_CHAT_INTENT_PROVIDER=deterministic
109
+ # Required only when PLACES_CHAT_INTENT_PROVIDER=bert. The model must expose
110
+ # IOB token labels for CATEGORY/PREFERENCE/EXCLUSION/LOCATION/REFERENCE/RADIUS.
111
+ PLACES_CHAT_BERT_MODEL_PATH=
112
+ PLACES_CHAT_BERT_MODEL_VERSION=
113
+ PLACES_CHAT_BERT_DEVICE=cpu
114
+ PLACES_CHAT_BERT_MIN_TOKEN_CONFIDENCE=0.60
115
  PLACES_CHAT_TAXONOMY_VERSION=places-taxonomy-v1
116
 
117
  LOG_LEVEL=INFO
Dockerfile CHANGED
@@ -4,6 +4,7 @@ ENV PYTHONDONTWRITEBYTECODE=1
4
  ENV PYTHONUNBUFFERED=1
5
  ENV PORT=7860
6
  ENV FASTTEXT_MODEL_PATH=/opt/models/fasttext-es/model.bin
 
7
 
8
  WORKDIR /app
9
 
@@ -15,13 +16,15 @@ COPY requirements.txt .
15
  RUN pip install --no-cache-dir --upgrade pip \
16
  && pip install --no-cache-dir -r requirements.txt
17
 
18
- # Keep the 300-dimensional Spanish FastText model in a cached image layer.
19
- # Normal source-code changes do not download the multi-GB model again.
20
  COPY app/shared/nlp/embeddings/download_fasttext_model.py /tmp/download_fasttext_model.py
21
- RUN HF_HOME=/tmp/hf-cache python /tmp/download_fasttext_model.py \
22
- --repo-id facebook/fasttext-es-vectors \
23
- --filename model.bin \
24
- --destination ${FASTTEXT_MODEL_PATH} \
 
 
25
  && rm -rf /tmp/hf-cache /tmp/download_fasttext_model.py
26
 
27
  COPY app ./app
 
4
  ENV PYTHONUNBUFFERED=1
5
  ENV PORT=7860
6
  ENV FASTTEXT_MODEL_PATH=/opt/models/fasttext-es/model.bin
7
+ ARG DOWNLOAD_FASTTEXT_MODEL=true
8
 
9
  WORKDIR /app
10
 
 
16
  RUN pip install --no-cache-dir --upgrade pip \
17
  && pip install --no-cache-dir -r requirements.txt
18
 
19
+ # Keep the rollback FastText model in a cached layer unless a BERT-only image
20
+ # is requested with --build-arg DOWNLOAD_FASTTEXT_MODEL=false.
21
  COPY app/shared/nlp/embeddings/download_fasttext_model.py /tmp/download_fasttext_model.py
22
+ RUN if [ "${DOWNLOAD_FASTTEXT_MODEL}" = "true" ]; then \
23
+ HF_HOME=/tmp/hf-cache python /tmp/download_fasttext_model.py \
24
+ --repo-id facebook/fasttext-es-vectors \
25
+ --filename model.bin \
26
+ --destination ${FASTTEXT_MODEL_PATH}; \
27
+ fi \
28
  && rm -rf /tmp/hf-cache /tmp/download_fasttext_model.py
29
 
30
  COPY app ./app
README.md CHANGED
@@ -197,25 +197,28 @@ y ubicacion actual a la API principal; Go llama `POST /internal/places/chat`, hi
197
  IDs devueltos, calcula distancias con PostGIS y aplica el orden geografico final.
198
 
199
  NLP separa categoria, preferencias, exclusiones, referencia y alcance geografico antes
200
- de buscar. La categoria y `is_active=true` son filtros inviolables. Por eso una consulta
201
- de cafeterias no puede devolver parques aunque el texto incluya "cerca del parque".
 
 
202
  Las ambiguedades que cambiarian los resultados devuelven `action=clarification` y un
203
  `state_patch` con `pending_clarification`; el siguiente turno puede resolverlo con frases
204
  como "la primera opcion" o "la segunda, cerca de mi".
205
 
206
- El chat no exige que el usuario nombre siempre una categoria. Primero aplica defaults
207
- deterministas y, para expresiones no literales, compara el mensaje con prototipos de
208
- actividad mediante el mismo FastText local usado por retrieval. Por ejemplo, "quiero
209
- comer algo" busca restaurantes, "quiero hacer ejercicio" busca opciones deportivas y
210
- "donde dormir" busca hospedaje. El clasificador se abstiene si la similitud es baja o
211
- dos categorias quedan demasiado cerca; no consume Llama ni otra API. Una categoria
212
- explicita siempre prevalece. Una intencion nueva y clara tambien cancela cualquier
213
- aclaracion pendiente anterior. Solo se pregunta cuando las interpretaciones plausibles
214
- cambiarian materialmente los resultados; las preferencias faltantes no bloquean una
215
- primera recomendacion.
216
-
217
- La recuperacion combina FastText/pgvector, BM25 y coincidencias de facetas. NLP devuelve
218
- solo candidatos tecnicos y `content_score`; no incorpora GPS al score. El flag inicial es
 
219
  `PLACES_CHAT_V2_ENABLED=false` y debe activarse despues de desplegar en Go tanto el proxy
220
  de chat como `/api/v1/internal/places/resolve-anchor`.
221
 
@@ -346,23 +349,68 @@ Si el score maximo no supera `SEMANTIC_NO_MATCH_THRESHOLD`, envia a Llama el mod
346
  `SEMANTIC_RELEVANCE_THRESHOLD` usa `low_confidence`; por encima usa `confident`.
347
  Llama solo embellece el tono y recibe exclusivamente los lugares seleccionados.
348
 
349
- ### Documento Semantico Ponderado De Lugares
350
 
351
  Los IDs numericos de tags devueltos por la API principal se resuelven mediante el
352
  catalogo versionado en `app/modules/places/infrastructure/place_tag_catalog.json`.
353
- El documento que se envia a FastText contiene exclusivamente señales semanticas y
354
- aplica estos pesos mediante repeticion antes del promedio de embeddings:
355
 
356
  ```text
357
- tags x6, category x4, description x3, name x1
358
  ```
359
 
360
- Las categorias generales se expanden con terminos de intencion en espanol. Direccion,
361
- ciudad, estado, `source`, precio e IDs desconocidos permanecen fuera del embedding;
362
- siguen disponibles como metadatos o filtros cuando corresponde. La version interna
363
- `weighted-tags-v2` forma parte del hash del documento, por lo que ejecutar nuevamente
364
- `initial_load_place_embeddings` actualiza todas las filas por `UPSERT` sin truncar la
365
- tabla ni cambiar `VECTOR(300)`.
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
366
 
367
  `GET` o `POST /places/search/metrics?k=5` conserva un benchmark offline separado llamado `built_in_places_v3_bm25`. Contiene doce lugares controlados, diez consultas y qrels graduados para calcular honestamente `Precision@k`, `Recall@k`, `MRR`, `MAP` y `nDCG@k`. Estas metricas requieren juicios de relevancia y por eso no se presentan como si midieran una consulta arbitraria de produccion.
368
 
 
197
  IDs devueltos, calcula distancias con PostGIS y aplica el orden geografico final.
198
 
199
  NLP separa categoria, preferencias, exclusiones, referencia y alcance geografico antes
200
+ de buscar. `is_active=true`, ciudad/estado confirmados y los IDs obtenidos por un filtro
201
+ geografico explicito son restricciones duras; la categoria es una hipotesis de ranking.
202
+ Esto permite recuperar vocabulario nuevo o categorias distintas entre sistemas sin
203
+ confundir una referencia como "cerca del parque" con el tipo de resultado solicitado.
204
  Las ambiguedades que cambiarian los resultados devuelven `action=clarification` y un
205
  `state_patch` con `pending_clarification`; el siguiente turno puede resolverlo con frases
206
  como "la primera opcion" o "la segunda, cerca de mi".
207
 
208
+ El chat no exige que el usuario nombre siempre una categoria. Puede habilitar un BERT
209
+ fine-tuneado de token classification para extraer valores abiertos de categoria,
210
+ preferencia, exclusion, ubicacion, referencia y radio. Esos textos se alinean despues
211
+ contra un catalogo dinamico mediante embeddings; no se convierten con aliases dentro del
212
+ adaptador BERT. Las reglas lexicas existentes quedan como fallback de despliegue y no
213
+ bloquean retrieval. El clasificador se abstiene si la similitud es baja o dos conceptos
214
+ quedan demasiado cerca. Las aclaraciones usan hipotesis con evidencia o facetas de los
215
+ candidatos recuperados, no un menu fijo.
216
+
217
+ La recuperacion combina dense retrieval (FastText de rollback o SentenceTransformer),
218
+ BM25 y coincidencias de facetas. La categoria y las exclusiones aportan señales positivas
219
+ o negativas; no eliminan candidatos por una coincidencia textual aislada. NLP devuelve
220
+ candidatos tecnicos y `content_score`; el GPS se aplica como filtro explicito de IDs antes
221
+ del ranking cuando el proveedor de lugares cercanos esta configurado. El flag inicial es
222
  `PLACES_CHAT_V2_ENABLED=false` y debe activarse despues de desplegar en Go tanto el proxy
223
  de chat como `/api/v1/internal/places/resolve-anchor`.
224
 
 
349
  `SEMANTIC_RELEVANCE_THRESHOLD` usa `low_confidence`; por encima usa `confident`.
350
  Llama solo embellece el tono y recibe exclusivamente los lugares seleccionados.
351
 
352
+ ### Documento Semantico Estructurado De Lugares
353
 
354
  Los IDs numericos de tags devueltos por la API principal se resuelven mediante el
355
  catalogo versionado en `app/modules/places/infrastructure/place_tag_catalog.json`.
356
+ El documento de Places contiene cada señal una sola vez y explicita el rol de cada
357
+ campo. Esto evita que la repeticion manual distorsione un encoder BERT:
358
 
359
  ```text
360
+ Nombre: ... Tipo registrado: ... Descripcion: ... Etiquetas: ...
361
  ```
362
 
363
+ No se expanden categorias mediante diccionarios de sinonimos. Direccion, ciudad,
364
+ estado, `source`, precio e IDs desconocidos permanecen fuera del embedding; siguen
365
+ disponibles como metadatos o filtros. La version `structured-place-v3` forma parte del
366
+ hash y fuerza un re-embedding seguro cuando cambia el documento.
367
+
368
+ ### Migracion BERT/Sentence-Transformer exclusiva de Places
369
+
370
+ La migracion es aditiva y no cambia los vectores de posts, perfiles o feed:
371
+
372
+ 1. Ejecuta `sql/migrations/20260716_02_places_semantic_v1.sql` con el rol DBA.
373
+ 2. Configura temporalmente el perfil BERT mostrado en `.env.example`.
374
+ 3. Ejecuta `python -m app.jobs.sync_place_embeddings` para backfill de la tabla
375
+ `place_embeddings_semantic_v1`.
376
+ 4. Ejecuta `sql/verify_places_semantic_v1.sql` y revisa que el plan use HNSW con
377
+ un volumen representativo.
378
+ 5. Activa `match_places_semantic_v1` y `search_places_semantic_v1` primero en shadow.
379
+ 6. Conserva `match_places` y la tabla FastText para rollback.
380
+
381
+ `/places/chat` ya no requiere una categoria canonica para recuperar candidatos. La
382
+ categoria inferida solo aporta afinidad al ranking; la union SQL obtiene pools dense y
383
+ lexical independientes. Un cliente puede optar al contrato conversacional estructurado
384
+ enviando `conversation_id`, `conversation_state`, `clarification_choice` o
385
+ `user_location` mientras `PLACES_CHAT_V2_ENABLED=true`.
386
+
387
+ Para fine-tuning, `scripts/train_place_retriever.py` acepta JSONL con `query`,
388
+ `positive` y `hard_negatives`. El artefacto resultante se configura mediante
389
+ `PLACES_EMBEDDING_MODEL`; no se incluye un modelo ficticio preentrenado en el repo.
390
+
391
+ El extractor de intencion se entrena por separado con
392
+ `scripts/train_place_intent_bert.py`. Su JSONL contiene `text` y spans abiertos
393
+ `{start, end, slot}`; los slots permitidos son `CATEGORY`, `PREFERENCE`,
394
+ `EXCLUSION`, `LOCATION`, `REFERENCE` y `RADIUS`. Los valores concretos (por ejemplo
395
+ "donas artesanales") nunca se convierten en labels del modelo:
396
+
397
+ ```powershell
398
+ python -m pip install -r requirements-training.txt
399
+ python scripts/train_place_intent_bert.py `
400
+ --train-file data/places-intent-train.jsonl `
401
+ --validation-file data/places-intent-validation.jsonl `
402
+ --output-dir .models/places-intent-bert
403
+ ```
404
+
405
+ Para activarlo, configura `PLACES_CHAT_INTENT_PROVIDER=bert`,
406
+ `PLACES_CHAT_BERT_MODEL_PATH=.models/places-intent-bert` y una version inmutable en
407
+ `PLACES_CHAT_BERT_MODEL_VERSION`. El parser determinista queda como fallback si el
408
+ modelo no puede cargarse; cuando BERT responde, no se vuelven a aplicar aliases de
409
+ categoria ni implicaciones manuales sobre sus spans.
410
+
411
+ Una imagen que ya no necesite el artefacto de rollback FastText puede construirse con
412
+ `docker build --build-arg DOWNLOAD_FASTTEXT_MODEL=false .`. Conserva el valor por
413
+ defecto durante el shadow/canary para permitir rollback inmediato.
414
 
415
  `GET` o `POST /places/search/metrics?k=5` conserva un benchmark offline separado llamado `built_in_places_v3_bm25`. Contiene doce lugares controlados, diez consultas y qrels graduados para calcular honestamente `Precision@k`, `Recall@k`, `MRR`, `MAP` y `nDCG@k`. Estas metricas requieren juicios de relevancia y por eso no se presentan como si midieran una consulta arbitraria de produccion.
416
 
app/jobs/sync_place_embeddings.py CHANGED
@@ -9,7 +9,7 @@ from app.modules.places.infrastructure.main_api_place_source import (
9
  from app.shared.config.settings import Settings, get_settings
10
  from app.shared.logging.config import configure_logging, get_logger
11
  from app.shared.nlp.embeddings.base import EmbeddingProvider
12
- from app.shared.nlp.embeddings.factory import create_embedding_provider
13
  from app.shared.nlp.embeddings.versioning import versioned_embedding_hash
14
  from app.shared.vector_store.aws_pgvector import AwsPgvectorClient
15
  from app.shared.vector_store.models import VectorUpsertRecord
@@ -35,12 +35,17 @@ async def main() -> None:
35
 
36
  source = MainApiPlacesClient(settings)
37
  vector_client = AwsPgvectorClient(settings, role="writer")
38
- embedding_provider = create_embedding_provider(settings)
39
  counters = SyncCounters()
40
  batch: list[PlaceSourceRecord] = []
41
 
42
  logger.info("Starting place embedding sync")
43
- logger.info("Embedding model=%s version=%s", settings.embedding_model, settings.embedding_version)
 
 
 
 
 
44
 
45
  async for place in source.iter_places(
46
  page_limit=args.page_limit,
@@ -93,14 +98,15 @@ async def _flush_batch(
93
  counters.processed += len(batch)
94
  try:
95
  existing_hashes = await vector_client.fetch_place_content_hashes(
96
- [record.id for record in batch]
 
97
  )
98
  expected_hashes = {
99
  record.id: versioned_embedding_hash(
100
  source_content_hash=record.content_hash,
101
- model=settings.embedding_model,
102
- version=settings.embedding_version,
103
- dimension=settings.embedding_dimension,
104
  )
105
  for record in batch
106
  }
@@ -130,7 +136,12 @@ async def _flush_batch(
130
  if dry_run:
131
  logger.info("Dry run: prepared %s place upserts", len(upserts))
132
  else:
133
- await vector_client.upsert_place_embeddings(upserts)
 
 
 
 
 
134
  counters.upserted += len(upserts)
135
  except Exception:
136
  counters.errors += len(batch)
 
9
  from app.shared.config.settings import Settings, get_settings
10
  from app.shared.logging.config import configure_logging, get_logger
11
  from app.shared.nlp.embeddings.base import EmbeddingProvider
12
+ from app.shared.nlp.embeddings.factory import create_place_embedding_provider
13
  from app.shared.nlp.embeddings.versioning import versioned_embedding_hash
14
  from app.shared.vector_store.aws_pgvector import AwsPgvectorClient
15
  from app.shared.vector_store.models import VectorUpsertRecord
 
35
 
36
  source = MainApiPlacesClient(settings)
37
  vector_client = AwsPgvectorClient(settings, role="writer")
38
+ embedding_provider = create_place_embedding_provider(settings, text_role="passage")
39
  counters = SyncCounters()
40
  batch: list[PlaceSourceRecord] = []
41
 
42
  logger.info("Starting place embedding sync")
43
+ logger.info(
44
+ "Places embedding model=%s version=%s dimension=%s",
45
+ settings.places_embedding_model,
46
+ settings.places_embedding_version,
47
+ settings.places_embedding_dimension,
48
+ )
49
 
50
  async for place in source.iter_places(
51
  page_limit=args.page_limit,
 
98
  counters.processed += len(batch)
99
  try:
100
  existing_hashes = await vector_client.fetch_place_content_hashes(
101
+ [record.id for record in batch],
102
+ function_name=settings.places_pgvector_hash_function,
103
  )
104
  expected_hashes = {
105
  record.id: versioned_embedding_hash(
106
  source_content_hash=record.content_hash,
107
+ model=settings.places_embedding_model,
108
+ version=settings.places_embedding_version,
109
+ dimension=settings.places_embedding_dimension,
110
  )
111
  for record in batch
112
  }
 
136
  if dry_run:
137
  logger.info("Dry run: prepared %s place upserts", len(upserts))
138
  else:
139
+ await vector_client.upsert_place_embeddings(
140
+ upserts,
141
+ function_name=settings.places_pgvector_upsert_function,
142
+ embedding_model=settings.places_embedding_model,
143
+ embedding_version=settings.places_embedding_version,
144
+ )
145
  counters.upserted += len(upserts)
146
  except Exception:
147
  counters.errors += len(batch)
app/modules/places/api/dependencies.py CHANGED
@@ -13,6 +13,9 @@ from app.modules.places.infrastructure.aws_pgvector_place_repository import (
13
  AwsPgvectorPlaceRepository,
14
  )
15
  from app.modules.places.infrastructure.bm25_place_ranker import Bm25PlaceRanker
 
 
 
16
  from app.modules.places.infrastructure.main_api_nearby_place_provider import (
17
  MainApiNearbyPlaceProvider,
18
  )
@@ -33,13 +36,21 @@ from app.modules.places.infrastructure.place_search_benchmark import (
33
  QRELS_SOURCE,
34
  get_default_place_search_benchmark,
35
  )
36
- from app.modules.places.infrastructure.semantic_place_ranker import SemanticPlaceRanker
37
- from app.modules.places.infrastructure.semantic_activity_classifier import (
38
- SemanticPlaceActivityClassifier,
39
  )
 
 
 
 
 
40
  from app.shared.cache.memory import SimpleTTLCache
41
  from app.shared.config.settings import get_settings
42
- from app.shared.dependencies import get_embedding_provider, get_llm_provider
 
 
 
 
43
  from app.shared.nlp.embeddings.mock import MockEmbeddingProvider
44
  from app.shared.nlp.llm.output_guard import PlaceChatOutputGuard
45
  from app.shared.vector_store.aws_pgvector import AwsPgvectorClient
@@ -49,14 +60,21 @@ from app.shared.vector_store.aws_pgvector import AwsPgvectorClient
49
  def get_place_repository() -> MockPlaceVectorRepository | AwsPgvectorPlaceRepository:
50
  settings = get_settings()
51
  if settings.vector_store_provider == "aws_pgvector":
52
- return AwsPgvectorPlaceRepository(vector_client=AwsPgvectorClient(settings, role="reader"))
53
- return MockPlaceVectorRepository(embedding_provider=get_embedding_provider())
 
 
 
 
54
 
55
 
56
  @lru_cache
57
  def get_place_ranker() -> SemanticPlaceRanker:
58
  settings = get_settings()
59
- return SemanticPlaceRanker(dimension=settings.embedding_dimension)
 
 
 
60
 
61
 
62
  @lru_cache
@@ -79,7 +97,7 @@ def get_nearby_place_provider() -> MainApiNearbyPlaceProvider:
79
  @lru_cache
80
  def get_search_places_use_case() -> SearchPlacesUseCase:
81
  return SearchPlacesUseCase(
82
- embedding_provider=get_embedding_provider(),
83
  place_repository=get_place_repository(),
84
  ranker=get_place_ranker(),
85
  cache=get_place_search_cache(),
@@ -101,7 +119,9 @@ def get_recommend_places_use_case() -> RecommendPlacesUseCase:
101
  @lru_cache
102
  def get_evaluate_place_search_use_case() -> EvaluatePlaceSearchUseCase:
103
  settings = get_settings()
104
- embedding_provider = MockEmbeddingProvider(dimension=settings.embedding_dimension)
 
 
105
  benchmark_search = SearchPlacesUseCase(
106
  embedding_provider=embedding_provider,
107
  place_repository=MockPlaceVectorRepository(embedding_provider),
@@ -133,14 +153,42 @@ def get_place_chat_intent_parser() -> DeterministicPlaceChatIntentParser:
133
  raise RuntimeError(
134
  "PLACES_CHAT_TAXONOMY_VERSION does not match the bundled taxonomy"
135
  )
 
 
 
 
136
  activity_classifier = (
137
- SemanticPlaceActivityClassifier(get_embedding_provider())
138
- if settings.embedding_provider.casefold() != "mock"
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
139
  else None
140
  )
141
  return DeterministicPlaceChatIntentParser(
142
  taxonomy=taxonomy,
143
  activity_classifier=activity_classifier,
 
144
  )
145
 
146
 
@@ -156,7 +204,7 @@ def get_place_anchor_resolver() -> MainApiPlaceAnchorResolver | MockPlaceAnchorR
156
  def get_hybrid_place_chat_retriever() -> HybridContentPlaceChatRetriever:
157
  settings = get_settings()
158
  return HybridContentPlaceChatRetriever(
159
- embedding_provider=get_embedding_provider(),
160
  place_repository=get_place_repository(),
161
  minimum_content_score=settings.places_chat_min_content_score,
162
  k1=settings.bm25_k1,
@@ -179,4 +227,14 @@ def get_chat_place_recommendations_use_case() -> ChatPlaceRecommendationsUseCase
179
  llm_enabled=settings.places_chat_llm_enabled,
180
  anchor_ambiguity_delta=settings.places_chat_ambiguity_delta,
181
  minimum_intent_confidence=settings.places_chat_intent_min_confidence,
 
 
 
 
 
 
 
 
 
 
182
  )
 
13
  AwsPgvectorPlaceRepository,
14
  )
15
  from app.modules.places.infrastructure.bm25_place_ranker import Bm25PlaceRanker
16
+ from app.modules.places.infrastructure.bert_intent_extractor import (
17
+ BertPlaceIntentExtractor,
18
+ )
19
  from app.modules.places.infrastructure.main_api_nearby_place_provider import (
20
  MainApiNearbyPlaceProvider,
21
  )
 
36
  QRELS_SOURCE,
37
  get_default_place_search_benchmark,
38
  )
39
+ from app.modules.places.infrastructure.open_vocabulary_category_classifier import (
40
+ OpenVocabularyPlaceCategoryClassifier,
 
41
  )
42
+ from app.modules.places.infrastructure.place_category_catalog import (
43
+ load_place_category_concepts,
44
+ )
45
+ from app.modules.places.infrastructure.place_semantic_document import place_tag_catalog
46
+ from app.modules.places.infrastructure.semantic_place_ranker import SemanticPlaceRanker
47
  from app.shared.cache.memory import SimpleTTLCache
48
  from app.shared.config.settings import get_settings
49
+ from app.shared.dependencies import (
50
+ get_llm_provider,
51
+ get_place_embedding_provider,
52
+ get_place_passage_embedding_provider,
53
+ )
54
  from app.shared.nlp.embeddings.mock import MockEmbeddingProvider
55
  from app.shared.nlp.llm.output_guard import PlaceChatOutputGuard
56
  from app.shared.vector_store.aws_pgvector import AwsPgvectorClient
 
60
  def get_place_repository() -> MockPlaceVectorRepository | AwsPgvectorPlaceRepository:
61
  settings = get_settings()
62
  if settings.vector_store_provider == "aws_pgvector":
63
+ return AwsPgvectorPlaceRepository(
64
+ vector_client=AwsPgvectorClient(settings, role="reader"),
65
+ match_function=settings.places_pgvector_match_function,
66
+ hybrid_function=settings.places_pgvector_hybrid_function,
67
+ )
68
+ return MockPlaceVectorRepository(embedding_provider=get_place_embedding_provider())
69
 
70
 
71
  @lru_cache
72
  def get_place_ranker() -> SemanticPlaceRanker:
73
  settings = get_settings()
74
+ return SemanticPlaceRanker(
75
+ dimension=settings.places_embedding_dimension,
76
+ model_name=settings.places_embedding_model,
77
+ )
78
 
79
 
80
  @lru_cache
 
97
  @lru_cache
98
  def get_search_places_use_case() -> SearchPlacesUseCase:
99
  return SearchPlacesUseCase(
100
+ embedding_provider=get_place_embedding_provider(),
101
  place_repository=get_place_repository(),
102
  ranker=get_place_ranker(),
103
  cache=get_place_search_cache(),
 
119
  @lru_cache
120
  def get_evaluate_place_search_use_case() -> EvaluatePlaceSearchUseCase:
121
  settings = get_settings()
122
+ embedding_provider = MockEmbeddingProvider(
123
+ dimension=settings.places_embedding_dimension
124
+ )
125
  benchmark_search = SearchPlacesUseCase(
126
  embedding_provider=embedding_provider,
127
  place_repository=MockPlaceVectorRepository(embedding_provider),
 
153
  raise RuntimeError(
154
  "PLACES_CHAT_TAXONOMY_VERSION does not match the bundled taxonomy"
155
  )
156
+ concepts = load_place_category_concepts(
157
+ settings.places_category_catalog_path,
158
+ fallback_tags=place_tag_catalog().values(),
159
+ )
160
  activity_classifier = (
161
+ OpenVocabularyPlaceCategoryClassifier(
162
+ concepts=concepts,
163
+ embedding_provider=get_place_embedding_provider(),
164
+ concept_embedding_provider=(
165
+ get_place_passage_embedding_provider()
166
+ if settings.places_embedding_provider
167
+ in {"sentence_transformer", "bert"}
168
+ else get_place_embedding_provider()
169
+ ),
170
+ minimum_similarity=settings.places_category_min_similarity,
171
+ minimum_margin=settings.places_category_min_margin,
172
+ )
173
+ if settings.places_embedding_provider.casefold() != "mock" and concepts
174
+ else None
175
+ )
176
+ contextual_extractor = (
177
+ BertPlaceIntentExtractor(
178
+ settings.places_chat_bert_model_path or "",
179
+ model_version=settings.places_chat_bert_model_version,
180
+ device=settings.places_chat_bert_device,
181
+ minimum_token_confidence=(
182
+ settings.places_chat_bert_min_token_confidence
183
+ ),
184
+ )
185
+ if settings.places_chat_intent_provider == "bert"
186
  else None
187
  )
188
  return DeterministicPlaceChatIntentParser(
189
  taxonomy=taxonomy,
190
  activity_classifier=activity_classifier,
191
+ contextual_extractor=contextual_extractor,
192
  )
193
 
194
 
 
204
  def get_hybrid_place_chat_retriever() -> HybridContentPlaceChatRetriever:
205
  settings = get_settings()
206
  return HybridContentPlaceChatRetriever(
207
+ embedding_provider=get_place_embedding_provider(),
208
  place_repository=get_place_repository(),
209
  minimum_content_score=settings.places_chat_min_content_score,
210
  k1=settings.bm25_k1,
 
227
  llm_enabled=settings.places_chat_llm_enabled,
228
  anchor_ambiguity_delta=settings.places_chat_ambiguity_delta,
229
  minimum_intent_confidence=settings.places_chat_intent_min_confidence,
230
+ minimum_hypothesis_confidence=(
231
+ settings.places_chat_hypothesis_min_confidence
232
+ ),
233
+ maximum_hypothesis_gap=settings.places_chat_hypothesis_max_gap,
234
+ nearby_place_provider=(
235
+ get_nearby_place_provider()
236
+ if settings.vector_store_provider != "mock"
237
+ else None
238
+ ),
239
+ default_radius_meters=settings.places_chat_default_radius_meters,
240
  )
app/modules/places/api/internal_chat_schemas.py CHANGED
@@ -65,7 +65,7 @@ class PendingClarificationOptionStateSchema(BaseModel):
65
  model_config = ConfigDict(extra="forbid")
66
 
67
  id: str = Field(..., min_length=1, max_length=64, pattern=r"^[a-zA-Z0-9_-]+$")
68
- value: str = Field(..., min_length=1, max_length=160)
69
  label: str = Field(..., min_length=1, max_length=160)
70
  place_id: str | None = Field(default=None, max_length=100)
71
  attributes: list[str] = Field(default_factory=list, max_length=30)
@@ -114,7 +114,7 @@ class PendingClarificationStateSchema(BaseModel):
114
  class ConversationStateSchema(BaseModel):
115
  model_config = ConfigDict(extra="forbid")
116
 
117
- target_category: str | None = Field(default=None, max_length=80)
118
  hard_filters: dict[str, Any] = Field(default_factory=dict)
119
  soft_preferences: list[str] = Field(default_factory=list, max_length=30)
120
  exclusions: list[str] = Field(default_factory=list, max_length=30)
@@ -240,6 +240,7 @@ class InternalPlaceChatResponse(BaseModel):
240
  ranking_version: str
241
  taxonomy_version: str
242
  trace_id: str
 
243
  metadata: dict[str, Any] = Field(default_factory=dict)
244
 
245
 
@@ -247,6 +248,16 @@ def internal_chat_result_to_schema(
247
  result: ChatPlaceRecommendationsResult,
248
  ) -> InternalPlaceChatResponse:
249
  directive = result.location_directive
 
 
 
 
 
 
 
 
 
 
250
  return InternalPlaceChatResponse(
251
  action=result.action,
252
  message=result.message,
@@ -292,9 +303,35 @@ def internal_chat_result_to_schema(
292
  ranking_version=result.ranking_version,
293
  taxonomy_version=result.taxonomy_version,
294
  trace_id=result.trace_id,
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
295
  metadata={
296
  "used_llm": result.used_llm,
297
  "guard_reason": result.guard_reason,
298
  "category_source": result.category_source,
 
 
299
  },
300
  )
 
65
  model_config = ConfigDict(extra="forbid")
66
 
67
  id: str = Field(..., min_length=1, max_length=64, pattern=r"^[a-zA-Z0-9_-]+$")
68
+ value: str = Field(..., min_length=1, max_length=500)
69
  label: str = Field(..., min_length=1, max_length=160)
70
  place_id: str | None = Field(default=None, max_length=100)
71
  attributes: list[str] = Field(default_factory=list, max_length=30)
 
114
  class ConversationStateSchema(BaseModel):
115
  model_config = ConfigDict(extra="forbid")
116
 
117
+ target_category: str | None = Field(default=None, max_length=500)
118
  hard_filters: dict[str, Any] = Field(default_factory=dict)
119
  soft_preferences: list[str] = Field(default_factory=list, max_length=30)
120
  exclusions: list[str] = Field(default_factory=list, max_length=30)
 
240
  ranking_version: str
241
  taxonomy_version: str
242
  trace_id: str
243
+ uncertainty: dict[str, Any]
244
  metadata: dict[str, Any] = Field(default_factory=dict)
245
 
246
 
 
248
  result: ChatPlaceRecommendationsResult,
249
  ) -> InternalPlaceChatResponse:
250
  directive = result.location_directive
251
+ hypotheses = list(result.category_hypotheses)
252
+ margin = (
253
+ max(
254
+ 0.0,
255
+ float(hypotheses[0]["probability"])
256
+ - float(hypotheses[1]["probability"]),
257
+ )
258
+ if len(hypotheses) >= 2
259
+ else None
260
+ )
261
  return InternalPlaceChatResponse(
262
  action=result.action,
263
  message=result.message,
 
303
  ranking_version=result.ranking_version,
304
  taxonomy_version=result.taxonomy_version,
305
  trace_id=result.trace_id,
306
+ uncertainty={
307
+ "decision": (
308
+ "review"
309
+ if result.action == "recommendations" and result.unresolved
310
+ else {
311
+ "recommendations": "auto",
312
+ "clarification": "clarify",
313
+ "no_match": "abstain",
314
+ }[result.action]
315
+ ),
316
+ "reason": (
317
+ result.unresolved[0]
318
+ if result.unresolved
319
+ else (
320
+ "sufficient_evidence"
321
+ if result.action == "recommendations"
322
+ else "catalog_exhausted"
323
+ )
324
+ ),
325
+ "top_probability": round(result.intent_confidence, 6),
326
+ "category_hypotheses": hypotheses,
327
+ "category_margin": round(margin, 6) if margin is not None else None,
328
+ "calibration_version": "uncalibrated-shadow-v1",
329
+ },
330
  metadata={
331
  "used_llm": result.used_llm,
332
  "guard_reason": result.guard_reason,
333
  "category_source": result.category_source,
334
+ "raw_category_phrase": result.raw_category_phrase,
335
+ "intent_model_version": result.intent_model_version,
336
  },
337
  )
app/modules/places/api/router.py CHANGED
@@ -1,16 +1,24 @@
1
- from fastapi import APIRouter, Depends, Query
 
 
 
2
 
3
  from app.modules.places.api.dependencies import (
4
  get_chat_places_use_case,
 
5
  get_evaluate_place_search_use_case,
6
  get_recommend_places_use_case,
7
  get_search_places_use_case,
8
  )
 
 
 
9
  from app.modules.places.api.schemas import (
10
  PlaceChatRequest,
11
  PlaceChatResponse,
12
  PlaceRecommendationRequest,
13
  PlaceRecommendationResponse,
 
14
  PlaceSearchMetricsResponse,
15
  PlaceSearchRequest,
16
  PlaceSearchResponse,
@@ -19,12 +27,19 @@ from app.modules.places.api.schemas import (
19
  search_metrics_result_to_schema,
20
  )
21
  from app.modules.places.application.use_cases.chat_places import ChatPlacesUseCase
 
 
 
 
 
22
  from app.modules.places.application.use_cases.evaluate_place_search import (
23
  EvaluatePlaceSearchUseCase,
24
  )
25
  from app.modules.places.application.use_cases.recommend_places import RecommendPlacesUseCase
26
  from app.modules.places.application.use_cases.search_places import SearchPlacesUseCase
27
  from app.shared.security.rate_limit import rate_limit_placeholder
 
 
28
 
29
  router = APIRouter(
30
  prefix="/places",
@@ -85,12 +100,163 @@ async def recommend_places(
85
  )
86
 
87
 
88
- @router.post("/chat", response_model=PlaceChatResponse)
 
 
 
 
89
  async def chat_places(
90
  payload: PlaceChatRequest,
91
- use_case: ChatPlacesUseCase = Depends(get_chat_places_use_case),
 
 
 
92
  ) -> PlaceChatResponse:
93
- result = await use_case.execute(
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
94
  message=payload.message,
95
  filters=payload.to_domain_filters(),
96
  limit=payload.limit,
 
1
+ import asyncio
2
+ from dataclasses import replace
3
+
4
+ from fastapi import APIRouter, Depends, HTTPException, Query
5
 
6
  from app.modules.places.api.dependencies import (
7
  get_chat_places_use_case,
8
+ get_chat_place_recommendations_use_case,
9
  get_evaluate_place_search_use_case,
10
  get_recommend_places_use_case,
11
  get_search_places_use_case,
12
  )
13
+ from app.modules.places.api.internal_chat_schemas import (
14
+ internal_chat_result_to_schema,
15
+ )
16
  from app.modules.places.api.schemas import (
17
  PlaceChatRequest,
18
  PlaceChatResponse,
19
  PlaceRecommendationRequest,
20
  PlaceRecommendationResponse,
21
+ PlaceResultSchema,
22
  PlaceSearchMetricsResponse,
23
  PlaceSearchRequest,
24
  PlaceSearchResponse,
 
27
  search_metrics_result_to_schema,
28
  )
29
  from app.modules.places.application.use_cases.chat_places import ChatPlacesUseCase
30
+ from app.modules.places.application.use_cases.chat_place_recommendations import (
31
+ ChatPlaceRecommendationsUseCase,
32
+ )
33
+ from app.modules.places.domain.chat_intent import ConversationState
34
+ from app.modules.places.domain.errors import ClarificationStateMismatchError
35
  from app.modules.places.application.use_cases.evaluate_place_search import (
36
  EvaluatePlaceSearchUseCase,
37
  )
38
  from app.modules.places.application.use_cases.recommend_places import RecommendPlacesUseCase
39
  from app.modules.places.application.use_cases.search_places import SearchPlacesUseCase
40
  from app.shared.security.rate_limit import rate_limit_placeholder
41
+ from app.shared.config.settings import get_settings
42
+ from app.shared.tracing import new_response_id
43
 
44
  router = APIRouter(
45
  prefix="/places",
 
100
  )
101
 
102
 
103
+ @router.post(
104
+ "/chat",
105
+ response_model=PlaceChatResponse,
106
+ response_model_exclude_none=True,
107
+ )
108
  async def chat_places(
109
  payload: PlaceChatRequest,
110
+ legacy_use_case: ChatPlacesUseCase = Depends(get_chat_places_use_case),
111
+ semantic_use_case: ChatPlaceRecommendationsUseCase = Depends(
112
+ get_chat_place_recommendations_use_case
113
+ ),
114
  ) -> PlaceChatResponse:
115
+ settings = get_settings()
116
+ use_semantic_chat = settings.places_chat_v2_enabled and any(
117
+ (
118
+ payload.conversation_id is not None,
119
+ payload.conversation_state is not None,
120
+ payload.clarification_choice is not None,
121
+ payload.user_location is not None,
122
+ )
123
+ )
124
+ if use_semantic_chat:
125
+ state = (
126
+ payload.conversation_state.to_domain()
127
+ if payload.conversation_state
128
+ else ConversationState()
129
+ )
130
+ if (
131
+ state.taxonomy_version is not None
132
+ and state.taxonomy_version != settings.places_chat_taxonomy_version
133
+ ):
134
+ raise HTTPException(
135
+ status_code=409,
136
+ detail="Conversation taxonomy version is incompatible",
137
+ )
138
+ request_filters = payload.to_domain_filters()
139
+ hard_filters = dict(state.hard_filters)
140
+ for key, value in {
141
+ "city": request_filters.city,
142
+ "state": request_filters.state,
143
+ "price_range": request_filters.price_range,
144
+ "occasion": request_filters.occasion,
145
+ }.items():
146
+ if value is not None:
147
+ hard_filters[key] = value
148
+ state = replace(
149
+ state,
150
+ target_category=state.target_category or request_filters.category,
151
+ hard_filters=hard_filters,
152
+ city=state.city or request_filters.city,
153
+ state=state.state or request_filters.state,
154
+ )
155
+ location = payload.user_location
156
+ candidate_limit = (
157
+ payload.candidate_limit or settings.places_chat_candidate_limit
158
+ )
159
+ if candidate_limit > settings.places_chat_candidate_limit:
160
+ raise HTTPException(
161
+ status_code=422,
162
+ detail=(
163
+ "candidate_limit exceeds the configured service maximum of "
164
+ f"{settings.places_chat_candidate_limit}"
165
+ ),
166
+ )
167
+ try:
168
+ async with asyncio.timeout(settings.request_timeout_seconds):
169
+ semantic_result = await semantic_use_case.execute(
170
+ message=payload.message,
171
+ state=state,
172
+ user_latitude=location.lat if location else None,
173
+ user_longitude=location.lng if location else None,
174
+ candidate_limit=candidate_limit,
175
+ result_limit=payload.limit,
176
+ clarification_choice=(
177
+ payload.clarification_choice.to_domain()
178
+ if payload.clarification_choice
179
+ else None
180
+ ),
181
+ )
182
+ except ClarificationStateMismatchError as exc:
183
+ raise HTTPException(
184
+ status_code=409,
185
+ detail="Clarification choice does not match the current state",
186
+ ) from exc
187
+ except TimeoutError as exc:
188
+ raise HTTPException(
189
+ status_code=503,
190
+ detail="Places chat timed out",
191
+ ) from exc
192
+ structured = internal_chat_result_to_schema(semantic_result)
193
+ decision = (
194
+ "review"
195
+ if semantic_result.action == "recommendations"
196
+ and semantic_result.unresolved
197
+ else {
198
+ "recommendations": "auto",
199
+ "clarification": "clarify",
200
+ "no_match": "abstain",
201
+ }[semantic_result.action]
202
+ )
203
+ reason = (
204
+ semantic_result.unresolved[0]
205
+ if semantic_result.unresolved
206
+ else (
207
+ "sufficient_evidence"
208
+ if semantic_result.action == "recommendations"
209
+ else "catalog_exhausted"
210
+ )
211
+ )
212
+ return PlaceChatResponse(
213
+ response_id=new_response_id(),
214
+ nlp_trace_id=semantic_result.trace_id,
215
+ action=semantic_result.action,
216
+ message=semantic_result.message,
217
+ places=[
218
+ PlaceResultSchema(
219
+ id=candidate.place_id,
220
+ name=candidate.name,
221
+ score=round(candidate.content_score, 4),
222
+ category=candidate.category,
223
+ city=candidate.metadata.get("city"),
224
+ state=candidate.metadata.get("state"),
225
+ metadata={
226
+ **candidate.metadata,
227
+ "semantic_score": candidate.semantic_score,
228
+ "lexical_score": candidate.lexical_score,
229
+ "match_level": candidate.match_level,
230
+ "matched_reasons": list(candidate.matched_reasons),
231
+ },
232
+ )
233
+ for candidate in semantic_result.candidates[: payload.limit]
234
+ ],
235
+ state_patch=semantic_result.state_patch,
236
+ location_directive=structured.location_directive,
237
+ clarification=structured.clarification,
238
+ unresolved=list(semantic_result.unresolved),
239
+ intent_confidence=semantic_result.intent_confidence,
240
+ ranking_version=semantic_result.ranking_version,
241
+ taxonomy_version=semantic_result.taxonomy_version,
242
+ uncertainty={
243
+ **structured.uncertainty,
244
+ "decision": decision,
245
+ "reason": reason,
246
+ },
247
+ metadata={
248
+ **structured.metadata,
249
+ "pipeline": "places-chat-semantic-v2",
250
+ "conversation_id": (
251
+ str(payload.conversation_id)
252
+ if payload.conversation_id is not None
253
+ else None
254
+ ),
255
+ "turn": payload.turn,
256
+ },
257
+ )
258
+
259
+ result = await legacy_use_case.execute(
260
  message=payload.message,
261
  filters=payload.to_domain_filters(),
262
  limit=payload.limit,
app/modules/places/api/schemas.py CHANGED
@@ -1,10 +1,18 @@
1
- from typing import Any
 
2
 
3
  from pydantic import BaseModel, ConfigDict, Field, model_validator
4
 
5
  from app.modules.places.application.use_cases.evaluate_place_search import (
6
  EvaluatePlaceSearchResult,
7
  )
 
 
 
 
 
 
 
8
  from app.modules.places.domain.models import PlaceCandidate, PlaceFilters
9
  from app.modules.places.domain.search_metrics import (
10
  SearchEngineMetrics,
@@ -64,6 +72,12 @@ class PlaceChatRequest(BaseModel):
64
  state: str | None = Field(default=None, max_length=80)
65
  filters: PlaceFiltersSchema = Field(default_factory=PlaceFiltersSchema)
66
  limit: int = Field(default=5, ge=1, le=8)
 
 
 
 
 
 
67
 
68
  def to_domain_filters(self) -> PlaceFilters:
69
  return PlaceFilters(
@@ -168,6 +182,15 @@ class PlaceChatResponse(BaseModel):
168
  message: str
169
  places: list[PlaceResultSchema]
170
  metadata: dict[str, Any]
 
 
 
 
 
 
 
 
 
171
 
172
 
173
  def place_to_schema(place: PlaceCandidate) -> PlaceResultSchema:
 
1
+ from typing import Any, Literal
2
+ from uuid import UUID
3
 
4
  from pydantic import BaseModel, ConfigDict, Field, model_validator
5
 
6
  from app.modules.places.application.use_cases.evaluate_place_search import (
7
  EvaluatePlaceSearchResult,
8
  )
9
+ from app.modules.places.api.internal_chat_schemas import (
10
+ ClarificationChoiceSchema,
11
+ ClarificationSchema,
12
+ ConversationStateSchema,
13
+ PlaceChatLocationDirectiveSchema,
14
+ UserLocationSchema,
15
+ )
16
  from app.modules.places.domain.models import PlaceCandidate, PlaceFilters
17
  from app.modules.places.domain.search_metrics import (
18
  SearchEngineMetrics,
 
72
  state: str | None = Field(default=None, max_length=80)
73
  filters: PlaceFiltersSchema = Field(default_factory=PlaceFiltersSchema)
74
  limit: int = Field(default=5, ge=1, le=8)
75
+ conversation_id: UUID | None = None
76
+ turn: int = Field(default=1, ge=1)
77
+ conversation_state: ConversationStateSchema | None = None
78
+ clarification_choice: ClarificationChoiceSchema | None = None
79
+ user_location: UserLocationSchema | None = None
80
+ candidate_limit: int | None = Field(default=None, ge=1, le=40)
81
 
82
  def to_domain_filters(self) -> PlaceFilters:
83
  return PlaceFilters(
 
182
  message: str
183
  places: list[PlaceResultSchema]
184
  metadata: dict[str, Any]
185
+ action: Literal["recommendations", "clarification", "no_match"] | None = None
186
+ state_patch: dict[str, Any] | None = None
187
+ location_directive: PlaceChatLocationDirectiveSchema | None = None
188
+ clarification: ClarificationSchema | None = None
189
+ unresolved: list[str] | None = None
190
+ intent_confidence: float | None = Field(default=None, ge=0, le=1)
191
+ ranking_version: str | None = None
192
+ taxonomy_version: str | None = None
193
+ uncertainty: dict[str, Any] | None = None
194
 
195
 
196
  def place_to_schema(place: PlaceCandidate) -> PlaceResultSchema:
app/modules/places/application/use_cases/chat_place_recommendations.py CHANGED
@@ -1,3 +1,4 @@
 
1
  from dataclasses import dataclass, replace
2
  import logging
3
  import math
@@ -7,6 +8,9 @@ from app.modules.places.application.ports.chat_retriever import (
7
  HybridPlaceChatRetriever,
8
  )
9
  from app.modules.places.application.ports.intent_parser import PlaceChatIntentParser
 
 
 
10
  from app.modules.places.application.ports.place_anchor_resolver import (
11
  PlaceAnchorResolver,
12
  )
@@ -36,6 +40,13 @@ from app.shared.tracing import new_trace_id
36
 
37
  logger = logging.getLogger(__name__)
38
 
 
 
 
 
 
 
 
39
 
40
  @dataclass(frozen=True)
41
  class ChatPlaceRecommendationsResult:
@@ -53,6 +64,9 @@ class ChatPlaceRecommendationsResult:
53
  category_source: str = "unresolved"
54
  used_llm: bool = False
55
  guard_reason: str | None = None
 
 
 
56
 
57
 
58
  class ChatPlaceRecommendationsUseCase:
@@ -68,7 +82,21 @@ class ChatPlaceRecommendationsUseCase:
68
  llm_enabled: bool = True,
69
  anchor_ambiguity_delta: float = 0.15,
70
  minimum_intent_confidence: float = 0.70,
 
 
 
 
71
  ) -> None:
 
 
 
 
 
 
 
 
 
 
72
  self._intent_parser = intent_parser
73
  self._anchor_resolver = anchor_resolver
74
  self._retriever = retriever
@@ -79,44 +107,37 @@ class ChatPlaceRecommendationsUseCase:
79
  self._llm_enabled = llm_enabled
80
  self._anchor_ambiguity_delta = anchor_ambiguity_delta
81
  self._minimum_intent_confidence = minimum_intent_confidence
 
 
 
 
82
 
83
  async def execute(
84
  self,
85
  message: str,
86
  state: ConversationState,
87
- user_latitude: float,
88
- user_longitude: float,
89
  candidate_limit: int,
90
  result_limit: int,
91
  clarification_choice: ClarificationChoice | None = None,
92
  ) -> ChatPlaceRecommendationsResult:
93
- del user_latitude, user_longitude
 
 
94
  trace_id = new_trace_id()
95
- intent = self._intent_parser.parse(
96
- message=message,
97
- state=state,
98
- has_user_location=True,
99
- clarification_choice=clarification_choice,
 
 
 
 
100
  )
101
  if intent.action == "clarification":
102
  return self._clarification_result(intent, trace_id)
103
- if intent.confidence < self._minimum_intent_confidence:
104
- categories = tuple(
105
- dict.fromkeys(
106
- (
107
- intent.target_category or "restaurant",
108
- "restaurant",
109
- "cafe",
110
- "park",
111
- )
112
- )
113
- )
114
- clarified = self._with_pending_clarification(
115
- intent,
116
- new_category_clarification(categories, kind="intent_category"),
117
- unresolved=("intent_confidence",),
118
- )
119
- return self._clarification_result(clarified, trace_id)
120
 
121
  resolved = await self._resolve_entities(intent, state)
122
  if isinstance(resolved, ChatPlaceRecommendationsResult):
@@ -127,13 +148,129 @@ class ChatPlaceRecommendationsUseCase:
127
 
128
  directive = self._location_directive(intent)
129
  if directive.source == "unresolved":
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
130
  return self._result(
131
  action="no_match",
132
- message="No pude identificar una ubicacion util para esta busqueda.",
133
  intent=intent,
134
  directive=directive,
135
  candidates=(),
136
- unresolved=(),
137
  trace_id=trace_id,
138
  )
139
 
@@ -159,13 +296,18 @@ class ChatPlaceRecommendationsUseCase:
159
  candidates=candidates[:result_limit],
160
  state=state,
161
  )
 
 
 
162
  return self._result(
163
  action="recommendations",
164
  message=final_message,
165
  intent=intent,
166
  directive=directive,
167
  candidates=candidates,
168
- unresolved=(),
 
 
169
  trace_id=trace_id,
170
  used_llm=used_llm,
171
  guard_reason=guard_reason,
@@ -222,7 +364,12 @@ class ChatPlaceRecommendationsUseCase:
222
  )
223
  return self._clarification_result(clarified, "")
224
  anchor = anchors[0]
225
- resolved_location = replace(location, resolved_place_id=anchor.place_id)
 
 
 
 
 
226
  explicit = ExplicitTargetLocation(
227
  anchor_text=location.anchor_text,
228
  place_id=anchor.place_id,
@@ -238,6 +385,36 @@ class ChatPlaceRecommendationsUseCase:
238
  explicit_target_location=explicit,
239
  ),
240
  )
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
241
 
242
  reference = resolved_intent.reference
243
  if reference and reference.entity and not reference.place_id:
@@ -415,13 +592,138 @@ class ChatPlaceRecommendationsUseCase:
415
  strict_radius=location.strict_radius,
416
  )
417
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
418
  async def _compose_message(
419
  self,
420
  intent: ParsedPlaceChatIntent,
421
  candidates: Sequence[PlaceChatCandidate],
422
  state: ConversationState,
423
  ) -> tuple[str, bool, str | None]:
424
- fallback = self._template_message(intent, candidates)
 
 
 
 
 
 
 
 
 
 
 
425
  if not self._llm_enabled:
426
  return fallback, False, "llm_disabled"
427
  try:
@@ -432,7 +734,7 @@ class ChatPlaceRecommendationsUseCase:
432
  ),
433
  region=state.city or state.state,
434
  places=[self._candidate_context(candidate) for candidate in candidates],
435
- response_mode="confident",
436
  )
437
  if any(
438
  candidate.name.casefold() in result.message.casefold()
@@ -443,6 +745,7 @@ class ChatPlaceRecommendationsUseCase:
443
  guarded = self._output_guard.validate(
444
  message=result.message,
445
  allowed_place_names=[candidate.name for candidate in candidates],
 
446
  )
447
  if guarded.used_fallback:
448
  return fallback, False, guarded.reason
@@ -454,7 +757,14 @@ class ChatPlaceRecommendationsUseCase:
454
  def _template_message(
455
  intent: ParsedPlaceChatIntent,
456
  candidates: Sequence[PlaceChatCandidate],
 
457
  ) -> str:
 
 
 
 
 
 
458
  exact = sum(candidate.match_level == "exact" for candidate in candidates)
459
  family = sum(candidate.match_level == "family" for candidate in candidates)
460
  if exact:
@@ -464,6 +774,13 @@ class ChatPlaceRecommendationsUseCase:
464
  category = intent.target_category or "lugar"
465
  return f"Encontre opciones de {category} que pueden encajar con tu solicitud."
466
 
 
 
 
 
 
 
 
467
  @staticmethod
468
  def _candidate_context(candidate: PlaceChatCandidate) -> dict[str, Any]:
469
  return {
@@ -501,17 +818,34 @@ class ChatPlaceRecommendationsUseCase:
501
  unresolved: tuple[str, ...],
502
  ) -> ParsedPlaceChatIntent:
503
  clarification = to_public_clarification(pending)
 
504
  return replace(
505
  intent,
506
  action="clarification",
507
- semantic_query="",
508
  confidence=min(intent.confidence, 0.75),
509
  state_patch=replace(
510
- intent.state_patch,
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
511
  pending_clarification=pending,
512
  ),
513
  clarification=clarification,
514
- alternatives=(),
515
  unresolved=unresolved,
516
  clarification_message=clarification.prompt,
517
  )
@@ -545,6 +879,17 @@ class ChatPlaceRecommendationsUseCase:
545
  category_source=intent.category_source,
546
  used_llm=used_llm,
547
  guard_reason=guard_reason,
 
 
 
 
 
 
 
 
 
 
 
548
  )
549
  if trace_id:
550
  self._log_result(result, intent)
 
1
+ import asyncio
2
  from dataclasses import dataclass, replace
3
  import logging
4
  import math
 
8
  HybridPlaceChatRetriever,
9
  )
10
  from app.modules.places.application.ports.intent_parser import PlaceChatIntentParser
11
+ from app.modules.places.application.ports.nearby_place_provider import (
12
+ NearbyPlaceProvider,
13
+ )
14
  from app.modules.places.application.ports.place_anchor_resolver import (
15
  PlaceAnchorResolver,
16
  )
 
40
 
41
  logger = logging.getLogger(__name__)
42
 
43
+ _NON_CATEGORY_ALTERNATIVE_KEYS = {
44
+ "location_scope",
45
+ "reference_entity",
46
+ "target_results",
47
+ "user_current_location",
48
+ }
49
+
50
 
51
  @dataclass(frozen=True)
52
  class ChatPlaceRecommendationsResult:
 
64
  category_source: str = "unresolved"
65
  used_llm: bool = False
66
  guard_reason: str | None = None
67
+ category_hypotheses: tuple[dict[str, Any], ...] = ()
68
+ raw_category_phrase: str | None = None
69
+ intent_model_version: str = "deterministic-open-v2"
70
 
71
 
72
  class ChatPlaceRecommendationsUseCase:
 
82
  llm_enabled: bool = True,
83
  anchor_ambiguity_delta: float = 0.15,
84
  minimum_intent_confidence: float = 0.70,
85
+ minimum_hypothesis_confidence: float = 0.60,
86
+ maximum_hypothesis_gap: float = 0.15,
87
+ nearby_place_provider: NearbyPlaceProvider | None = None,
88
+ default_radius_meters: int = 5_000,
89
  ) -> None:
90
+ if not 0.0 <= minimum_intent_confidence <= 1.0:
91
+ raise ValueError("minimum_intent_confidence must be between zero and one")
92
+ if not 0.0 <= minimum_hypothesis_confidence <= 1.0:
93
+ raise ValueError(
94
+ "minimum_hypothesis_confidence must be between zero and one"
95
+ )
96
+ if not 0.0 <= maximum_hypothesis_gap <= 1.0:
97
+ raise ValueError("maximum_hypothesis_gap must be between zero and one")
98
+ if not 1 <= default_radius_meters <= 50_000:
99
+ raise ValueError("default_radius_meters must be between 1 and 50000")
100
  self._intent_parser = intent_parser
101
  self._anchor_resolver = anchor_resolver
102
  self._retriever = retriever
 
107
  self._llm_enabled = llm_enabled
108
  self._anchor_ambiguity_delta = anchor_ambiguity_delta
109
  self._minimum_intent_confidence = minimum_intent_confidence
110
+ self._minimum_hypothesis_confidence = minimum_hypothesis_confidence
111
+ self._maximum_hypothesis_gap = maximum_hypothesis_gap
112
+ self._nearby_place_provider = nearby_place_provider
113
+ self._default_radius_meters = default_radius_meters
114
 
115
  async def execute(
116
  self,
117
  message: str,
118
  state: ConversationState,
119
+ user_latitude: float | None,
120
+ user_longitude: float | None,
121
  candidate_limit: int,
122
  result_limit: int,
123
  clarification_choice: ClarificationChoice | None = None,
124
  ) -> ChatPlaceRecommendationsResult:
125
+ if (user_latitude is None) != (user_longitude is None):
126
+ raise ValueError("user_latitude and user_longitude must be provided together")
127
+ has_user_location = user_latitude is not None
128
  trace_id = new_trace_id()
129
+ # Both the open-vocabulary classifier and the optional BERT extractor
130
+ # are CPU-bound and load lazily. Keep their first inference off the
131
+ # async request loop.
132
+ intent = await asyncio.to_thread(
133
+ self._intent_parser.parse,
134
+ message,
135
+ state,
136
+ has_user_location,
137
+ clarification_choice,
138
  )
139
  if intent.action == "clarification":
140
  return self._clarification_result(intent, trace_id)
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
141
 
142
  resolved = await self._resolve_entities(intent, state)
143
  if isinstance(resolved, ChatPlaceRecommendationsResult):
 
148
 
149
  directive = self._location_directive(intent)
150
  if directive.source == "unresolved":
151
+ region = state.city or state.state
152
+ if not region:
153
+ return self._result(
154
+ action="no_match",
155
+ message="No pude identificar una ubicacion util para esta busqueda.",
156
+ intent=intent,
157
+ directive=directive,
158
+ candidates=(),
159
+ unresolved=("location",),
160
+ trace_id=trace_id,
161
+ )
162
+ directive = PlaceChatLocationDirective(
163
+ source="state_anchor",
164
+ scope="target_results",
165
+ anchor_text=region,
166
+ )
167
+
168
+ geographic_latitude = (
169
+ user_latitude
170
+ if directive.source == "user_current"
171
+ else intent.location.latitude
172
+ )
173
+ geographic_longitude = (
174
+ user_longitude
175
+ if directive.source == "user_current"
176
+ else intent.location.longitude
177
+ )
178
+ if (
179
+ directive.source in {"user_current", "explicit_anchor", "state_anchor"}
180
+ and self._nearby_place_provider is not None
181
+ and geographic_latitude is not None
182
+ and geographic_longitude is not None
183
+ ):
184
+ nearby_ids = await self._nearby_place_provider.get_nearby_place_ids(
185
+ latitude=geographic_latitude,
186
+ longitude=geographic_longitude,
187
+ radius_meters=(
188
+ directive.radius_meters or self._default_radius_meters
189
+ ),
190
+ )
191
+ if not nearby_ids:
192
+ return self._result(
193
+ action="no_match",
194
+ message="No encontre lugares cercanos dentro del radio solicitado.",
195
+ intent=intent,
196
+ directive=directive,
197
+ candidates=(),
198
+ unresolved=(),
199
+ trace_id=trace_id,
200
+ )
201
+ intent = replace(
202
+ intent,
203
+ hard_filters={
204
+ **intent.hard_filters,
205
+ "place_ids": tuple(sorted(nearby_ids)),
206
+ },
207
+ )
208
+
209
+ if intent.confidence < self._minimum_intent_confidence:
210
+ evidence_candidates = tuple(
211
+ await self._retriever.retrieve(
212
+ intent=intent,
213
+ limit=min(candidate_limit, 12),
214
+ )
215
+ )
216
+ pending = (
217
+ self._category_clarification_from_hypotheses(intent)
218
+ or self._category_clarification_from_candidates(evidence_candidates)
219
+ )
220
+ if pending is not None:
221
+ clarified = self._with_pending_clarification(
222
+ intent,
223
+ pending,
224
+ unresolved=("intent_confidence",),
225
+ )
226
+ return self._clarification_result(clarified, trace_id)
227
+
228
+ sufficient_candidates = tuple(
229
+ candidate
230
+ for candidate in evidence_candidates
231
+ if candidate.metadata.get("retrieval_diagnostics", {}).get(
232
+ "meets_minimum_content_score"
233
+ )
234
+ )
235
+ if evidence_candidates:
236
+ review_candidates = (
237
+ sufficient_candidates or evidence_candidates
238
+ )
239
+ final_message, used_llm, guard_reason = await self._compose_message(
240
+ intent=intent,
241
+ candidates=review_candidates[:result_limit],
242
+ state=state,
243
+ )
244
+ return self._result(
245
+ action="recommendations",
246
+ message=final_message,
247
+ intent=intent,
248
+ directive=directive,
249
+ candidates=review_candidates,
250
+ unresolved=tuple(
251
+ dict.fromkeys(
252
+ (
253
+ *intent.unresolved,
254
+ "intent_confidence",
255
+ *(
256
+ ()
257
+ if sufficient_candidates
258
+ else ("retrieval_evidence",)
259
+ ),
260
+ )
261
+ )
262
+ ),
263
+ trace_id=trace_id,
264
+ used_llm=used_llm,
265
+ guard_reason=guard_reason,
266
+ )
267
  return self._result(
268
  action="no_match",
269
+ message=self._uncertain_intent_message(intent, evidence_candidates),
270
  intent=intent,
271
  directive=directive,
272
  candidates=(),
273
+ unresolved=("intent_confidence",),
274
  trace_id=trace_id,
275
  )
276
 
 
296
  candidates=candidates[:result_limit],
297
  state=state,
298
  )
299
+ has_sufficient_evidence = any(
300
+ self._meets_content_threshold(candidate) for candidate in candidates
301
+ )
302
  return self._result(
303
  action="recommendations",
304
  message=final_message,
305
  intent=intent,
306
  directive=directive,
307
  candidates=candidates,
308
+ unresolved=(
309
+ () if has_sufficient_evidence else ("retrieval_evidence",)
310
+ ),
311
  trace_id=trace_id,
312
  used_llm=used_llm,
313
  guard_reason=guard_reason,
 
364
  )
365
  return self._clarification_result(clarified, "")
366
  anchor = anchors[0]
367
+ resolved_location = replace(
368
+ location,
369
+ resolved_place_id=anchor.place_id,
370
+ latitude=anchor.latitude,
371
+ longitude=anchor.longitude,
372
+ )
373
  explicit = ExplicitTargetLocation(
374
  anchor_text=location.anchor_text,
375
  place_id=anchor.place_id,
 
385
  explicit_target_location=explicit,
386
  ),
387
  )
388
+ elif (
389
+ location.anchor_text
390
+ and location.resolved_place_id
391
+ and (location.latitude is None or location.longitude is None)
392
+ ):
393
+ anchors = list(
394
+ await self._anchor_resolver.resolve(
395
+ text=location.anchor_text,
396
+ city=state.city,
397
+ state=state.state,
398
+ limit=5,
399
+ )
400
+ )
401
+ selected = next(
402
+ (
403
+ anchor
404
+ for anchor in anchors
405
+ if anchor.place_id == location.resolved_place_id
406
+ ),
407
+ None,
408
+ )
409
+ if selected is not None:
410
+ resolved_intent = replace(
411
+ resolved_intent,
412
+ location=replace(
413
+ location,
414
+ latitude=selected.latitude,
415
+ longitude=selected.longitude,
416
+ ),
417
+ )
418
 
419
  reference = resolved_intent.reference
420
  if reference and reference.entity and not reference.place_id:
 
592
  strict_radius=location.strict_radius,
593
  )
594
 
595
+ def _category_clarification_from_hypotheses(
596
+ self,
597
+ intent: ParsedPlaceChatIntent,
598
+ ) -> PendingClarification | None:
599
+ scores: dict[str, float] = {}
600
+ labels: dict[str, str] = {}
601
+ if intent.target_category:
602
+ scores[intent.target_category] = intent.confidence
603
+
604
+ for alternative in intent.alternatives:
605
+ key = alternative.key.strip()
606
+ if not key or key in _NON_CATEGORY_ALTERNATIVE_KEYS:
607
+ continue
608
+ scores[key] = max(scores.get(key, 0.0), alternative.confidence)
609
+ if alternative.description.strip():
610
+ labels[key] = alternative.description.strip()
611
+
612
+ ranked = sorted(scores.items(), key=lambda item: (-item[1], item[0]))
613
+ if not ranked or ranked[0][1] < self._minimum_hypothesis_confidence:
614
+ return None
615
+ top_score = ranked[0][1]
616
+ ordered = tuple(
617
+ category
618
+ for category, score in ranked
619
+ if score >= self._minimum_hypothesis_confidence
620
+ and top_score - score <= self._maximum_hypothesis_gap
621
+ )[:5]
622
+ if len(ordered) < 2:
623
+ return None
624
+ return new_category_clarification(
625
+ ordered,
626
+ kind="intent_category",
627
+ labels=labels,
628
+ )
629
+
630
+ @staticmethod
631
+ def _category_clarification_from_candidates(
632
+ candidates: Sequence[PlaceChatCandidate],
633
+ ) -> PendingClarification | None:
634
+ counts: dict[str, int] = {}
635
+ best_scores: dict[str, float] = {}
636
+ for candidate in candidates:
637
+ diagnostics = candidate.metadata.get("retrieval_diagnostics", {})
638
+ if diagnostics.get("meets_minimum_content_score") is False:
639
+ continue
640
+ category = (candidate.category or "").strip()
641
+ if not category:
642
+ continue
643
+ counts[category] = counts.get(category, 0) + 1
644
+ best_scores[category] = max(
645
+ best_scores.get(category, 0.0),
646
+ candidate.content_score,
647
+ )
648
+ ordered = tuple(
649
+ category
650
+ for category, _ in sorted(
651
+ best_scores.items(),
652
+ key=lambda item: (-item[1], -counts[item[0]], item[0]),
653
+ )[:3]
654
+ )
655
+ if len(ordered) < 2:
656
+ return None
657
+ labels = {
658
+ category: (
659
+ f"{category.replace('_', ' ').title()} "
660
+ f"({counts[category]} opciones encontradas)"
661
+ )
662
+ for category in ordered
663
+ }
664
+ return new_category_clarification(
665
+ ordered,
666
+ kind="intent_category",
667
+ labels=labels,
668
+ )
669
+
670
+ @staticmethod
671
+ def _uncertain_intent_message(
672
+ intent: ParsedPlaceChatIntent,
673
+ candidates: Sequence[PlaceChatCandidate],
674
+ ) -> str:
675
+ categories = tuple(
676
+ dict.fromkeys(
677
+ candidate.category.replace("_", " ").strip().title()
678
+ for candidate in candidates
679
+ if candidate.category and candidate.category.strip()
680
+ )
681
+ )[:3]
682
+ if len(categories) > 1:
683
+ evidence = ", ".join(categories[:-1]) + f" y {categories[-1]}"
684
+ return (
685
+ f"Encontre senales relacionadas con {evidence}, pero no pude "
686
+ "determinar con suficiente confianza cual describe tu plan. "
687
+ "Cuentame que actividad quieres hacer."
688
+ )
689
+ if categories:
690
+ return (
691
+ f"La busqueda apunta a {categories[0]}, pero la intencion sigue "
692
+ "siendo ambigua. Cuentame que actividad quieres hacer para afinarla."
693
+ )
694
+ if intent.soft_preferences:
695
+ preferences = ", ".join(
696
+ preference.replace("_", " ")
697
+ for preference in intent.soft_preferences[:3]
698
+ )
699
+ return (
700
+ f"Entendi que buscas algo {preferences}, pero no pude determinar "
701
+ "con suficiente confianza el tipo de lugar. Describe la actividad "
702
+ "que tienes en mente."
703
+ )
704
+ return (
705
+ "No pude determinar con suficiente confianza el tipo de lugar. "
706
+ "Describe la actividad o el plan que tienes en mente."
707
+ )
708
+
709
  async def _compose_message(
710
  self,
711
  intent: ParsedPlaceChatIntent,
712
  candidates: Sequence[PlaceChatCandidate],
713
  state: ConversationState,
714
  ) -> tuple[str, bool, str | None]:
715
+ has_sufficient_evidence = any(
716
+ self._meets_content_threshold(candidate) for candidate in candidates
717
+ )
718
+ response_mode = (
719
+ "low_confidence"
720
+ if (
721
+ intent.confidence < self._minimum_intent_confidence
722
+ or not has_sufficient_evidence
723
+ )
724
+ else "confident"
725
+ )
726
+ fallback = self._template_message(intent, candidates, response_mode)
727
  if not self._llm_enabled:
728
  return fallback, False, "llm_disabled"
729
  try:
 
734
  ),
735
  region=state.city or state.state,
736
  places=[self._candidate_context(candidate) for candidate in candidates],
737
+ response_mode=response_mode,
738
  )
739
  if any(
740
  candidate.name.casefold() in result.message.casefold()
 
745
  guarded = self._output_guard.validate(
746
  message=result.message,
747
  allowed_place_names=[candidate.name for candidate in candidates],
748
+ response_mode=response_mode,
749
  )
750
  if guarded.used_fallback:
751
  return fallback, False, guarded.reason
 
757
  def _template_message(
758
  intent: ParsedPlaceChatIntent,
759
  candidates: Sequence[PlaceChatCandidate],
760
+ response_mode: str = "confident",
761
  ) -> str:
762
+ if response_mode == "low_confidence":
763
+ return (
764
+ "Encontre opciones semanticamente relacionadas, aunque la evidencia "
765
+ "todavia es debil. Revisalas como sugerencias y ajusta tu busqueda "
766
+ "si no representan el plan que tienes en mente."
767
+ )
768
  exact = sum(candidate.match_level == "exact" for candidate in candidates)
769
  family = sum(candidate.match_level == "family" for candidate in candidates)
770
  if exact:
 
774
  category = intent.target_category or "lugar"
775
  return f"Encontre opciones de {category} que pueden encajar con tu solicitud."
776
 
777
+ @staticmethod
778
+ def _meets_content_threshold(candidate: PlaceChatCandidate) -> bool:
779
+ diagnostics = candidate.metadata.get("retrieval_diagnostics", {})
780
+ if "meets_minimum_content_score" in diagnostics:
781
+ return bool(diagnostics["meets_minimum_content_score"])
782
+ return candidate.content_score > 0.0
783
+
784
  @staticmethod
785
  def _candidate_context(candidate: PlaceChatCandidate) -> dict[str, Any]:
786
  return {
 
818
  unresolved: tuple[str, ...],
819
  ) -> ParsedPlaceChatIntent:
820
  clarification = to_public_clarification(pending)
821
+ patch = intent.state_patch
822
  return replace(
823
  intent,
824
  action="clarification",
 
825
  confidence=min(intent.confidence, 0.75),
826
  state_patch=replace(
827
+ patch,
828
+ target_category=(
829
+ patch.target_category or intent.target_category
830
+ ),
831
+ hard_filters=(
832
+ patch.hard_filters
833
+ if patch.hard_filters is not None
834
+ else (dict(intent.hard_filters) if intent.hard_filters else None)
835
+ ),
836
+ soft_preferences=(
837
+ patch.soft_preferences
838
+ if patch.soft_preferences is not None
839
+ else (intent.soft_preferences or None)
840
+ ),
841
+ exclusions=(
842
+ patch.exclusions
843
+ if patch.exclusions is not None
844
+ else (intent.exclusions or None)
845
+ ),
846
  pending_clarification=pending,
847
  ),
848
  clarification=clarification,
 
849
  unresolved=unresolved,
850
  clarification_message=clarification.prompt,
851
  )
 
879
  category_source=intent.category_source,
880
  used_llm=used_llm,
881
  guard_reason=guard_reason,
882
+ category_hypotheses=tuple(
883
+ {
884
+ "id": alternative.key,
885
+ "label": alternative.description,
886
+ "probability": round(alternative.confidence, 6),
887
+ }
888
+ for alternative in intent.alternatives
889
+ if alternative.key not in _NON_CATEGORY_ALTERNATIVE_KEYS
890
+ )[:5],
891
+ raw_category_phrase=intent.raw_category_phrase,
892
+ intent_model_version=intent.intent_model_version,
893
  )
894
  if trace_id:
895
  self._log_result(result, intent)
app/modules/places/application/use_cases/chat_places.py CHANGED
@@ -42,7 +42,12 @@ class ChatPlacesUseCase:
42
  filters=filters,
43
  limit=limit,
44
  )
45
- places = search_result.places
 
 
 
 
 
46
  context_places = [place.to_llm_context() for place in places]
47
 
48
  llm_provider = self._llm_provider.provider_name
@@ -55,18 +60,23 @@ class ChatPlacesUseCase:
55
  user_intent=search_result.normalized_query,
56
  region=filters.city or filters.state,
57
  places=context_places,
 
58
  )
59
  llm_provider = llm_result.provider
60
  llm_model = llm_result.model
61
  guarded = self._output_guard.validate(
62
  message=llm_result.message,
63
  allowed_place_names=[place.name for place in places],
 
64
  )
65
  used_llm = not guarded.used_fallback
66
  guard_reason = guarded.reason
67
  final_message = guarded.message
68
  except Exception as exc:
69
- guarded = self._output_guard.fallback(reason=exc.__class__.__name__)
 
 
 
70
  final_message = guarded.message
71
  guard_reason = guarded.reason
72
 
@@ -80,6 +90,7 @@ class ChatPlacesUseCase:
80
  "llm_model": llm_model,
81
  "used_llm": used_llm,
82
  "guard_reason": guard_reason,
 
83
  "places_used_as_context": [place.id for place in places],
84
  "timestamp": datetime.now(UTC).isoformat(),
85
  },
 
42
  filters=filters,
43
  limit=limit,
44
  )
45
+ response_mode = search_result.metrics.match_quality
46
+ places = (
47
+ []
48
+ if response_mode == "no_match"
49
+ else [place for place in search_result.places if place.score > 0]
50
+ )
51
  context_places = [place.to_llm_context() for place in places]
52
 
53
  llm_provider = self._llm_provider.provider_name
 
60
  user_intent=search_result.normalized_query,
61
  region=filters.city or filters.state,
62
  places=context_places,
63
+ response_mode=response_mode,
64
  )
65
  llm_provider = llm_result.provider
66
  llm_model = llm_result.model
67
  guarded = self._output_guard.validate(
68
  message=llm_result.message,
69
  allowed_place_names=[place.name for place in places],
70
+ response_mode=response_mode,
71
  )
72
  used_llm = not guarded.used_fallback
73
  guard_reason = guarded.reason
74
  final_message = guarded.message
75
  except Exception as exc:
76
+ guarded = self._output_guard.fallback(
77
+ reason=exc.__class__.__name__,
78
+ response_mode=response_mode,
79
+ )
80
  final_message = guarded.message
81
  guard_reason = guarded.reason
82
 
 
90
  "llm_model": llm_model,
91
  "used_llm": used_llm,
92
  "guard_reason": guard_reason,
93
+ "response_mode": response_mode,
94
  "places_used_as_context": [place.id for place in places],
95
  "timestamp": datetime.now(UTC).isoformat(),
96
  },
app/modules/places/application/use_cases/search_places.py CHANGED
@@ -1,4 +1,5 @@
1
  from dataclasses import dataclass, replace
 
2
  import json
3
  from typing import Sequence
4
 
@@ -84,7 +85,10 @@ class SearchPlacesUseCase:
84
  place_ids=tuple(sorted(nearby_ids)),
85
  )
86
 
87
- query_embedding = self._embedding_provider.embed_text(normalized_query)
 
 
 
88
  candidates = await self._place_repository.search(
89
  embedding=query_embedding,
90
  filters=effective_filters,
 
1
  from dataclasses import dataclass, replace
2
+ import asyncio
3
  import json
4
  from typing import Sequence
5
 
 
85
  place_ids=tuple(sorted(nearby_ids)),
86
  )
87
 
88
+ query_embedding = await asyncio.to_thread(
89
+ self._embedding_provider.embed_text,
90
+ normalized_query,
91
+ )
92
  candidates = await self._place_repository.search(
93
  embedding=query_embedding,
94
  filters=effective_filters,
app/modules/places/domain/chat_intent.py CHANGED
@@ -113,6 +113,8 @@ class LocationIntent:
113
  resolved_place_id: str | None = None
114
  radius_meters: int | None = None
115
  strict_radius: bool = False
 
 
116
 
117
 
118
  @dataclass(frozen=True)
@@ -127,6 +129,8 @@ class PlaceCategoryInference:
127
  category: str
128
  confidence: float
129
  source: Literal["lexical_activity", "semantic_activity"]
 
 
130
 
131
 
132
  @dataclass(frozen=True)
@@ -220,6 +224,8 @@ class ParsedPlaceChatIntent:
220
  alternatives: tuple[IntentAlternative, ...] = ()
221
  unresolved: tuple[str, ...] = ()
222
  clarification_message: str | None = None
 
 
223
 
224
 
225
  @dataclass(frozen=True)
 
113
  resolved_place_id: str | None = None
114
  radius_meters: int | None = None
115
  strict_radius: bool = False
116
+ latitude: float | None = None
117
+ longitude: float | None = None
118
 
119
 
120
  @dataclass(frozen=True)
 
129
  category: str
130
  confidence: float
131
  source: Literal["lexical_activity", "semantic_activity"]
132
+ category_values: tuple[str, ...] = ()
133
+ label: str | None = None
134
 
135
 
136
  @dataclass(frozen=True)
 
224
  alternatives: tuple[IntentAlternative, ...] = ()
225
  unresolved: tuple[str, ...] = ()
226
  clarification_message: str | None = None
227
+ raw_category_phrase: str | None = None
228
+ intent_model_version: str = "deterministic-open-v2"
229
 
230
 
231
  @dataclass(frozen=True)
app/modules/places/domain/clarifications.py CHANGED
@@ -1,4 +1,6 @@
1
- from collections.abc import Sequence
 
 
2
  from uuid import uuid4
3
 
4
  from app.modules.places.domain.chat_intent import (
@@ -11,36 +13,33 @@ from app.modules.places.domain.chat_intent import (
11
  )
12
 
13
 
14
- _CATEGORY_LABELS = {
15
- "restaurant": "Restaurantes",
16
- "cafe": "Cafeterias",
17
- "park": "Parques",
18
- "nightlife": "Fiesta y vida nocturna",
19
- "sports": "Ejercicio y deporte",
20
- "cinema": "Cines",
21
- "shopping": "Compras",
22
- "lodging": "Hospedaje",
23
- }
24
-
25
-
26
  def new_category_clarification(
27
  categories: Sequence[str],
28
  *,
29
  kind: ClarificationKind = "target_category",
 
30
  ) -> PendingClarification:
31
  unique = tuple(dict.fromkeys(category for category in categories if category))[:5]
32
  if len(unique) < 2:
33
  raise ValueError("a category clarification requires at least two options")
 
 
34
  return PendingClarification(
35
  clarification_id=str(uuid4()),
36
  kind=kind,
37
  options=tuple(
38
  PendingClarificationOption(
39
- option_id=category,
40
  value=category,
41
- label=_CATEGORY_LABELS.get(category, category.replace("_", " ").title()),
 
 
 
 
 
 
42
  )
43
- for category in unique
44
  ),
45
  )
46
 
@@ -156,3 +155,27 @@ def _bounded_text(value: str, maximum: int) -> str:
156
  if len(value) <= maximum:
157
  return value
158
  return value[: maximum - 1].rstrip() + "…"
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ from collections.abc import Mapping, Sequence
2
+ import re
3
+ import unicodedata
4
  from uuid import uuid4
5
 
6
  from app.modules.places.domain.chat_intent import (
 
13
  )
14
 
15
 
 
 
 
 
 
 
 
 
 
 
 
 
16
  def new_category_clarification(
17
  categories: Sequence[str],
18
  *,
19
  kind: ClarificationKind = "target_category",
20
+ labels: Mapping[str, str] | None = None,
21
  ) -> PendingClarification:
22
  unique = tuple(dict.fromkeys(category for category in categories if category))[:5]
23
  if len(unique) < 2:
24
  raise ValueError("a category clarification requires at least two options")
25
+ option_labels = labels or {}
26
+ option_ids = _category_option_ids(unique)
27
  return PendingClarification(
28
  clarification_id=str(uuid4()),
29
  kind=kind,
30
  options=tuple(
31
  PendingClarificationOption(
32
+ option_id=option_id,
33
  value=category,
34
+ label=_bounded_text(
35
+ option_labels.get(
36
+ category,
37
+ category.replace("_", " ").title(),
38
+ ),
39
+ 160,
40
+ ),
41
  )
42
+ for category, option_id in zip(unique, option_ids)
43
  ),
44
  )
45
 
 
155
  if len(value) <= maximum:
156
  return value
157
  return value[: maximum - 1].rstrip() + "…"
158
+
159
+
160
+ def _category_option_ids(values: Sequence[str]) -> tuple[str, ...]:
161
+ seen: set[str] = set()
162
+ option_ids: list[str] = []
163
+ for index, value in enumerate(values, start=1):
164
+ option_id = _category_option_id(value, index)
165
+ if option_id in seen:
166
+ suffix = f"_{index}"
167
+ option_id = option_id[: 64 - len(suffix)].rstrip("_-") + suffix
168
+ seen.add(option_id)
169
+ option_ids.append(option_id)
170
+ return tuple(option_ids)
171
+
172
+
173
+ def _category_option_id(value: str, index: int) -> str:
174
+ if re.fullmatch(r"[a-zA-Z0-9_-]{1,64}", value):
175
+ return value
176
+ ascii_value = unicodedata.normalize("NFKD", value).encode(
177
+ "ascii", "ignore"
178
+ ).decode("ascii")
179
+ slug = re.sub(r"[^a-zA-Z0-9_-]+", "_", ascii_value).strip("_-").lower()
180
+ slug = slug[:54].rstrip("_-") or "category"
181
+ return f"{slug}_{index}"
app/modules/places/infrastructure/aws_pgvector_place_repository.py CHANGED
@@ -9,8 +9,15 @@ from app.shared.vector_store.models import VectorMatch
9
  class AwsPgvectorPlaceRepository(PlaceVectorRepository):
10
  source_name = "pgvector"
11
 
12
- def __init__(self, vector_client: AwsPgvectorClient) -> None:
 
 
 
 
 
13
  self._vector_client = vector_client
 
 
14
 
15
  async def search(
16
  self,
@@ -24,12 +31,39 @@ class AwsPgvectorPlaceRepository(PlaceVectorRepository):
24
  embedding=embedding,
25
  filters=metadata_filter,
26
  limit=limit,
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
27
  )
28
  return [_match_to_candidate(match) for match in matches]
29
 
30
 
31
  def _match_to_candidate(match: VectorMatch) -> PlaceCandidate:
32
  metadata = dict(match.metadata)
 
 
 
 
33
  return PlaceCandidate(
34
  id=match.id,
35
  name=str(metadata.get("name") or match.id),
 
9
  class AwsPgvectorPlaceRepository(PlaceVectorRepository):
10
  source_name = "pgvector"
11
 
12
+ def __init__(
13
+ self,
14
+ vector_client: AwsPgvectorClient,
15
+ match_function: str = "match_places",
16
+ hybrid_function: str | None = None,
17
+ ) -> None:
18
  self._vector_client = vector_client
19
+ self._match_function = match_function
20
+ self._hybrid_function = hybrid_function
21
 
22
  async def search(
23
  self,
 
31
  embedding=embedding,
32
  filters=metadata_filter,
33
  limit=limit,
34
+ function_name=self._match_function,
35
+ )
36
+ return [_match_to_candidate(match) for match in matches]
37
+
38
+ async def search_hybrid(
39
+ self,
40
+ query_text: str,
41
+ embedding: list[float],
42
+ filters: PlaceFilters,
43
+ limit: int,
44
+ ) -> Sequence[PlaceCandidate]:
45
+ """Use the versioned SQL hybrid contract when it is configured."""
46
+
47
+ if not self._hybrid_function:
48
+ return await self.search(embedding=embedding, filters=filters, limit=limit)
49
+ metadata_filter = filters.as_metadata_filter()
50
+ metadata_filter["is_active"] = True
51
+ matches = await self._vector_client.search_places_hybrid(
52
+ query_text=query_text,
53
+ embedding=embedding,
54
+ filters=metadata_filter,
55
+ limit=limit,
56
+ function_name=self._hybrid_function,
57
  )
58
  return [_match_to_candidate(match) for match in matches]
59
 
60
 
61
  def _match_to_candidate(match: VectorMatch) -> PlaceCandidate:
62
  metadata = dict(match.metadata)
63
+ if match.semantic_score is not None:
64
+ metadata["semantic_score"] = match.semantic_score
65
+ if match.lexical_score is not None:
66
+ metadata["lexical_score"] = match.lexical_score
67
  return PlaceCandidate(
68
  id=match.id,
69
  name=str(metadata.get("name") or match.id),
app/modules/places/infrastructure/bert_intent_extractor.py ADDED
@@ -0,0 +1,595 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """Lazy BERT token-classification adapter for open place-chat slots.
2
+
3
+ The extractor deliberately returns raw concepts instead of canonical place
4
+ categories. Category alignment belongs to the semantic catalog/retrieval
5
+ stage; doing it here would recreate a closed taxonomy in the model adapter.
6
+
7
+ The default loader imports ``transformers`` only on the first non-empty call.
8
+ Tests and alternative serving runtimes can inject a callable classifier or a
9
+ loader, so importing this module never requires the optional ML dependency.
10
+ """
11
+
12
+ from __future__ import annotations
13
+
14
+ import math
15
+ import threading
16
+ from collections.abc import Callable, Mapping, Sequence
17
+ from dataclasses import dataclass
18
+ from typing import Any, Literal, Protocol, TypeAlias, cast
19
+
20
+
21
+ SlotType = Literal[
22
+ "CATEGORY",
23
+ "PREFERENCE",
24
+ "EXCLUSION",
25
+ "LOCATION",
26
+ "REFERENCE",
27
+ "RADIUS",
28
+ ]
29
+ SpanPolarity = Literal["positive", "negative", "neutral"]
30
+ IOBPrefix = Literal["B", "I"]
31
+
32
+ _SLOT_TYPES: frozenset[str] = frozenset(
33
+ {
34
+ "CATEGORY",
35
+ "PREFERENCE",
36
+ "EXCLUSION",
37
+ "LOCATION",
38
+ "REFERENCE",
39
+ "RADIUS",
40
+ }
41
+ )
42
+ _POLARITIES: frozenset[str] = frozenset({"positive", "negative", "neutral"})
43
+
44
+
45
+ class BertIntentExtractionError(RuntimeError):
46
+ """Base error raised by the contextual intent extractor."""
47
+
48
+
49
+ class BertIntentModelLoadError(BertIntentExtractionError):
50
+ """The configured token-classification model could not be loaded."""
51
+
52
+
53
+ class BertIntentInferenceError(BertIntentExtractionError):
54
+ """The loaded model failed while processing a message."""
55
+
56
+
57
+ class BertIntentOutputError(BertIntentExtractionError):
58
+ """The model returned an invalid token-classification payload."""
59
+
60
+
61
+ @dataclass(frozen=True)
62
+ class SlotLabelDefinition:
63
+ """Maps one model label to a domain slot and its semantic polarity.
64
+
65
+ Mapping is intentionally configuration-driven. For example, a model can
66
+ expose ``B-AMENITY`` and map ``AMENITY`` to a positive ``PREFERENCE``, or
67
+ expose ``B-NEGATIVE_AMENITY`` and map it to a negative ``PREFERENCE``.
68
+ """
69
+
70
+ slot_type: SlotType
71
+ polarity: SpanPolarity = "neutral"
72
+
73
+ def __post_init__(self) -> None:
74
+ slot_type = str(self.slot_type).strip().upper()
75
+ polarity = str(self.polarity).strip().lower()
76
+ if slot_type not in _SLOT_TYPES:
77
+ raise ValueError(
78
+ "slot_type must be one of " + ", ".join(sorted(_SLOT_TYPES))
79
+ )
80
+ if polarity not in _POLARITIES:
81
+ raise ValueError(
82
+ "polarity must be one of " + ", ".join(sorted(_POLARITIES))
83
+ )
84
+ object.__setattr__(self, "slot_type", cast(SlotType, slot_type))
85
+ object.__setattr__(self, "polarity", cast(SpanPolarity, polarity))
86
+
87
+
88
+ @dataclass(frozen=True)
89
+ class IntentSpan:
90
+ """One contextual slot using character offsets into the original message."""
91
+
92
+ slot_type: SlotType
93
+ text: str
94
+ start: int
95
+ end: int
96
+ polarity: SpanPolarity
97
+ confidence: float
98
+ token_count: int = 1
99
+
100
+ def __post_init__(self) -> None:
101
+ if self.slot_type not in _SLOT_TYPES:
102
+ raise ValueError(f"unsupported slot_type: {self.slot_type!r}")
103
+ if self.polarity not in _POLARITIES:
104
+ raise ValueError(f"unsupported polarity: {self.polarity!r}")
105
+ if self.start < 0 or self.end <= self.start:
106
+ raise ValueError("span offsets must satisfy 0 <= start < end")
107
+ if not isinstance(self.text, str) or not self.text:
108
+ raise ValueError("span text must not be empty")
109
+ if not math.isfinite(self.confidence) or not 0.0 <= self.confidence <= 1.0:
110
+ raise ValueError("span confidence must be finite and between 0 and 1")
111
+ if self.token_count <= 0:
112
+ raise ValueError("span token_count must be positive")
113
+
114
+
115
+ @dataclass(frozen=True)
116
+ class IntentFrame:
117
+ """Open contextual interpretation of one unmodified user message."""
118
+
119
+ raw_text: str
120
+ spans: tuple[IntentSpan, ...]
121
+ confidence: float
122
+ model_name: str
123
+ model_version: str
124
+
125
+ def __post_init__(self) -> None:
126
+ if not isinstance(self.raw_text, str):
127
+ raise TypeError("raw_text must be str")
128
+ if not math.isfinite(self.confidence) or not 0.0 <= self.confidence <= 1.0:
129
+ raise ValueError("frame confidence must be finite and between 0 and 1")
130
+ for span in self.spans:
131
+ if span.end > len(self.raw_text):
132
+ raise ValueError("span extends beyond raw_text")
133
+ if self.raw_text[span.start : span.end] != span.text:
134
+ raise ValueError("span text must match raw_text character offsets")
135
+
136
+ def by_type(self, slot_type: SlotType) -> tuple[IntentSpan, ...]:
137
+ """Return all spans of a type without normalizing their raw values."""
138
+
139
+ return tuple(span for span in self.spans if span.slot_type == slot_type)
140
+
141
+ @property
142
+ def categories(self) -> tuple[IntentSpan, ...]:
143
+ return self.by_type("CATEGORY")
144
+
145
+ @property
146
+ def preferences(self) -> tuple[IntentSpan, ...]:
147
+ return tuple(
148
+ span
149
+ for span in self.spans
150
+ if span.slot_type == "PREFERENCE" and span.polarity == "positive"
151
+ )
152
+
153
+ @property
154
+ def exclusions(self) -> tuple[IntentSpan, ...]:
155
+ return tuple(
156
+ span
157
+ for span in self.spans
158
+ if span.slot_type == "EXCLUSION" or span.polarity == "negative"
159
+ )
160
+
161
+
162
+ class TokenClassifier(Protocol):
163
+ """Minimal shape shared by a Hugging Face pipeline and test doubles."""
164
+
165
+ def __call__(self, text: str) -> Sequence[Mapping[str, Any]]:
166
+ """Return token rows containing an IOB label, score, start and end."""
167
+
168
+
169
+ TokenClassifierLoader: TypeAlias = Callable[
170
+ [str, str | None, int | str | None], TokenClassifier
171
+ ]
172
+ LabelDefinitionInput: TypeAlias = (
173
+ SlotLabelDefinition | SlotType | tuple[SlotType, SpanPolarity]
174
+ )
175
+
176
+
177
+ DEFAULT_LABEL_DEFINITIONS: Mapping[str, SlotLabelDefinition] = {
178
+ "CATEGORY": SlotLabelDefinition("CATEGORY", "positive"),
179
+ "PREFERENCE": SlotLabelDefinition("PREFERENCE", "positive"),
180
+ "EXCLUSION": SlotLabelDefinition("EXCLUSION", "negative"),
181
+ "LOCATION": SlotLabelDefinition("LOCATION", "neutral"),
182
+ "REFERENCE": SlotLabelDefinition("REFERENCE", "neutral"),
183
+ "RADIUS": SlotLabelDefinition("RADIUS", "neutral"),
184
+ }
185
+
186
+
187
+ @dataclass(frozen=True)
188
+ class _TokenPrediction:
189
+ prefix: IOBPrefix
190
+ definition: SlotLabelDefinition
191
+ start: int
192
+ end: int
193
+ score: float
194
+ order: int
195
+
196
+
197
+ @dataclass
198
+ class _SpanBuilder:
199
+ definition: SlotLabelDefinition
200
+ start: int
201
+ end: int
202
+ weighted_score: float
203
+ score_weight: int
204
+ token_count: int
205
+ last_order: int
206
+
207
+ @classmethod
208
+ def from_token(cls, token: _TokenPrediction) -> _SpanBuilder:
209
+ weight = max(1, token.end - token.start)
210
+ return cls(
211
+ definition=token.definition,
212
+ start=token.start,
213
+ end=token.end,
214
+ weighted_score=token.score * weight,
215
+ score_weight=weight,
216
+ token_count=1,
217
+ last_order=token.order,
218
+ )
219
+
220
+ def append(self, token: _TokenPrediction) -> None:
221
+ weight = max(1, token.end - token.start)
222
+ self.end = max(self.end, token.end)
223
+ self.weighted_score += token.score * weight
224
+ self.score_weight += weight
225
+ self.token_count += 1
226
+ self.last_order = token.order
227
+
228
+ def build(self, raw_text: str) -> IntentSpan:
229
+ return IntentSpan(
230
+ slot_type=self.definition.slot_type,
231
+ text=raw_text[self.start : self.end],
232
+ start=self.start,
233
+ end=self.end,
234
+ polarity=self.definition.polarity,
235
+ confidence=self.weighted_score / self.score_weight,
236
+ token_count=self.token_count,
237
+ )
238
+
239
+
240
+ class BertPlaceIntentExtractor:
241
+ """Extract contextual, open-value slots with a fine-tuned BERT model.
242
+
243
+ ``label_definitions`` is merged over the standard six slot labels. Keys
244
+ refer to model labels *without* the IOB prefix. Unknown labels are ignored,
245
+ which lets one model expose additional tasks without coupling Places to
246
+ them. Values can be ``SlotLabelDefinition``, a canonical slot string, or a
247
+ ``(slot, polarity)`` tuple.
248
+ """
249
+
250
+ def __init__(
251
+ self,
252
+ model_name_or_path: str,
253
+ *,
254
+ model_version: str | None = None,
255
+ device: int | str | None = None,
256
+ label_definitions: Mapping[str, LabelDefinitionInput] | None = None,
257
+ minimum_token_confidence: float = 0.0,
258
+ classifier: TokenClassifier | None = None,
259
+ model_loader: TokenClassifierLoader | None = None,
260
+ ) -> None:
261
+ model_name = _required_text(model_name_or_path, "model_name_or_path")
262
+ if not math.isfinite(minimum_token_confidence) or not (
263
+ 0.0 <= minimum_token_confidence <= 1.0
264
+ ):
265
+ raise ValueError(
266
+ "minimum_token_confidence must be finite and between 0 and 1"
267
+ )
268
+ if classifier is not None and model_loader is not None:
269
+ raise ValueError("provide classifier or model_loader, not both")
270
+ if classifier is not None and not callable(classifier):
271
+ raise TypeError("classifier must be callable")
272
+
273
+ definitions = dict(DEFAULT_LABEL_DEFINITIONS)
274
+ for raw_label, raw_definition in (label_definitions or {}).items():
275
+ label = _normalized_model_label(raw_label)
276
+ definitions[label] = _coerce_label_definition(raw_definition)
277
+
278
+ self._model_name = model_name
279
+ self._model_version = (
280
+ _required_text(model_version, "model_version")
281
+ if model_version is not None
282
+ else "unspecified"
283
+ )
284
+ self._revision = model_version
285
+ self._device = device
286
+ self._definitions = definitions
287
+ self._minimum_token_confidence = minimum_token_confidence
288
+ self._classifier = classifier
289
+ self._model_loader = model_loader or _load_transformers_classifier
290
+ self._load_lock = threading.Lock()
291
+ self._inference_lock = threading.Lock()
292
+
293
+ @property
294
+ def is_loaded(self) -> bool:
295
+ return self._classifier is not None
296
+
297
+ @property
298
+ def model_name(self) -> str:
299
+ return self._model_name
300
+
301
+ @property
302
+ def model_version(self) -> str:
303
+ return self._model_version
304
+
305
+ def extract(self, text: str) -> IntentFrame:
306
+ """Extract an immutable frame while preserving ``text`` byte-for-byte."""
307
+
308
+ if not isinstance(text, str):
309
+ raise TypeError(f"text must be str, got {type(text).__name__}")
310
+ if not text.strip():
311
+ return IntentFrame(
312
+ raw_text=text,
313
+ spans=(),
314
+ confidence=0.0,
315
+ model_name=self._model_name,
316
+ model_version=self._model_version,
317
+ )
318
+
319
+ classifier = self._ensure_classifier()
320
+ try:
321
+ # Hugging Face pipelines and GPU modules are not guaranteed to be
322
+ # safe under concurrent calls. Parsing already runs in a worker
323
+ # thread, so serializing one model does not block the event loop.
324
+ with self._inference_lock:
325
+ raw_predictions = classifier(text)
326
+ except Exception as exc:
327
+ raise BertIntentInferenceError(
328
+ f"token classification failed for {self._model_name!r}: {exc}"
329
+ ) from exc
330
+
331
+ predictions = self._prepare_predictions(raw_predictions, text)
332
+ spans = _decode_iob(predictions, text)
333
+ return IntentFrame(
334
+ raw_text=text,
335
+ spans=spans,
336
+ confidence=_frame_confidence(spans),
337
+ model_name=self._model_name,
338
+ model_version=self._model_version,
339
+ )
340
+
341
+ def _ensure_classifier(self) -> TokenClassifier:
342
+ classifier = self._classifier
343
+ if classifier is not None:
344
+ return classifier
345
+
346
+ with self._load_lock:
347
+ classifier = self._classifier
348
+ if classifier is not None:
349
+ return classifier
350
+ try:
351
+ classifier = self._model_loader(
352
+ self._model_name,
353
+ self._revision,
354
+ self._device,
355
+ )
356
+ except BertIntentModelLoadError:
357
+ raise
358
+ except Exception as exc:
359
+ raise BertIntentModelLoadError(
360
+ f"could not load token-classification model "
361
+ f"{self._model_name!r}: {exc}"
362
+ ) from exc
363
+ if not callable(classifier):
364
+ raise BertIntentModelLoadError(
365
+ f"loader for {self._model_name!r} did not return a callable"
366
+ )
367
+ self._classifier = classifier
368
+ return classifier
369
+
370
+ def _prepare_predictions(
371
+ self,
372
+ raw_predictions: Sequence[Mapping[str, Any]],
373
+ text: str,
374
+ ) -> tuple[_TokenPrediction, ...]:
375
+ if isinstance(raw_predictions, (str, bytes, Mapping)) or not isinstance(
376
+ raw_predictions, Sequence
377
+ ):
378
+ raise BertIntentOutputError(
379
+ "token classifier output must be a sequence of mappings"
380
+ )
381
+
382
+ predictions: list[_TokenPrediction] = []
383
+ for order, row in enumerate(raw_predictions):
384
+ if not isinstance(row, Mapping):
385
+ raise BertIntentOutputError(
386
+ f"token classifier row {order} must be a mapping"
387
+ )
388
+ label_value = row.get("entity", row.get("entity_group", row.get("label")))
389
+ if not isinstance(label_value, str):
390
+ raise BertIntentOutputError(
391
+ f"token classifier row {order} has no string entity label"
392
+ )
393
+ prefix, base_label = _split_iob_label(label_value)
394
+ if base_label is None:
395
+ continue
396
+ definition = self._definitions.get(base_label)
397
+ if definition is None:
398
+ continue
399
+
400
+ score = _finite_score(row.get("score"), order)
401
+ if score < self._minimum_token_confidence:
402
+ continue
403
+ start = _integer_offset(row.get("start"), "start", order)
404
+ end = _integer_offset(row.get("end"), "end", order)
405
+ if start < 0 or end <= start or end > len(text):
406
+ raise BertIntentOutputError(
407
+ f"token classifier row {order} has invalid offsets "
408
+ f"start={start}, end={end}, text_length={len(text)}"
409
+ )
410
+ token_order = _token_order(row.get("index"), order)
411
+ predictions.append(
412
+ _TokenPrediction(
413
+ prefix=prefix,
414
+ definition=definition,
415
+ start=start,
416
+ end=end,
417
+ score=score,
418
+ order=token_order,
419
+ )
420
+ )
421
+ return tuple(
422
+ sorted(
423
+ predictions,
424
+ key=lambda token: (token.start, token.end, token.order),
425
+ )
426
+ )
427
+
428
+
429
+ def _decode_iob(
430
+ predictions: Sequence[_TokenPrediction],
431
+ raw_text: str,
432
+ ) -> tuple[IntentSpan, ...]:
433
+ spans: list[IntentSpan] = []
434
+ current: _SpanBuilder | None = None
435
+
436
+ for token in predictions:
437
+ continues_current = bool(
438
+ token.prefix == "I"
439
+ and current is not None
440
+ and current.definition == token.definition
441
+ and token.start >= current.end
442
+ and token.order == current.last_order + 1
443
+ )
444
+ if continues_current:
445
+ assert current is not None
446
+ current.append(token)
447
+ continue
448
+
449
+ if current is not None:
450
+ spans.append(current.build(raw_text))
451
+ # A stray I-label begins a recoverable new span instead of losing the
452
+ # user's concept because of one imperfect model transition.
453
+ current = _SpanBuilder.from_token(token)
454
+
455
+ if current is not None:
456
+ spans.append(current.build(raw_text))
457
+ return tuple(spans)
458
+
459
+
460
+ def _frame_confidence(spans: Sequence[IntentSpan]) -> float:
461
+ if not spans:
462
+ return 0.0
463
+ weights = [max(1, span.end - span.start) for span in spans]
464
+ return sum(
465
+ span.confidence * weight for span, weight in zip(spans, weights)
466
+ ) / sum(weights)
467
+
468
+
469
+ def _split_iob_label(raw_label: str) -> tuple[IOBPrefix, str | None]:
470
+ label = raw_label.strip().upper()
471
+ if not label:
472
+ raise BertIntentOutputError("token classifier returned an empty label")
473
+ if label == "O":
474
+ return "B", None
475
+ if len(label) > 2 and label[0] in {"B", "I"} and label[1] in {"-", "_"}:
476
+ return cast(IOBPrefix, label[0]), _normalized_model_label(label[2:])
477
+ # Aggregated Hugging Face pipelines may return CATEGORY instead of
478
+ # B-CATEGORY. Treat an unprefixed label as one complete span.
479
+ return "B", _normalized_model_label(label)
480
+
481
+
482
+ def _coerce_label_definition(value: LabelDefinitionInput) -> SlotLabelDefinition:
483
+ if isinstance(value, SlotLabelDefinition):
484
+ return value
485
+ if isinstance(value, str):
486
+ slot_type = value.strip().upper()
487
+ default_definition = DEFAULT_LABEL_DEFINITIONS.get(slot_type)
488
+ if default_definition is not None:
489
+ return default_definition
490
+ return SlotLabelDefinition(cast(SlotType, slot_type))
491
+ if isinstance(value, tuple) and len(value) == 2:
492
+ return SlotLabelDefinition(
493
+ cast(SlotType, value[0]),
494
+ cast(SpanPolarity, value[1]),
495
+ )
496
+ raise TypeError(
497
+ "label definition must be SlotLabelDefinition, a slot string, or a "
498
+ "(slot, polarity) tuple"
499
+ )
500
+
501
+
502
+ def _normalized_model_label(value: Any) -> str:
503
+ if not isinstance(value, str):
504
+ raise TypeError(f"model label must be str, got {type(value).__name__}")
505
+ label = value.strip().upper()
506
+ if not label:
507
+ raise ValueError("model label must not be empty")
508
+ return label
509
+
510
+
511
+ def _required_text(value: Any, field_name: str) -> str:
512
+ if not isinstance(value, str):
513
+ raise TypeError(f"{field_name} must be str, got {type(value).__name__}")
514
+ cleaned = value.strip()
515
+ if not cleaned:
516
+ raise ValueError(f"{field_name} must not be empty")
517
+ return cleaned
518
+
519
+
520
+ def _finite_score(value: Any, row: int) -> float:
521
+ if isinstance(value, bool):
522
+ raise BertIntentOutputError(f"token classifier row {row} has invalid score")
523
+ try:
524
+ score = float(value)
525
+ except (TypeError, ValueError) as exc:
526
+ raise BertIntentOutputError(
527
+ f"token classifier row {row} has no numeric score"
528
+ ) from exc
529
+ if not math.isfinite(score) or not 0.0 <= score <= 1.0:
530
+ raise BertIntentOutputError(
531
+ f"token classifier row {row} score must be finite and between 0 and 1"
532
+ )
533
+ return score
534
+
535
+
536
+ def _integer_offset(value: Any, name: str, row: int) -> int:
537
+ if isinstance(value, bool) or not isinstance(value, int):
538
+ raise BertIntentOutputError(
539
+ f"token classifier row {row} has no integer {name} offset"
540
+ )
541
+ return value
542
+
543
+
544
+ def _token_order(value: Any, fallback: int) -> int:
545
+ """Preserve Hugging Face token adjacency after ignored labels are removed."""
546
+
547
+ if value is None:
548
+ return fallback
549
+ if isinstance(value, bool) or not isinstance(value, int) or value < 0:
550
+ raise BertIntentOutputError(
551
+ "token classifier output has an invalid token index"
552
+ )
553
+ return value
554
+
555
+
556
+ def _load_transformers_classifier(
557
+ model_name_or_path: str,
558
+ revision: str | None,
559
+ device: int | str | None,
560
+ ) -> TokenClassifier:
561
+ """Load the optional Hugging Face implementation on first inference."""
562
+
563
+ try:
564
+ from transformers import ( # type: ignore[import-not-found]
565
+ AutoModelForTokenClassification,
566
+ AutoTokenizer,
567
+ pipeline,
568
+ )
569
+ except ImportError as exc:
570
+ raise BertIntentModelLoadError(
571
+ "transformers is required to load the BERT intent extractor; "
572
+ "install it or inject classifier/model_loader"
573
+ ) from exc
574
+
575
+ pretrained_kwargs: dict[str, Any] = {"trust_remote_code": False}
576
+ if revision is not None:
577
+ pretrained_kwargs["revision"] = revision
578
+ tokenizer = AutoTokenizer.from_pretrained(
579
+ model_name_or_path,
580
+ **pretrained_kwargs,
581
+ )
582
+ model = AutoModelForTokenClassification.from_pretrained(
583
+ model_name_or_path,
584
+ **pretrained_kwargs,
585
+ )
586
+ pipeline_kwargs: dict[str, Any] = {
587
+ "task": "token-classification",
588
+ "model": model,
589
+ "tokenizer": tokenizer,
590
+ "aggregation_strategy": "none",
591
+ "ignore_labels": ["O"],
592
+ }
593
+ if device is not None:
594
+ pipeline_kwargs["device"] = device
595
+ return cast(TokenClassifier, pipeline(**pipeline_kwargs))
app/modules/places/infrastructure/deterministic_intent_parser.py CHANGED
@@ -1,6 +1,7 @@
1
  from dataclasses import dataclass, replace
2
  from functools import lru_cache
3
  import json
 
4
  from pathlib import Path
5
  import re
6
  from typing import Any, Iterable
@@ -28,6 +29,12 @@ from app.modules.places.domain.chat_intent import (
28
  PlaceReference,
29
  )
30
  from app.modules.places.domain.errors import ClarificationStateMismatchError
 
 
 
 
 
 
31
  from app.shared.nlp.preprocessing.text import (
32
  prepare_for_embedding,
33
  tokenize_for_embeddings,
@@ -50,9 +57,15 @@ _EXCLUSION_PATTERN = re.compile(
50
  r"\b(?:sin|excepto|evita(?:r)?|no\s+quiero|que\s+no\s+(?:sea|tenga))\s+"
51
  r"(?P<value>.+?)(?="
52
  r"\s+cerca\s+(?:de\s+la|del|de)\b|[,;]|"
53
- r"\s+(?:pero|aunque)\b|\s+y\s+(?:con|que|quiero|busco)\b|$)"
54
  )
55
  _CURRENT_LOCATION_VALUES = {"mi", "aqui", "donde estoy", "mi ubicacion"}
 
 
 
 
 
 
56
  _GENERIC_REQUEST_TOKENS = {
57
  "dame",
58
  "favor",
@@ -171,14 +184,29 @@ class PlaceChatTaxonomy:
171
  )
172
 
173
 
 
 
 
 
 
 
 
 
 
 
 
 
 
174
  class DeterministicPlaceChatIntentParser:
175
  def __init__(
176
  self,
177
  taxonomy: PlaceChatTaxonomy | None = None,
178
  activity_classifier: PlaceActivityClassifier | None = None,
 
179
  ) -> None:
180
  self._taxonomy = taxonomy or load_place_chat_taxonomy()
181
  self._activity_classifier = activity_classifier
 
182
 
183
  def parse(
184
  self,
@@ -188,19 +216,30 @@ class DeterministicPlaceChatIntentParser:
188
  clarification_choice: ClarificationChoice | None = None,
189
  ) -> ParsedPlaceChatIntent:
190
  normalized = prepare_for_embedding(message)
191
- target_clause = self._target_clause(normalized)
192
  if clarification_choice is not None:
193
  return self._resolve_structured_clarification(
194
  choice=clarification_choice,
195
  state=state,
196
  has_user_location=has_user_location,
197
  )
 
 
 
 
 
 
 
 
 
198
  if state.pending_clarification:
199
  pending = ensure_legacy_pending_options(state.pending_clarification)
200
  state = replace(state, pending_clarification=pending)
201
- categories = self._matched_categories(target_clause)
202
  inferred = (
203
- self._inferred_activity_category(target_clause)
 
 
 
204
  if not categories
205
  else None
206
  )
@@ -220,15 +259,25 @@ class DeterministicPlaceChatIntentParser:
220
  clear_pending_clarification=True,
221
  ),
222
  )
223
- return self._resolve_pending_clarification(
224
- normalized=normalized,
225
- state=state,
226
- has_user_location=has_user_location,
 
 
 
227
  )
228
 
229
- reference_text = self._extract_reference(normalized)
230
- location_text = self._extract_location(normalized)
231
- radius_meters = self._extract_radius_meters(normalized)
 
 
 
 
 
 
 
232
  excluded_texts = tuple(
233
  match.group("value").strip(" ,.;")
234
  for match in _EXCLUSION_PATTERN.finditer(normalized)
@@ -237,22 +286,33 @@ class DeterministicPlaceChatIntentParser:
237
  # Category words inside references, exclusions, or geographic anchors
238
  # do not describe the requested result type. In "cafeteria cerca del
239
  # parque", only cafe is a hard category.
240
- categories = self._matched_categories(target_clause)
241
  if len(categories) > 1:
242
  pending = new_category_clarification(categories)
243
- return self._clarification(
244
- pending=pending,
245
- unresolved=("target_category",),
246
- state=state,
247
- has_user_location=has_user_location,
 
 
 
248
  )
249
 
250
  explicit_category = categories[0] if categories else None
251
  inferred = (
252
- self._inferred_activity_category(target_clause)
 
 
 
253
  if explicit_category is None
254
  else None
255
  )
 
 
 
 
 
256
  inferred_category = inferred.category if inferred else None
257
  requested_category = explicit_category or inferred_category
258
  target_category = requested_category or state.target_category
@@ -273,11 +333,36 @@ class DeterministicPlaceChatIntentParser:
273
  if location_text:
274
  location_text = _RADIUS_PATTERN.sub("", location_text).strip(" ,.;")
275
 
276
- detected_preferences = self._matched_preferences(normalized)
277
- explicit_exclusions = _ordered_unique(
278
- exclusion
 
 
 
 
 
 
279
  for excluded_text in excluded_texts
280
- for exclusion in self._matched_preferences(excluded_text)
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
281
  )
282
  positive_preferences = tuple(
283
  preference
@@ -318,25 +403,45 @@ class DeterministicPlaceChatIntentParser:
318
  else inherited_reference
319
  )
320
  hard_filters = dict(state.hard_filters)
 
 
 
 
321
  if "economico" in merged_preferences or _contains_any(
322
  target_clause,
323
  ("mas barato", "economico", "barato"),
324
  ):
325
  hard_filters["price_preference"] = "lower"
326
 
327
- if target_category is None:
328
- pending = new_category_clarification(
329
- ("restaurant", "cafe", "park", "nightlife")
330
- )
331
- return self._clarification(
332
- pending=pending,
333
- unresolved=("target_category",),
334
- state=state,
335
- has_user_location=has_user_location,
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
336
  )
337
-
338
- category = self._taxonomy.category(target_category)
339
- category_values = category.storage_values if category else (target_category,)
340
  compatible_category_values = (
341
  category.compatible_storage_values if category else ()
342
  )
@@ -345,24 +450,27 @@ class DeterministicPlaceChatIntentParser:
345
  if (
346
  reference_text
347
  and location_text
348
- and location_text not in _CURRENT_LOCATION_VALUES
349
  ):
350
- return self._location_scope_clarification(
351
- target_category=target_category,
352
- category_values=category_values,
353
- compatible_category_values=compatible_category_values,
354
- category_evidence_terms=category_evidence_terms,
355
- explicit_category=requested_category,
356
- category_source=category_source,
357
- hard_filters=hard_filters,
358
- preferences=merged_preferences,
359
- exclusions=merged_exclusions,
360
- reference=reference,
361
- reference_text=reference_text,
362
- location_text=location_text,
363
- radius_meters=radius_meters,
364
- state=state,
365
- has_user_location=has_user_location,
 
 
 
366
  )
367
 
368
  location, explicit_location = self._location_intent(
@@ -383,14 +491,18 @@ class DeterministicPlaceChatIntentParser:
383
  )
384
  category_query_term = self._category_query_term(target_clause, category)
385
  semantic_parts = [
386
- target_category,
387
  category_query_term,
388
  semantic_target,
389
  *merged_preferences,
390
  ]
391
  if reference and reference.entity:
392
  semantic_parts.append(reference.entity)
393
- semantic_query = " ".join(_ordered_unique(semantic_parts)).strip()
 
 
 
 
394
 
395
  patch = ConversationStatePatch(
396
  target_category=requested_category,
@@ -408,10 +520,17 @@ class DeterministicPlaceChatIntentParser:
408
  clear_reference=category_changed and reference_text is None,
409
  taxonomy_version=self._taxonomy.version,
410
  )
411
- if explicit_category:
 
 
412
  confidence = 0.96
413
  elif inferred_category:
414
  confidence = inferred.confidence if inferred else 0.88
 
 
 
 
 
415
  else:
416
  confidence = 0.84
417
  if reference_text:
@@ -431,6 +550,18 @@ class DeterministicPlaceChatIntentParser:
431
  compatible_category_values=compatible_category_values,
432
  category_evidence_terms=category_evidence_terms,
433
  category_source=category_source,
 
 
 
 
 
 
 
 
 
 
 
 
434
  )
435
 
436
  def _location_scope_clarification(
@@ -543,8 +674,10 @@ class DeterministicPlaceChatIntentParser:
543
  base_state = replace(state, pending_clarification=None)
544
  if pending.kind in ("target_category", "intent_category"):
545
  if self._taxonomy.category(selected.value) is None:
546
- raise ClarificationStateMismatchError(
547
- "clarification category is not supported"
 
 
548
  )
549
  resolved = self.parse(
550
  message=self._category_message(selected.value),
@@ -780,6 +913,61 @@ class DeterministicPlaceChatIntentParser:
780
  return definition.aliases[0]
781
  return category or ""
782
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
783
  def _location_intent(
784
  self,
785
  location_text: str | None,
@@ -787,7 +975,10 @@ class DeterministicPlaceChatIntentParser:
787
  state: ConversationState,
788
  has_user_location: bool,
789
  ) -> tuple[LocationIntent, ExplicitTargetLocation | None]:
790
- if location_text and location_text in _CURRENT_LOCATION_VALUES:
 
 
 
791
  return (
792
  LocationIntent(
793
  scope="user_current_location",
@@ -831,11 +1022,123 @@ class DeterministicPlaceChatIntentParser:
831
  LocationIntent(
832
  scope="user_current_location",
833
  source="user_current",
 
 
834
  ),
835
  None,
836
  )
837
  return (LocationIntent(scope="unresolved", source="none"), None)
838
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
839
  def _matched_categories(self, normalized: str) -> tuple[str, ...]:
840
  return _ordered_unique(
841
  category.canonical
@@ -864,20 +1167,44 @@ class DeterministicPlaceChatIntentParser:
864
  def _inferred_activity_category(
865
  self,
866
  normalized: str,
 
 
867
  ) -> PlaceCategoryInference | None:
868
- for category, patterns in _ACTIVITY_CATEGORY_DEFAULTS:
869
- if _contains_any(normalized, patterns):
870
- return PlaceCategoryInference(
871
- category=category,
872
- confidence=0.88,
873
- source="lexical_activity",
874
- )
 
 
 
 
 
 
 
 
 
 
 
 
 
875
  if self._activity_classifier is None:
876
- return None
877
- inferred = self._activity_classifier.classify(normalized)
878
- if inferred is None or self._taxonomy.category(inferred.category) is None:
879
- return None
880
- return inferred
 
 
 
 
 
 
 
 
 
881
 
882
  @staticmethod
883
  def _target_clause(normalized: str) -> str:
@@ -1023,6 +1350,50 @@ def _contains_phrase(text: str, phrase: str) -> bool:
1023
  return bool(re.search(rf"(?<!\w){re.escape(phrase)}(?!\w)", text))
1024
 
1025
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1026
  def _contains_any(text: str, values: tuple[str, ...]) -> bool:
1027
  return any(
1028
  _contains_phrase(text, prepare_for_embedding(value))
 
1
  from dataclasses import dataclass, replace
2
  from functools import lru_cache
3
  import json
4
+ import logging
5
  from pathlib import Path
6
  import re
7
  from typing import Any, Iterable
 
29
  PlaceReference,
30
  )
31
  from app.modules.places.domain.errors import ClarificationStateMismatchError
32
+ from app.modules.places.infrastructure.bert_intent_extractor import (
33
+ BertIntentExtractionError,
34
+ BertPlaceIntentExtractor,
35
+ IntentFrame,
36
+ IntentSpan,
37
+ )
38
  from app.shared.nlp.preprocessing.text import (
39
  prepare_for_embedding,
40
  tokenize_for_embeddings,
 
57
  r"\b(?:sin|excepto|evita(?:r)?|no\s+quiero|que\s+no\s+(?:sea|tenga))\s+"
58
  r"(?P<value>.+?)(?="
59
  r"\s+cerca\s+(?:de\s+la|del|de)\b|[,;]|"
60
+ r"\s+(?:pero|aunque|con)\b|\s+y\s+(?:con|que|quiero|busco)\b|$)"
61
  )
62
  _CURRENT_LOCATION_VALUES = {"mi", "aqui", "donde estoy", "mi ubicacion"}
63
+ _DETERMINISTIC_INTENT_VERSION = "deterministic-open-v2"
64
+ _RADIUS_VALUE_PATTERN = re.compile(
65
+ r"(?P<value>\d+(?:[.,]\d+)?)\s*"
66
+ r"(?P<unit>km|kilometros?|m|metros?)\b"
67
+ )
68
+ _LOGGER = logging.getLogger(__name__)
69
  _GENERIC_REQUEST_TOKENS = {
70
  "dame",
71
  "favor",
 
184
  )
185
 
186
 
187
+ @dataclass(frozen=True)
188
+ class _ContextualSignals:
189
+ frame: IntentFrame | None
190
+ category_phrases: tuple[str, ...] = ()
191
+ preferences: tuple[str, ...] = ()
192
+ exclusions: tuple[str, ...] = ()
193
+ location: str | None = None
194
+ reference: str | None = None
195
+ radius_meters: int | None = None
196
+ category_confidence: float | None = None
197
+ intent_model_version: str = _DETERMINISTIC_INTENT_VERSION
198
+
199
+
200
  class DeterministicPlaceChatIntentParser:
201
  def __init__(
202
  self,
203
  taxonomy: PlaceChatTaxonomy | None = None,
204
  activity_classifier: PlaceActivityClassifier | None = None,
205
+ contextual_extractor: BertPlaceIntentExtractor | None = None,
206
  ) -> None:
207
  self._taxonomy = taxonomy or load_place_chat_taxonomy()
208
  self._activity_classifier = activity_classifier
209
+ self._contextual_extractor = contextual_extractor
210
 
211
  def parse(
212
  self,
 
216
  clarification_choice: ClarificationChoice | None = None,
217
  ) -> ParsedPlaceChatIntent:
218
  normalized = prepare_for_embedding(message)
 
219
  if clarification_choice is not None:
220
  return self._resolve_structured_clarification(
221
  choice=clarification_choice,
222
  state=state,
223
  has_user_location=has_user_location,
224
  )
225
+ contextual = self._contextual_signals(message)
226
+ # Token classification is multi-label and a valid frame can still be
227
+ # incomplete. Strip both model-detected context spans and any remaining
228
+ # high-precision legacy context before building the retrieval query. The
229
+ # category itself stays model/open-vocabulary first; taxonomy aliases are
230
+ # deliberately not restored for a successful BERT frame.
231
+ target_clause = self._target_clause(
232
+ _remove_contextual_spans(normalized, contextual.frame)
233
+ )
234
  if state.pending_clarification:
235
  pending = ensure_legacy_pending_options(state.pending_clarification)
236
  state = replace(state, pending_clarification=pending)
237
+ categories = self._contextual_categories(contextual, target_clause)
238
  inferred = (
239
+ self._inferred_activity_category(
240
+ target_clause,
241
+ allow_lexical_defaults=contextual.frame is None,
242
+ )
243
  if not categories
244
  else None
245
  )
 
259
  clear_pending_clarification=True,
260
  ),
261
  )
262
+ return replace(
263
+ self._resolve_pending_clarification(
264
+ normalized=normalized,
265
+ state=state,
266
+ has_user_location=has_user_location,
267
+ ),
268
+ intent_model_version=contextual.intent_model_version,
269
  )
270
 
271
+ # Fuse slots independently. A CATEGORY-only frame must not erase a
272
+ # LOCATION, RADIUS, REFERENCE, preference, or exclusion that the model
273
+ # omitted. These narrow extractors are rollout fallbacks, not gates.
274
+ reference_text = contextual.reference or self._extract_reference(normalized)
275
+ location_text = contextual.location or self._extract_location(normalized)
276
+ radius_meters = (
277
+ contextual.radius_meters
278
+ if contextual.radius_meters is not None
279
+ else self._extract_radius_meters(normalized)
280
+ )
281
  excluded_texts = tuple(
282
  match.group("value").strip(" ,.;")
283
  for match in _EXCLUSION_PATTERN.finditer(normalized)
 
286
  # Category words inside references, exclusions, or geographic anchors
287
  # do not describe the requested result type. In "cafeteria cerca del
288
  # parque", only cafe is a hard category.
289
+ categories = self._contextual_categories(contextual, target_clause)
290
  if len(categories) > 1:
291
  pending = new_category_clarification(categories)
292
+ return replace(
293
+ self._clarification(
294
+ pending=pending,
295
+ unresolved=("target_category",),
296
+ state=state,
297
+ has_user_location=has_user_location,
298
+ ),
299
+ intent_model_version=contextual.intent_model_version,
300
  )
301
 
302
  explicit_category = categories[0] if categories else None
303
  inferred = (
304
+ self._inferred_activity_category(
305
+ target_clause,
306
+ allow_lexical_defaults=contextual.frame is None,
307
+ )
308
  if explicit_category is None
309
  else None
310
  )
311
+ category_alternatives = (
312
+ self._ranked_activity_alternatives(target_clause)
313
+ if explicit_category is None
314
+ else ()
315
+ )
316
  inferred_category = inferred.category if inferred else None
317
  requested_category = explicit_category or inferred_category
318
  target_category = requested_category or state.target_category
 
333
  if location_text:
334
  location_text = _RADIUS_PATTERN.sub("", location_text).strip(" ,.;")
335
 
336
+ legacy_preference_text = normalized if not contextual.preferences else ""
337
+ detected_preferences = _ordered_unique(
338
+ (
339
+ *contextual.preferences,
340
+ *self._matched_preferences(legacy_preference_text),
341
+ )
342
+ )
343
+ legacy_excluded_texts = tuple(
344
+ excluded_text
345
  for excluded_text in excluded_texts
346
+ if not any(
347
+ _contains_phrase(
348
+ prepare_for_embedding(excluded_text),
349
+ prepare_for_embedding(contextual_exclusion),
350
+ )
351
+ for contextual_exclusion in contextual.exclusions
352
+ )
353
+ )
354
+ explicit_exclusions = _ordered_unique(
355
+ (
356
+ *contextual.exclusions,
357
+ *(
358
+ exclusion
359
+ for excluded_text in legacy_excluded_texts
360
+ for exclusion in (
361
+ self._matched_preferences(excluded_text)
362
+ or (prepare_for_embedding(excluded_text),)
363
+ )
364
+ ),
365
+ )
366
  )
367
  positive_preferences = tuple(
368
  preference
 
403
  else inherited_reference
404
  )
405
  hard_filters = dict(state.hard_filters)
406
+ if state.city and "city" not in hard_filters:
407
+ hard_filters["city"] = state.city
408
+ if state.state and "state" not in hard_filters:
409
+ hard_filters["state"] = state.state
410
  if "economico" in merged_preferences or _contains_any(
411
  target_clause,
412
  ("mas barato", "economico", "barato"),
413
  ):
414
  hard_filters["price_preference"] = "lower"
415
 
416
+ # An unresolved category is not a reason to stop retrieval. Preserve the
417
+ # user's open-ended concept in ``semantic_query`` and let dense/lexical
418
+ # retrieval provide evidence. A downstream decision policy may still ask
419
+ # a clarification, but its options must come from model hypotheses or real
420
+ # candidates rather than a fixed menu.
421
+ category = (
422
+ self._taxonomy.category(target_category)
423
+ if target_category is not None
424
+ else None
425
+ )
426
+ inferred_category_values = (
427
+ tuple(inferred.category_values)
428
+ if inferred is not None
429
+ and inferred.category == target_category
430
+ and inferred.category_values
431
+ else ()
432
+ )
433
+ rehydrated_category_values = self._activity_category_values(
434
+ target_category
435
+ )
436
+ category_values = (
437
+ category.storage_values
438
+ if category is not None
439
+ else (
440
+ inferred_category_values
441
+ or rehydrated_category_values
442
+ or ((target_category,) if target_category else ())
443
  )
444
+ )
 
 
445
  compatible_category_values = (
446
  category.compatible_storage_values if category else ()
447
  )
 
450
  if (
451
  reference_text
452
  and location_text
453
+ and prepare_for_embedding(location_text) not in _CURRENT_LOCATION_VALUES
454
  ):
455
+ return replace(
456
+ self._location_scope_clarification(
457
+ target_category=target_category,
458
+ category_values=category_values,
459
+ compatible_category_values=compatible_category_values,
460
+ category_evidence_terms=category_evidence_terms,
461
+ explicit_category=requested_category,
462
+ category_source=category_source,
463
+ hard_filters=hard_filters,
464
+ preferences=merged_preferences,
465
+ exclusions=merged_exclusions,
466
+ reference=reference,
467
+ reference_text=reference_text,
468
+ location_text=location_text,
469
+ radius_meters=radius_meters,
470
+ state=state,
471
+ has_user_location=has_user_location,
472
+ ),
473
+ intent_model_version=contextual.intent_model_version,
474
  )
475
 
476
  location, explicit_location = self._location_intent(
 
491
  )
492
  category_query_term = self._category_query_term(target_clause, category)
493
  semantic_parts = [
494
+ target_category or "",
495
  category_query_term,
496
  semantic_target,
497
  *merged_preferences,
498
  ]
499
  if reference and reference.entity:
500
  semantic_parts.append(reference.entity)
501
+ semantic_query = (
502
+ " ".join(_ordered_unique(semantic_parts)).strip()
503
+ or target_clause
504
+ or normalized
505
+ )
506
 
507
  patch = ConversationStatePatch(
508
  target_category=requested_category,
 
520
  clear_reference=category_changed and reference_text is None,
521
  taxonomy_version=self._taxonomy.version,
522
  )
523
+ if explicit_category and contextual.category_phrases:
524
+ confidence = contextual.category_confidence or contextual.frame.confidence
525
+ elif explicit_category:
526
  confidence = 0.96
527
  elif inferred_category:
528
  confidence = inferred.confidence if inferred else 0.88
529
+ elif target_category is None:
530
+ # This confidence describes category resolution, not whether the
531
+ # query is searchable. Keeping it below the automatic-decision band
532
+ # allows candidate-derived clarification without suppressing recall.
533
+ confidence = 0.55
534
  else:
535
  confidence = 0.84
536
  if reference_text:
 
550
  compatible_category_values=compatible_category_values,
551
  category_evidence_terms=category_evidence_terms,
552
  category_source=category_source,
553
+ alternatives=category_alternatives,
554
+ unresolved=("target_category",) if target_category is None else (),
555
+ raw_category_phrase=(
556
+ contextual.category_phrases[0]
557
+ if contextual.category_phrases
558
+ else (
559
+ semantic_target or target_clause
560
+ if explicit_category is None
561
+ else explicit_category
562
+ )
563
+ ),
564
+ intent_model_version=contextual.intent_model_version,
565
  )
566
 
567
  def _location_scope_clarification(
 
674
  base_state = replace(state, pending_clarification=None)
675
  if pending.kind in ("target_category", "intent_category"):
676
  if self._taxonomy.category(selected.value) is None:
677
+ return self._resolve_open_category_choice(
678
+ value=selected.value,
679
+ state=base_state,
680
+ has_user_location=has_user_location,
681
  )
682
  resolved = self.parse(
683
  message=self._category_message(selected.value),
 
913
  return definition.aliases[0]
914
  return category or ""
915
 
916
+ def _resolve_open_category_choice(
917
+ self,
918
+ value: str,
919
+ state: ConversationState,
920
+ has_user_location: bool,
921
+ ) -> ParsedPlaceChatIntent:
922
+ category = " ".join(value.split()).strip()
923
+ if not category:
924
+ raise ClarificationStateMismatchError(
925
+ "clarification category must not be empty"
926
+ )
927
+ hard_filters = dict(state.hard_filters)
928
+ if state.city and "city" not in hard_filters:
929
+ hard_filters["city"] = state.city
930
+ if state.state and "state" not in hard_filters:
931
+ hard_filters["state"] = state.state
932
+ location, _ = self._location_intent(
933
+ location_text=None,
934
+ radius_meters=None,
935
+ state=state,
936
+ has_user_location=has_user_location,
937
+ )
938
+ semantic_parts = [category, *state.soft_preferences]
939
+ if state.reference and state.reference.entity:
940
+ semantic_parts.append(state.reference.entity)
941
+ category_values = _ordered_unique(
942
+ (category, *self._activity_category_values(category))
943
+ )
944
+ return ParsedPlaceChatIntent(
945
+ action="recommendations",
946
+ target_category=category,
947
+ category_values=category_values,
948
+ hard_filters=hard_filters,
949
+ soft_preferences=state.soft_preferences,
950
+ exclusions=state.exclusions,
951
+ reference=state.reference,
952
+ location=location,
953
+ semantic_query=" ".join(_ordered_unique(semantic_parts)).strip(),
954
+ confidence=1.0,
955
+ state_patch=ConversationStatePatch(
956
+ target_category=category,
957
+ hard_filters=(
958
+ hard_filters if hard_filters != state.hard_filters else None
959
+ ),
960
+ clear_pending_clarification=True,
961
+ taxonomy_version=self._taxonomy.version,
962
+ ),
963
+ category_evidence_terms=(category,),
964
+ category_source="explicit",
965
+ raw_category_phrase=category,
966
+ intent_model_version=(
967
+ f"{_DETERMINISTIC_INTENT_VERSION}+dynamic-clarification-v1"
968
+ ),
969
+ )
970
+
971
  def _location_intent(
972
  self,
973
  location_text: str | None,
 
975
  state: ConversationState,
976
  has_user_location: bool,
977
  ) -> tuple[LocationIntent, ExplicitTargetLocation | None]:
978
+ if (
979
+ location_text
980
+ and prepare_for_embedding(location_text) in _CURRENT_LOCATION_VALUES
981
+ ):
982
  return (
983
  LocationIntent(
984
  scope="user_current_location",
 
1022
  LocationIntent(
1023
  scope="user_current_location",
1024
  source="user_current",
1025
+ radius_meters=radius_meters,
1026
+ strict_radius=radius_meters is not None,
1027
  ),
1028
  None,
1029
  )
1030
  return (LocationIntent(scope="unresolved", source="none"), None)
1031
 
1032
+ def _contextual_signals(self, message: str) -> _ContextualSignals:
1033
+ extractor = self._contextual_extractor
1034
+ if extractor is None:
1035
+ return _ContextualSignals(frame=None)
1036
+
1037
+ try:
1038
+ frame = extractor.extract(message)
1039
+ except BertIntentExtractionError as exc:
1040
+ configured_version = getattr(
1041
+ extractor,
1042
+ "model_version",
1043
+ "unspecified",
1044
+ )
1045
+ _LOGGER.warning(
1046
+ "BERT place-intent extraction failed; using deterministic "
1047
+ "fallback (model_version=%s): %s",
1048
+ configured_version,
1049
+ exc,
1050
+ )
1051
+ return _ContextualSignals(
1052
+ frame=None,
1053
+ intent_model_version=(
1054
+ f"{_DETERMINISTIC_INTENT_VERSION}+"
1055
+ f"bert-fallback:{configured_version}"
1056
+ ),
1057
+ )
1058
+
1059
+ category_spans = tuple(
1060
+ span
1061
+ for span in frame.by_type("CATEGORY")
1062
+ if span.polarity != "negative"
1063
+ )
1064
+ positive_preferences = tuple(
1065
+ span
1066
+ for span in frame.by_type("PREFERENCE")
1067
+ if span.polarity == "positive"
1068
+ )
1069
+ negative_spans = tuple(
1070
+ span
1071
+ for span in frame.spans
1072
+ if span.slot_type == "EXCLUSION" or span.polarity == "negative"
1073
+ )
1074
+ locations = frame.by_type("LOCATION")
1075
+ references = frame.by_type("REFERENCE")
1076
+ radii = frame.by_type("RADIUS")
1077
+ version = (
1078
+ f"bert-token:{frame.model_name}@{frame.model_version}+"
1079
+ f"{_DETERMINISTIC_INTENT_VERSION}"
1080
+ )
1081
+ return _ContextualSignals(
1082
+ frame=frame,
1083
+ category_phrases=_span_values(category_spans),
1084
+ preferences=_span_values(positive_preferences),
1085
+ exclusions=_span_values(negative_spans),
1086
+ location=_first_span_value(locations),
1087
+ reference=_first_span_value(references),
1088
+ radius_meters=(
1089
+ _radius_meters_from_text(radii[0].text) if radii else None
1090
+ ),
1091
+ category_confidence=(
1092
+ max(span.confidence for span in category_spans)
1093
+ if category_spans
1094
+ else None
1095
+ ),
1096
+ intent_model_version=version,
1097
+ )
1098
+
1099
+ def _contextual_categories(
1100
+ self,
1101
+ contextual: _ContextualSignals,
1102
+ target_clause: str,
1103
+ ) -> tuple[str, ...]:
1104
+ if not contextual.category_phrases:
1105
+ return (
1106
+ self._matched_categories(target_clause)
1107
+ if contextual.frame is None
1108
+ else ()
1109
+ )
1110
+
1111
+ categories: list[str] = []
1112
+ for phrase in contextual.category_phrases:
1113
+ aligned = (
1114
+ self._activity_classifier.classify(phrase)
1115
+ if self._activity_classifier is not None
1116
+ else None
1117
+ )
1118
+ # Semantic alignment is optional and abstaining. Unknown values
1119
+ # pass through unchanged; no lexical alias table is consulted once
1120
+ # the contextual model has supplied a category span.
1121
+ categories.append(aligned.category if aligned is not None else phrase)
1122
+ return _ordered_unique(categories)
1123
+
1124
+ def _activity_category_values(
1125
+ self,
1126
+ target_category: str | None,
1127
+ ) -> tuple[str, ...]:
1128
+ """Rehydrate an open catalog concept on conversation continuations."""
1129
+
1130
+ if target_category is None or self._activity_classifier is None:
1131
+ return ()
1132
+ get_concept = getattr(self._activity_classifier, "get_concept", None)
1133
+ if callable(get_concept):
1134
+ concept = get_concept(target_category)
1135
+ if concept is not None:
1136
+ return tuple(getattr(concept, "storage_values", ()) or ())
1137
+ for concept in getattr(self._activity_classifier, "concepts", ()):
1138
+ if getattr(concept, "id", None) == target_category:
1139
+ return tuple(getattr(concept, "storage_values", ()) or ())
1140
+ return ()
1141
+
1142
  def _matched_categories(self, normalized: str) -> tuple[str, ...]:
1143
  return _ordered_unique(
1144
  category.canonical
 
1167
  def _inferred_activity_category(
1168
  self,
1169
  normalized: str,
1170
+ *,
1171
+ allow_lexical_defaults: bool = True,
1172
  ) -> PlaceCategoryInference | None:
1173
+ # Prefer the injected semantic/open-vocabulary model. Lexical activity
1174
+ # rules remain a compatibility fallback during rollout, never a gate.
1175
+ if self._activity_classifier is not None:
1176
+ inferred = self._activity_classifier.classify(normalized)
1177
+ if inferred is not None:
1178
+ return inferred
1179
+ if allow_lexical_defaults:
1180
+ for category, patterns in _ACTIVITY_CATEGORY_DEFAULTS:
1181
+ if _contains_any(normalized, patterns):
1182
+ return PlaceCategoryInference(
1183
+ category=category,
1184
+ confidence=0.88,
1185
+ source="lexical_activity",
1186
+ )
1187
+ return None
1188
+
1189
+ def _ranked_activity_alternatives(
1190
+ self,
1191
+ normalized: str,
1192
+ ) -> tuple[IntentAlternative, ...]:
1193
  if self._activity_classifier is None:
1194
+ return ()
1195
+ rank = getattr(self._activity_classifier, "rank", None)
1196
+ if not callable(rank):
1197
+ return ()
1198
+ matches = rank(normalized, limit=5)
1199
+ return tuple(
1200
+ IntentAlternative(
1201
+ key=str(match.concept_id),
1202
+ description=str(match.label),
1203
+ confidence=max(0.0, min(1.0, (float(match.score) + 1.0) / 2.0)),
1204
+ )
1205
+ for match in matches
1206
+ if getattr(match, "concept_id", None)
1207
+ )
1208
 
1209
  @staticmethod
1210
  def _target_clause(normalized: str) -> str:
 
1350
  return bool(re.search(rf"(?<!\w){re.escape(phrase)}(?!\w)", text))
1351
 
1352
 
1353
+ def _remove_contextual_spans(
1354
+ target_clause: str,
1355
+ frame: IntentFrame | None,
1356
+ ) -> str:
1357
+ if frame is None:
1358
+ return target_clause
1359
+ cleaned = target_clause
1360
+ for span in frame.spans:
1361
+ if (
1362
+ span.slot_type not in {"EXCLUSION", "LOCATION", "REFERENCE", "RADIUS"}
1363
+ and span.polarity != "negative"
1364
+ ):
1365
+ continue
1366
+ normalized_value = prepare_for_embedding(span.text)
1367
+ if normalized_value:
1368
+ cleaned = re.sub(
1369
+ rf"(?<!\w){re.escape(normalized_value)}(?!\w)",
1370
+ " ",
1371
+ cleaned,
1372
+ )
1373
+ return " ".join(cleaned.split())
1374
+
1375
+
1376
+ def _span_values(spans: Iterable[IntentSpan]) -> tuple[str, ...]:
1377
+ return _ordered_unique(
1378
+ " ".join(span.text.split()).strip(" ,.;") for span in spans
1379
+ )
1380
+
1381
+
1382
+ def _first_span_value(spans: tuple[IntentSpan, ...]) -> str | None:
1383
+ values = _span_values(spans[:1])
1384
+ return values[0] if values else None
1385
+
1386
+
1387
+ def _radius_meters_from_text(value: str) -> int | None:
1388
+ match = _RADIUS_VALUE_PATTERN.search(prepare_for_embedding(value))
1389
+ if match is None:
1390
+ return None
1391
+ distance = float(match.group("value").replace(",", "."))
1392
+ if match.group("unit").startswith(("km", "kilometro")):
1393
+ distance *= 1000
1394
+ return max(1, min(50_000, int(round(distance))))
1395
+
1396
+
1397
  def _contains_any(text: str, values: tuple[str, ...]) -> bool:
1398
  return any(
1399
  _contains_phrase(text, prepare_for_embedding(value))
app/modules/places/infrastructure/hybrid_chat_retriever.py CHANGED
@@ -1,5 +1,7 @@
 
1
  import json
2
  import math
 
3
  from typing import Sequence
4
 
5
  from app.modules.places.application.ports.place_repository import PlaceVectorRepository
@@ -23,7 +25,12 @@ _EXACT_ENTITY_PREFERENCES = {"hello_kitty"}
23
 
24
 
25
  class HybridContentPlaceChatRetriever:
26
- """Fuse semantic, lexical and facet signals over a bounded vector candidate pool."""
 
 
 
 
 
27
 
28
  def __init__(
29
  self,
@@ -50,7 +57,7 @@ class HybridContentPlaceChatRetriever:
50
  intent: ParsedPlaceChatIntent,
51
  limit: int,
52
  ) -> Sequence[PlaceChatCandidate]:
53
- if intent.action != "recommendations" or not intent.target_category:
54
  return []
55
  if limit < 1:
56
  return []
@@ -62,36 +69,40 @@ class HybridContentPlaceChatRetriever:
62
  if cached is not None:
63
  return cached
64
 
65
- embedding = self._embedding_provider.embed_text(semantic_query)
66
- repository_categories = tuple(
67
- dict.fromkeys(
68
- (
69
- *intent.category_values,
70
- *intent.compatible_category_values,
71
- )
72
- )
73
  )
74
  filters = PlaceFilters(
75
- city=None,
76
- state=None,
77
- categories=repository_categories,
 
 
 
78
  price_range=_optional_string(intent.hard_filters.get("price_range")),
79
  occasion=_optional_string(intent.hard_filters.get("occasion")),
 
 
 
 
 
 
 
 
 
80
  is_active=True,
81
  )
 
82
  raw_candidates = list(
83
- await self._place_repository.search(
 
84
  embedding=embedding,
85
  filters=filters,
86
- limit=min(max(limit * 3, limit), 120),
87
  )
88
  )
89
- candidates = [
90
- candidate
91
- for candidate in raw_candidates
92
- if self._matches_hard_category(candidate, intent)
93
- and not self._matches_exclusions(candidate, intent.exclusions)
94
- ]
95
  if not candidates:
96
  return []
97
 
@@ -111,26 +122,68 @@ class HybridContentPlaceChatRetriever:
111
  b=self._b,
112
  )
113
  lexical = normalize_bm25(raw_lexical)
114
- semantic = _unit_score(candidate.score)
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
115
  theme_score, match_level, reasons = self._facet_match(
116
  candidate,
117
  intent,
118
  document_tokens,
 
 
119
  )
120
- content_score = self._weights.score(
121
  semantic_score=semantic,
122
  lexical_score=lexical,
123
  theme_or_reference_score=theme_score,
124
  )
125
- requires_content_evidence = bool(
126
- intent.soft_preferences
127
- or (intent.reference and intent.reference.entity)
128
  )
129
- if (
130
- requires_content_evidence
131
- and content_score < self._minimum_content_score
132
- ):
133
- continue
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
134
  ranked.append(
135
  PlaceChatCandidate(
136
  place_id=candidate.id,
@@ -141,16 +194,16 @@ class HybridContentPlaceChatRetriever:
141
  lexical_score=lexical,
142
  match_level=match_level,
143
  matched_reasons=reasons,
144
- metadata=dict(candidate.metadata),
145
  )
146
  )
147
 
148
  ranked.sort(
149
  key=lambda item: (
150
- _match_level_priority(item.match_level),
151
  -item.content_score,
152
  -item.semantic_score,
153
  -item.lexical_score,
 
154
  item.place_id,
155
  )
156
  )
@@ -168,6 +221,34 @@ class HybridContentPlaceChatRetriever:
168
  self._cache.set(cache_key, result)
169
  return result
170
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
171
  @staticmethod
172
  def _cache_key(
173
  intent: ParsedPlaceChatIntent,
@@ -176,6 +257,7 @@ class HybridContentPlaceChatRetriever:
176
  ) -> str:
177
  payload = {
178
  "query": semantic_query,
 
179
  "categories": intent.category_values,
180
  "compatible_categories": intent.compatible_category_values,
181
  "category_evidence_terms": intent.category_evidence_terms,
@@ -191,59 +273,88 @@ class HybridContentPlaceChatRetriever:
191
  )
192
 
193
  @staticmethod
194
- def _matches_hard_category(
195
  candidate: PlaceCandidate,
196
  intent: ParsedPlaceChatIntent,
197
- ) -> bool:
198
- if not candidate.category:
199
- return False
200
- actual = prepare_for_embedding(candidate.category).replace(" ", "_")
201
  exact = {
202
- prepare_for_embedding(value).replace(" ", "_")
203
- for value in intent.category_values
 
204
  }
205
- if actual in exact:
206
- return True
207
  compatible = {
208
- prepare_for_embedding(value).replace(" ", "_")
209
  for value in intent.compatible_category_values
 
210
  }
211
- if actual not in compatible:
212
- return False
213
- document_tokens = _category_evidence_tokens(candidate)
214
- return any(
215
- evidence_tokens and evidence_tokens <= document_tokens
 
 
 
 
 
 
216
  for term in intent.category_evidence_terms
217
- if (evidence_tokens := set(tokenize(term)))
218
  )
 
 
 
 
 
 
 
219
 
220
  @staticmethod
221
- def _matches_exclusions(
222
  candidate: PlaceCandidate,
223
  exclusions: tuple[str, ...],
224
- ) -> bool:
225
  if not exclusions:
226
- return False
227
- document_tokens = set(place_tokens(candidate))
228
- return any(
229
- exclusion_tokens
230
- and exclusion_tokens <= document_tokens
231
- for exclusion in exclusions
232
- if (
233
- exclusion_tokens := set(
234
- tokenize(exclusion.replace("_", " "))
 
235
  )
 
236
  )
237
  )
 
 
 
 
 
 
 
 
 
 
 
 
238
 
239
  @staticmethod
240
  def _facet_match(
241
  candidate: PlaceCandidate,
242
  intent: ParsedPlaceChatIntent,
243
  document_tokens: list[str],
 
 
244
  ) -> tuple[float, str, tuple[str, ...]]:
245
  token_set = set(document_tokens)
246
- reasons: list[str] = [intent.target_category or "place"]
 
 
247
  matched_preferences = 0
248
  exact_preference = False
249
  for preference in intent.soft_preferences:
@@ -268,11 +379,16 @@ class HybridContentPlaceChatRetriever:
268
  if reference_exact:
269
  reasons.append(intent.reference.entity)
270
 
271
- if reference_exact or exact_preference:
272
- return 1.0, "exact", tuple(dict.fromkeys(reasons))
273
- if matched_preferences:
274
- return preference_score, "family", tuple(dict.fromkeys(reasons))
275
- return 0.0, "broad", tuple(dict.fromkeys(reasons))
 
 
 
 
 
276
 
277
 
278
  def _match_level_priority(level: str) -> int:
@@ -286,6 +402,12 @@ def _optional_string(value: object) -> str | None:
286
  return normalized or None
287
 
288
 
 
 
 
 
 
 
289
  def _category_evidence_tokens(candidate: PlaceCandidate) -> set[str]:
290
  # Do not inspect candidate.document here: indexed documents intentionally
291
  # contain broad category profiles (for example, all `entertainment` records
@@ -310,6 +432,22 @@ def _as_evidence_text(value: object) -> str:
310
  return str(value)
311
 
312
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
313
  def _unit_score(value: object) -> float:
314
  try:
315
  score = float(value)
@@ -318,3 +456,13 @@ def _unit_score(value: object) -> float:
318
  if not math.isfinite(score):
319
  return 0.0
320
  return max(0.0, min(1.0, score))
 
 
 
 
 
 
 
 
 
 
 
1
+ import asyncio
2
  import json
3
  import math
4
+ import re
5
  from typing import Sequence
6
 
7
  from app.modules.places.application.ports.place_repository import PlaceVectorRepository
 
25
 
26
 
27
  class HybridContentPlaceChatRetriever:
28
+ """Fuse semantic, lexical and facet signals over a bounded vector candidate pool.
29
+
30
+ Categories are deliberately treated as ranking evidence, not repository
31
+ filters. This keeps open-vocabulary queries useful even when the parser
32
+ cannot map the user's wording to a canonical category.
33
+ """
34
 
35
  def __init__(
36
  self,
 
57
  intent: ParsedPlaceChatIntent,
58
  limit: int,
59
  ) -> Sequence[PlaceChatCandidate]:
60
+ if intent.action != "recommendations":
61
  return []
62
  if limit < 1:
63
  return []
 
69
  if cached is not None:
70
  return cached
71
 
72
+ embedding = await asyncio.to_thread(
73
+ self._embedding_provider.embed_text,
74
+ semantic_query,
 
 
 
 
 
75
  )
76
  filters = PlaceFilters(
77
+ city=_optional_string(intent.hard_filters.get("city")),
78
+ state=_optional_string(intent.hard_filters.get("state")),
79
+ # A canonical category is a hypothesis, not a hard constraint. A
80
+ # category filter here would prevent semantic retrieval from ever
81
+ # seeing useful places whose source taxonomy differs from ours.
82
+ categories=None,
83
  price_range=_optional_string(intent.hard_filters.get("price_range")),
84
  occasion=_optional_string(intent.hard_filters.get("occasion")),
85
+ place_ids=(
86
+ tuple(
87
+ str(place_id)
88
+ for place_id in intent.hard_filters.get("place_ids", ())
89
+ if str(place_id).strip()
90
+ )
91
+ if intent.hard_filters.get("place_ids") is not None
92
+ else None
93
+ ),
94
  is_active=True,
95
  )
96
+ candidate_pool_limit = min(max(limit * 8, 40), 120)
97
  raw_candidates = list(
98
+ await self._search_repository(
99
+ query_text=intent.semantic_query.strip(),
100
  embedding=embedding,
101
  filters=filters,
102
+ limit=candidate_pool_limit,
103
  )
104
  )
105
+ candidates = raw_candidates
 
 
 
 
 
106
  if not candidates:
107
  return []
108
 
 
122
  b=self._b,
123
  )
124
  lexical = normalize_bm25(raw_lexical)
125
+ repository_semantic = _optional_finite_score(
126
+ candidate.metadata.get("semantic_score")
127
+ )
128
+ repository_lexical = _optional_finite_score(
129
+ candidate.metadata.get("lexical_score")
130
+ )
131
+ has_repository_channel_scores = (
132
+ repository_semantic is not None or repository_lexical is not None
133
+ )
134
+ semantic = _unit_score(
135
+ repository_semantic
136
+ if has_repository_channel_scores
137
+ else candidate.score
138
+ )
139
+ if repository_lexical is not None:
140
+ lexical = max(lexical, normalize_bm25(repository_lexical))
141
+ category_score, category_match, category_reason = (
142
+ self._category_affinity(candidate, intent)
143
+ )
144
  theme_score, match_level, reasons = self._facet_match(
145
  candidate,
146
  intent,
147
  document_tokens,
148
+ category_score=category_score,
149
+ category_reason=category_reason,
150
  )
151
+ base_content_score = self._weights.score(
152
  semantic_score=semantic,
153
  lexical_score=lexical,
154
  theme_or_reference_score=theme_score,
155
  )
156
+ exclusion_affinity, exclusion_matches = self._exclusion_affinity(
157
+ candidate,
158
+ intent.exclusions,
159
  )
160
+ # Exclusions are negative ranking evidence, not a boolean gate. A
161
+ # source description may mention an excluded concept in a negated
162
+ # form ("sin ruido"), and removing that row would invert intent.
163
+ content_score = max(
164
+ 0.0,
165
+ base_content_score - 0.35 * exclusion_affinity,
166
+ )
167
+ # Keep weak candidates so short or novel queries do not collapse to
168
+ # zero results. The old minimum remains a quality diagnostic for
169
+ # downstream confidence/clarification policy, rather than a gate.
170
+ metadata = dict(candidate.metadata)
171
+ metadata["retrieval_diagnostics"] = {
172
+ "category_affinity": round(category_score, 6),
173
+ "category_match": category_match,
174
+ "exclusion_affinity": round(exclusion_affinity, 6),
175
+ "exclusion_matches": list(exclusion_matches),
176
+ "content_quality": (
177
+ "sufficient"
178
+ if content_score >= self._minimum_content_score
179
+ else "weak"
180
+ ),
181
+ "meets_minimum_content_score": (
182
+ content_score >= self._minimum_content_score
183
+ ),
184
+ "minimum_content_score": self._minimum_content_score,
185
+ "query_token_count": len(query_tokens),
186
+ }
187
  ranked.append(
188
  PlaceChatCandidate(
189
  place_id=candidate.id,
 
194
  lexical_score=lexical,
195
  match_level=match_level,
196
  matched_reasons=reasons,
197
+ metadata=metadata,
198
  )
199
  )
200
 
201
  ranked.sort(
202
  key=lambda item: (
 
203
  -item.content_score,
204
  -item.semantic_score,
205
  -item.lexical_score,
206
+ _match_level_priority(item.match_level),
207
  item.place_id,
208
  )
209
  )
 
221
  self._cache.set(cache_key, result)
222
  return result
223
 
224
+ async def _search_repository(
225
+ self,
226
+ query_text: str,
227
+ embedding: list[float],
228
+ filters: PlaceFilters,
229
+ limit: int,
230
+ ) -> Sequence[PlaceCandidate]:
231
+ """Prefer an optional independent dense+lexical repository search.
232
+
233
+ ``PlaceVectorRepository`` intentionally keeps its existing contract.
234
+ Repositories can opt into hybrid candidate generation duck-typically,
235
+ while mocks and current adapters continue through ``search``.
236
+ """
237
+
238
+ hybrid_search = getattr(self._place_repository, "search_hybrid", None)
239
+ if callable(hybrid_search):
240
+ return await hybrid_search(
241
+ query_text=query_text,
242
+ embedding=embedding,
243
+ filters=filters,
244
+ limit=limit,
245
+ )
246
+ return await self._place_repository.search(
247
+ embedding=embedding,
248
+ filters=filters,
249
+ limit=limit,
250
+ )
251
+
252
  @staticmethod
253
  def _cache_key(
254
  intent: ParsedPlaceChatIntent,
 
257
  ) -> str:
258
  payload = {
259
  "query": semantic_query,
260
+ "target_category": intent.target_category,
261
  "categories": intent.category_values,
262
  "compatible_categories": intent.compatible_category_values,
263
  "category_evidence_terms": intent.category_evidence_terms,
 
273
  )
274
 
275
  @staticmethod
276
+ def _category_affinity(
277
  candidate: PlaceCandidate,
278
  intent: ParsedPlaceChatIntent,
279
+ ) -> tuple[float, str, str | None]:
280
+ actual = _normalized_category(candidate.category)
 
 
281
  exact = {
282
+ normalized
283
+ for value in (intent.target_category, *intent.category_values)
284
+ if (normalized := _normalized_category(value))
285
  }
 
 
286
  compatible = {
287
+ normalized
288
  for value in intent.compatible_category_values
289
+ if (normalized := _normalized_category(value))
290
  }
291
+ has_category_hypothesis = bool(exact or compatible)
292
+ if not has_category_hypothesis:
293
+ return 0.0, "not_requested", None
294
+
295
+ reason = intent.target_category or next(iter(intent.category_values), None)
296
+ if actual and actual in exact:
297
+ return 1.0, "exact", reason or candidate.category
298
+
299
+ candidate_evidence_tokens = _category_evidence_tokens(candidate)
300
+ has_specific_evidence = any(
301
+ term_tokens and term_tokens <= candidate_evidence_tokens
302
  for term in intent.category_evidence_terms
303
+ if (term_tokens := set(tokenize(term)))
304
  )
305
+ if actual and actual in compatible:
306
+ if has_specific_evidence:
307
+ return 0.80, "compatible_with_evidence", reason
308
+ return 0.35, "compatible", reason
309
+ if has_specific_evidence:
310
+ return 0.65, "textual_evidence", reason
311
+ return 0.0, "none", None
312
 
313
  @staticmethod
314
+ def _exclusion_affinity(
315
  candidate: PlaceCandidate,
316
  exclusions: tuple[str, ...],
317
+ ) -> tuple[float, tuple[str, ...]]:
318
  if not exclusions:
319
+ return 0.0, ()
320
+ evidence = prepare_for_embedding(
321
+ " ".join(
322
+ value
323
+ for value in (
324
+ candidate.name,
325
+ candidate.category or "",
326
+ candidate.document or "",
327
+ _as_evidence_text(candidate.metadata.get("tags")),
328
+ _as_evidence_text(candidate.metadata.get("short_description")),
329
  )
330
+ if value
331
  )
332
  )
333
+ document_tokens = set(tokenize(evidence))
334
+ matches: list[str] = []
335
+ for exclusion in exclusions:
336
+ normalized = prepare_for_embedding(exclusion.replace("_", " "))
337
+ exclusion_tokens = set(tokenize(normalized))
338
+ if not exclusion_tokens or not exclusion_tokens <= document_tokens:
339
+ continue
340
+ if _is_negated_in_evidence(evidence, normalized):
341
+ continue
342
+ matches.append(exclusion)
343
+ unique_matches = tuple(dict.fromkeys(matches))
344
+ return len(unique_matches) / len(exclusions), unique_matches
345
 
346
  @staticmethod
347
  def _facet_match(
348
  candidate: PlaceCandidate,
349
  intent: ParsedPlaceChatIntent,
350
  document_tokens: list[str],
351
+ category_score: float = 0.0,
352
+ category_reason: str | None = None,
353
  ) -> tuple[float, str, tuple[str, ...]]:
354
  token_set = set(document_tokens)
355
+ reasons: list[str] = []
356
+ if category_score > 0 and category_reason:
357
+ reasons.append(category_reason)
358
  matched_preferences = 0
359
  exact_preference = False
360
  for preference in intent.soft_preferences:
 
379
  if reference_exact:
380
  reasons.append(intent.reference.entity)
381
 
382
+ facet_score = max(
383
+ category_score,
384
+ preference_score,
385
+ 1.0 if reference_exact else 0.0,
386
+ )
387
+ if reference_exact or exact_preference or category_score >= 1.0:
388
+ return facet_score, "exact", tuple(dict.fromkeys(reasons))
389
+ if matched_preferences or category_score >= 0.50:
390
+ return facet_score, "family", tuple(dict.fromkeys(reasons))
391
+ return facet_score, "broad", tuple(dict.fromkeys(reasons))
392
 
393
 
394
  def _match_level_priority(level: str) -> int:
 
402
  return normalized or None
403
 
404
 
405
+ def _normalized_category(value: object) -> str:
406
+ if value is None:
407
+ return ""
408
+ return prepare_for_embedding(str(value)).replace(" ", "_")
409
+
410
+
411
  def _category_evidence_tokens(candidate: PlaceCandidate) -> set[str]:
412
  # Do not inspect candidate.document here: indexed documents intentionally
413
  # contain broad category profiles (for example, all `entertainment` records
 
432
  return str(value)
433
 
434
 
435
+ def _is_negated_in_evidence(evidence: str, concept: str) -> bool:
436
+ """Detect common source-side negation without business-category rules."""
437
+
438
+ escaped = re.escape(concept).replace(r"\ ", r"\s+")
439
+ negation = (
440
+ r"(?:sin|libre\s+de|no\s+(?:hay|tiene|ofrece)|"
441
+ r"evita(?:r)?|prohibid[oa]s?)"
442
+ )
443
+ return bool(
444
+ re.search(
445
+ rf"\b{negation}\s+(?:\w+\s+){{0,2}}{escaped}\b",
446
+ evidence,
447
+ )
448
+ )
449
+
450
+
451
  def _unit_score(value: object) -> float:
452
  try:
453
  score = float(value)
 
456
  if not math.isfinite(score):
457
  return 0.0
458
  return max(0.0, min(1.0, score))
459
+
460
+
461
+ def _optional_finite_score(value: object) -> float | None:
462
+ if value is None:
463
+ return None
464
+ try:
465
+ score = float(value)
466
+ except (TypeError, ValueError):
467
+ return None
468
+ return score if math.isfinite(score) else None
app/modules/places/infrastructure/open_vocabulary_category_classifier.py ADDED
@@ -0,0 +1,368 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """Open-vocabulary place category alignment over a dynamic concept catalog."""
2
+
3
+ from __future__ import annotations
4
+
5
+ import math
6
+ import threading
7
+ from collections.abc import Mapping, Sequence
8
+ from dataclasses import dataclass
9
+ from typing import Any
10
+
11
+ from app.modules.places.domain.chat_intent import PlaceCategoryInference
12
+ from app.shared.nlp.embeddings.base import EmbeddingProvider
13
+
14
+
15
+ @dataclass(frozen=True)
16
+ class PlaceCategoryConcept:
17
+ """A category concept supplied by configuration, a database, or an API."""
18
+
19
+ id: str
20
+ label: str
21
+ description: str
22
+ examples: tuple[str, ...] = ()
23
+ storage_values: tuple[str, ...] = ()
24
+
25
+ def __post_init__(self) -> None:
26
+ object.__setattr__(self, "id", _required_text(self.id, "id"))
27
+ object.__setattr__(self, "label", _required_text(self.label, "label"))
28
+ object.__setattr__(
29
+ self,
30
+ "description",
31
+ _required_text(self.description, "description"),
32
+ )
33
+ object.__setattr__(
34
+ self,
35
+ "examples",
36
+ _clean_text_values(self.examples, "examples"),
37
+ )
38
+ object.__setattr__(
39
+ self,
40
+ "storage_values",
41
+ _clean_text_values(self.storage_values, "storage_values"),
42
+ )
43
+
44
+
45
+ @dataclass(frozen=True)
46
+ class PlaceCategoryMatch:
47
+ """One semantic catalog match, including separation from the next match."""
48
+
49
+ concept_id: str
50
+ label: str
51
+ description: str
52
+ storage_values: tuple[str, ...]
53
+ score: float
54
+ margin: float
55
+
56
+
57
+ @dataclass(frozen=True)
58
+ class _IndexedConcept:
59
+ concept: PlaceCategoryConcept
60
+ vectors: tuple[tuple[float, ...], ...]
61
+
62
+
63
+ ConceptInput = PlaceCategoryConcept | Mapping[str, Any]
64
+
65
+
66
+ class OpenVocabularyPlaceCategoryClassifier:
67
+ """Ranks injected concepts without a closed classifier label head.
68
+
69
+ Concept vectors are built on first use, so constructing this component does
70
+ not force a transformer model to load during application startup. ``rank``
71
+ always exposes evidence; ``classify`` applies confidence thresholds and is
72
+ compatible with ``PlaceActivityClassifier``.
73
+ """
74
+
75
+ def __init__(
76
+ self,
77
+ concepts: Sequence[ConceptInput],
78
+ embedding_provider: EmbeddingProvider,
79
+ *,
80
+ concept_embedding_provider: EmbeddingProvider | None = None,
81
+ minimum_similarity: float = 0.44,
82
+ minimum_margin: float = 0.04,
83
+ ) -> None:
84
+ if not -1.0 <= minimum_similarity <= 1.0:
85
+ raise ValueError("minimum_similarity must be between -1 and 1")
86
+ if not 0.0 <= minimum_margin <= 2.0:
87
+ raise ValueError("minimum_margin must be between 0 and 2")
88
+
89
+ prepared_concepts = tuple(_coerce_concept(concept) for concept in concepts)
90
+ duplicate_ids = _duplicate_concept_ids(prepared_concepts)
91
+ if duplicate_ids:
92
+ raise ValueError(
93
+ "concept ids must be unique (case-insensitive): "
94
+ + ", ".join(duplicate_ids)
95
+ )
96
+
97
+ self._concepts = prepared_concepts
98
+ self._concepts_by_id = {
99
+ concept.id.casefold(): concept for concept in prepared_concepts
100
+ }
101
+ self._embedding_provider = embedding_provider
102
+ self._concept_embedding_provider = (
103
+ concept_embedding_provider or embedding_provider
104
+ )
105
+ self._minimum_similarity = minimum_similarity
106
+ self._minimum_margin = minimum_margin
107
+ self._index: tuple[_IndexedConcept, ...] | None = None
108
+ self._embedding_dimension: int | None = None
109
+ self._index_lock = threading.Lock()
110
+
111
+ @property
112
+ def concepts(self) -> tuple[PlaceCategoryConcept, ...]:
113
+ return self._concepts
114
+
115
+ @property
116
+ def is_indexed(self) -> bool:
117
+ return self._index is not None
118
+
119
+ def get_concept(self, concept_id: str) -> PlaceCategoryConcept | None:
120
+ """Resolve persisted concept IDs without rerunning classification."""
121
+
122
+ if not isinstance(concept_id, str):
123
+ return None
124
+ return self._concepts_by_id.get(concept_id.strip().casefold())
125
+
126
+ def rank(self, text: str, limit: int = 3) -> tuple[PlaceCategoryMatch, ...]:
127
+ """Return semantic top-k matches without suppressing low-confidence rows."""
128
+
129
+ if isinstance(limit, bool) or not isinstance(limit, int) or limit <= 0:
130
+ raise ValueError("limit must be a positive integer")
131
+ if not isinstance(text, str):
132
+ raise TypeError(f"text must be str, got {type(text).__name__}")
133
+ query_text = text.strip()
134
+ if not query_text or not self._concepts:
135
+ return ()
136
+
137
+ query_vector = _validated_vector(
138
+ self._embedding_provider.embed_text(query_text),
139
+ context="query",
140
+ )
141
+ if query_vector is None:
142
+ return ()
143
+
144
+ index = self._ensure_index()
145
+ if not index:
146
+ return ()
147
+ if self._embedding_dimension != len(query_vector):
148
+ raise ValueError(
149
+ "query and concept embedding dimensions differ: "
150
+ f"query={len(query_vector)}, concepts={self._embedding_dimension}"
151
+ )
152
+
153
+ scores = sorted(
154
+ (
155
+ (
156
+ entry,
157
+ max(
158
+ _cosine_similarity(query_vector, prototype)
159
+ for prototype in entry.vectors
160
+ ),
161
+ )
162
+ for entry in index
163
+ if entry.vectors
164
+ ),
165
+ key=lambda item: (-item[1], item[0].concept.id.casefold()),
166
+ )
167
+ matches: list[PlaceCategoryMatch] = []
168
+ for position, (entry, score) in enumerate(scores[:limit]):
169
+ next_score = scores[position + 1][1] if position + 1 < len(scores) else -1.0
170
+ concept = entry.concept
171
+ matches.append(
172
+ PlaceCategoryMatch(
173
+ concept_id=concept.id,
174
+ label=concept.label,
175
+ description=concept.description,
176
+ storage_values=concept.storage_values,
177
+ score=score,
178
+ margin=max(0.0, score - next_score),
179
+ )
180
+ )
181
+ return tuple(matches)
182
+
183
+ def classify(self, text: str) -> PlaceCategoryInference | None:
184
+ """Return the best concept only when its score and margin are sufficient."""
185
+
186
+ matches = self.rank(text, limit=2)
187
+ if not matches:
188
+ return None
189
+ best = matches[0]
190
+ if (
191
+ best.score < self._minimum_similarity
192
+ or best.margin < self._minimum_margin
193
+ ):
194
+ return None
195
+
196
+ semantic_confidence = (best.score + 1.0) / 2.0
197
+ separation_confidence = min(1.0, best.margin / 0.5)
198
+ confidence = max(
199
+ 0.0,
200
+ min(0.99, semantic_confidence * 0.85 + separation_confidence * 0.15),
201
+ )
202
+ return PlaceCategoryInference(
203
+ category=best.concept_id,
204
+ confidence=confidence,
205
+ source="semantic_activity",
206
+ category_values=best.storage_values,
207
+ label=best.label,
208
+ )
209
+
210
+ def _ensure_index(self) -> tuple[_IndexedConcept, ...]:
211
+ index = self._index
212
+ if index is not None:
213
+ return index
214
+
215
+ with self._index_lock:
216
+ index = self._index
217
+ if index is not None:
218
+ return index
219
+
220
+ texts: list[str] = []
221
+ owners: list[int] = []
222
+ for concept_index, concept in enumerate(self._concepts):
223
+ for semantic_text in _semantic_texts(concept):
224
+ texts.append(semantic_text)
225
+ owners.append(concept_index)
226
+
227
+ raw_vectors = self._concept_embedding_provider.embed_batch(texts)
228
+ if len(raw_vectors) != len(texts):
229
+ raise ValueError(
230
+ "embedding provider returned an unexpected number of concept "
231
+ f"vectors: returned={len(raw_vectors)}, expected={len(texts)}"
232
+ )
233
+
234
+ vectors_by_concept: list[list[tuple[float, ...]]] = [
235
+ [] for _ in self._concepts
236
+ ]
237
+ embedding_dimension: int | None = None
238
+ for text_index, (owner, raw_vector) in enumerate(
239
+ zip(owners, raw_vectors)
240
+ ):
241
+ vector = _validated_vector(
242
+ raw_vector,
243
+ context=f"concept text {text_index}",
244
+ )
245
+ if vector is None:
246
+ continue
247
+ if embedding_dimension is None:
248
+ embedding_dimension = len(vector)
249
+ elif len(vector) != embedding_dimension:
250
+ raise ValueError(
251
+ "concept embedding dimensions differ: "
252
+ f"expected={embedding_dimension}, returned={len(vector)}, "
253
+ f"text_index={text_index}"
254
+ )
255
+ vectors_by_concept[owner].append(vector)
256
+
257
+ index = tuple(
258
+ _IndexedConcept(concept=concept, vectors=tuple(vectors))
259
+ for concept, vectors in zip(self._concepts, vectors_by_concept)
260
+ if vectors
261
+ )
262
+ self._embedding_dimension = embedding_dimension
263
+ self._index = index
264
+ return index
265
+
266
+
267
+ def _coerce_concept(value: ConceptInput) -> PlaceCategoryConcept:
268
+ if isinstance(value, PlaceCategoryConcept):
269
+ return value
270
+ if not isinstance(value, Mapping):
271
+ raise TypeError(
272
+ "concept must be PlaceCategoryConcept or a mapping, got "
273
+ f"{type(value).__name__}"
274
+ )
275
+ missing = [key for key in ("id", "label", "description") if key not in value]
276
+ if missing:
277
+ raise ValueError("concept is missing required fields: " + ", ".join(missing))
278
+ return PlaceCategoryConcept(
279
+ id=value["id"],
280
+ label=value["label"],
281
+ description=value["description"],
282
+ examples=_value_sequence(value.get("examples", ()), "examples"),
283
+ storage_values=_value_sequence(
284
+ value.get("storage_values", ()), "storage_values"
285
+ ),
286
+ )
287
+
288
+
289
+ def _value_sequence(value: Any, field_name: str) -> tuple[str, ...]:
290
+ if value is None:
291
+ return ()
292
+ if isinstance(value, str):
293
+ return (value,)
294
+ if not isinstance(value, Sequence):
295
+ raise TypeError(f"{field_name} must be a string or sequence of strings")
296
+ return tuple(value)
297
+
298
+
299
+ def _required_text(value: Any, field_name: str) -> str:
300
+ if not isinstance(value, str):
301
+ raise TypeError(f"{field_name} must be str, got {type(value).__name__}")
302
+ cleaned = " ".join(value.split())
303
+ if not cleaned:
304
+ raise ValueError(f"{field_name} must not be empty")
305
+ return cleaned
306
+
307
+
308
+ def _clean_text_values(values: Sequence[str], field_name: str) -> tuple[str, ...]:
309
+ cleaned: list[str] = []
310
+ seen: set[str] = set()
311
+ for index, value in enumerate(values):
312
+ item = _required_text(value, f"{field_name}[{index}]")
313
+ key = item.casefold()
314
+ if key not in seen:
315
+ seen.add(key)
316
+ cleaned.append(item)
317
+ return tuple(cleaned)
318
+
319
+
320
+ def _duplicate_concept_ids(
321
+ concepts: Sequence[PlaceCategoryConcept],
322
+ ) -> tuple[str, ...]:
323
+ seen: set[str] = set()
324
+ duplicates: list[str] = []
325
+ for concept in concepts:
326
+ key = concept.id.casefold()
327
+ if key in seen:
328
+ duplicates.append(concept.id)
329
+ seen.add(key)
330
+ return tuple(duplicates)
331
+
332
+
333
+ def _semantic_texts(concept: PlaceCategoryConcept) -> tuple[str, ...]:
334
+ candidates = (
335
+ f"{concept.label}. {concept.description}",
336
+ concept.label,
337
+ concept.description,
338
+ *concept.examples,
339
+ )
340
+ return _clean_text_values(candidates, "semantic_texts")
341
+
342
+
343
+ def _validated_vector(
344
+ raw_vector: Sequence[float],
345
+ *,
346
+ context: str,
347
+ ) -> tuple[float, ...] | None:
348
+ try:
349
+ vector = tuple(float(value) for value in raw_vector)
350
+ except (TypeError, ValueError) as exc:
351
+ raise ValueError(f"embedding for {context} is not numeric") from exc
352
+ if not vector or not any(value != 0.0 for value in vector):
353
+ return None
354
+ if not all(math.isfinite(value) for value in vector):
355
+ raise ValueError(f"embedding for {context} contains NaN or infinity")
356
+ return vector
357
+
358
+
359
+ def _cosine_similarity(
360
+ left: Sequence[float],
361
+ right: Sequence[float],
362
+ ) -> float:
363
+ dot = sum(a * b for a, b in zip(left, right))
364
+ left_norm = math.sqrt(sum(value * value for value in left))
365
+ right_norm = math.sqrt(sum(value * value for value in right))
366
+ if left_norm == 0.0 or right_norm == 0.0:
367
+ return -1.0
368
+ return max(-1.0, min(1.0, dot / (left_norm * right_norm)))
app/modules/places/infrastructure/place_category_catalog.py ADDED
@@ -0,0 +1,87 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """Load an open-vocabulary place concept catalog from data, not code rules."""
2
+
3
+ from __future__ import annotations
4
+
5
+ import json
6
+ from collections.abc import Iterable, Mapping
7
+ from pathlib import Path
8
+ from typing import Any
9
+
10
+ from app.modules.places.infrastructure.open_vocabulary_category_classifier import (
11
+ PlaceCategoryConcept,
12
+ )
13
+ from app.modules.places.infrastructure.place_semantic_document import PlaceTag
14
+ from app.shared.nlp.preprocessing.text import prepare_for_embedding
15
+
16
+
17
+ def load_place_category_concepts(
18
+ catalog_path: str | None,
19
+ *,
20
+ fallback_tags: Iterable[PlaceTag] = (),
21
+ ) -> tuple[PlaceCategoryConcept, ...]:
22
+ """Load concepts exported by the source system, or derive them from tag data.
23
+
24
+ The external catalog format is ``{"concepts": [...]}`` (a bare list is also
25
+ accepted). It can be refreshed independently of an application deployment.
26
+ The bundled tag inventory is only a backwards-compatible data fallback; no
27
+ category names, aliases or linguistic rules live in this module.
28
+ """
29
+
30
+ if catalog_path:
31
+ path = Path(catalog_path).expanduser()
32
+ if not path.is_file():
33
+ raise FileNotFoundError(f"Places category catalog not found at {path}")
34
+ payload = json.loads(path.read_text(encoding="utf-8"))
35
+ records = payload.get("concepts") if isinstance(payload, Mapping) else payload
36
+ if not isinstance(records, list):
37
+ raise ValueError("Places category catalog must contain a concepts list")
38
+ return tuple(_concept_from_record(record) for record in records)
39
+
40
+ return _concepts_from_tags(fallback_tags)
41
+
42
+
43
+ def _concepts_from_tags(tags: Iterable[PlaceTag]) -> tuple[PlaceCategoryConcept, ...]:
44
+ concepts: list[PlaceCategoryConcept] = []
45
+ seen: set[str] = set()
46
+ for tag in tags:
47
+ if prepare_for_embedding(tag.category).replace(" ", "_") != "place_category":
48
+ continue
49
+ normalized = prepare_for_embedding(tag.name)
50
+ concept_id = normalized.replace(" ", "_")
51
+ if not concept_id or concept_id in seen:
52
+ continue
53
+ seen.add(concept_id)
54
+ concepts.append(
55
+ PlaceCategoryConcept(
56
+ id=concept_id,
57
+ label=tag.name,
58
+ description=f"Tipo de lugar registrado: {tag.name}",
59
+ storage_values=tuple(dict.fromkeys((tag.name, normalized, concept_id))),
60
+ )
61
+ )
62
+ return tuple(concepts)
63
+
64
+
65
+ def _concept_from_record(record: Any) -> PlaceCategoryConcept:
66
+ if not isinstance(record, Mapping):
67
+ raise ValueError("Each Places category concept must be an object")
68
+ try:
69
+ return PlaceCategoryConcept(
70
+ id=str(record["id"]),
71
+ label=str(record["label"]),
72
+ description=str(record["description"]),
73
+ examples=_text_tuple(record.get("examples")),
74
+ storage_values=_text_tuple(record.get("storage_values")),
75
+ )
76
+ except KeyError as exc:
77
+ raise ValueError(f"Missing Places category concept field: {exc.args[0]}") from exc
78
+
79
+
80
+ def _text_tuple(value: Any) -> tuple[str, ...]:
81
+ if value is None:
82
+ return ()
83
+ if isinstance(value, str):
84
+ return (value,)
85
+ if not isinstance(value, list):
86
+ raise ValueError("Concept examples and storage_values must be lists")
87
+ return tuple(str(item) for item in value)
app/modules/places/infrastructure/place_semantic_document.py CHANGED
@@ -4,42 +4,16 @@ import json
4
  from pathlib import Path
5
  from typing import Any
6
 
7
- from app.shared.nlp.embeddings.weighted_document import build_weighted_document
8
  from app.shared.nlp.preprocessing.text import clean_text
9
 
10
 
11
  PLACE_SEMANTIC_FIELD_WEIGHTS = {
12
- "tags": 6,
13
- "category": 4,
14
- "description": 3,
15
  "name": 1,
16
  }
17
- PLACE_SEMANTIC_DOCUMENT_VERSION = "weighted-tags-v2"
18
-
19
-
20
- # Broad categories from the main API are expanded into Spanish intent terms so
21
- # sparse OSM records still have a useful semantic anchor.
22
- CATEGORY_SEMANTIC_PROFILES = {
23
- "restaurant": "restaurant restaurante comida gastronomia comer cena almuerzo desayuno",
24
- "cafe": "cafe cafeteria bebidas desayuno postres conversar",
25
- "bar": "bar bebidas cocteles cerveza amigos musica noche",
26
- "nightlife": "nightlife vida nocturna noche baile musica bar fiesta",
27
- "shopping": "shopping compras tiendas ropa calzado productos mercado centro comercial",
28
- "lodging": "lodging alojamiento hotel hospedaje hostal dormir turismo viaje",
29
- "park": "park parque naturaleza caminar paseo aire libre mascotas ejercicio",
30
- "culture": "culture cultura museo arte historia biblioteca exposicion lectura",
31
- "tourism": "tourism turismo atraccion visitar explorar paseo historia",
32
- "sports": "sports deporte ejercicio entrenamiento gimnasio actividad fisica",
33
- "community": "community comunidad convivencia reuniones centro comunitario actividades",
34
- "family": "family familia ninos juegos convivencia actividades familiares",
35
- "entertainment": "entertainment entretenimiento diversion juegos cine actividades",
36
- "cinema": "cinema cine pelicula estreno sala de cine entretenimiento",
37
- "library": "library biblioteca libros lectura estudio cultura",
38
- "bakery": "bakery panaderia pan pasteleria reposteria",
39
- "ice_cream": "ice cream heladeria helado postres dessert",
40
- "market": "market mercado tianguis compras productos locales",
41
- "outdoors": "outdoors aire libre mirador sendero naturaleza aventura",
42
- }
43
 
44
 
45
  @dataclass(frozen=True)
@@ -63,26 +37,28 @@ def build_place_semantic_document(
63
  description: str,
64
  resolved_tags: ResolvedPlaceTags,
65
  ) -> str:
66
- tags_text = " ".join(resolved_tags.names)
67
- return build_weighted_document(
68
- [
69
- (name, PLACE_SEMANTIC_FIELD_WEIGHTS["name"]),
70
- (
71
- semantic_category_text(category),
72
- PLACE_SEMANTIC_FIELD_WEIGHTS["category"],
73
- ),
74
- (description, PLACE_SEMANTIC_FIELD_WEIGHTS["description"]),
75
- (tags_text, PLACE_SEMANTIC_FIELD_WEIGHTS["tags"]),
76
- ]
77
  )
 
 
 
 
 
78
 
79
 
80
  def semantic_category_text(category: Any) -> str:
81
  raw_category = clean_text(str(category or "")).casefold().replace("_", " ")
82
  if not raw_category:
83
  return ""
84
- profile_key = raw_category.replace(" ", "_")
85
- return CATEGORY_SEMANTIC_PROFILES.get(profile_key, raw_category)
86
 
87
 
88
  def resolve_place_tags(value: Any) -> ResolvedPlaceTags:
 
4
  from pathlib import Path
5
  from typing import Any
6
 
 
7
  from app.shared.nlp.preprocessing.text import clean_text
8
 
9
 
10
  PLACE_SEMANTIC_FIELD_WEIGHTS = {
11
+ "tags": 1,
12
+ "category": 1,
13
+ "description": 1,
14
  "name": 1,
15
  }
16
+ PLACE_SEMANTIC_DOCUMENT_VERSION = "structured-place-v3"
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
17
 
18
 
19
  @dataclass(frozen=True)
 
37
  description: str,
38
  resolved_tags: ResolvedPlaceTags,
39
  ) -> str:
40
+ # Transformer encoders use context and sentence structure; repeating tokens
41
+ # to simulate weights (the old FastText strategy) distorts that context.
42
+ # Keep every source value once and expose its role explicitly instead.
43
+ fields = (
44
+ ("Nombre", clean_text(name)),
45
+ ("Tipo registrado", semantic_category_text(category)),
46
+ ("Descripcion", clean_text(description)),
47
+ ("Etiquetas", " ".join(resolved_tags.names)),
48
+ ("Familias de etiquetas", " ".join(resolved_tags.categories)),
 
 
49
  )
50
+ return " ".join(
51
+ f"{label}: {value}."
52
+ for label, value in fields
53
+ if value
54
+ ).strip()
55
 
56
 
57
  def semantic_category_text(category: Any) -> str:
58
  raw_category = clean_text(str(category or "")).casefold().replace("_", " ")
59
  if not raw_category:
60
  return ""
61
+ return raw_category
 
62
 
63
 
64
  def resolve_place_tags(value: Any) -> ResolvedPlaceTags:
app/modules/places/infrastructure/semantic_activity_classifier.py CHANGED
@@ -1,204 +1,56 @@
1
- import math
2
- from typing import Sequence
3
 
4
- from app.modules.places.domain.chat_intent import PlaceCategoryInference
5
- from app.shared.nlp.embeddings.base import EmbeddingProvider
6
- from app.shared.nlp.preprocessing.text import tokenize_for_embeddings
 
7
 
 
8
 
9
- _CATEGORY_PROTOTYPES: dict[str, tuple[str, ...]] = {
10
- "restaurant": (
11
- "comer comida restaurante",
12
- "hambre tacos pizza sushi",
13
- "desayunar almorzar cenar",
14
- "antojo platillo cocina",
15
- ),
16
- "cafe": (
17
- "cafe cafeteria capuchino espresso",
18
- "tomar cafe merendar conversar",
19
- "trabajar laptop cafe",
20
- ),
21
- "park": (
22
- "parque picnic caminar relajarse",
23
- "juegos infantiles areas verdes",
24
- "pasear familia cesped",
25
- ),
26
- "bar": (
27
- "cerveza cocteles tragos bar",
28
- "beber copas cantina",
29
- ),
30
- "nightlife": (
31
- "bailar fiesta discoteca antro",
32
- "fiesta musica vida nocturna",
33
- ),
34
- "culture": (
35
- "museo arte exposicion cultura",
36
- "historia galeria centro cultural",
37
- ),
38
- "shopping": (
39
- "comprar tiendas centro comercial",
40
- "ropa regalos compras",
41
- ),
42
- "sports": (
43
- "ejercicio entrenar gimnasio deporte",
44
- "futbol cancha nadar fitness",
45
- ),
46
- "bakery": (
47
- "pan pasteles panaderia reposteria",
48
- "comprar pan pastel",
49
- ),
50
- "ice_cream": (
51
- "helado postre heladeria dulce",
52
- "comer helado nieve",
53
- ),
54
- "cinema": (
55
- "pelicula cine estreno",
56
- "ver pelicula pantalla",
57
- ),
58
- "library": (
59
- "leer estudiar libros biblioteca",
60
- "lectura investigacion biblioteca",
61
- ),
62
- "market": (
63
- "mercado tianguis productos locales",
64
- "puestos comprar alimentos mercado",
65
- ),
66
- "outdoors": (
67
- "senderismo naturaleza montana mirador",
68
- "aventura aire libre paisaje",
69
- ),
70
- "lodging": (
71
- "dormir hotel hospedaje alojamiento",
72
- "pasar noche hostal",
73
- ),
74
- }
75
 
76
- _GENERIC_SINGLE_TOKEN_INTENTS = {
77
- "divertirme",
78
- "pasear",
79
- "relajarme",
80
- "salir",
81
- }
 
82
 
83
 
84
  class SemanticPlaceActivityClassifier:
85
- """Nearest-prototype classifier over the already-loaded local embeddings."""
86
 
87
  def __init__(
88
  self,
89
  embedding_provider: EmbeddingProvider,
 
 
 
90
  minimum_similarity: float = 0.44,
91
  minimum_margin: float = 0.04,
92
- window_size: int = 8,
93
  ) -> None:
94
- if not -1.0 <= minimum_similarity <= 1.0:
95
- raise ValueError("minimum_similarity must be between -1 and 1")
96
- if not 0.0 <= minimum_margin <= 2.0:
97
- raise ValueError("minimum_margin must be between 0 and 2")
98
- if window_size < 2:
99
  raise ValueError("window_size must be at least 2")
100
-
101
- self._embedding_provider = embedding_provider
102
- self._minimum_similarity = minimum_similarity
103
- self._minimum_margin = minimum_margin
104
- self._window_size = window_size
105
- self._prototype_vectors = {
106
- category: tuple(
107
- vector
108
- for vector in embedding_provider.embed_batch(list(prototypes))
109
- if _has_magnitude(vector)
110
- )
111
- for category, prototypes in _CATEGORY_PROTOTYPES.items()
112
- }
113
-
114
- def classify(self, text: str) -> PlaceCategoryInference | None:
115
- tokens = tokenize_for_embeddings(text)
116
- if not tokens:
117
- return None
118
- if len(tokens) == 1 and tokens[0] in _GENERIC_SINGLE_TOKEN_INTENTS:
119
- return None
120
-
121
- query_vectors = tuple(
122
- vector
123
- for vector in self._embedding_provider.embed_batch(
124
- self._query_segments(tokens)
125
- )
126
- if _has_magnitude(vector)
127
  )
128
- if not query_vectors:
129
- return None
130
 
131
- scores = sorted(
132
- (
133
- (
134
- category,
135
- max(
136
- _cosine_similarity(query, prototype)
137
- for query in query_vectors
138
- for prototype in prototypes
139
- ),
140
- )
141
- for category, prototypes in self._prototype_vectors.items()
142
- if prototypes
143
- ),
144
- key=lambda item: (-item[1], item[0]),
145
- )
146
- if not scores:
147
- return None
148
 
149
- category, best_score = scores[0]
150
- second_score = scores[1][1] if len(scores) > 1 else -1.0
151
- margin = best_score - second_score
152
- if (
153
- best_score < self._minimum_similarity
154
- or margin < self._minimum_margin
155
- ):
156
- return None
157
-
158
- confidence = min(
159
- 0.92,
160
- max(
161
- 0.74,
162
- 0.74
163
- + (best_score - self._minimum_similarity) * 0.30
164
- + min(margin, 0.20) * 0.35,
165
- ),
166
- )
167
- return PlaceCategoryInference(
168
- category=category,
169
- confidence=confidence,
170
- source="semantic_activity",
171
- )
172
-
173
- def _query_segments(self, tokens: list[str]) -> list[str]:
174
- full_text = " ".join(tokens)
175
- if len(tokens) <= self._window_size:
176
- return [full_text]
177
-
178
- segments = [full_text]
179
- stride = max(2, self._window_size // 2)
180
- for start in range(0, len(tokens), stride):
181
- window = tokens[start : start + self._window_size]
182
- if len(window) >= 2:
183
- segments.append(" ".join(window))
184
- if start + self._window_size >= len(tokens):
185
- break
186
- return list(dict.fromkeys(segments))
187
-
188
-
189
- def _has_magnitude(vector: Sequence[float]) -> bool:
190
- return any(float(value) != 0.0 for value in vector)
191
 
 
 
192
 
193
- def _cosine_similarity(
194
- left: Sequence[float],
195
- right: Sequence[float],
196
- ) -> float:
197
- if len(left) != len(right) or not left:
198
- return -1.0
199
- dot = sum(float(a) * float(b) for a, b in zip(left, right))
200
- left_norm = math.sqrt(sum(float(value) ** 2 for value in left))
201
- right_norm = math.sqrt(sum(float(value) ** 2 for value in right))
202
- if left_norm == 0.0 or right_norm == 0.0:
203
- return -1.0
204
- return max(-1.0, min(1.0, dot / (left_norm * right_norm)))
 
1
+ """Backward-compatible facade for the data-driven category aligner.
 
2
 
3
+ The previous implementation embedded a fixed category/prototype dictionary in
4
+ Python. This facade keeps the import path for callers while requiring concepts
5
+ to come from configuration, the source API, or a catalog export.
6
+ """
7
 
8
+ from __future__ import annotations
9
 
10
+ from collections.abc import Mapping, Sequence
11
+ from typing import Any
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
12
 
13
+ from app.modules.places.domain.chat_intent import PlaceCategoryInference
14
+ from app.modules.places.infrastructure.open_vocabulary_category_classifier import (
15
+ OpenVocabularyPlaceCategoryClassifier,
16
+ PlaceCategoryConcept,
17
+ PlaceCategoryMatch,
18
+ )
19
+ from app.shared.nlp.embeddings.base import EmbeddingProvider
20
 
21
 
22
  class SemanticPlaceActivityClassifier:
23
+ """Compatibility adapter with no built-in categories or lexical rules."""
24
 
25
  def __init__(
26
  self,
27
  embedding_provider: EmbeddingProvider,
28
+ concepts: Sequence[PlaceCategoryConcept | Mapping[str, Any]],
29
+ *,
30
+ concept_embedding_provider: EmbeddingProvider | None = None,
31
  minimum_similarity: float = 0.44,
32
  minimum_margin: float = 0.04,
33
+ window_size: int | None = None,
34
  ) -> None:
35
+ if window_size is not None and window_size < 2:
 
 
 
 
36
  raise ValueError("window_size must be at least 2")
37
+ self._delegate = OpenVocabularyPlaceCategoryClassifier(
38
+ concepts=concepts,
39
+ embedding_provider=embedding_provider,
40
+ concept_embedding_provider=concept_embedding_provider,
41
+ minimum_similarity=minimum_similarity,
42
+ minimum_margin=minimum_margin,
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
43
  )
 
 
44
 
45
+ @property
46
+ def concepts(self) -> tuple[PlaceCategoryConcept, ...]:
47
+ return self._delegate.concepts
 
 
 
 
 
 
 
 
 
 
 
 
 
 
48
 
49
+ def get_concept(self, concept_id: str) -> PlaceCategoryConcept | None:
50
+ return self._delegate.get_concept(concept_id)
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
51
 
52
+ def classify(self, text: str) -> PlaceCategoryInference | None:
53
+ return self._delegate.classify(text)
54
 
55
+ def rank(self, text: str, limit: int = 3) -> tuple[PlaceCategoryMatch, ...]:
56
+ return self._delegate.rank(text, limit=limit)
 
 
 
 
 
 
 
 
 
 
app/modules/places/infrastructure/semantic_place_ranker.py CHANGED
@@ -10,11 +10,16 @@ from app.modules.places.infrastructure.place_semantic_document import (
10
  class SemanticPlaceRanker(PlaceRanker):
11
  """Preserve the cosine-similarity order returned by PGVector."""
12
 
13
- engine_name = "fasttext_mean_embeddings"
14
  score_metric = "cosine_similarity"
15
  field_weights = PLACE_SEMANTIC_FIELD_WEIGHTS
16
 
17
- def __init__(self, dimension: int = 300) -> None:
 
 
 
 
 
18
  self.ranking_parameters = {"dimension": float(dimension)}
19
 
20
  def rank(
 
10
  class SemanticPlaceRanker(PlaceRanker):
11
  """Preserve the cosine-similarity order returned by PGVector."""
12
 
13
+ engine_name = "semantic_embeddings"
14
  score_metric = "cosine_similarity"
15
  field_weights = PLACE_SEMANTIC_FIELD_WEIGHTS
16
 
17
+ def __init__(
18
+ self,
19
+ dimension: int = 300,
20
+ model_name: str = "semantic_embeddings",
21
+ ) -> None:
22
+ self.engine_name = model_name
23
  self.ranking_parameters = {"dimension": float(dimension)}
24
 
25
  def rank(
app/shared/config/settings.py CHANGED
@@ -167,6 +167,82 @@ class Settings(BaseSettings):
167
  alias="FASTTEXT_AUTO_DOWNLOAD",
168
  )
169
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
170
  bm25_k1: float = Field(default=1.5, gt=0, alias="BM25_K1")
171
  bm25_b: float = Field(default=0.75, ge=0, le=1, alias="BM25_B")
172
  bm25_relevance_threshold: float = Field(
@@ -218,6 +294,24 @@ class Settings(BaseSettings):
218
  le=1.0,
219
  alias="PLACES_CHAT_AMBIGUITY_DELTA",
220
  )
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
221
  places_chat_ranking_version: str = Field(
222
  default="places-chat-v2",
223
  min_length=1,
@@ -230,6 +324,31 @@ class Settings(BaseSettings):
230
  max_length=64,
231
  alias="PLACES_CHAT_TAXONOMY_VERSION",
232
  )
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
233
 
234
  log_level: str = Field(default="INFO", alias="LOG_LEVEL")
235
  request_timeout_seconds: int = Field(
@@ -291,6 +410,61 @@ class Settings(BaseSettings):
291
  @model_validator(mode="after")
292
  def validate_post_feed_security(self) -> "Settings":
293
  self.vector_store_provider = self.vector_store_provider.strip().lower()
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
294
  allowed_resources = {"places", "posts", "users", "clubs", "groups", "events"}
295
  for resource_type, thresholds in self.global_search_resource_thresholds.items():
296
  if resource_type not in allowed_resources:
 
167
  alias="FASTTEXT_AUTO_DOWNLOAD",
168
  )
169
 
170
+ # Places can migrate independently from posts, global search and feed
171
+ # embeddings. Defaults preserve the current FastText contract; enabling a
172
+ # Sentence-Transformer is an explicit, reversible deployment choice.
173
+ places_embedding_provider: str = Field(
174
+ default="fasttext",
175
+ alias="PLACES_EMBEDDING_PROVIDER",
176
+ )
177
+ places_embedding_dimension: int = Field(
178
+ default=300,
179
+ gt=0,
180
+ alias="PLACES_EMBEDDING_DIMENSION",
181
+ )
182
+ places_embedding_model: str = Field(
183
+ default="facebook/fasttext-es-vectors",
184
+ min_length=1,
185
+ alias="PLACES_EMBEDDING_MODEL",
186
+ )
187
+ places_embedding_version: str = Field(
188
+ default="common-crawl-300-v1",
189
+ min_length=1,
190
+ alias="PLACES_EMBEDDING_VERSION",
191
+ )
192
+ places_embedding_query_prefix: str = Field(
193
+ default="",
194
+ alias="PLACES_EMBEDDING_QUERY_PREFIX",
195
+ )
196
+ places_embedding_passage_prefix: str = Field(
197
+ default="",
198
+ alias="PLACES_EMBEDDING_PASSAGE_PREFIX",
199
+ )
200
+ places_embedding_batch_size: int = Field(
201
+ default=32,
202
+ ge=1,
203
+ le=512,
204
+ alias="PLACES_EMBEDDING_BATCH_SIZE",
205
+ )
206
+ places_embedding_device: str | None = Field(
207
+ default=None,
208
+ alias="PLACES_EMBEDDING_DEVICE",
209
+ )
210
+ places_category_catalog_path: str | None = Field(
211
+ default=None,
212
+ alias="PLACES_CATEGORY_CATALOG_PATH",
213
+ )
214
+ places_category_min_similarity: float = Field(
215
+ default=0.44,
216
+ ge=-1.0,
217
+ le=1.0,
218
+ alias="PLACES_CATEGORY_MIN_SIMILARITY",
219
+ )
220
+ places_category_min_margin: float = Field(
221
+ default=0.04,
222
+ ge=0.0,
223
+ le=2.0,
224
+ alias="PLACES_CATEGORY_MIN_MARGIN",
225
+ )
226
+ places_pgvector_match_function: str = Field(
227
+ default="match_places",
228
+ min_length=1,
229
+ alias="PLACES_PGVECTOR_MATCH_FUNCTION",
230
+ )
231
+ places_pgvector_hybrid_function: str | None = Field(
232
+ default=None,
233
+ alias="PLACES_PGVECTOR_HYBRID_FUNCTION",
234
+ )
235
+ places_pgvector_upsert_function: str = Field(
236
+ default="upsert_place_embedding",
237
+ min_length=1,
238
+ alias="PLACES_PGVECTOR_UPSERT_FUNCTION",
239
+ )
240
+ places_pgvector_hash_function: str = Field(
241
+ default="get_place_content_hashes",
242
+ min_length=1,
243
+ alias="PLACES_PGVECTOR_HASH_FUNCTION",
244
+ )
245
+
246
  bm25_k1: float = Field(default=1.5, gt=0, alias="BM25_K1")
247
  bm25_b: float = Field(default=0.75, ge=0, le=1, alias="BM25_B")
248
  bm25_relevance_threshold: float = Field(
 
294
  le=1.0,
295
  alias="PLACES_CHAT_AMBIGUITY_DELTA",
296
  )
297
+ places_chat_hypothesis_min_confidence: float = Field(
298
+ default=0.60,
299
+ ge=0.0,
300
+ le=1.0,
301
+ alias="PLACES_CHAT_HYPOTHESIS_MIN_CONFIDENCE",
302
+ )
303
+ places_chat_hypothesis_max_gap: float = Field(
304
+ default=0.15,
305
+ ge=0.0,
306
+ le=1.0,
307
+ alias="PLACES_CHAT_HYPOTHESIS_MAX_GAP",
308
+ )
309
+ places_chat_default_radius_meters: int = Field(
310
+ default=5_000,
311
+ ge=1,
312
+ le=50_000,
313
+ alias="PLACES_CHAT_DEFAULT_RADIUS_METERS",
314
+ )
315
  places_chat_ranking_version: str = Field(
316
  default="places-chat-v2",
317
  min_length=1,
 
324
  max_length=64,
325
  alias="PLACES_CHAT_TAXONOMY_VERSION",
326
  )
327
+ # Contextual intent extraction is opt-in. ``disabled`` is accepted as an
328
+ # operational alias for the deterministic-only path so deployments can
329
+ # explicitly turn the optional model off without changing code.
330
+ places_chat_intent_provider: str = Field(
331
+ default="deterministic",
332
+ alias="PLACES_CHAT_INTENT_PROVIDER",
333
+ )
334
+ places_chat_bert_model_path: str | None = Field(
335
+ default=None,
336
+ alias="PLACES_CHAT_BERT_MODEL_PATH",
337
+ )
338
+ places_chat_bert_model_version: str | None = Field(
339
+ default=None,
340
+ alias="PLACES_CHAT_BERT_MODEL_VERSION",
341
+ )
342
+ places_chat_bert_device: str | None = Field(
343
+ default=None,
344
+ alias="PLACES_CHAT_BERT_DEVICE",
345
+ )
346
+ places_chat_bert_min_token_confidence: float = Field(
347
+ default=0.60,
348
+ ge=0.0,
349
+ le=1.0,
350
+ alias="PLACES_CHAT_BERT_MIN_TOKEN_CONFIDENCE",
351
+ )
352
 
353
  log_level: str = Field(default="INFO", alias="LOG_LEVEL")
354
  request_timeout_seconds: int = Field(
 
410
  @model_validator(mode="after")
411
  def validate_post_feed_security(self) -> "Settings":
412
  self.vector_store_provider = self.vector_store_provider.strip().lower()
413
+ self.places_embedding_provider = self.places_embedding_provider.strip().lower()
414
+ if self.places_embedding_device is not None:
415
+ self.places_embedding_device = (
416
+ self.places_embedding_device.strip() or None
417
+ )
418
+ if self.places_category_catalog_path is not None:
419
+ self.places_category_catalog_path = (
420
+ self.places_category_catalog_path.strip() or None
421
+ )
422
+ if self.places_pgvector_hybrid_function is not None:
423
+ self.places_pgvector_hybrid_function = (
424
+ self.places_pgvector_hybrid_function.strip() or None
425
+ )
426
+ if self.places_embedding_provider not in {
427
+ "fasttext",
428
+ "mock",
429
+ "sentence_transformer",
430
+ "bert",
431
+ }:
432
+ raise ValueError(
433
+ "PLACES_EMBEDDING_PROVIDER debe ser fasttext, mock, "
434
+ "sentence_transformer o bert"
435
+ )
436
+ self.places_chat_intent_provider = (
437
+ self.places_chat_intent_provider.strip().lower()
438
+ )
439
+ if self.places_chat_intent_provider not in {
440
+ "disabled",
441
+ "deterministic",
442
+ "bert",
443
+ }:
444
+ raise ValueError(
445
+ "PLACES_CHAT_INTENT_PROVIDER debe ser disabled, "
446
+ "deterministic o bert"
447
+ )
448
+ if self.places_chat_bert_model_path is not None:
449
+ self.places_chat_bert_model_path = (
450
+ self.places_chat_bert_model_path.strip() or None
451
+ )
452
+ if self.places_chat_bert_model_version is not None:
453
+ self.places_chat_bert_model_version = (
454
+ self.places_chat_bert_model_version.strip() or None
455
+ )
456
+ if self.places_chat_bert_device is not None:
457
+ self.places_chat_bert_device = (
458
+ self.places_chat_bert_device.strip() or None
459
+ )
460
+ if (
461
+ self.places_chat_intent_provider == "bert"
462
+ and self.places_chat_bert_model_path is None
463
+ ):
464
+ raise ValueError(
465
+ "PLACES_CHAT_BERT_MODEL_PATH es obligatorio cuando "
466
+ "PLACES_CHAT_INTENT_PROVIDER=bert"
467
+ )
468
  allowed_resources = {"places", "posts", "users", "clubs", "groups", "events"}
469
  for resource_type, thresholds in self.global_search_resource_thresholds.items():
470
  if resource_type not in allowed_resources:
app/shared/dependencies.py CHANGED
@@ -4,7 +4,10 @@ from app.shared.cache.memory import SimpleTTLCache
4
  from app.shared.config.settings import get_settings
5
  from app.shared.nlp.embeddings.base import EmbeddingProvider
6
  from app.shared.nlp.embeddings.cached import CachedEmbeddingProvider
7
- from app.shared.nlp.embeddings.factory import create_embedding_provider
 
 
 
8
  from app.shared.nlp.llm.base import LLMProvider
9
  from app.shared.nlp.llm.groq_llama import GroqLlamaProvider
10
  from app.shared.nlp.llm.mock import MockLLMProvider
@@ -19,6 +22,32 @@ def get_embedding_provider() -> EmbeddingProvider:
19
  )
20
 
21
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
22
  @lru_cache
23
  def get_llm_provider() -> LLMProvider:
24
  settings = get_settings()
 
4
  from app.shared.config.settings import get_settings
5
  from app.shared.nlp.embeddings.base import EmbeddingProvider
6
  from app.shared.nlp.embeddings.cached import CachedEmbeddingProvider
7
+ from app.shared.nlp.embeddings.factory import (
8
+ create_embedding_provider,
9
+ create_place_embedding_provider,
10
+ )
11
  from app.shared.nlp.llm.base import LLMProvider
12
  from app.shared.nlp.llm.groq_llama import GroqLlamaProvider
13
  from app.shared.nlp.llm.mock import MockLLMProvider
 
22
  )
23
 
24
 
25
+ @lru_cache
26
+ def get_place_embedding_provider() -> EmbeddingProvider:
27
+ """Return the query encoder dedicated to Places.
28
+
29
+ It intentionally has a separate cache and configuration so a 768-dimensional
30
+ BERT rollout cannot invalidate or break the global 300-dimensional indexes.
31
+ """
32
+
33
+ settings = get_settings()
34
+ return CachedEmbeddingProvider(
35
+ provider=create_place_embedding_provider(settings, text_role="query"),
36
+ cache=SimpleTTLCache(default_ttl_seconds=settings.embedding_cache_ttl_seconds),
37
+ )
38
+
39
+
40
+ @lru_cache
41
+ def get_place_passage_embedding_provider() -> EmbeddingProvider:
42
+ """Return the Places passage encoder (for E5-style asymmetric models)."""
43
+
44
+ settings = get_settings()
45
+ return CachedEmbeddingProvider(
46
+ provider=create_place_embedding_provider(settings, text_role="passage"),
47
+ cache=SimpleTTLCache(default_ttl_seconds=settings.embedding_cache_ttl_seconds),
48
+ )
49
+
50
+
51
  @lru_cache
52
  def get_llm_provider() -> LLMProvider:
53
  settings = get_settings()
app/shared/nlp/embeddings/cached.py CHANGED
@@ -22,4 +22,32 @@ class CachedEmbeddingProvider(EmbeddingProvider):
22
  return embedding
23
 
24
  def embed_batch(self, texts: list[str]) -> list[list[float]]:
25
- return [self.embed_text(text) for text in texts]
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
22
  return embedding
23
 
24
  def embed_batch(self, texts: list[str]) -> list[list[float]]:
25
+ if not texts:
26
+ return []
27
+
28
+ results: list[list[float] | None] = [None] * len(texts)
29
+ missing_positions: dict[str, list[int]] = {}
30
+ for index, text in enumerate(texts):
31
+ cache_key = f"embedding:{text}"
32
+ cached = self._cache.get(cache_key)
33
+ if cached is not None:
34
+ results[index] = cached
35
+ else:
36
+ missing_positions.setdefault(text, []).append(index)
37
+
38
+ missing_texts = list(missing_positions)
39
+ if missing_texts:
40
+ generated = self._provider.embed_batch(missing_texts)
41
+ if len(generated) != len(missing_texts):
42
+ raise ValueError(
43
+ "Embedding provider returned an unexpected batch size: "
44
+ f"returned={len(generated)}, expected={len(missing_texts)}"
45
+ )
46
+ for text, embedding in zip(missing_texts, generated):
47
+ self._cache.set(f"embedding:{text}", embedding)
48
+ for index in missing_positions[text]:
49
+ results[index] = embedding
50
+
51
+ if any(result is None for result in results):
52
+ raise RuntimeError("Embedding batch cache left unresolved positions")
53
+ return [result for result in results if result is not None]
app/shared/nlp/embeddings/factory.py CHANGED
@@ -2,6 +2,9 @@ from app.shared.config.settings import Settings
2
  from app.shared.nlp.embeddings.base import EmbeddingProvider
3
  from app.shared.nlp.embeddings.fasttext import FastTextEmbeddingProvider
4
  from app.shared.nlp.embeddings.mock import MockEmbeddingProvider
 
 
 
5
 
6
 
7
  def create_embedding_provider(settings: Settings) -> EmbeddingProvider:
@@ -17,3 +20,50 @@ def create_embedding_provider(settings: Settings) -> EmbeddingProvider:
17
  if provider == "mock":
18
  return MockEmbeddingProvider(dimension=settings.embedding_dimension)
19
  raise ValueError(f"Unsupported EMBEDDING_PROVIDER: {settings.embedding_provider}")
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
2
  from app.shared.nlp.embeddings.base import EmbeddingProvider
3
  from app.shared.nlp.embeddings.fasttext import FastTextEmbeddingProvider
4
  from app.shared.nlp.embeddings.mock import MockEmbeddingProvider
5
+ from app.shared.nlp.embeddings.sentence_transformer import (
6
+ SentenceTransformerEmbeddingProvider,
7
+ )
8
 
9
 
10
  def create_embedding_provider(settings: Settings) -> EmbeddingProvider:
 
20
  if provider == "mock":
21
  return MockEmbeddingProvider(dimension=settings.embedding_dimension)
22
  raise ValueError(f"Unsupported EMBEDDING_PROVIDER: {settings.embedding_provider}")
23
+
24
+
25
+ def create_place_embedding_provider(
26
+ settings: Settings,
27
+ *,
28
+ text_role: str = "query",
29
+ ) -> EmbeddingProvider:
30
+ """Create the Places-only embedding provider.
31
+
32
+ Keeping this separate from ``create_embedding_provider`` prevents a Places
33
+ migration from changing post, feed or global-search vector dimensions.
34
+ ``text_role`` selects the query/passage prefix required by retrieval models
35
+ such as E5 while sharing the same model weights.
36
+ """
37
+
38
+ if text_role not in {"query", "passage"}:
39
+ raise ValueError("text_role must be 'query' or 'passage'")
40
+
41
+ provider = settings.places_embedding_provider.casefold()
42
+ if provider == "fasttext":
43
+ return FastTextEmbeddingProvider(
44
+ model_path=settings.fasttext_model_path,
45
+ expected_dimension=settings.places_embedding_dimension,
46
+ repo_id=settings.fasttext_model_repo_id,
47
+ filename=settings.fasttext_model_filename,
48
+ auto_download=settings.fasttext_auto_download,
49
+ )
50
+ if provider == "mock":
51
+ return MockEmbeddingProvider(dimension=settings.places_embedding_dimension)
52
+ if provider in {"sentence_transformer", "bert"}:
53
+ prefix = (
54
+ settings.places_embedding_query_prefix
55
+ if text_role == "query"
56
+ else settings.places_embedding_passage_prefix
57
+ )
58
+ return SentenceTransformerEmbeddingProvider(
59
+ model_name_or_path=settings.places_embedding_model,
60
+ expected_dimension=settings.places_embedding_dimension,
61
+ batch_size=settings.places_embedding_batch_size,
62
+ device=settings.places_embedding_device,
63
+ text_prefix=prefix,
64
+ normalize_embeddings=True,
65
+ )
66
+ raise ValueError(
67
+ "Unsupported PLACES_EMBEDDING_PROVIDER: "
68
+ f"{settings.places_embedding_provider}"
69
+ )
app/shared/nlp/embeddings/sentence_transformer.py ADDED
@@ -0,0 +1,288 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """Lazy Sentence-Transformer embeddings with strict output validation.
2
+
3
+ The optional ``sentence-transformers`` dependency is intentionally imported only
4
+ when the first non-empty text is embedded. This keeps application startup and
5
+ test discovery independent from heavyweight model loading.
6
+ """
7
+
8
+ from __future__ import annotations
9
+
10
+ import math
11
+ import threading
12
+ from collections.abc import Callable, Sequence
13
+ from typing import Any, Protocol
14
+
15
+ from app.shared.nlp.embeddings.base import EmbeddingProvider
16
+
17
+
18
+ class SentenceTransformerModel(Protocol):
19
+ """Small portion of the SentenceTransformer API used by this provider."""
20
+
21
+ def encode(self, sentences: Sequence[str], **kwargs: Any) -> Any: ...
22
+
23
+ def get_sentence_embedding_dimension(self) -> int | None: ...
24
+
25
+
26
+ ModelLoader = Callable[[str, str | None], SentenceTransformerModel]
27
+
28
+
29
+ _SHARED_MODELS: dict[tuple[str, str | None], SentenceTransformerModel] = {}
30
+ _SHARED_MODELS_LOCK = threading.Lock()
31
+ _MODEL_INFERENCE_LOCKS: dict[int, threading.Lock] = {}
32
+
33
+
34
+ class SentenceTransformerEmbeddingError(RuntimeError):
35
+ """Base error raised when a sentence embedding cannot be produced safely."""
36
+
37
+
38
+ class SentenceTransformerModelLoadError(SentenceTransformerEmbeddingError):
39
+ """Raised when the configured model or its runtime cannot be loaded."""
40
+
41
+
42
+ class SentenceTransformerInferenceError(SentenceTransformerEmbeddingError):
43
+ """Raised when model inference fails."""
44
+
45
+
46
+ class SentenceTransformerDimensionError(ValueError):
47
+ """Raised when a model returns a vector with an unexpected dimension."""
48
+
49
+
50
+ class SentenceTransformerEmbeddingProvider(EmbeddingProvider):
51
+ """Batch Sentence-Transformer provider with lazy, thread-safe model loading.
52
+
53
+ ``text_prefix`` supports retrieval models such as E5, whose query and
54
+ passage encoders use the same weights but require different input prefixes.
55
+ Configure separate provider instances for queries and documents when those
56
+ prefixes differ.
57
+ """
58
+
59
+ def __init__(
60
+ self,
61
+ model_name_or_path: str,
62
+ expected_dimension: int,
63
+ *,
64
+ batch_size: int = 32,
65
+ device: str | None = None,
66
+ text_prefix: str = "",
67
+ normalize_embeddings: bool = True,
68
+ model_loader: ModelLoader | None = None,
69
+ ) -> None:
70
+ if not model_name_or_path.strip():
71
+ raise ValueError("model_name_or_path must not be empty")
72
+ if expected_dimension <= 0:
73
+ raise ValueError("expected_dimension must be greater than zero")
74
+ if batch_size <= 0:
75
+ raise ValueError("batch_size must be greater than zero")
76
+
77
+ self.model_name_or_path = model_name_or_path
78
+ self.dimension = int(expected_dimension)
79
+ self.batch_size = int(batch_size)
80
+ self.device = device
81
+ self.text_prefix = text_prefix
82
+ self.normalize_embeddings = normalize_embeddings
83
+
84
+ self._model_loader = model_loader or _load_sentence_transformer
85
+ self._model: SentenceTransformerModel | None = None
86
+ self._model_lock = threading.Lock()
87
+
88
+ @property
89
+ def is_loaded(self) -> bool:
90
+ """Whether the heavyweight model has already been initialized."""
91
+
92
+ return self._model is not None
93
+
94
+ def embed_text(self, text: str) -> list[float]:
95
+ return self.embed_batch([text])[0]
96
+
97
+ def embed_batch(self, texts: list[str]) -> list[list[float]]:
98
+ if not texts:
99
+ return []
100
+
101
+ prepared: list[str] = []
102
+ positions: list[int] = []
103
+ for index, text in enumerate(texts):
104
+ if not isinstance(text, str):
105
+ raise TypeError(
106
+ f"texts[{index}] must be str, got {type(text).__name__}"
107
+ )
108
+ stripped = text.strip()
109
+ if stripped:
110
+ positions.append(index)
111
+ prepared.append(f"{self.text_prefix}{stripped}")
112
+
113
+ embeddings = [[0.0] * self.dimension for _ in texts]
114
+ if not prepared:
115
+ return embeddings
116
+
117
+ model = self._get_model()
118
+ try:
119
+ with _inference_lock_for(model):
120
+ raw_embeddings = model.encode(
121
+ prepared,
122
+ batch_size=self.batch_size,
123
+ convert_to_numpy=True,
124
+ normalize_embeddings=False,
125
+ show_progress_bar=False,
126
+ )
127
+ except Exception as exc:
128
+ raise SentenceTransformerInferenceError(
129
+ "Sentence-Transformer inference failed for "
130
+ f"model {self.model_name_or_path!r}: {exc}"
131
+ ) from exc
132
+
133
+ vectors = self._validate_and_convert(raw_embeddings, len(prepared))
134
+ for position, vector in zip(positions, vectors):
135
+ embeddings[position] = vector
136
+ return embeddings
137
+
138
+ def _get_model(self) -> SentenceTransformerModel:
139
+ model = self._model
140
+ if model is not None:
141
+ return model
142
+
143
+ with self._model_lock:
144
+ model = self._model
145
+ if model is not None:
146
+ return model
147
+ try:
148
+ model = self._model_loader(self.model_name_or_path, self.device)
149
+ except SentenceTransformerEmbeddingError:
150
+ raise
151
+ except Exception as exc:
152
+ raise SentenceTransformerModelLoadError(
153
+ "Could not load Sentence-Transformer model "
154
+ f"{self.model_name_or_path!r}: {exc}"
155
+ ) from exc
156
+
157
+ self._validate_model_dimension(model)
158
+ self._model = model
159
+ return model
160
+
161
+ def _validate_model_dimension(self, model: SentenceTransformerModel) -> None:
162
+ dimension_getter = getattr(
163
+ model, "get_sentence_embedding_dimension", None
164
+ )
165
+ if not callable(dimension_getter):
166
+ return
167
+ try:
168
+ reported_dimension = dimension_getter()
169
+ except Exception as exc:
170
+ raise SentenceTransformerModelLoadError(
171
+ "Could not inspect the embedding dimension for "
172
+ f"model {self.model_name_or_path!r}: {exc}"
173
+ ) from exc
174
+ if reported_dimension is None:
175
+ return
176
+ if int(reported_dimension) != self.dimension:
177
+ raise SentenceTransformerDimensionError(
178
+ "Sentence-Transformer model dimension does not match the "
179
+ "configured embedding dimension: "
180
+ f"model={reported_dimension}, configured={self.dimension}, "
181
+ f"name={self.model_name_or_path!r}"
182
+ )
183
+
184
+ def _validate_and_convert(
185
+ self,
186
+ raw_embeddings: Any,
187
+ expected_count: int,
188
+ ) -> list[list[float]]:
189
+ converted = (
190
+ raw_embeddings.tolist()
191
+ if hasattr(raw_embeddings, "tolist")
192
+ else raw_embeddings
193
+ )
194
+ try:
195
+ rows = list(converted)
196
+ except (TypeError, ValueError) as exc:
197
+ raise SentenceTransformerInferenceError(
198
+ "Sentence-Transformer returned a non-iterable embedding result"
199
+ ) from exc
200
+
201
+ # Some compatible runtimes collapse a one-item batch to one dimension.
202
+ if expected_count == 1 and rows and _is_scalar(rows[0]):
203
+ rows = [rows]
204
+ if len(rows) != expected_count:
205
+ raise SentenceTransformerInferenceError(
206
+ "Sentence-Transformer returned an unexpected number of vectors: "
207
+ f"returned={len(rows)}, expected={expected_count}"
208
+ )
209
+
210
+ vectors: list[list[float]] = []
211
+ for row_index, row in enumerate(rows):
212
+ values_source = row.tolist() if hasattr(row, "tolist") else row
213
+ try:
214
+ values = [float(value) for value in values_source]
215
+ except (TypeError, ValueError) as exc:
216
+ raise SentenceTransformerInferenceError(
217
+ "Sentence-Transformer returned a non-numeric vector at "
218
+ f"batch index {row_index}"
219
+ ) from exc
220
+ if len(values) != self.dimension:
221
+ raise SentenceTransformerDimensionError(
222
+ "Sentence-Transformer returned an unexpected vector "
223
+ f"dimension at batch index {row_index}: "
224
+ f"returned={len(values)}, expected={self.dimension}"
225
+ )
226
+ if not all(math.isfinite(value) for value in values):
227
+ raise SentenceTransformerInferenceError(
228
+ "Sentence-Transformer returned NaN or infinity at "
229
+ f"batch index {row_index}"
230
+ )
231
+ if self.normalize_embeddings:
232
+ values = _l2_normalize_nonzero(values, row_index)
233
+ vectors.append(values)
234
+ return vectors
235
+
236
+
237
+ def _load_sentence_transformer(
238
+ model_name_or_path: str,
239
+ device: str | None,
240
+ ) -> SentenceTransformerModel:
241
+ cache_key = (model_name_or_path, device)
242
+ cached = _SHARED_MODELS.get(cache_key)
243
+ if cached is not None:
244
+ return cached
245
+ with _SHARED_MODELS_LOCK:
246
+ cached = _SHARED_MODELS.get(cache_key)
247
+ if cached is not None:
248
+ return cached
249
+ try:
250
+ from sentence_transformers import SentenceTransformer
251
+ except ImportError as exc:
252
+ raise SentenceTransformerModelLoadError(
253
+ "sentence-transformers is required for BERT embeddings. "
254
+ "Install the project's optional Sentence-Transformer dependencies."
255
+ ) from exc
256
+
257
+ try:
258
+ model = SentenceTransformer(model_name_or_path, device=device)
259
+ except Exception as exc:
260
+ raise SentenceTransformerModelLoadError(
261
+ "Could not initialize Sentence-Transformer model "
262
+ f"{model_name_or_path!r}: {exc}"
263
+ ) from exc
264
+ _SHARED_MODELS[cache_key] = model
265
+ return model
266
+
267
+
268
+ def _is_scalar(value: Any) -> bool:
269
+ return isinstance(value, (int, float, complex))
270
+
271
+
272
+ def _inference_lock_for(model: SentenceTransformerModel) -> threading.Lock:
273
+ model_id = id(model)
274
+ with _SHARED_MODELS_LOCK:
275
+ return _MODEL_INFERENCE_LOCKS.setdefault(model_id, threading.Lock())
276
+
277
+
278
+ def _l2_normalize_nonzero(
279
+ vector: list[float],
280
+ row_index: int,
281
+ ) -> list[float]:
282
+ norm = math.sqrt(sum(value * value for value in vector))
283
+ if norm == 0:
284
+ raise SentenceTransformerInferenceError(
285
+ "Sentence-Transformer returned a zero-norm vector for non-empty "
286
+ f"text at batch index {row_index}"
287
+ )
288
+ return [value / norm for value in vector]
app/shared/vector_store/aws_pgvector.py CHANGED
@@ -53,14 +53,48 @@ class AwsPgvectorClient:
53
  embedding: list[float],
54
  filters: dict[str, Any],
55
  limit: int,
 
56
  ) -> list[VectorMatch]:
57
  return await self._match(
58
- function_name="match_places",
59
  embedding=embedding,
60
  filters=filters,
61
  limit=limit,
62
  )
63
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
64
  async def match_posts(
65
  self,
66
  embedding: list[float],
@@ -108,6 +142,7 @@ class AwsPgvectorClient:
108
 
109
  async def check_read_contract(self) -> dict[str, Any]:
110
  """Check that read-only pgvector functions are visible and executable."""
 
111
  try:
112
  async with self.connection() as connection:
113
  vector_row = await connection.fetchrow(
@@ -120,10 +155,10 @@ class AwsPgvectorClient:
120
  "exists": False,
121
  "executable": False,
122
  }
123
- for function_name, signature in READ_CONTRACT_SIGNATURES.items()
124
  }
125
  if vector_available:
126
- for function_name, signature in READ_CONTRACT_SIGNATURES.items():
127
  row = await connection.fetchrow(
128
  """
129
  SELECT
@@ -169,9 +204,13 @@ class AwsPgvectorClient:
169
  "functions": functions,
170
  }
171
 
172
- async def fetch_place_content_hashes(self, ids: Iterable[str]) -> dict[str, str]:
 
 
 
 
173
  return await self._fetch_content_hashes(
174
- function_name="get_place_content_hashes",
175
  ids=ids,
176
  )
177
 
@@ -205,10 +244,15 @@ class AwsPgvectorClient:
205
  async def upsert_place_embeddings(
206
  self,
207
  records: list[VectorUpsertRecord],
 
 
 
208
  ) -> None:
209
  await self._upsert_records(
210
- function_name="upsert_place_embedding",
211
  records=records,
 
 
212
  )
213
 
214
  async def upsert_post_embeddings(
@@ -323,6 +367,8 @@ class AwsPgvectorClient:
323
  self,
324
  function_name: str,
325
  records: list[VectorUpsertRecord],
 
 
326
  ) -> None:
327
  if not records:
328
  return
@@ -346,8 +392,8 @@ class AwsPgvectorClient:
346
  json.dumps(record.metadata, ensure_ascii=False),
347
  vector_literal(record.embedding),
348
  record.content_hash,
349
- self._settings.embedding_model,
350
- self._settings.embedding_version,
351
  record.is_active,
352
  )
353
  for record in records
@@ -425,6 +471,18 @@ def _read_contract_is_ready(
425
  )
426
 
427
 
 
 
 
 
 
 
 
 
 
 
 
 
428
  def _credentials_for_role(settings: Settings, role: str) -> tuple[str | None, str | None]:
429
  if role == "writer":
430
  return (
 
53
  embedding: list[float],
54
  filters: dict[str, Any],
55
  limit: int,
56
+ function_name: str = "match_places",
57
  ) -> list[VectorMatch]:
58
  return await self._match(
59
+ function_name=function_name,
60
  embedding=embedding,
61
  filters=filters,
62
  limit=limit,
63
  )
64
 
65
+ async def search_places_hybrid(
66
+ self,
67
+ query_text: str,
68
+ embedding: list[float],
69
+ filters: dict[str, Any],
70
+ limit: int,
71
+ function_name: str,
72
+ ) -> list[VectorMatch]:
73
+ """Search a versioned Places index using independent dense and lexical pools."""
74
+
75
+ query = (
76
+ f"SELECT * FROM {quote_identifier(function_name)}("
77
+ "$1::text, $2::vector, $3::integer, $4::jsonb)"
78
+ )
79
+ try:
80
+ async with self.connection() as connection:
81
+ rows = await connection.fetch(
82
+ query,
83
+ query_text,
84
+ vector_literal(embedding),
85
+ limit,
86
+ json.dumps(filters, ensure_ascii=False),
87
+ )
88
+ except asyncpg.exceptions.UndefinedFunctionError as exc:
89
+ raise AppError(
90
+ "Places hybrid SQL contract is missing. Run the versioned "
91
+ "Places semantic embedding migration and grant EXECUTE to the "
92
+ "reader role.",
93
+ code="pgvector_places_hybrid_contract_missing",
94
+ status_code=503,
95
+ ) from exc
96
+ return [_row_to_vector_match(row) for row in rows]
97
+
98
  async def match_posts(
99
  self,
100
  embedding: list[float],
 
142
 
143
  async def check_read_contract(self) -> dict[str, Any]:
144
  """Check that read-only pgvector functions are visible and executable."""
145
+ signatures = _configured_read_contract_signatures(self._settings)
146
  try:
147
  async with self.connection() as connection:
148
  vector_row = await connection.fetchrow(
 
155
  "exists": False,
156
  "executable": False,
157
  }
158
+ for function_name, signature in signatures.items()
159
  }
160
  if vector_available:
161
+ for function_name, signature in signatures.items():
162
  row = await connection.fetchrow(
163
  """
164
  SELECT
 
204
  "functions": functions,
205
  }
206
 
207
+ async def fetch_place_content_hashes(
208
+ self,
209
+ ids: Iterable[str],
210
+ function_name: str = "get_place_content_hashes",
211
+ ) -> dict[str, str]:
212
  return await self._fetch_content_hashes(
213
+ function_name=function_name,
214
  ids=ids,
215
  )
216
 
 
244
  async def upsert_place_embeddings(
245
  self,
246
  records: list[VectorUpsertRecord],
247
+ function_name: str = "upsert_place_embedding",
248
+ embedding_model: str | None = None,
249
+ embedding_version: str | None = None,
250
  ) -> None:
251
  await self._upsert_records(
252
+ function_name=function_name,
253
  records=records,
254
+ embedding_model=embedding_model,
255
+ embedding_version=embedding_version,
256
  )
257
 
258
  async def upsert_post_embeddings(
 
367
  self,
368
  function_name: str,
369
  records: list[VectorUpsertRecord],
370
+ embedding_model: str | None = None,
371
+ embedding_version: str | None = None,
372
  ) -> None:
373
  if not records:
374
  return
 
392
  json.dumps(record.metadata, ensure_ascii=False),
393
  vector_literal(record.embedding),
394
  record.content_hash,
395
+ embedding_model or self._settings.embedding_model,
396
+ embedding_version or self._settings.embedding_version,
397
  record.is_active,
398
  )
399
  for record in records
 
471
  )
472
 
473
 
474
+ def _configured_read_contract_signatures(
475
+ settings: Settings,
476
+ ) -> dict[str, str]:
477
+ signatures = dict(READ_CONTRACT_SIGNATURES)
478
+ match_name = settings.places_pgvector_match_function
479
+ signatures[match_name] = f"{match_name}(vector, integer, jsonb)"
480
+ hybrid_name = settings.places_pgvector_hybrid_function
481
+ if hybrid_name:
482
+ signatures[hybrid_name] = f"{hybrid_name}(text, vector, integer, jsonb)"
483
+ return signatures
484
+
485
+
486
  def _credentials_for_role(settings: Settings, role: str) -> tuple[str | None, str | None]:
487
  if role == "writer":
488
  return (
requirements-training.txt ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ -r requirements.txt
2
+ accelerate>=0.33,<2.0
3
+ torch>=2.1,<3.0
requirements.txt CHANGED
@@ -10,5 +10,7 @@ huggingface-hub>=0.34,<2.0
10
  numpy>=1.26,<2.0
11
  scipy>=1.10,<1.15
12
  scikit-learn>=1.5,<1.7
 
 
13
  pytest>=8.0,<9.0
14
  pytest-asyncio>=0.23,<1.0
 
10
  numpy>=1.26,<2.0
11
  scipy>=1.10,<1.15
12
  scikit-learn>=1.5,<1.7
13
+ sentence-transformers>=3.0,<6.0
14
+ transformers>=4.44,<5.0
15
  pytest>=8.0,<9.0
16
  pytest-asyncio>=0.23,<1.0
scripts/train_place_intent_bert.py ADDED
@@ -0,0 +1,501 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """Fine-tune a Hugging Face token classifier for open place-chat slots.
2
+
3
+ The input files are JSONL. Each non-empty line has this shape::
4
+
5
+ {
6
+ "text": "Quiero donas artesanales sin ruido",
7
+ "spans": [
8
+ {"start": 7, "end": 25, "slot": "CATEGORY"},
9
+ {"start": 30, "end": 35, "slot": "EXCLUSION"}
10
+ ]
11
+ }
12
+
13
+ Offsets use Python's half-open character convention: ``text[start:end]``.
14
+ Only domain-independent slot types are accepted. Category values remain raw
15
+ text, so this training path does not recreate a closed business taxonomy.
16
+
17
+ Imports for Transformers and PyTorch are deliberately lazy. Importing this
18
+ module, validating data, and running ``--help`` do not require the optional ML
19
+ runtime.
20
+ """
21
+
22
+ from __future__ import annotations
23
+
24
+ import argparse
25
+ import inspect
26
+ import json
27
+ import math
28
+ from dataclasses import dataclass
29
+ from datetime import datetime, timezone
30
+ from pathlib import Path
31
+ from typing import Any, Mapping, Sequence
32
+
33
+
34
+ SLOT_TYPES: tuple[str, ...] = (
35
+ "CATEGORY",
36
+ "PREFERENCE",
37
+ "EXCLUSION",
38
+ "LOCATION",
39
+ "REFERENCE",
40
+ "RADIUS",
41
+ )
42
+ LABELS: tuple[str, ...] = (
43
+ "O",
44
+ *(label for slot in SLOT_TYPES for label in (f"B-{slot}", f"I-{slot}")),
45
+ )
46
+ LABEL_TO_ID: Mapping[str, int] = {label: index for index, label in enumerate(LABELS)}
47
+ ID_TO_LABEL: Mapping[int, str] = {index: label for label, index in LABEL_TO_ID.items()}
48
+ IGNORED_LABEL_ID = -100
49
+
50
+
51
+ @dataclass(frozen=True)
52
+ class LabeledSpan:
53
+ """One validated, open-value slot annotation."""
54
+
55
+ start: int
56
+ end: int
57
+ slot: str
58
+
59
+
60
+ @dataclass(frozen=True)
61
+ class IntentTrainingExample:
62
+ """One validated token-classification example."""
63
+
64
+ text: str
65
+ spans: tuple[LabeledSpan, ...]
66
+
67
+
68
+ def validate_training_example(
69
+ payload: Any,
70
+ *,
71
+ context: str = "example",
72
+ ) -> IntentTrainingExample:
73
+ """Validate one JSON-compatible example and normalize slot casing.
74
+
75
+ Spans must be disjoint. This is stricter than silently selecting one label
76
+ for an overlapping token and makes annotation errors fail before training.
77
+ """
78
+
79
+ if not isinstance(payload, Mapping):
80
+ raise ValueError(f"{context}: expected a JSON object")
81
+
82
+ text = payload.get("text")
83
+ if not isinstance(text, str) or not text.strip():
84
+ raise ValueError(f"{context}: text must be a non-empty string")
85
+
86
+ raw_spans = payload.get("spans")
87
+ if not isinstance(raw_spans, list):
88
+ raise ValueError(f"{context}: spans must be a list")
89
+
90
+ spans: list[LabeledSpan] = []
91
+ for index, raw_span in enumerate(raw_spans):
92
+ span_context = f"{context}, span {index}"
93
+ if not isinstance(raw_span, Mapping):
94
+ raise ValueError(f"{span_context}: expected a JSON object")
95
+
96
+ start = _integer_offset(raw_span.get("start"), "start", span_context)
97
+ end = _integer_offset(raw_span.get("end"), "end", span_context)
98
+ if start < 0 or end <= start or end > len(text):
99
+ raise ValueError(
100
+ f"{span_context}: offsets must satisfy "
101
+ f"0 <= start < end <= {len(text)}; got start={start}, end={end}"
102
+ )
103
+ if not text[start:end].strip():
104
+ raise ValueError(f"{span_context}: annotated text cannot be blank")
105
+
106
+ raw_slot = raw_span.get("slot", raw_span.get("label"))
107
+ if not isinstance(raw_slot, str) or not raw_slot.strip():
108
+ raise ValueError(f"{span_context}: slot must be a non-empty string")
109
+ slot = raw_slot.strip().upper()
110
+ if slot not in SLOT_TYPES:
111
+ raise ValueError(
112
+ f"{span_context}: unsupported slot {raw_slot!r}; expected one of "
113
+ + ", ".join(SLOT_TYPES)
114
+ )
115
+ spans.append(LabeledSpan(start=start, end=end, slot=slot))
116
+
117
+ spans.sort(key=lambda span: (span.start, span.end, span.slot))
118
+ for previous, current in zip(spans, spans[1:]):
119
+ if current.start < previous.end:
120
+ raise ValueError(
121
+ f"{context}: overlapping spans "
122
+ f"[{previous.start}, {previous.end}) and "
123
+ f"[{current.start}, {current.end})"
124
+ )
125
+
126
+ return IntentTrainingExample(text=text, spans=tuple(spans))
127
+
128
+
129
+ def read_jsonl(path: Path) -> list[IntentTrainingExample]:
130
+ """Read and validate a UTF-8 JSONL dataset with contextual errors."""
131
+
132
+ if not path.is_file():
133
+ raise FileNotFoundError(f"Dataset not found: {path}")
134
+
135
+ examples: list[IntentTrainingExample] = []
136
+ with path.open("r", encoding="utf-8") as source:
137
+ for line_number, raw_line in enumerate(source, start=1):
138
+ if not raw_line.strip():
139
+ continue
140
+ try:
141
+ payload = json.loads(raw_line)
142
+ except json.JSONDecodeError as exc:
143
+ raise ValueError(
144
+ f"{path}, line {line_number}: invalid JSON: {exc.msg}"
145
+ ) from exc
146
+ examples.append(
147
+ validate_training_example(
148
+ payload,
149
+ context=f"{path}, line {line_number}",
150
+ )
151
+ )
152
+
153
+ if not examples:
154
+ raise ValueError(f"Dataset has no examples: {path}")
155
+ return examples
156
+
157
+
158
+ def align_spans_to_token_offsets(
159
+ text: str,
160
+ spans: Sequence[LabeledSpan],
161
+ token_offsets: Sequence[Sequence[int]],
162
+ *,
163
+ label_to_id: Mapping[str, int] = LABEL_TO_ID,
164
+ ) -> list[int]:
165
+ """Align character spans to tokenizer offsets using IOB labels.
166
+
167
+ Any token with a non-empty intersection with a span receives that slot.
168
+ This handles subword tokenizers while preserving character-level source
169
+ annotations. Special/padding tokens, represented by ``(0, 0)``, receive
170
+ the standard ``-100`` ignore label.
171
+
172
+ Every annotated span must be covered by at least one token. Consequently,
173
+ truncation cannot silently turn a positive span into ``O``.
174
+ """
175
+
176
+ if not isinstance(text, str):
177
+ raise TypeError("text must be str")
178
+
179
+ _validate_label_mapping(label_to_id)
180
+ normalized_spans = tuple(spans)
181
+ _validate_span_objects(text, normalized_spans)
182
+
183
+ seen_span_indexes: set[int] = set()
184
+ labels: list[int] = []
185
+ previous_token_start = -1
186
+ for token_index, raw_offset in enumerate(token_offsets):
187
+ token_start, token_end = _token_offset(raw_offset, token_index, len(text))
188
+ if token_start == token_end == 0:
189
+ labels.append(IGNORED_LABEL_ID)
190
+ continue
191
+ if token_start < previous_token_start:
192
+ raise ValueError("token offsets must be ordered by start position")
193
+ previous_token_start = token_start
194
+
195
+ matching = [
196
+ span_index
197
+ for span_index, span in enumerate(normalized_spans)
198
+ if token_start < span.end and span.start < token_end
199
+ ]
200
+ if len(matching) > 1:
201
+ raise ValueError(
202
+ f"token {token_index} [{token_start}, {token_end}) intersects "
203
+ "multiple annotated spans"
204
+ )
205
+ if not matching:
206
+ labels.append(label_to_id["O"])
207
+ continue
208
+
209
+ span_index = matching[0]
210
+ span = normalized_spans[span_index]
211
+ prefix = "I" if span_index in seen_span_indexes else "B"
212
+ labels.append(label_to_id[f"{prefix}-{span.slot}"])
213
+ seen_span_indexes.add(span_index)
214
+
215
+ missing = [
216
+ span
217
+ for span_index, span in enumerate(normalized_spans)
218
+ if span_index not in seen_span_indexes
219
+ ]
220
+ if missing:
221
+ details = ", ".join(
222
+ f"{span.slot}[{span.start}, {span.end})" for span in missing
223
+ )
224
+ raise ValueError(
225
+ "annotated spans were not covered by tokenizer offsets "
226
+ f"(possibly truncated): {details}"
227
+ )
228
+ return labels
229
+
230
+
231
+ def encode_examples(
232
+ examples: Sequence[IntentTrainingExample],
233
+ tokenizer: Any,
234
+ *,
235
+ max_length: int,
236
+ ) -> list[dict[str, Any]]:
237
+ """Tokenize and align examples without depending on a dataset library."""
238
+
239
+ if isinstance(max_length, bool) or not isinstance(max_length, int) or max_length <= 0:
240
+ raise ValueError("max_length must be a positive integer")
241
+
242
+ encoded_examples: list[dict[str, Any]] = []
243
+ for example_index, example in enumerate(examples):
244
+ encoding = tokenizer(
245
+ example.text,
246
+ truncation=True,
247
+ max_length=max_length,
248
+ return_offsets_mapping=True,
249
+ )
250
+ if not isinstance(encoding, Mapping):
251
+ raise TypeError(
252
+ f"tokenizer output for example {example_index} must be a mapping"
253
+ )
254
+ if "offset_mapping" not in encoding:
255
+ raise ValueError(
256
+ "tokenizer did not return offset_mapping; a fast tokenizer is required"
257
+ )
258
+
259
+ offsets = encoding["offset_mapping"]
260
+ if not isinstance(offsets, Sequence) or isinstance(offsets, (str, bytes)):
261
+ raise ValueError(
262
+ f"tokenizer offset_mapping for example {example_index} "
263
+ "must be a sequence"
264
+ )
265
+ feature = {
266
+ key: value for key, value in encoding.items() if key != "offset_mapping"
267
+ }
268
+ feature["labels"] = align_spans_to_token_offsets(
269
+ example.text,
270
+ example.spans,
271
+ offsets,
272
+ )
273
+ input_ids = feature.get("input_ids")
274
+ if not isinstance(input_ids, Sequence) or isinstance(input_ids, (str, bytes)):
275
+ raise ValueError(
276
+ f"tokenizer output for example {example_index} has no input_ids sequence"
277
+ )
278
+ if len(input_ids) != len(feature["labels"]):
279
+ raise ValueError(
280
+ f"tokenizer output for example {example_index} has mismatched "
281
+ "input_ids and offset_mapping lengths"
282
+ )
283
+ encoded_examples.append(feature)
284
+ return encoded_examples
285
+
286
+
287
+ def main(argv: Sequence[str] | None = None) -> None:
288
+ args = build_parser().parse_args(argv)
289
+ _run_training(args)
290
+
291
+
292
+ def build_parser() -> argparse.ArgumentParser:
293
+ parser = argparse.ArgumentParser(description=__doc__)
294
+ parser.add_argument("--train-file", required=True, help="Training JSONL path")
295
+ parser.add_argument(
296
+ "--validation-file",
297
+ help="Optional validation JSONL path",
298
+ )
299
+ parser.add_argument("--output-dir", required=True)
300
+ parser.add_argument(
301
+ "--base-model",
302
+ default="dccuchile/bert-base-spanish-wwm-cased",
303
+ help="Hugging Face model id or local model directory",
304
+ )
305
+ parser.add_argument("--epochs", type=_positive_float, default=3.0)
306
+ parser.add_argument("--batch-size", type=_positive_int, default=16)
307
+ parser.add_argument("--learning-rate", type=_positive_float, default=2e-5)
308
+ parser.add_argument("--max-length", type=_positive_int, default=256)
309
+ parser.add_argument("--seed", type=int, default=42)
310
+ return parser
311
+
312
+
313
+ def _run_training(args: argparse.Namespace) -> None:
314
+ # Optional heavyweight imports stay behind argument parsing so ``--help``
315
+ # remains available in API and CI environments without the ML toolchain.
316
+ try:
317
+ import transformers
318
+ from transformers import (
319
+ AutoModelForTokenClassification,
320
+ AutoTokenizer,
321
+ DataCollatorForTokenClassification,
322
+ Trainer,
323
+ TrainingArguments,
324
+ set_seed,
325
+ )
326
+ except ImportError as exc:
327
+ raise RuntimeError(
328
+ "Training requires transformers with its PyTorch backend. "
329
+ "Install the optional ML dependencies before running this script."
330
+ ) from exc
331
+
332
+ train_path = Path(args.train_file)
333
+ validation_path = Path(args.validation_file) if args.validation_file else None
334
+ train_examples = read_jsonl(train_path)
335
+ validation_examples = read_jsonl(validation_path) if validation_path else []
336
+
337
+ set_seed(args.seed)
338
+ tokenizer = AutoTokenizer.from_pretrained(args.base_model, use_fast=True)
339
+ if not getattr(tokenizer, "is_fast", False):
340
+ raise RuntimeError(
341
+ "The selected model does not provide a fast tokenizer with offsets"
342
+ )
343
+
344
+ encoded_train = encode_examples(
345
+ train_examples,
346
+ tokenizer,
347
+ max_length=args.max_length,
348
+ )
349
+ encoded_validation = encode_examples(
350
+ validation_examples,
351
+ tokenizer,
352
+ max_length=args.max_length,
353
+ )
354
+
355
+ model = AutoModelForTokenClassification.from_pretrained(
356
+ args.base_model,
357
+ num_labels=len(LABELS),
358
+ label2id=dict(LABEL_TO_ID),
359
+ id2label=dict(ID_TO_LABEL),
360
+ ignore_mismatched_sizes=True,
361
+ )
362
+
363
+ output_dir = Path(args.output_dir)
364
+ output_dir.mkdir(parents=True, exist_ok=True)
365
+ has_validation = bool(encoded_validation)
366
+ strategy = "epoch" if has_validation else "no"
367
+ training_kwargs: dict[str, Any] = {
368
+ "output_dir": str(output_dir),
369
+ "num_train_epochs": args.epochs,
370
+ "per_device_train_batch_size": args.batch_size,
371
+ "per_device_eval_batch_size": args.batch_size,
372
+ "learning_rate": args.learning_rate,
373
+ "seed": args.seed,
374
+ "data_seed": args.seed,
375
+ "save_strategy": "epoch",
376
+ "logging_strategy": "steps",
377
+ "logging_steps": 25,
378
+ "report_to": [],
379
+ "load_best_model_at_end": has_validation,
380
+ }
381
+ parameter_names = inspect.signature(TrainingArguments.__init__).parameters
382
+ strategy_parameter = (
383
+ "eval_strategy" if "eval_strategy" in parameter_names else "evaluation_strategy"
384
+ )
385
+ training_kwargs[strategy_parameter] = strategy
386
+
387
+ trainer = Trainer(
388
+ model=model,
389
+ args=TrainingArguments(**training_kwargs),
390
+ train_dataset=encoded_train,
391
+ eval_dataset=encoded_validation if has_validation else None,
392
+ data_collator=DataCollatorForTokenClassification(tokenizer=tokenizer),
393
+ tokenizer=tokenizer,
394
+ )
395
+ train_result = trainer.train()
396
+ trainer.save_model(str(output_dir))
397
+ tokenizer.save_pretrained(str(output_dir))
398
+
399
+ manifest = {
400
+ "schema_version": 1,
401
+ "task": "token-classification",
402
+ "architecture": "bert-open-place-intent",
403
+ "base_model": args.base_model,
404
+ "slots": list(SLOT_TYPES),
405
+ "labels": list(LABELS),
406
+ "label_to_id": dict(LABEL_TO_ID),
407
+ "training_examples": len(train_examples),
408
+ "validation_examples": len(validation_examples),
409
+ "epochs": args.epochs,
410
+ "batch_size": args.batch_size,
411
+ "learning_rate": args.learning_rate,
412
+ "max_length": args.max_length,
413
+ "seed": args.seed,
414
+ "train_loss": _finite_or_none(getattr(train_result, "training_loss", None)),
415
+ "transformers_version": getattr(transformers, "__version__", "unknown"),
416
+ "trained_at": datetime.now(timezone.utc).isoformat(),
417
+ }
418
+ (output_dir / "place_intent_training_manifest.json").write_text(
419
+ json.dumps(manifest, indent=2, ensure_ascii=False) + "\n",
420
+ encoding="utf-8",
421
+ )
422
+
423
+
424
+ def _integer_offset(value: Any, field: str, context: str) -> int:
425
+ if isinstance(value, bool) or not isinstance(value, int):
426
+ raise ValueError(f"{context}: {field} must be an integer")
427
+ return value
428
+
429
+
430
+ def _validate_span_objects(text: str, spans: Sequence[LabeledSpan]) -> None:
431
+ previous: LabeledSpan | None = None
432
+ for index, span in enumerate(spans):
433
+ if not isinstance(span, LabeledSpan):
434
+ raise TypeError(f"span {index} must be LabeledSpan")
435
+ if span.slot not in SLOT_TYPES:
436
+ raise ValueError(f"span {index} has unsupported slot {span.slot!r}")
437
+ if span.start < 0 or span.end <= span.start or span.end > len(text):
438
+ raise ValueError(f"span {index} has invalid offsets")
439
+ if previous is not None and span.start < previous.end:
440
+ raise ValueError("spans must be sorted and non-overlapping")
441
+ previous = span
442
+
443
+
444
+ def _validate_label_mapping(label_to_id: Mapping[str, int]) -> None:
445
+ missing = [label for label in LABELS if label not in label_to_id]
446
+ if missing:
447
+ raise ValueError("label_to_id is missing labels: " + ", ".join(missing))
448
+
449
+
450
+ def _token_offset(
451
+ raw_offset: Sequence[int],
452
+ token_index: int,
453
+ text_length: int,
454
+ ) -> tuple[int, int]:
455
+ if (
456
+ isinstance(raw_offset, (str, bytes))
457
+ or not isinstance(raw_offset, Sequence)
458
+ or len(raw_offset) != 2
459
+ ):
460
+ raise ValueError(f"token offset {token_index} must contain start and end")
461
+ start, end = raw_offset
462
+ if (
463
+ isinstance(start, bool)
464
+ or isinstance(end, bool)
465
+ or not isinstance(start, int)
466
+ or not isinstance(end, int)
467
+ ):
468
+ raise ValueError(f"token offset {token_index} must contain integers")
469
+ if start == end == 0:
470
+ return 0, 0
471
+ if start < 0 or end <= start or end > text_length:
472
+ raise ValueError(
473
+ f"token offset {token_index} is invalid for text length {text_length}"
474
+ )
475
+ return start, end
476
+
477
+
478
+ def _positive_int(value: str) -> int:
479
+ parsed = int(value)
480
+ if parsed <= 0:
481
+ raise argparse.ArgumentTypeError("must be greater than zero")
482
+ return parsed
483
+
484
+
485
+ def _positive_float(value: str) -> float:
486
+ parsed = float(value)
487
+ if not math.isfinite(parsed) or parsed <= 0:
488
+ raise argparse.ArgumentTypeError("must be a finite value greater than zero")
489
+ return parsed
490
+
491
+
492
+ def _finite_or_none(value: Any) -> float | None:
493
+ try:
494
+ parsed = float(value)
495
+ except (TypeError, ValueError):
496
+ return None
497
+ return parsed if math.isfinite(parsed) else None
498
+
499
+
500
+ if __name__ == "__main__":
501
+ main()
scripts/train_place_retriever.py ADDED
@@ -0,0 +1,147 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """Fine-tune a bi-encoder for open-vocabulary place retrieval.
2
+
3
+ Input is JSONL with at least ``query`` and ``positive``. ``hard_negatives`` may
4
+ contain semantically close but incorrect place documents. The script keeps all
5
+ runtime imports lazy so the API can still run with FastText during rollout.
6
+ """
7
+
8
+ from __future__ import annotations
9
+
10
+ import argparse
11
+ import json
12
+ import random
13
+ from pathlib import Path
14
+ from typing import Any
15
+
16
+
17
+ def main() -> None:
18
+ args = _parse_args()
19
+ random.seed(args.seed)
20
+
21
+ try:
22
+ from sentence_transformers import InputExample, SentenceTransformer, losses
23
+ from torch.utils.data import DataLoader
24
+ except ImportError as exc:
25
+ raise RuntimeError(
26
+ "Install sentence-transformers and its PyTorch runtime before training"
27
+ ) from exc
28
+
29
+ rows = _read_training_rows(Path(args.train_file))
30
+ examples = [
31
+ InputExample(
32
+ texts=[
33
+ f"query: {row['query']}",
34
+ f"passage: {row['positive']}",
35
+ *(
36
+ f"passage: {negative}"
37
+ for negative in row["hard_negatives"][: args.max_hard_negatives]
38
+ ),
39
+ ]
40
+ )
41
+ for row in rows
42
+ ]
43
+ random.shuffle(examples)
44
+
45
+ model = SentenceTransformer(args.base_model, device=args.device)
46
+ data_loader = DataLoader(
47
+ examples,
48
+ shuffle=True,
49
+ batch_size=args.batch_size,
50
+ drop_last=len(examples) >= args.batch_size,
51
+ )
52
+ cached_loss = getattr(losses, "CachedMultipleNegativesRankingLoss", None)
53
+ if cached_loss is not None:
54
+ train_loss = cached_loss(model, mini_batch_size=args.mini_batch_size)
55
+ else:
56
+ train_loss = losses.MultipleNegativesRankingLoss(model)
57
+
58
+ output = Path(args.output_dir)
59
+ output.mkdir(parents=True, exist_ok=True)
60
+ warmup_steps = max(1, int(len(data_loader) * args.epochs * args.warmup_ratio))
61
+ model.fit(
62
+ train_objectives=[(data_loader, train_loss)],
63
+ epochs=args.epochs,
64
+ warmup_steps=warmup_steps,
65
+ optimizer_params={"lr": args.learning_rate},
66
+ output_path=str(output),
67
+ show_progress_bar=True,
68
+ )
69
+ (output / "places_training_manifest.json").write_text(
70
+ json.dumps(
71
+ {
72
+ "base_model": args.base_model,
73
+ "training_examples": len(examples),
74
+ "epochs": args.epochs,
75
+ "batch_size": args.batch_size,
76
+ "learning_rate": args.learning_rate,
77
+ "query_prefix": "query: ",
78
+ "passage_prefix": "passage: ",
79
+ "objective": type(train_loss).__name__,
80
+ "seed": args.seed,
81
+ },
82
+ indent=2,
83
+ ensure_ascii=False,
84
+ ),
85
+ encoding="utf-8",
86
+ )
87
+
88
+
89
+ def _read_training_rows(path: Path) -> list[dict[str, Any]]:
90
+ if not path.is_file():
91
+ raise FileNotFoundError(f"Training dataset not found: {path}")
92
+ rows: list[dict[str, Any]] = []
93
+ for line_number, raw_line in enumerate(
94
+ path.read_text(encoding="utf-8").splitlines(),
95
+ start=1,
96
+ ):
97
+ if not raw_line.strip():
98
+ continue
99
+ payload = json.loads(raw_line)
100
+ query = _required_text(payload.get("query"), line_number, "query")
101
+ positive = _required_text(payload.get("positive"), line_number, "positive")
102
+ raw_negatives = payload.get("hard_negatives", [])
103
+ if isinstance(raw_negatives, str):
104
+ raw_negatives = [raw_negatives]
105
+ if not isinstance(raw_negatives, list):
106
+ raise ValueError(
107
+ f"Line {line_number}: hard_negatives must be a list of strings"
108
+ )
109
+ negatives = [
110
+ _required_text(value, line_number, "hard_negatives")
111
+ for value in raw_negatives
112
+ ]
113
+ rows.append(
114
+ {"query": query, "positive": positive, "hard_negatives": negatives}
115
+ )
116
+ if len(rows) < 2:
117
+ raise ValueError("At least two training examples are required")
118
+ return rows
119
+
120
+
121
+ def _required_text(value: Any, line_number: int, field: str) -> str:
122
+ if not isinstance(value, str) or not value.strip():
123
+ raise ValueError(f"Line {line_number}: {field} must be a non-empty string")
124
+ return " ".join(value.split())
125
+
126
+
127
+ def _parse_args() -> argparse.Namespace:
128
+ parser = argparse.ArgumentParser(description=__doc__)
129
+ parser.add_argument("--train-file", required=True)
130
+ parser.add_argument("--output-dir", required=True)
131
+ parser.add_argument(
132
+ "--base-model",
133
+ default="intfloat/multilingual-e5-base",
134
+ )
135
+ parser.add_argument("--device", default=None)
136
+ parser.add_argument("--epochs", type=int, default=2)
137
+ parser.add_argument("--batch-size", type=int, default=16)
138
+ parser.add_argument("--mini-batch-size", type=int, default=8)
139
+ parser.add_argument("--max-hard-negatives", type=int, default=3)
140
+ parser.add_argument("--learning-rate", type=float, default=2e-5)
141
+ parser.add_argument("--warmup-ratio", type=float, default=0.1)
142
+ parser.add_argument("--seed", type=int, default=42)
143
+ return parser.parse_args()
144
+
145
+
146
+ if __name__ == "__main__":
147
+ main()
sql/migrations/20260716_02_places_semantic_v1.sql ADDED
@@ -0,0 +1,255 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ -- Additive Places-only Sentence-Transformer index (multilingual-e5-base: 768d).
2
+ -- This migration never alters or drops the existing FastText VECTOR(300) table.
3
+
4
+ BEGIN;
5
+
6
+ CREATE EXTENSION IF NOT EXISTS vector;
7
+
8
+ CREATE TABLE IF NOT EXISTS public.place_embeddings_semantic_v1 (
9
+ external_id TEXT PRIMARY KEY,
10
+ document TEXT NOT NULL,
11
+ metadata JSONB NOT NULL DEFAULT '{}'::jsonb,
12
+ embedding VECTOR(768) NOT NULL,
13
+ content_hash TEXT NOT NULL,
14
+ embedding_model TEXT NOT NULL,
15
+ embedding_version TEXT NOT NULL,
16
+ is_active BOOLEAN NOT NULL DEFAULT true,
17
+ textsearch TSVECTOR GENERATED ALWAYS AS (
18
+ to_tsvector('simple', document)
19
+ ) STORED,
20
+ updated_at TIMESTAMPTZ NOT NULL DEFAULT now()
21
+ );
22
+
23
+ CREATE INDEX IF NOT EXISTS place_embeddings_semantic_v1_embedding_hnsw_idx
24
+ ON public.place_embeddings_semantic_v1
25
+ USING hnsw (embedding vector_cosine_ops);
26
+
27
+ CREATE INDEX IF NOT EXISTS place_embeddings_semantic_v1_textsearch_gin_idx
28
+ ON public.place_embeddings_semantic_v1 USING gin (textsearch);
29
+
30
+ CREATE INDEX IF NOT EXISTS place_embeddings_semantic_v1_metadata_gin_idx
31
+ ON public.place_embeddings_semantic_v1 USING gin (metadata);
32
+
33
+ CREATE OR REPLACE FUNCTION public.match_places_semantic_v1(
34
+ query_embedding VECTOR(768),
35
+ match_count INTEGER,
36
+ filters JSONB DEFAULT '{}'::jsonb
37
+ )
38
+ RETURNS TABLE (
39
+ external_id TEXT,
40
+ document TEXT,
41
+ metadata JSONB,
42
+ score DOUBLE PRECISION
43
+ )
44
+ LANGUAGE sql
45
+ STABLE
46
+ SECURITY DEFINER
47
+ SET search_path = public
48
+ AS $$
49
+ SELECT
50
+ place.external_id,
51
+ place.document,
52
+ place.metadata,
53
+ 1 - (place.embedding <=> query_embedding) AS score
54
+ FROM public.place_embeddings_semantic_v1 AS place
55
+ WHERE place.is_active = true
56
+ AND COALESCE((filters->>'is_active')::boolean, true) = true
57
+ AND ((filters ? 'city') IS FALSE OR lower(place.metadata->>'city') = lower(filters->>'city'))
58
+ AND ((filters ? 'state') IS FALSE OR lower(place.metadata->>'state') = lower(filters->>'state'))
59
+ AND ((filters ? 'category') IS FALSE OR lower(place.metadata->>'category') = lower(filters->>'category'))
60
+ AND ((filters ? 'price_range') IS FALSE OR place.metadata->>'price_range' = filters->>'price_range')
61
+ AND ((filters ? 'occasion') IS FALSE OR place.metadata->>'occasion' ILIKE ('%' || (filters->>'occasion') || '%'))
62
+ AND (
63
+ (filters ? 'place_ids') IS FALSE
64
+ OR place.external_id IN (SELECT jsonb_array_elements_text(filters->'place_ids'))
65
+ )
66
+ ORDER BY place.embedding <=> query_embedding, place.external_id
67
+ LIMIT GREATEST(match_count, 0);
68
+ $$;
69
+
70
+ CREATE OR REPLACE FUNCTION public.search_places_semantic_v1(
71
+ query_text TEXT,
72
+ query_embedding VECTOR(768),
73
+ match_count INTEGER,
74
+ filters JSONB DEFAULT '{}'::jsonb
75
+ )
76
+ RETURNS TABLE (
77
+ external_id TEXT,
78
+ document TEXT,
79
+ metadata JSONB,
80
+ score DOUBLE PRECISION,
81
+ semantic_score DOUBLE PRECISION,
82
+ lexical_score DOUBLE PRECISION
83
+ )
84
+ LANGUAGE sql
85
+ STABLE
86
+ SECURITY DEFINER
87
+ SET search_path = public
88
+ AS $$
89
+ WITH dense AS (
90
+ SELECT
91
+ place.external_id,
92
+ 1 - (place.embedding <=> query_embedding) AS semantic_score,
93
+ row_number() OVER (
94
+ ORDER BY place.embedding <=> query_embedding, place.external_id
95
+ ) AS dense_rank
96
+ FROM public.place_embeddings_semantic_v1 AS place
97
+ WHERE place.is_active = true
98
+ AND COALESCE((filters->>'is_active')::boolean, true) = true
99
+ AND ((filters ? 'city') IS FALSE OR lower(place.metadata->>'city') = lower(filters->>'city'))
100
+ AND ((filters ? 'state') IS FALSE OR lower(place.metadata->>'state') = lower(filters->>'state'))
101
+ AND ((filters ? 'category') IS FALSE OR lower(place.metadata->>'category') = lower(filters->>'category'))
102
+ AND ((filters ? 'price_range') IS FALSE OR place.metadata->>'price_range' = filters->>'price_range')
103
+ AND ((filters ? 'occasion') IS FALSE OR place.metadata->>'occasion' ILIKE ('%' || (filters->>'occasion') || '%'))
104
+ AND (
105
+ (filters ? 'place_ids') IS FALSE
106
+ OR place.external_id IN (SELECT jsonb_array_elements_text(filters->'place_ids'))
107
+ )
108
+ ORDER BY place.embedding <=> query_embedding, place.external_id
109
+ LIMIT GREATEST(match_count * 4, 100)
110
+ ),
111
+ lexical AS (
112
+ SELECT
113
+ place.external_id,
114
+ ts_rank_cd(
115
+ place.textsearch,
116
+ websearch_to_tsquery('simple', COALESCE(query_text, ''))
117
+ )::DOUBLE PRECISION AS lexical_score,
118
+ row_number() OVER (
119
+ ORDER BY
120
+ ts_rank_cd(
121
+ place.textsearch,
122
+ websearch_to_tsquery('simple', COALESCE(query_text, ''))
123
+ ) DESC,
124
+ place.external_id
125
+ ) AS lexical_rank
126
+ FROM public.place_embeddings_semantic_v1 AS place
127
+ WHERE place.is_active = true
128
+ AND COALESCE((filters->>'is_active')::boolean, true) = true
129
+ AND ((filters ? 'city') IS FALSE OR lower(place.metadata->>'city') = lower(filters->>'city'))
130
+ AND ((filters ? 'state') IS FALSE OR lower(place.metadata->>'state') = lower(filters->>'state'))
131
+ AND ((filters ? 'category') IS FALSE OR lower(place.metadata->>'category') = lower(filters->>'category'))
132
+ AND ((filters ? 'price_range') IS FALSE OR place.metadata->>'price_range' = filters->>'price_range')
133
+ AND ((filters ? 'occasion') IS FALSE OR place.metadata->>'occasion' ILIKE ('%' || (filters->>'occasion') || '%'))
134
+ AND (
135
+ (filters ? 'place_ids') IS FALSE
136
+ OR place.external_id IN (SELECT jsonb_array_elements_text(filters->'place_ids'))
137
+ )
138
+ AND query_text IS NOT NULL
139
+ AND btrim(query_text) <> ''
140
+ AND place.textsearch @@ websearch_to_tsquery('simple', query_text)
141
+ ORDER BY lexical_score DESC, place.external_id
142
+ LIMIT GREATEST(match_count * 4, 100)
143
+ ),
144
+ unioned AS (
145
+ SELECT
146
+ COALESCE(dense.external_id, lexical.external_id) AS external_id,
147
+ dense.semantic_score,
148
+ lexical.lexical_score,
149
+ dense.dense_rank,
150
+ lexical.lexical_rank
151
+ FROM dense
152
+ FULL OUTER JOIN lexical USING (external_id)
153
+ ),
154
+ scored AS (
155
+ SELECT
156
+ unioned.*,
157
+ (
158
+ COALESCE(1.0 / (60.0 + unioned.dense_rank), 0.0)
159
+ + COALESCE(1.0 / (60.0 + unioned.lexical_rank), 0.0)
160
+ ) AS rrf_score
161
+ FROM unioned
162
+ )
163
+ SELECT
164
+ place.external_id,
165
+ place.document,
166
+ place.metadata,
167
+ LEAST(
168
+ 1.0,
169
+ 0.75 * GREATEST(COALESCE(scored.semantic_score, 0.0), 0.0)
170
+ + 0.15 * LEAST(COALESCE(scored.lexical_score, 0.0), 1.0)
171
+ + 0.10 * LEAST(scored.rrf_score * 31.0, 1.0)
172
+ ) AS score,
173
+ scored.semantic_score,
174
+ scored.lexical_score
175
+ FROM scored
176
+ JOIN public.place_embeddings_semantic_v1 AS place USING (external_id)
177
+ ORDER BY score DESC, place.external_id
178
+ LIMIT GREATEST(match_count, 0);
179
+ $$;
180
+
181
+ CREATE OR REPLACE FUNCTION public.upsert_place_embedding_semantic_v1(
182
+ p_external_id TEXT,
183
+ p_document TEXT,
184
+ p_metadata JSONB,
185
+ p_embedding VECTOR(768),
186
+ p_content_hash TEXT,
187
+ p_embedding_model TEXT,
188
+ p_embedding_version TEXT,
189
+ p_is_active BOOLEAN
190
+ )
191
+ RETURNS VOID
192
+ LANGUAGE sql
193
+ SECURITY DEFINER
194
+ SET search_path = public
195
+ AS $$
196
+ INSERT INTO public.place_embeddings_semantic_v1 (
197
+ external_id, document, metadata, embedding, content_hash,
198
+ embedding_model, embedding_version, is_active, updated_at
199
+ ) VALUES (
200
+ p_external_id, p_document, p_metadata, p_embedding, p_content_hash,
201
+ p_embedding_model, p_embedding_version, p_is_active, now()
202
+ )
203
+ ON CONFLICT (external_id) DO UPDATE SET
204
+ document = EXCLUDED.document,
205
+ metadata = EXCLUDED.metadata,
206
+ embedding = EXCLUDED.embedding,
207
+ content_hash = EXCLUDED.content_hash,
208
+ embedding_model = EXCLUDED.embedding_model,
209
+ embedding_version = EXCLUDED.embedding_version,
210
+ is_active = EXCLUDED.is_active,
211
+ updated_at = now();
212
+ $$;
213
+
214
+ CREATE OR REPLACE FUNCTION public.get_place_content_hashes_semantic_v1(
215
+ p_external_ids TEXT[]
216
+ )
217
+ RETURNS TABLE (external_id TEXT, content_hash TEXT)
218
+ LANGUAGE sql
219
+ STABLE
220
+ SECURITY DEFINER
221
+ SET search_path = public
222
+ AS $$
223
+ SELECT place.external_id, place.content_hash
224
+ FROM public.place_embeddings_semantic_v1 AS place
225
+ WHERE place.external_id = ANY(p_external_ids);
226
+ $$;
227
+
228
+ REVOKE ALL ON TABLE public.place_embeddings_semantic_v1 FROM PUBLIC;
229
+ REVOKE ALL ON FUNCTION public.match_places_semantic_v1(VECTOR, INTEGER, JSONB) FROM PUBLIC;
230
+ REVOKE ALL ON FUNCTION public.search_places_semantic_v1(TEXT, VECTOR, INTEGER, JSONB) FROM PUBLIC;
231
+ REVOKE ALL ON FUNCTION public.upsert_place_embedding_semantic_v1(TEXT, TEXT, JSONB, VECTOR, TEXT, TEXT, TEXT, BOOLEAN) FROM PUBLIC;
232
+ REVOKE ALL ON FUNCTION public.get_place_content_hashes_semantic_v1(TEXT[]) FROM PUBLIC;
233
+
234
+ DO $$
235
+ BEGIN
236
+ IF EXISTS (SELECT 1 FROM pg_roles WHERE rolname = 'nlp_reader') THEN
237
+ EXECUTE 'GRANT USAGE ON SCHEMA public TO nlp_reader';
238
+ EXECUTE 'GRANT EXECUTE ON FUNCTION public.match_places_semantic_v1(VECTOR, INTEGER, JSONB) TO nlp_reader';
239
+ EXECUTE 'GRANT EXECUTE ON FUNCTION public.search_places_semantic_v1(TEXT, VECTOR, INTEGER, JSONB) TO nlp_reader';
240
+ END IF;
241
+ IF EXISTS (SELECT 1 FROM pg_roles WHERE rolname = 'nlp_writer') THEN
242
+ EXECUTE 'GRANT USAGE ON SCHEMA public TO nlp_writer';
243
+ EXECUTE 'GRANT EXECUTE ON FUNCTION public.upsert_place_embedding_semantic_v1(TEXT, TEXT, JSONB, VECTOR, TEXT, TEXT, TEXT, BOOLEAN) TO nlp_writer';
244
+ EXECUTE 'GRANT EXECUTE ON FUNCTION public.get_place_content_hashes_semantic_v1(TEXT[]) TO nlp_writer';
245
+ END IF;
246
+ END
247
+ $$;
248
+
249
+ -- Execute after creating the least-privilege roles used by the service:
250
+ -- GRANT EXECUTE ON FUNCTION public.match_places_semantic_v1(VECTOR, INTEGER, JSONB) TO nlp_reader;
251
+ -- GRANT EXECUTE ON FUNCTION public.search_places_semantic_v1(TEXT, VECTOR, INTEGER, JSONB) TO nlp_reader;
252
+ -- GRANT EXECUTE ON FUNCTION public.upsert_place_embedding_semantic_v1(TEXT, TEXT, JSONB, VECTOR, TEXT, TEXT, TEXT, BOOLEAN) TO nlp_writer;
253
+ -- GRANT EXECUTE ON FUNCTION public.get_place_content_hashes_semantic_v1(TEXT[]) TO nlp_writer;
254
+
255
+ COMMIT;
sql/verify_places_semantic_v1.sql ADDED
@@ -0,0 +1,46 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ BEGIN;
2
+ SET TRANSACTION READ ONLY;
3
+
4
+ DO $$
5
+ DECLARE
6
+ embedding_type TEXT;
7
+ BEGIN
8
+ SELECT format_type(attribute.atttypid, attribute.atttypmod)
9
+ INTO embedding_type
10
+ FROM pg_attribute AS attribute
11
+ WHERE attribute.attrelid = 'public.place_embeddings_semantic_v1'::regclass
12
+ AND attribute.attname = 'embedding'
13
+ AND NOT attribute.attisdropped;
14
+
15
+ IF embedding_type IS DISTINCT FROM 'vector(768)' THEN
16
+ RAISE EXCEPTION 'Expected place_embeddings_semantic_v1.embedding vector(768), got %', embedding_type;
17
+ END IF;
18
+ IF to_regprocedure('public.match_places_semantic_v1(vector,integer,jsonb)') IS NULL THEN
19
+ RAISE EXCEPTION 'match_places_semantic_v1 is missing';
20
+ END IF;
21
+ IF to_regprocedure('public.search_places_semantic_v1(text,vector,integer,jsonb)') IS NULL THEN
22
+ RAISE EXCEPTION 'search_places_semantic_v1 is missing';
23
+ END IF;
24
+ IF NOT EXISTS (
25
+ SELECT 1
26
+ FROM pg_indexes
27
+ WHERE schemaname = 'public'
28
+ AND tablename = 'place_embeddings_semantic_v1'
29
+ AND indexdef ILIKE '%USING hnsw%'
30
+ ) THEN
31
+ RAISE EXCEPTION 'Places semantic HNSW index is missing';
32
+ END IF;
33
+ END
34
+ $$;
35
+
36
+ -- Inspect this plan after a representative backfill. With enough rows, the
37
+ -- nearest-neighbor branch should use the HNSW index rather than materializing
38
+ -- the entire eligible corpus before ordering.
39
+ EXPLAIN (ANALYZE, BUFFERS, COSTS, VERBOSE)
40
+ SELECT place.external_id
41
+ FROM public.place_embeddings_semantic_v1 AS place
42
+ WHERE place.is_active = true
43
+ ORDER BY place.embedding <=> (array_fill(0::real, ARRAY[768])::vector)
44
+ LIMIT 20;
45
+
46
+ ROLLBACK;
tests/conftest.py CHANGED
@@ -5,6 +5,10 @@ os.environ["EMBEDDING_PROVIDER"] = "mock"
5
  os.environ["EMBEDDING_DIMENSION"] = "16"
6
  os.environ["EMBEDDING_MODEL"] = "mock-embedding"
7
  os.environ["EMBEDDING_VERSION"] = "test-v1"
 
 
 
 
8
  os.environ["GROQ_API_KEY"] = ""
9
  os.environ["SEARCH_INTERNAL_TOKEN"] = "test-search-token"
10
  os.environ["NLP_SERVICE_TOKEN"] = "test-nlp-service-token"
 
5
  os.environ["EMBEDDING_DIMENSION"] = "16"
6
  os.environ["EMBEDDING_MODEL"] = "mock-embedding"
7
  os.environ["EMBEDDING_VERSION"] = "test-v1"
8
+ os.environ["PLACES_EMBEDDING_PROVIDER"] = "mock"
9
+ os.environ["PLACES_EMBEDDING_DIMENSION"] = "16"
10
+ os.environ["PLACES_EMBEDDING_MODEL"] = "mock-place-embedding"
11
+ os.environ["PLACES_EMBEDDING_VERSION"] = "test-places-v1"
12
  os.environ["GROQ_API_KEY"] = ""
13
  os.environ["SEARCH_INTERNAL_TOKEN"] = "test-search-token"
14
  os.environ["NLP_SERVICE_TOKEN"] = "test-nlp-service-token"
tests/test_api_endpoints.py CHANGED
@@ -20,14 +20,14 @@ def test_places_search_endpoint() -> None:
20
  payload = response.json()
21
  assert payload["query"] == "lugares tranquilos para cenar"
22
  assert payload["places"]
23
- assert payload["metrics"]["engine"] == "fasttext_mean_embeddings"
24
  assert payload["metrics"]["candidate_retrieval"] == "mock_embeddings"
25
  assert payload["metrics"]["score_metric"] == "cosine_similarity"
26
  assert payload["metrics"]["ranking_parameters"] == {"dimension": 16.0}
27
  assert payload["metrics"]["field_weights"] == {
28
- "tags": 6,
29
- "category": 4,
30
- "description": 3,
31
  "name": 1,
32
  }
33
  assert payload["metrics"]["returned_count"] == len(payload["places"])
@@ -76,7 +76,7 @@ def test_places_chat_endpoint_returns_trace_and_structured_places() -> None:
76
  response = client.post(
77
  "/places/chat",
78
  json={
79
- "message": "quiero una cena tranquila con mi pareja",
80
  "city": "Tuxtla Gutierrez",
81
  "filters": {"occasion": "pareja", "is_active": True},
82
  "limit": 3,
@@ -92,6 +92,72 @@ def test_places_chat_endpoint_returns_trace_and_structured_places() -> None:
92
  assert payload["metadata"]["places_used_as_context"]
93
 
94
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
95
  def test_places_recommendations_returns_llm_message_and_semantic_metadata() -> None:
96
  client = TestClient(create_app())
97
 
@@ -109,7 +175,7 @@ def test_places_recommendations_returns_llm_message_and_semantic_metadata() -> N
109
  payload = response.json()
110
  assert payload["message"]
111
  assert payload["places"]
112
- assert payload["metrics"]["engine"] == "fasttext_mean_embeddings"
113
  assert payload["metrics"]["score_metric"] == "cosine_similarity"
114
  assert payload["metrics"]["returned_count"] == len(payload["places"])
115
  assert payload["metrics"]["candidate_retrieval"] == "mock_embeddings"
@@ -118,7 +184,7 @@ def test_places_recommendations_returns_llm_message_and_semantic_metadata() -> N
118
  assert payload["metrics"]["scope"] == "current_query"
119
  assert payload["metrics"]["ground_truth_available"] is False
120
  assert "evaluation_metrics" not in payload
121
- assert payload["metadata"]["ranking"] == "fasttext_mean_embeddings"
122
  assert payload["metadata"]["response_mode"] == "confident"
123
  assert payload["metadata"]["used_llm"] is True
124
 
 
20
  payload = response.json()
21
  assert payload["query"] == "lugares tranquilos para cenar"
22
  assert payload["places"]
23
+ assert payload["metrics"]["engine"] == "mock-place-embedding"
24
  assert payload["metrics"]["candidate_retrieval"] == "mock_embeddings"
25
  assert payload["metrics"]["score_metric"] == "cosine_similarity"
26
  assert payload["metrics"]["ranking_parameters"] == {"dimension": 16.0}
27
  assert payload["metrics"]["field_weights"] == {
28
+ "tags": 1,
29
+ "category": 1,
30
+ "description": 1,
31
  "name": 1,
32
  }
33
  assert payload["metrics"]["returned_count"] == len(payload["places"])
 
76
  response = client.post(
77
  "/places/chat",
78
  json={
79
+ "message": "cafe tranquilo postres platica",
80
  "city": "Tuxtla Gutierrez",
81
  "filters": {"occasion": "pareja", "is_active": True},
82
  "limit": 3,
 
92
  assert payload["metadata"]["places_used_as_context"]
93
 
94
 
95
+ def test_places_chat_opt_in_uses_semantic_conversation_contract() -> None:
96
+ client = TestClient(create_app())
97
+
98
+ response = client.post(
99
+ "/places/chat",
100
+ json={
101
+ "conversation_id": "3a4723f6-260d-4c11-b9d6-089f07a4f338",
102
+ "turn": 1,
103
+ "message": "una cafeteria tranquila",
104
+ "conversation_state": {
105
+ "city": "Tuxtla Gutierrez",
106
+ "taxonomy_version": "places-taxonomy-v1",
107
+ },
108
+ "user_location": {"lat": 16.7531, "lng": -93.1156},
109
+ "filters": {"is_active": True},
110
+ "candidate_limit": 5,
111
+ "limit": 1,
112
+ },
113
+ )
114
+
115
+ assert response.status_code == 200
116
+ payload = response.json()
117
+ assert payload["action"] == "recommendations"
118
+ assert payload["state_patch"]["target_category"] == "cafe"
119
+ assert payload["location_directive"]["source"] == "user_current"
120
+ assert payload["uncertainty"]["decision"] == "auto"
121
+ assert payload["metadata"]["pipeline"] == "places-chat-semantic-v2"
122
+ assert len(payload["places"]) == 1
123
+
124
+
125
+ def test_places_chat_opt_in_rejects_incompatible_conversation_state() -> None:
126
+ client = TestClient(create_app())
127
+
128
+ response = client.post(
129
+ "/places/chat",
130
+ json={
131
+ "conversation_id": "3a4723f6-260d-4c11-b9d6-089f07a4f338",
132
+ "message": "donas",
133
+ "conversation_state": {"taxonomy_version": "obsolete-v0"},
134
+ "user_location": {"lat": 16.7531, "lng": -93.1156},
135
+ },
136
+ )
137
+
138
+ assert response.status_code == 409
139
+
140
+
141
+ def test_places_chat_opt_in_maps_stale_clarification_to_conflict() -> None:
142
+ client = TestClient(create_app())
143
+
144
+ response = client.post(
145
+ "/places/chat",
146
+ json={
147
+ "conversation_id": "3a4723f6-260d-4c11-b9d6-089f07a4f338",
148
+ "message": "donas",
149
+ "conversation_state": {"taxonomy_version": "places-taxonomy-v1"},
150
+ "clarification_choice": {
151
+ "clarification_id": "00000000-0000-4000-8000-000000000000",
152
+ "option_id": "bakery",
153
+ },
154
+ "user_location": {"lat": 16.7531, "lng": -93.1156},
155
+ },
156
+ )
157
+
158
+ assert response.status_code == 409
159
+
160
+
161
  def test_places_recommendations_returns_llm_message_and_semantic_metadata() -> None:
162
  client = TestClient(create_app())
163
 
 
175
  payload = response.json()
176
  assert payload["message"]
177
  assert payload["places"]
178
+ assert payload["metrics"]["engine"] == "mock-place-embedding"
179
  assert payload["metrics"]["score_metric"] == "cosine_similarity"
180
  assert payload["metrics"]["returned_count"] == len(payload["places"])
181
  assert payload["metrics"]["candidate_retrieval"] == "mock_embeddings"
 
184
  assert payload["metrics"]["scope"] == "current_query"
185
  assert payload["metrics"]["ground_truth_available"] is False
186
  assert "evaluation_metrics" not in payload
187
+ assert payload["metadata"]["ranking"] == "mock-place-embedding"
188
  assert payload["metadata"]["response_mode"] == "confident"
189
  assert payload["metadata"]["used_llm"] is True
190
 
tests/test_bert_place_intent_extractor.py ADDED
@@ -0,0 +1,277 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ import math
2
+
3
+ import pytest
4
+
5
+ from app.modules.places.infrastructure.bert_intent_extractor import (
6
+ BertIntentInferenceError,
7
+ BertIntentModelLoadError,
8
+ BertIntentOutputError,
9
+ BertPlaceIntentExtractor,
10
+ )
11
+
12
+
13
+ class FakeTokenClassifier:
14
+ def __init__(self, predictions: list[dict[str, object]]) -> None:
15
+ self.predictions = predictions
16
+ self.calls: list[str] = []
17
+
18
+ def __call__(self, text: str) -> list[dict[str, object]]:
19
+ self.calls.append(text)
20
+ return list(self.predictions)
21
+
22
+
23
+ def _prediction(
24
+ text: str,
25
+ value: str,
26
+ entity: str,
27
+ score: float,
28
+ *,
29
+ after: int = 0,
30
+ ) -> dict[str, object]:
31
+ start = text.index(value, after)
32
+ return {
33
+ "entity": entity,
34
+ "score": score,
35
+ "start": start,
36
+ "end": start + len(value),
37
+ "word": value,
38
+ }
39
+
40
+
41
+ def test_extractor_loads_lazily_and_preserves_raw_open_value_spans() -> None:
42
+ text = (
43
+ "Quiero donas artesanales, sin ruido, cerca del Parque Central a 2 km."
44
+ )
45
+ predictions = [
46
+ _prediction(text, "donas", "B-CATEGORY", 0.96),
47
+ _prediction(text, "artesanales", "I-CATEGORY", 0.90),
48
+ _prediction(text, "ruido", "B-EXCLUSION", 0.94),
49
+ _prediction(text, "Parque", "B-LOCATION", 0.91),
50
+ _prediction(text, "Central", "I-LOCATION", 0.89),
51
+ _prediction(text, "2", "B-RADIUS", 0.88),
52
+ _prediction(text, "km", "I-RADIUS", 0.86),
53
+ ]
54
+ classifier = FakeTokenClassifier(predictions)
55
+ loader_calls: list[tuple[str, str | None, int | str | None]] = []
56
+
57
+ def loader(
58
+ model_name: str,
59
+ revision: str | None,
60
+ device: int | str | None,
61
+ ) -> FakeTokenClassifier:
62
+ loader_calls.append((model_name, revision, device))
63
+ return classifier
64
+
65
+ extractor = BertPlaceIntentExtractor(
66
+ "frimeet/places-intent-bert",
67
+ model_version="2026-07-16",
68
+ device="cpu",
69
+ model_loader=loader,
70
+ )
71
+
72
+ assert extractor.is_loaded is False
73
+ assert loader_calls == []
74
+
75
+ frame = extractor.extract(text)
76
+
77
+ assert extractor.is_loaded is True
78
+ assert loader_calls == [
79
+ ("frimeet/places-intent-bert", "2026-07-16", "cpu")
80
+ ]
81
+ assert frame.raw_text == text
82
+ assert frame.model_name == "frimeet/places-intent-bert"
83
+ assert frame.model_version == "2026-07-16"
84
+ assert [span.text for span in frame.spans] == [
85
+ "donas artesanales",
86
+ "ruido",
87
+ "Parque Central",
88
+ "2 km",
89
+ ]
90
+ assert frame.categories[0].text == "donas artesanales"
91
+ assert frame.categories[0].polarity == "positive"
92
+ assert frame.categories[0].token_count == 2
93
+ assert frame.exclusions[0].text == "ruido"
94
+ assert frame.exclusions[0].polarity == "negative"
95
+ assert frame.by_type("LOCATION")[0].text == "Parque Central"
96
+ assert frame.by_type("RADIUS")[0].text == "2 km"
97
+ assert 0.86 <= frame.confidence <= 0.96
98
+ for span in frame.spans:
99
+ assert text[span.start : span.end] == span.text
100
+
101
+ extractor.extract(text)
102
+ assert len(loader_calls) == 1
103
+ assert classifier.calls == [text, text]
104
+
105
+
106
+ def test_custom_model_labels_map_to_slots_and_contextual_polarity() -> None:
107
+ text = "Busco cafecito con terraza pero evito ruido"
108
+ classifier = FakeTokenClassifier(
109
+ [
110
+ _prediction(text, "cafecito", "B-TARGET", 0.93),
111
+ _prediction(text, "terraza", "B-AMENITY", 0.91),
112
+ _prediction(text, "ruido", "B-NEGATIVE_AMENITY", 0.95),
113
+ _prediction(text, "Busco", "B-UNRELATED_HEAD", 0.99),
114
+ ]
115
+ )
116
+ extractor = BertPlaceIntentExtractor(
117
+ "injected-model",
118
+ classifier=classifier,
119
+ label_definitions={
120
+ "TARGET": "CATEGORY",
121
+ "AMENITY": ("PREFERENCE", "positive"),
122
+ "NEGATIVE_AMENITY": ("PREFERENCE", "negative"),
123
+ },
124
+ )
125
+
126
+ frame = extractor.extract(text)
127
+
128
+ assert [span.text for span in frame.categories] == ["cafecito"]
129
+ assert [span.text for span in frame.preferences] == ["terraza"]
130
+ assert [span.text for span in frame.exclusions] == ["ruido"]
131
+ assert frame.exclusions[0].slot_type == "PREFERENCE"
132
+ assert "UNRELATED_HEAD" not in {span.slot_type for span in frame.spans}
133
+
134
+
135
+ def test_stray_i_label_is_recovered_but_a_skipped_token_breaks_the_span() -> None:
136
+ text = "terraza muy tranquila"
137
+ classifier = FakeTokenClassifier(
138
+ [
139
+ _prediction(text, "terraza", "I-PREFERENCE", 0.92),
140
+ _prediction(text, "muy", "I-PREFERENCE", 0.30),
141
+ _prediction(text, "tranquila", "I-PREFERENCE", 0.90),
142
+ ]
143
+ )
144
+ extractor = BertPlaceIntentExtractor(
145
+ "injected-model",
146
+ classifier=classifier,
147
+ minimum_token_confidence=0.8,
148
+ )
149
+
150
+ frame = extractor.extract(text)
151
+
152
+ assert [span.text for span in frame.preferences] == ["terraza", "tranquila"]
153
+ assert all(span.token_count == 1 for span in frame.preferences)
154
+
155
+
156
+ def test_huggingface_token_index_breaks_span_across_ignored_o_tokens() -> None:
157
+ text = "cafe cerca del parque"
158
+ first = _prediction(text, "cafe", "B-CATEGORY", 0.94)
159
+ first["index"] = 1
160
+ second = _prediction(text, "parque", "I-CATEGORY", 0.91)
161
+ second["index"] = 5
162
+ classifier = FakeTokenClassifier([first, second])
163
+
164
+ frame = BertPlaceIntentExtractor(
165
+ "injected-model",
166
+ classifier=classifier,
167
+ ).extract(text)
168
+
169
+ assert [span.text for span in frame.categories] == ["cafe", "parque"]
170
+ assert all(span.token_count == 1 for span in frame.categories)
171
+
172
+
173
+ def test_empty_text_returns_an_empty_frame_without_loading_the_model() -> None:
174
+ def loader(
175
+ _model_name: str,
176
+ _revision: str | None,
177
+ _device: int | str | None,
178
+ ) -> FakeTokenClassifier:
179
+ raise AssertionError("empty text must not load transformers")
180
+
181
+ extractor = BertPlaceIntentExtractor("lazy-model", model_loader=loader)
182
+
183
+ frame = extractor.extract(" \t\n")
184
+
185
+ assert frame.raw_text == " \t\n"
186
+ assert frame.spans == ()
187
+ assert frame.confidence == 0.0
188
+ assert extractor.is_loaded is False
189
+
190
+
191
+ def test_injected_classifier_does_not_require_transformers_and_accepts_group_labels() -> None:
192
+ text = "Parque México"
193
+ classifier = FakeTokenClassifier(
194
+ [
195
+ {
196
+ "entity_group": "LOCATION",
197
+ "score": 0.97,
198
+ "start": 0,
199
+ "end": len(text),
200
+ }
201
+ ]
202
+ )
203
+ extractor = BertPlaceIntentExtractor("no-transformers-needed", classifier=classifier)
204
+
205
+ frame = extractor.extract(text)
206
+
207
+ assert frame.by_type("LOCATION")[0].text == text
208
+ assert frame.by_type("LOCATION")[0].confidence == pytest.approx(0.97)
209
+
210
+
211
+ def test_loader_inference_and_malformed_output_errors_have_context() -> None:
212
+ def broken_loader(
213
+ _model_name: str,
214
+ _revision: str | None,
215
+ _device: int | str | None,
216
+ ) -> FakeTokenClassifier:
217
+ raise OSError("model cache unavailable")
218
+
219
+ extractor = BertPlaceIntentExtractor("missing-model", model_loader=broken_loader)
220
+ with pytest.raises(
221
+ BertIntentModelLoadError,
222
+ match="missing-model.*model cache unavailable",
223
+ ):
224
+ extractor.extract("donas")
225
+
226
+ class BrokenClassifier:
227
+ def __call__(self, _text: str) -> list[dict[str, object]]:
228
+ raise RuntimeError("backend crashed")
229
+
230
+ extractor = BertPlaceIntentExtractor("broken-model", classifier=BrokenClassifier())
231
+ with pytest.raises(
232
+ BertIntentInferenceError,
233
+ match="broken-model.*backend crashed",
234
+ ):
235
+ extractor.extract("donas")
236
+
237
+ malformed = FakeTokenClassifier(
238
+ [{"entity": "B-CATEGORY", "score": 0.9, "start": 0}]
239
+ )
240
+ extractor = BertPlaceIntentExtractor("bad-output", classifier=malformed)
241
+ with pytest.raises(BertIntentOutputError, match="integer end offset"):
242
+ extractor.extract("donas")
243
+
244
+
245
+ def test_frame_and_span_confidence_are_length_weighted() -> None:
246
+ text = "cafe silencioso"
247
+ classifier = FakeTokenClassifier(
248
+ [
249
+ _prediction(text, "cafe", "B-CATEGORY", 1.0),
250
+ _prediction(text, "silencioso", "B-PREFERENCE", 0.5),
251
+ ]
252
+ )
253
+ frame = BertPlaceIntentExtractor("model", classifier=classifier).extract(text)
254
+
255
+ expected = (4 * 1.0 + 10 * 0.5) / 14
256
+ assert frame.confidence == pytest.approx(expected)
257
+ assert math.isfinite(frame.confidence)
258
+
259
+
260
+ def test_constructor_rejects_invalid_configuration() -> None:
261
+ classifier = FakeTokenClassifier([])
262
+ with pytest.raises(ValueError, match="model_name_or_path"):
263
+ BertPlaceIntentExtractor(" ")
264
+ with pytest.raises(ValueError, match="minimum_token_confidence"):
265
+ BertPlaceIntentExtractor("model", minimum_token_confidence=1.1)
266
+ with pytest.raises(ValueError, match="classifier or model_loader"):
267
+ BertPlaceIntentExtractor(
268
+ "model",
269
+ classifier=classifier,
270
+ model_loader=lambda _name, _revision, _device: classifier,
271
+ )
272
+ with pytest.raises(ValueError, match="slot_type"):
273
+ BertPlaceIntentExtractor(
274
+ "model",
275
+ classifier=classifier,
276
+ label_definitions={"CUSTOM": "NOT_A_SLOT"}, # type: ignore[dict-item]
277
+ )
tests/test_embeddings.py CHANGED
@@ -1,4 +1,7 @@
1
  from app.shared.nlp.embeddings.mock import MockEmbeddingProvider
 
 
 
2
 
3
 
4
  def test_mock_embedding_provider_is_deterministic() -> None:
@@ -9,3 +12,27 @@ def test_mock_embedding_provider_is_deterministic() -> None:
9
 
10
  assert first == second
11
  assert len(first) == provider.dimension
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
  from app.shared.nlp.embeddings.mock import MockEmbeddingProvider
2
+ from app.shared.nlp.embeddings.base import EmbeddingProvider
3
+ from app.shared.nlp.embeddings.cached import CachedEmbeddingProvider
4
+ from app.shared.cache.memory import SimpleTTLCache
5
 
6
 
7
  def test_mock_embedding_provider_is_deterministic() -> None:
 
12
 
13
  assert first == second
14
  assert len(first) == provider.dimension
15
+
16
+
17
+ class RecordingBatchProvider(EmbeddingProvider):
18
+ def __init__(self) -> None:
19
+ self.batches: list[list[str]] = []
20
+
21
+ def embed_text(self, text: str) -> list[float]:
22
+ raise AssertionError("batch cache should use embed_batch for misses")
23
+
24
+ def embed_batch(self, texts: list[str]) -> list[list[float]]:
25
+ self.batches.append(list(texts))
26
+ return [[float(len(text))] for text in texts]
27
+
28
+
29
+ def test_cached_provider_batches_unique_misses_and_preserves_order() -> None:
30
+ inner = RecordingBatchProvider()
31
+ provider = CachedEmbeddingProvider(inner, SimpleTTLCache())
32
+
33
+ first = provider.embed_batch(["donas", "cafe", "donas"])
34
+ second = provider.embed_batch(["cafe", "parque"])
35
+
36
+ assert first == [[5.0], [4.0], [5.0]]
37
+ assert second == [[4.0], [6.0]]
38
+ assert inner.batches == [["donas", "cafe"], ["parque"]]
tests/test_hybrid_chat_retriever.py ADDED
@@ -0,0 +1,232 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ import pytest
2
+
3
+ from app.modules.places.domain.chat_intent import (
4
+ ConversationStatePatch,
5
+ LocationIntent,
6
+ ParsedPlaceChatIntent,
7
+ )
8
+ from app.modules.places.domain.models import PlaceCandidate
9
+ from app.modules.places.infrastructure.hybrid_chat_retriever import (
10
+ HybridContentPlaceChatRetriever,
11
+ )
12
+ from app.shared.nlp.embeddings.mock import MockEmbeddingProvider
13
+
14
+
15
+ def _intent(
16
+ query: str,
17
+ *,
18
+ target_category: str | None = None,
19
+ category_values: tuple[str, ...] = (),
20
+ hard_filters: dict[str, object] | None = None,
21
+ exclusions: tuple[str, ...] = (),
22
+ ) -> ParsedPlaceChatIntent:
23
+ return ParsedPlaceChatIntent(
24
+ action="recommendations",
25
+ target_category=target_category,
26
+ category_values=category_values,
27
+ hard_filters=hard_filters or {},
28
+ soft_preferences=(),
29
+ exclusions=exclusions,
30
+ reference=None,
31
+ location=LocationIntent(
32
+ scope="target_results",
33
+ source="user_current",
34
+ ),
35
+ semantic_query=query,
36
+ confidence=0.90,
37
+ state_patch=ConversationStatePatch(),
38
+ )
39
+
40
+
41
+ class RecordingRepository:
42
+ source_name = "test"
43
+
44
+ def __init__(self, candidates: list[PlaceCandidate]) -> None:
45
+ self.candidates = candidates
46
+ self.calls: list[dict[str, object]] = []
47
+
48
+ async def search(self, embedding, filters, limit):
49
+ self.calls.append(
50
+ {
51
+ "embedding": embedding,
52
+ "filters": filters,
53
+ "limit": limit,
54
+ }
55
+ )
56
+ return self.candidates
57
+
58
+
59
+ @pytest.mark.asyncio
60
+ async def test_open_vocabulary_recommendation_retrieves_without_category() -> None:
61
+ repository = RecordingRepository(
62
+ [
63
+ PlaceCandidate(
64
+ id="donut_shop",
65
+ name="Dulce Circular",
66
+ category="bakery",
67
+ score=0.0,
68
+ metadata={"tags": "postres artesanales"},
69
+ )
70
+ ]
71
+ )
72
+ retriever = HybridContentPlaceChatRetriever(
73
+ embedding_provider=MockEmbeddingProvider(dimension=16),
74
+ place_repository=repository,
75
+ minimum_content_score=0.95,
76
+ )
77
+
78
+ candidates = await retriever.retrieve(
79
+ intent=_intent(
80
+ "donas",
81
+ hard_filters={"city": "Tuxtla Gutierrez", "state": "Chiapas"},
82
+ ),
83
+ limit=3,
84
+ )
85
+
86
+ assert [candidate.place_id for candidate in candidates] == ["donut_shop"]
87
+ assert repository.calls[0]["filters"].categories is None
88
+ assert repository.calls[0]["filters"].city == "Tuxtla Gutierrez"
89
+ assert repository.calls[0]["filters"].state == "Chiapas"
90
+ diagnostics = candidates[0].metadata["retrieval_diagnostics"]
91
+ assert diagnostics == {
92
+ "category_affinity": 0.0,
93
+ "category_match": "not_requested",
94
+ "exclusion_affinity": 0.0,
95
+ "exclusion_matches": [],
96
+ "content_quality": "weak",
97
+ "meets_minimum_content_score": False,
98
+ "minimum_content_score": 0.95,
99
+ "query_token_count": 1,
100
+ }
101
+
102
+
103
+ @pytest.mark.asyncio
104
+ async def test_category_boosts_ranking_but_does_not_remove_other_categories() -> None:
105
+ repository = RecordingRepository(
106
+ [
107
+ PlaceCandidate(
108
+ id="park",
109
+ name="Parque Vecinal",
110
+ category="park",
111
+ score=0.2,
112
+ ),
113
+ PlaceCandidate(
114
+ id="cafe",
115
+ name="Salon Urbano",
116
+ category="cafe",
117
+ score=0.2,
118
+ ),
119
+ ]
120
+ )
121
+ retriever = HybridContentPlaceChatRetriever(
122
+ embedding_provider=MockEmbeddingProvider(dimension=16),
123
+ place_repository=repository,
124
+ )
125
+
126
+ candidates = await retriever.retrieve(
127
+ intent=_intent(
128
+ "sitio agradable",
129
+ target_category="cafe",
130
+ category_values=("cafe", "cafeteria"),
131
+ ),
132
+ limit=5,
133
+ )
134
+
135
+ assert [candidate.place_id for candidate in candidates] == ["cafe", "park"]
136
+ assert candidates[0].metadata["retrieval_diagnostics"]["category_match"] == (
137
+ "exact"
138
+ )
139
+ assert candidates[1].metadata["retrieval_diagnostics"]["category_match"] == (
140
+ "none"
141
+ )
142
+
143
+
144
+ class HybridRepository:
145
+ source_name = "test_hybrid"
146
+
147
+ def __init__(self) -> None:
148
+ self.hybrid_call: dict[str, object] | None = None
149
+
150
+ async def search_hybrid(self, query_text, embedding, filters, limit):
151
+ self.hybrid_call = {
152
+ "query_text": query_text,
153
+ "embedding": embedding,
154
+ "filters": filters,
155
+ "limit": limit,
156
+ }
157
+ return [
158
+ PlaceCandidate(
159
+ id="hybrid_result",
160
+ name="Resultado Hibrido",
161
+ category="bakery",
162
+ # The fused repository score is not a semantic score. This
163
+ # row came from the independent lexical candidate pool.
164
+ score=0.99,
165
+ metadata={"lexical_score": 1.0},
166
+ )
167
+ ]
168
+
169
+ async def search(self, embedding, filters, limit):
170
+ del embedding, filters, limit
171
+ raise AssertionError("search must not run when search_hybrid is available")
172
+
173
+
174
+ @pytest.mark.asyncio
175
+ async def test_optional_repository_hybrid_search_is_preferred() -> None:
176
+ repository = HybridRepository()
177
+ retriever = HybridContentPlaceChatRetriever(
178
+ embedding_provider=MockEmbeddingProvider(dimension=16),
179
+ place_repository=repository,
180
+ )
181
+
182
+ candidates = await retriever.retrieve(
183
+ intent=_intent("Donas cerca"),
184
+ limit=3,
185
+ )
186
+
187
+ assert [candidate.place_id for candidate in candidates] == ["hybrid_result"]
188
+ assert repository.hybrid_call is not None
189
+ assert repository.hybrid_call["query_text"] == "Donas cerca"
190
+ assert repository.hybrid_call["filters"].categories is None
191
+ assert repository.hybrid_call["limit"] == 40
192
+ assert candidates[0].semantic_score == 0.0
193
+ assert candidates[0].lexical_score > 0.0
194
+
195
+
196
+ @pytest.mark.asyncio
197
+ async def test_exclusions_penalize_instead_of_dropping_and_respect_negation() -> None:
198
+ repository = RecordingRepository(
199
+ [
200
+ PlaceCandidate(
201
+ id="quiet",
202
+ name="Patio Sereno",
203
+ category="cafe",
204
+ score=0.5,
205
+ document="Un espacio tranquilo sin ruido exterior",
206
+ ),
207
+ PlaceCandidate(
208
+ id="noisy",
209
+ name="Foro Central",
210
+ category="cafe",
211
+ score=0.5,
212
+ document="Musica y ruido durante toda la noche",
213
+ ),
214
+ ]
215
+ )
216
+ retriever = HybridContentPlaceChatRetriever(
217
+ embedding_provider=MockEmbeddingProvider(dimension=16),
218
+ place_repository=repository,
219
+ )
220
+
221
+ candidates = await retriever.retrieve(
222
+ intent=_intent("lugar tranquilo", exclusions=("ruido",)),
223
+ limit=5,
224
+ )
225
+
226
+ assert [candidate.place_id for candidate in candidates] == ["quiet", "noisy"]
227
+ quiet_diagnostics = candidates[0].metadata["retrieval_diagnostics"]
228
+ noisy_diagnostics = candidates[1].metadata["retrieval_diagnostics"]
229
+ assert quiet_diagnostics["exclusion_affinity"] == 0.0
230
+ assert noisy_diagnostics["exclusion_affinity"] == 1.0
231
+ assert noisy_diagnostics["exclusion_matches"] == ["ruido"]
232
+ assert candidates[0].content_score > candidates[1].content_score
tests/test_internal_places_chat.py CHANGED
@@ -52,8 +52,9 @@ def test_internal_chat_returns_only_technical_candidates() -> None:
52
  "strict_radius": False,
53
  }
54
  assert payload["candidates"]
55
- assert {candidate["place_id"] for candidate in payload["candidates"]} == {
56
- "place_1"
 
57
  }
58
  assert set(payload["candidates"][0]) == {
59
  "place_id",
@@ -114,7 +115,7 @@ def test_internal_chat_recommends_for_short_explicit_categories(
114
  }
115
 
116
 
117
- def test_internal_chat_never_returns_parks_for_a_cafe_query() -> None:
118
  client = TestClient(create_app())
119
  request = {
120
  **BASE_REQUEST,
@@ -130,9 +131,7 @@ def test_internal_chat_never_returns_parks_for_a_cafe_query() -> None:
130
  assert response.status_code == 200
131
  payload = response.json()
132
  assert payload["action"] == "recommendations"
133
- assert {candidate["place_id"] for candidate in payload["candidates"]} == {
134
- "place_1"
135
- }
136
  assert payload["location_directive"]["source"] == "explicit_anchor"
137
  assert payload["location_directive"]["anchor_place_id"] == "place_6"
138
 
@@ -296,7 +295,7 @@ def test_internal_chat_rejects_a_stale_structured_choice() -> None:
296
  assert response.status_code == 409
297
 
298
 
299
- def test_internal_chat_applies_content_exclusions_before_ranking() -> None:
300
  client = TestClient(create_app())
301
  request = {
302
  **BASE_REQUEST,
@@ -311,8 +310,10 @@ def test_internal_chat_applies_content_exclusions_before_ranking() -> None:
311
 
312
  assert response.status_code == 200
313
  payload = response.json()
314
- assert payload["action"] == "no_match"
315
- assert payload["candidates"] == []
 
 
316
  assert payload["state_patch"]["exclusions"] == ["tranquilo"]
317
 
318
 
 
52
  "strict_radius": False,
53
  }
54
  assert payload["candidates"]
55
+ assert payload["candidates"][0]["place_id"] == "place_1"
56
+ assert "place_1" in {
57
+ candidate["place_id"] for candidate in payload["candidates"]
58
  }
59
  assert set(payload["candidates"][0]) == {
60
  "place_id",
 
115
  }
116
 
117
 
118
+ def test_internal_chat_soft_category_ranks_cafe_first_near_park_anchor() -> None:
119
  client = TestClient(create_app())
120
  request = {
121
  **BASE_REQUEST,
 
131
  assert response.status_code == 200
132
  payload = response.json()
133
  assert payload["action"] == "recommendations"
134
+ assert payload["candidates"][0]["place_id"] == "place_1"
 
 
135
  assert payload["location_directive"]["source"] == "explicit_anchor"
136
  assert payload["location_directive"]["anchor_place_id"] == "place_6"
137
 
 
295
  assert response.status_code == 409
296
 
297
 
298
+ def test_internal_chat_applies_content_exclusions_as_negative_ranking_evidence() -> None:
299
  client = TestClient(create_app())
300
  request = {
301
  **BASE_REQUEST,
 
310
 
311
  assert response.status_code == 200
312
  payload = response.json()
313
+ assert payload["action"] == "recommendations"
314
+ candidate_ids = [candidate["place_id"] for candidate in payload["candidates"]]
315
+ assert "place_1" in candidate_ids
316
+ assert candidate_ids[0] != "place_1"
317
  assert payload["state_patch"]["exclusions"] == ["tranquilo"]
318
 
319
 
tests/test_main_api_place_source.py CHANGED
@@ -31,7 +31,7 @@ def test_place_to_source_record_maps_api_place() -> None:
31
  assert len(record.content_hash) == 64
32
 
33
 
34
- def test_place_to_source_record_resolves_and_weights_numeric_tags() -> None:
35
  record = place_to_source_record(
36
  {
37
  "id": "place_weighted",
@@ -48,17 +48,17 @@ def test_place_to_source_record_resolves_and_weights_numeric_tags() -> None:
48
  assert record.metadata["tags"] == "Compras,Ropa barata"
49
  assert record.metadata["tag_ids"] == [29, 187, 9999]
50
  assert record.metadata["unknown_tag_ids"] == [9999]
51
- assert record.metadata["semantic_document_version"] == "weighted-tags-v2"
52
  assert record.document.count("Nombre Ambiguo") == 1
53
- assert record.document.count("Venta de prendas y accesorios.") == 3
54
- assert record.document.count("Compras Ropa barata") == 6
55
- assert record.document.count("compras tiendas ropa") == 4
56
  assert "osm" not in record.document
57
  assert "Direccion que no debe influir" not in record.document
58
  assert "9999" not in record.document
59
 
60
 
61
- def test_semantic_documents_keep_canonical_and_local_category_terms() -> None:
62
  park = place_to_source_record(
63
  {"id": "park", "name": "Area Uno", "category": "park"}
64
  )
@@ -66,8 +66,10 @@ def test_semantic_documents_keep_canonical_and_local_category_terms() -> None:
66
  {"id": "shopping", "name": "Area Dos", "category": "shopping"}
67
  )
68
 
69
- assert park is not None and "park parque" in park.document
70
- assert shopping is not None and "shopping compras" in shopping.document
 
 
71
 
72
 
73
  def test_place_tag_catalog_contains_complete_supplied_mapping() -> None:
 
31
  assert len(record.content_hash) == 64
32
 
33
 
34
+ def test_place_to_source_record_resolves_tags_without_token_repetition() -> None:
35
  record = place_to_source_record(
36
  {
37
  "id": "place_weighted",
 
48
  assert record.metadata["tags"] == "Compras,Ropa barata"
49
  assert record.metadata["tag_ids"] == [29, 187, 9999]
50
  assert record.metadata["unknown_tag_ids"] == [9999]
51
+ assert record.metadata["semantic_document_version"] == "structured-place-v3"
52
  assert record.document.count("Nombre Ambiguo") == 1
53
+ assert record.document.count("Venta de prendas y accesorios") == 1
54
+ assert record.document.count("Compras Ropa barata") == 1
55
+ assert "Tipo registrado: shopping." in record.document
56
  assert "osm" not in record.document
57
  assert "Direccion que no debe influir" not in record.document
58
  assert "9999" not in record.document
59
 
60
 
61
+ def test_semantic_documents_keep_source_category_without_manual_expansion() -> None:
62
  park = place_to_source_record(
63
  {"id": "park", "name": "Area Uno", "category": "park"}
64
  )
 
66
  {"id": "shopping", "name": "Area Dos", "category": "shopping"}
67
  )
68
 
69
+ assert park is not None and "Tipo registrado: park." in park.document
70
+ assert shopping is not None and "Tipo registrado: shopping." in shopping.document
71
+ assert "park parque naturaleza" not in park.document
72
+ assert "shopping compras tiendas" not in shopping.document
73
 
74
 
75
  def test_place_tag_catalog_contains_complete_supplied_mapping() -> None:
tests/test_open_vocabulary_category_classifier.py ADDED
@@ -0,0 +1,228 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ import math
2
+
3
+ import pytest
4
+
5
+ from app.modules.places.infrastructure.open_vocabulary_category_classifier import (
6
+ OpenVocabularyPlaceCategoryClassifier,
7
+ PlaceCategoryConcept,
8
+ )
9
+ from app.shared.nlp.embeddings.base import EmbeddingProvider
10
+
11
+
12
+ class ControlledEmbeddingProvider(EmbeddingProvider):
13
+ def __init__(self, vectors: dict[str, list[float]]) -> None:
14
+ self.vectors = vectors
15
+ self.text_calls: list[str] = []
16
+ self.batch_calls: list[list[str]] = []
17
+
18
+ def embed_text(self, text: str) -> list[float]:
19
+ self.text_calls.append(text)
20
+ return self.vectors.get(text, [0.0, 0.0, 0.0])
21
+
22
+ def embed_batch(self, texts: list[str]) -> list[list[float]]:
23
+ self.batch_calls.append(list(texts))
24
+ return [self.vectors.get(text, [0.0, 0.0, 0.0]) for text in texts]
25
+
26
+
27
+ def _catalog_vectors() -> dict[str, list[float]]:
28
+ return {
29
+ "Dulces horneados. Lugares con masas dulces y glaseadas": [1.0, 0.0, 0.0],
30
+ "Dulces horneados": [1.0, 0.0, 0.0],
31
+ "Lugares con masas dulces y glaseadas": [1.0, 0.0, 0.0],
32
+ "quiero una dona artesanal": [1.0, 0.0, 0.0],
33
+ "Naturaleza urbana. Espacios abiertos con vegetacion": [0.0, 1.0, 0.0],
34
+ "Naturaleza urbana": [0.0, 1.0, 0.0],
35
+ "Espacios abiertos con vegetacion": [0.0, 1.0, 0.0],
36
+ "quiero caminar entre arboles": [0.0, 1.0, 0.0],
37
+ "se me antojaron donitas": [0.9, 0.1, 0.0],
38
+ "quiero salir": [1.0, 1.0, 0.0],
39
+ "algo completamente distinto": [0.0, 0.0, 1.0],
40
+ }
41
+
42
+
43
+ def _concepts() -> list[PlaceCategoryConcept]:
44
+ return [
45
+ PlaceCategoryConcept(
46
+ id="sweet_baked_goods",
47
+ label="Dulces horneados",
48
+ description="Lugares con masas dulces y glaseadas",
49
+ examples=("quiero una dona artesanal",),
50
+ storage_values=("baked_goods", "pastry_vendor"),
51
+ ),
52
+ PlaceCategoryConcept(
53
+ id="urban_nature",
54
+ label="Naturaleza urbana",
55
+ description="Espacios abiertos con vegetacion",
56
+ examples=("quiero caminar entre arboles",),
57
+ storage_values=("green_area",),
58
+ ),
59
+ ]
60
+
61
+
62
+ def test_rank_uses_injected_catalog_and_exposes_score_margin_and_storage_values() -> None:
63
+ embeddings = ControlledEmbeddingProvider(_catalog_vectors())
64
+ classifier = OpenVocabularyPlaceCategoryClassifier(
65
+ _concepts(),
66
+ embeddings,
67
+ )
68
+
69
+ assert classifier.is_indexed is False
70
+ assert embeddings.batch_calls == []
71
+
72
+ matches = classifier.rank("se me antojaron donitas", limit=2)
73
+
74
+ assert classifier.is_indexed is True
75
+ assert len(embeddings.batch_calls) == 1
76
+ assert [match.concept_id for match in matches] == [
77
+ "sweet_baked_goods",
78
+ "urban_nature",
79
+ ]
80
+ assert matches[0].storage_values == ("baked_goods", "pastry_vendor")
81
+ assert matches[0].score == pytest.approx(0.9938837)
82
+ assert matches[0].margin > 0.88
83
+ assert matches[1].margin >= 1.0
84
+ assert classifier.get_concept("SWEET_BAKED_GOODS") == _concepts()[0]
85
+
86
+
87
+ def test_rank_limit_still_calculates_top_margin_against_runner_up() -> None:
88
+ classifier = OpenVocabularyPlaceCategoryClassifier(
89
+ _concepts(),
90
+ ControlledEmbeddingProvider(_catalog_vectors()),
91
+ )
92
+
93
+ only_match = classifier.rank("se me antojaron donitas", limit=1)
94
+
95
+ assert len(only_match) == 1
96
+ assert only_match[0].margin > 0.88
97
+
98
+
99
+ def test_query_and_concept_encoders_can_use_distinct_e5_prefix_roles() -> None:
100
+ vectors = _catalog_vectors()
101
+ query_embeddings = ControlledEmbeddingProvider(
102
+ {"se me antojaron donitas": vectors["se me antojaron donitas"]}
103
+ )
104
+ concept_embeddings = ControlledEmbeddingProvider(vectors)
105
+ classifier = OpenVocabularyPlaceCategoryClassifier(
106
+ _concepts(),
107
+ query_embeddings,
108
+ concept_embedding_provider=concept_embeddings,
109
+ )
110
+
111
+ matches = classifier.rank("se me antojaron donitas", limit=1)
112
+
113
+ assert matches[0].concept_id == "sweet_baked_goods"
114
+ assert query_embeddings.text_calls == ["se me antojaron donitas"]
115
+ assert query_embeddings.batch_calls == []
116
+ assert len(concept_embeddings.batch_calls) == 1
117
+
118
+
119
+ def test_classify_is_compatible_with_place_activity_classifier() -> None:
120
+ classifier = OpenVocabularyPlaceCategoryClassifier(
121
+ _concepts(),
122
+ ControlledEmbeddingProvider(_catalog_vectors()),
123
+ minimum_similarity=0.5,
124
+ minimum_margin=0.05,
125
+ )
126
+
127
+ result = classifier.classify("se me antojaron donitas")
128
+
129
+ assert result is not None
130
+ assert result.category == "sweet_baked_goods"
131
+ assert result.source == "semantic_activity"
132
+ assert result.category_values == ("baked_goods", "pastry_vendor")
133
+ assert result.label == "Dulces horneados"
134
+ assert 0.0 <= result.confidence <= 0.99
135
+
136
+
137
+ def test_classify_abstains_when_candidates_are_tied_or_similarity_is_low() -> None:
138
+ classifier = OpenVocabularyPlaceCategoryClassifier(
139
+ _concepts(),
140
+ ControlledEmbeddingProvider(_catalog_vectors()),
141
+ minimum_similarity=0.5,
142
+ minimum_margin=0.05,
143
+ )
144
+
145
+ assert classifier.classify("quiero salir") is None
146
+ assert classifier.classify("algo completamente distinto") is None
147
+
148
+
149
+ def test_mapping_catalog_is_supported_without_code_level_categories() -> None:
150
+ concept = {
151
+ "id": "orbital_archive",
152
+ "label": "Archivo orbital",
153
+ "description": "Colecciones documentales sobre misiones espaciales",
154
+ "examples": "quiero estudiar expediciones fuera de la Tierra",
155
+ "storage_values": ["space_records"],
156
+ }
157
+ semantic_text = (
158
+ "Archivo orbital. Colecciones documentales sobre misiones espaciales"
159
+ )
160
+ embeddings = ControlledEmbeddingProvider(
161
+ {
162
+ semantic_text: [1.0, 0.0, 0.0],
163
+ "Archivo orbital": [1.0, 0.0, 0.0],
164
+ "Colecciones documentales sobre misiones espaciales": [1.0, 0.0, 0.0],
165
+ "quiero estudiar expediciones fuera de la Tierra": [1.0, 0.0, 0.0],
166
+ "busco documentos de misiones espaciales": [1.0, 0.0, 0.0],
167
+ }
168
+ )
169
+ classifier = OpenVocabularyPlaceCategoryClassifier([concept], embeddings)
170
+
171
+ match = classifier.rank("busco documentos de misiones espaciales", limit=1)[0]
172
+
173
+ assert match.concept_id == "orbital_archive"
174
+ assert match.storage_values == ("space_records",)
175
+
176
+
177
+ def test_blank_or_zero_query_and_empty_catalog_return_no_matches() -> None:
178
+ embeddings = ControlledEmbeddingProvider(_catalog_vectors())
179
+ classifier = OpenVocabularyPlaceCategoryClassifier(_concepts(), embeddings)
180
+
181
+ assert classifier.rank(" ") == ()
182
+ assert classifier.rank("unknown") == ()
183
+ assert classifier.classify("unknown") is None
184
+ assert embeddings.batch_calls == []
185
+
186
+ empty = OpenVocabularyPlaceCategoryClassifier([], embeddings)
187
+ assert empty.rank("se me antojaron donitas") == ()
188
+
189
+
190
+ def test_catalog_and_rank_inputs_are_validated() -> None:
191
+ embeddings = ControlledEmbeddingProvider({})
192
+ with pytest.raises(ValueError, match="unique"):
193
+ OpenVocabularyPlaceCategoryClassifier(
194
+ [
195
+ {
196
+ "id": "Dynamic",
197
+ "label": "First",
198
+ "description": "First concept",
199
+ },
200
+ {
201
+ "id": "dynamic",
202
+ "label": "Second",
203
+ "description": "Second concept",
204
+ },
205
+ ],
206
+ embeddings,
207
+ )
208
+ with pytest.raises(ValueError, match="missing required fields"):
209
+ OpenVocabularyPlaceCategoryClassifier(
210
+ [{"id": "incomplete", "label": "Incomplete"}],
211
+ embeddings,
212
+ )
213
+
214
+ classifier = OpenVocabularyPlaceCategoryClassifier([], embeddings)
215
+ with pytest.raises(ValueError, match="positive integer"):
216
+ classifier.rank("query", limit=0)
217
+ with pytest.raises(TypeError, match="text must be str"):
218
+ classifier.rank(42) # type: ignore[arg-type]
219
+
220
+
221
+ def test_non_finite_vectors_are_rejected() -> None:
222
+ embeddings = ControlledEmbeddingProvider(
223
+ {"query": [math.nan, 0.0, 1.0]}
224
+ )
225
+ classifier = OpenVocabularyPlaceCategoryClassifier(_concepts(), embeddings)
226
+
227
+ with pytest.raises(ValueError, match="NaN or infinity"):
228
+ classifier.rank("query")
tests/test_pgvector_readiness.py CHANGED
@@ -1,4 +1,9 @@
1
- from app.shared.vector_store.aws_pgvector import _read_contract_is_ready
 
 
 
 
 
2
 
3
 
4
  def _functions(search_v2: bool) -> dict[str, dict[str, object]]:
@@ -20,3 +25,19 @@ def test_readiness_rejects_legacy_search_function() -> None:
20
 
21
  def test_readiness_accepts_search_candidate_v2_contract() -> None:
22
  assert _read_contract_is_ready(True, _functions(search_v2=True)) is True
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ from types import SimpleNamespace
2
+
3
+ from app.shared.vector_store.aws_pgvector import (
4
+ _configured_read_contract_signatures,
5
+ _read_contract_is_ready,
6
+ )
7
 
8
 
9
  def _functions(search_v2: bool) -> dict[str, dict[str, object]]:
 
25
 
26
  def test_readiness_accepts_search_candidate_v2_contract() -> None:
27
  assert _read_contract_is_ready(True, _functions(search_v2=True)) is True
28
+
29
+
30
+ def test_readiness_includes_configured_places_semantic_contract() -> None:
31
+ signatures = _configured_read_contract_signatures(
32
+ SimpleNamespace(
33
+ places_pgvector_match_function="match_places_semantic_v1",
34
+ places_pgvector_hybrid_function="search_places_semantic_v1",
35
+ )
36
+ )
37
+
38
+ assert signatures["match_places_semantic_v1"] == (
39
+ "match_places_semantic_v1(vector, integer, jsonb)"
40
+ )
41
+ assert signatures["search_places_semantic_v1"] == (
42
+ "search_places_semantic_v1(text, vector, integer, jsonb)"
43
+ )
tests/test_place_chat_intent_parser.py CHANGED
@@ -5,9 +5,13 @@ from app.modules.places.domain.chat_intent import (
5
  ConversationState,
6
  ExplicitTargetLocation,
7
  PendingClarification,
 
8
  PlaceCategoryInference,
9
  PlaceReference,
10
  )
 
 
 
11
  from app.modules.places.domain.errors import ClarificationStateMismatchError
12
  from app.modules.places.infrastructure.deterministic_intent_parser import (
13
  DeterministicPlaceChatIntentParser,
@@ -122,15 +126,31 @@ def test_explicit_radius_is_strict_and_removed_from_anchor_text() -> None:
122
  assert intent.location.strict_radius is True
123
 
124
 
125
- def test_missing_category_requires_clarification() -> None:
126
  intent = DeterministicPlaceChatIntentParser().parse(
127
  message="quiero un lugar bonito para salir",
128
  state=ConversationState(),
129
  has_user_location=True,
130
  )
131
 
132
- assert intent.action == "clarification"
 
 
133
  assert intent.unresolved == ("target_category",)
 
 
 
 
 
 
 
 
 
 
 
 
 
 
134
 
135
 
136
  def test_food_activity_defaults_to_restaurant_without_clarification() -> None:
@@ -353,7 +373,7 @@ def test_pending_scope_is_resolved_by_structured_choice_without_repeating() -> N
353
  def test_structured_choice_rejects_a_stale_clarification_id() -> None:
354
  parser = DeterministicPlaceChatIntentParser()
355
  first = parser.parse(
356
- message="quiero un lugar bonito para salir",
357
  state=ConversationState(),
358
  has_user_location=True,
359
  )
@@ -410,3 +430,336 @@ def test_exclusion_is_kept_out_of_positive_preferences() -> None:
410
  assert "musica" not in intent.soft_preferences
411
  assert intent.exclusions == ("musica",)
412
  assert intent.state_patch.as_dict()["exclusions"] == ["musica"]
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
5
  ConversationState,
6
  ExplicitTargetLocation,
7
  PendingClarification,
8
+ PendingClarificationOption,
9
  PlaceCategoryInference,
10
  PlaceReference,
11
  )
12
+ from app.modules.places.infrastructure.bert_intent_extractor import (
13
+ BertPlaceIntentExtractor,
14
+ )
15
  from app.modules.places.domain.errors import ClarificationStateMismatchError
16
  from app.modules.places.infrastructure.deterministic_intent_parser import (
17
  DeterministicPlaceChatIntentParser,
 
126
  assert intent.location.strict_radius is True
127
 
128
 
129
+ def test_missing_category_preserves_open_concept_for_retrieval() -> None:
130
  intent = DeterministicPlaceChatIntentParser().parse(
131
  message="quiero un lugar bonito para salir",
132
  state=ConversationState(),
133
  has_user_location=True,
134
  )
135
 
136
+ assert intent.action == "recommendations"
137
+ assert intent.target_category is None
138
+ assert "bonito" in intent.semantic_query
139
  assert intent.unresolved == ("target_category",)
140
+ assert intent.state_patch.pending_clarification is None
141
+
142
+
143
+ def test_unknown_category_word_is_not_dropped_before_dense_retrieval() -> None:
144
+ intent = DeterministicPlaceChatIntentParser().parse(
145
+ message="quiero donas artesanales",
146
+ state=ConversationState(),
147
+ has_user_location=True,
148
+ )
149
+
150
+ assert intent.action == "recommendations"
151
+ assert intent.target_category is None
152
+ assert "donas" in intent.semantic_query
153
+ assert intent.raw_category_phrase is not None
154
 
155
 
156
  def test_food_activity_defaults_to_restaurant_without_clarification() -> None:
 
373
  def test_structured_choice_rejects_a_stale_clarification_id() -> None:
374
  parser = DeterministicPlaceChatIntentParser()
375
  first = parser.parse(
376
+ message="cafeterias como la de Hello Kitty cerca del Parque Central",
377
  state=ConversationState(),
378
  has_user_location=True,
379
  )
 
430
  assert "musica" not in intent.soft_preferences
431
  assert intent.exclusions == ("musica",)
432
  assert intent.state_patch.as_dict()["exclusions"] == ["musica"]
433
+
434
+
435
+ def test_unknown_exclusion_is_preserved_and_does_not_consume_positive_context() -> None:
436
+ intent = DeterministicPlaceChatIntentParser().parse(
437
+ message="una cafeteria sin ruido con terraza",
438
+ state=ConversationState(),
439
+ has_user_location=True,
440
+ )
441
+
442
+ assert intent.action == "recommendations"
443
+ assert intent.exclusions == ("ruido",)
444
+ assert "terraza" in intent.semantic_query
445
+ assert "ruido" not in intent.semantic_query
446
+
447
+
448
+ def test_bert_slots_drive_open_category_exclusion_and_location_before_legacy() -> None:
449
+ message = "quiero donas artesanales sin ruido cerca de la plaza"
450
+
451
+ def token(value: str, entity: str, score: float) -> dict[str, object]:
452
+ start = message.index(value)
453
+ return {
454
+ "entity": entity,
455
+ "score": score,
456
+ "start": start,
457
+ "end": start + len(value),
458
+ }
459
+
460
+ class Classifier:
461
+ def __call__(self, _text: str) -> list[dict[str, object]]:
462
+ return [
463
+ token("donas", "B-CATEGORY", 0.96),
464
+ token("artesanales", "I-CATEGORY", 0.94),
465
+ token("ruido", "B-EXCLUSION", 0.95),
466
+ token("plaza", "B-LOCATION", 0.93),
467
+ ]
468
+
469
+ extractor = BertPlaceIntentExtractor(
470
+ "places-intent-test",
471
+ model_version="test-v1",
472
+ classifier=Classifier(),
473
+ )
474
+ intent = DeterministicPlaceChatIntentParser(
475
+ contextual_extractor=extractor,
476
+ ).parse(
477
+ message=message,
478
+ state=ConversationState(),
479
+ has_user_location=True,
480
+ )
481
+
482
+ assert intent.action == "recommendations"
483
+ assert intent.target_category == "donas artesanales"
484
+ assert intent.category_values == ("donas artesanales",)
485
+ assert intent.raw_category_phrase == "donas artesanales"
486
+ assert intent.exclusions == ("ruido",)
487
+ assert intent.location.scope == "target_results"
488
+ assert intent.location.anchor_text == "plaza"
489
+ assert "donas artesanales" in intent.semantic_query
490
+ assert "ruido" not in intent.semantic_query
491
+ assert intent.state_patch.target_category == "donas artesanales"
492
+ assert intent.intent_model_version == (
493
+ "bert-token:places-intent-test@test-v1+deterministic-open-v2"
494
+ )
495
+
496
+
497
+ def test_bert_raw_category_is_aligned_semantically_to_dynamic_storage_values() -> None:
498
+ message = "quiero donas artesanales"
499
+ start = message.index("donas")
500
+
501
+ class TokenClassifier:
502
+ def __call__(self, _text: str) -> list[dict[str, object]]:
503
+ return [
504
+ {
505
+ "entity": "B-CATEGORY",
506
+ "score": 0.96,
507
+ "start": start,
508
+ "end": len(message),
509
+ }
510
+ ]
511
+
512
+ class Concept:
513
+ id = "donut_shop"
514
+ storage_values = ("bakery", "dessert")
515
+
516
+ class ActivityClassifier:
517
+ concepts = (Concept(),)
518
+
519
+ def classify(self, text: str) -> PlaceCategoryInference | None:
520
+ assert text == "donas artesanales"
521
+ return PlaceCategoryInference(
522
+ category="donut_shop",
523
+ confidence=0.91,
524
+ source="semantic_activity",
525
+ category_values=("bakery", "dessert"),
526
+ label="Donas",
527
+ )
528
+
529
+ intent = DeterministicPlaceChatIntentParser(
530
+ activity_classifier=ActivityClassifier(),
531
+ contextual_extractor=BertPlaceIntentExtractor(
532
+ "places-intent-test",
533
+ classifier=TokenClassifier(),
534
+ ),
535
+ ).parse(
536
+ message=message,
537
+ state=ConversationState(),
538
+ has_user_location=True,
539
+ )
540
+
541
+ assert intent.target_category == "donut_shop"
542
+ assert intent.category_values == ("bakery", "dessert")
543
+ assert intent.raw_category_phrase == "donas artesanales"
544
+
545
+
546
+ def test_bert_failure_falls_open_and_exposes_fallback_model_version() -> None:
547
+ class BrokenClassifier:
548
+ def __call__(self, _text: str) -> list[dict[str, object]]:
549
+ raise RuntimeError("inference backend unavailable")
550
+
551
+ extractor = BertPlaceIntentExtractor(
552
+ "broken-intent-model",
553
+ model_version="broken-v1",
554
+ classifier=BrokenClassifier(),
555
+ )
556
+ intent = DeterministicPlaceChatIntentParser(
557
+ contextual_extractor=extractor,
558
+ ).parse(
559
+ message="recomiendame una cafeteria tranquila",
560
+ state=ConversationState(),
561
+ has_user_location=True,
562
+ )
563
+
564
+ assert intent.action == "recommendations"
565
+ assert intent.target_category == "cafe"
566
+ assert "tranquilo" in intent.soft_preferences
567
+ assert intent.intent_model_version == (
568
+ "deterministic-open-v2+bert-fallback:broken-v1"
569
+ )
570
+
571
+
572
+ def test_successful_bert_frame_does_not_reapply_manual_category_aliases() -> None:
573
+ extractor = BertPlaceIntentExtractor(
574
+ "places-intent-test",
575
+ classifier=lambda _text: [],
576
+ )
577
+
578
+ intent = DeterministicPlaceChatIntentParser(
579
+ contextual_extractor=extractor,
580
+ ).parse(
581
+ message="cafeteria con una variacion no etiquetada",
582
+ state=ConversationState(),
583
+ has_user_location=True,
584
+ )
585
+
586
+ assert intent.target_category is None
587
+ assert intent.category_values == ()
588
+ assert "cafeteria" in intent.semantic_query
589
+ assert intent.category_source == "unresolved"
590
+
591
+
592
+ def test_partial_bert_frame_fuses_missing_context_slots_independently() -> None:
593
+ message = "cafeteria tranquila sin ruido cerca del centro"
594
+ category_start = message.index("cafeteria")
595
+ extractor = BertPlaceIntentExtractor(
596
+ "places-intent-test",
597
+ classifier=lambda _text: [
598
+ {
599
+ "entity": "B-CATEGORY",
600
+ "score": 0.97,
601
+ "start": category_start,
602
+ "end": category_start + len("cafeteria"),
603
+ }
604
+ ],
605
+ )
606
+
607
+ intent = DeterministicPlaceChatIntentParser(
608
+ contextual_extractor=extractor,
609
+ ).parse(
610
+ message=message,
611
+ state=ConversationState(),
612
+ has_user_location=True,
613
+ )
614
+
615
+ # The model-provided open category is preserved (no taxonomy remap), while
616
+ # omitted slots use narrow rollout fallbacks instead of disappearing.
617
+ assert intent.target_category == "cafeteria"
618
+ assert intent.location.scope == "target_results"
619
+ assert intent.location.source == "current_message"
620
+ assert intent.location.anchor_text == "centro"
621
+ assert intent.exclusions == ("ruido",)
622
+ assert "tranquilo" in intent.soft_preferences
623
+ assert "centro" not in intent.semantic_query
624
+ assert "ruido" not in intent.semantic_query
625
+
626
+
627
+ def test_bert_preference_reference_and_radius_remain_open_raw_signals() -> None:
628
+ message = "busco salones de te estilo Casa Azul con terraza maximo 2 km"
629
+
630
+ def token(value: str, entity: str) -> dict[str, object]:
631
+ start = message.index(value)
632
+ return {
633
+ "entity": entity,
634
+ "score": 0.94,
635
+ "start": start,
636
+ "end": start + len(value),
637
+ }
638
+
639
+ class Classifier:
640
+ def __call__(self, _text: str) -> list[dict[str, object]]:
641
+ return [
642
+ token("salones", "B-CATEGORY"),
643
+ token("de", "I-CATEGORY"),
644
+ token("te", "I-CATEGORY"),
645
+ token("Casa", "B-REFERENCE"),
646
+ token("Azul", "I-REFERENCE"),
647
+ token("terraza", "B-PREFERENCE"),
648
+ token("2", "B-RADIUS"),
649
+ token("km", "I-RADIUS"),
650
+ ]
651
+
652
+ intent = DeterministicPlaceChatIntentParser(
653
+ contextual_extractor=BertPlaceIntentExtractor(
654
+ "places-intent-test",
655
+ classifier=Classifier(),
656
+ ),
657
+ ).parse(
658
+ message=message,
659
+ state=ConversationState(),
660
+ has_user_location=True,
661
+ )
662
+
663
+ assert intent.target_category == "salones de te"
664
+ assert intent.soft_preferences == ("terraza",)
665
+ assert intent.reference is not None
666
+ assert intent.reference.entity == "Casa Azul"
667
+ assert intent.location.source == "user_current"
668
+ assert intent.location.radius_meters == 2000
669
+ assert intent.location.strict_radius is True
670
+
671
+
672
+ def test_dynamic_category_clarification_selection_is_not_taxonomy_gated() -> None:
673
+ pending = PendingClarification(
674
+ clarification_id="dynamic-category-1",
675
+ kind="intent_category",
676
+ options=(
677
+ PendingClarificationOption(
678
+ option_id="donuts",
679
+ value="donas artesanales",
680
+ label="Donas artesanales",
681
+ ),
682
+ PendingClarificationOption(
683
+ option_id="desserts",
684
+ value="postres frios",
685
+ label="Postres frios",
686
+ ),
687
+ ),
688
+ )
689
+ state = ConversationState(
690
+ soft_preferences=("tranquilo",),
691
+ exclusions=("ruido",),
692
+ pending_clarification=pending,
693
+ city="Puebla",
694
+ )
695
+
696
+ intent = DeterministicPlaceChatIntentParser().parse(
697
+ message="Donas artesanales",
698
+ state=state,
699
+ has_user_location=True,
700
+ clarification_choice=ClarificationChoice(
701
+ clarification_id="dynamic-category-1",
702
+ option_id="donuts",
703
+ ),
704
+ )
705
+
706
+ assert intent.action == "recommendations"
707
+ assert intent.target_category == "donas artesanales"
708
+ assert intent.category_values == ("donas artesanales",)
709
+ assert intent.raw_category_phrase == "donas artesanales"
710
+ assert intent.soft_preferences == ("tranquilo",)
711
+ assert intent.exclusions == ("ruido",)
712
+ assert intent.hard_filters["city"] == "Puebla"
713
+ assert intent.state_patch.target_category == "donas artesanales"
714
+ assert intent.state_patch.clear_pending_clarification is True
715
+ assert "dynamic-clarification-v1" in intent.intent_model_version
716
+
717
+
718
+ def test_open_catalog_category_values_are_rehydrated_on_following_turn() -> None:
719
+ class Concept:
720
+ id = "donut_shop"
721
+ storage_values = ("bakery", "dessert")
722
+
723
+ class ActivityClassifier:
724
+ concepts = (Concept(),)
725
+
726
+ def classify(self, text: str) -> PlaceCategoryInference | None:
727
+ if "donas" in text:
728
+ return PlaceCategoryInference(
729
+ category="donut_shop",
730
+ confidence=0.91,
731
+ source="semantic_activity",
732
+ category_values=("bakery", "dessert"),
733
+ label="Donas",
734
+ )
735
+ return None
736
+
737
+ parser = DeterministicPlaceChatIntentParser(
738
+ activity_classifier=ActivityClassifier(),
739
+ )
740
+ first = parser.parse(
741
+ message="quiero donas glaseadas",
742
+ state=ConversationState(),
743
+ has_user_location=True,
744
+ )
745
+ continued = parser.parse(
746
+ message="mas barato",
747
+ state=ConversationState(target_category=first.target_category),
748
+ has_user_location=True,
749
+ )
750
+
751
+ assert first.target_category == "donut_shop"
752
+ assert first.category_values == ("bakery", "dessert")
753
+ assert continued.target_category == "donut_shop"
754
+ assert continued.category_values == ("bakery", "dessert")
755
+
756
+
757
+ def test_generic_short_request_never_produces_an_empty_embedding_query() -> None:
758
+ intent = DeterministicPlaceChatIntentParser().parse(
759
+ message="dame opciones",
760
+ state=ConversationState(),
761
+ has_user_location=True,
762
+ )
763
+
764
+ assert intent.action == "recommendations"
765
+ assert intent.semantic_query == "dame opciones"
tests/test_place_chat_recommendations_use_case.py CHANGED
@@ -1,3 +1,5 @@
 
 
1
  import pytest
2
 
3
  from app.modules.places.application.use_cases.chat_place_recommendations import (
@@ -10,8 +12,11 @@ from app.modules.places.domain.clarifications import (
10
  from app.modules.places.domain.chat_intent import (
11
  ClarificationChoice,
12
  ConversationState,
 
 
13
  PendingClarification,
14
  PendingClarificationOption,
 
15
  PlaceReference,
16
  ResolvedPlaceAnchor,
17
  )
@@ -33,21 +38,32 @@ from app.shared.nlp.llm.mock import MockLLMProvider
33
  from app.shared.nlp.llm.output_guard import PlaceChatOutputGuard
34
 
35
 
36
- def build_use_case(*, llm_enabled: bool, anchor_resolver=None):
 
 
 
 
 
 
 
37
  embedding = MockEmbeddingProvider(dimension=16)
38
  return ChatPlaceRecommendationsUseCase(
39
- intent_parser=DeterministicPlaceChatIntentParser(),
40
  anchor_resolver=anchor_resolver or MockPlaceAnchorResolver(),
41
- retriever=HybridContentPlaceChatRetriever(
42
- embedding_provider=embedding,
43
- place_repository=MockPlaceVectorRepository(embedding),
44
- minimum_content_score=0.20,
 
 
 
45
  ),
46
  llm_provider=MockLLMProvider(),
47
  output_guard=PlaceChatOutputGuard(),
48
  ranking_version="places-chat-v2",
49
  taxonomy_version="places-taxonomy-v1",
50
  llm_enabled=llm_enabled,
 
51
  )
52
 
53
 
@@ -68,6 +84,29 @@ def test_public_anchor_buttons_are_bounded_for_the_main_api_contract() -> None:
68
  assert len(clarification.options[0].message) <= 200
69
 
70
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
71
  @pytest.mark.asyncio
72
  async def test_llm_cannot_change_action_or_candidates() -> None:
73
  without_llm = await build_use_case(llm_enabled=False).execute(
@@ -95,6 +134,326 @@ async def test_llm_cannot_change_action_or_candidates() -> None:
95
  assert with_llm.guard_reason == "candidate_name_deferred_to_main_api"
96
 
97
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
98
  class AmbiguousAnchorResolver:
99
  async def resolve(self, text, city, state, limit=3):
100
  del text, city, state, limit
@@ -308,7 +667,7 @@ class MixedCategoryRepository:
308
  ("shopping", "shopping_exact"),
309
  ),
310
  )
311
- async def test_short_category_queries_survive_sparse_scores_without_family_noise(
312
  message: str,
313
  expected_id: str,
314
  ) -> None:
@@ -325,11 +684,23 @@ async def test_short_category_queries_survive_sparse_scores_without_family_noise
325
 
326
  candidates = await retriever.retrieve(intent=intent, limit=5)
327
 
328
- assert [candidate.place_id for candidate in candidates] == [expected_id]
 
 
 
 
 
 
 
 
 
 
 
 
329
 
330
 
331
  @pytest.mark.asyncio
332
- async def test_hard_category_and_controlled_theme_relaxation() -> None:
333
  embedding = MockEmbeddingProvider(dimension=16)
334
  retriever = HybridContentPlaceChatRetriever(
335
  embedding_provider=embedding,
@@ -344,16 +715,18 @@ async def test_hard_category_and_controlled_theme_relaxation() -> None:
344
 
345
  candidates = await retriever.retrieve(intent=intent, limit=10)
346
 
347
- assert [candidate.place_id for candidate in candidates] == [
 
348
  "exact",
349
  "family",
350
  "broad",
351
- ]
352
- assert [candidate.match_level for candidate in candidates] == [
353
- "exact",
354
- "family",
355
- "broad",
356
- ]
 
357
 
358
 
359
  class ReferenceLocationResolver:
 
1
+ from dataclasses import replace
2
+
3
  import pytest
4
 
5
  from app.modules.places.application.use_cases.chat_place_recommendations import (
 
12
  from app.modules.places.domain.chat_intent import (
13
  ClarificationChoice,
14
  ConversationState,
15
+ ExplicitTargetLocation,
16
+ IntentAlternative,
17
  PendingClarification,
18
  PendingClarificationOption,
19
+ PlaceChatCandidate,
20
  PlaceReference,
21
  ResolvedPlaceAnchor,
22
  )
 
38
  from app.shared.nlp.llm.output_guard import PlaceChatOutputGuard
39
 
40
 
41
+ def build_use_case(
42
+ *,
43
+ llm_enabled: bool,
44
+ anchor_resolver=None,
45
+ intent_parser=None,
46
+ retriever=None,
47
+ nearby_place_provider=None,
48
+ ):
49
  embedding = MockEmbeddingProvider(dimension=16)
50
  return ChatPlaceRecommendationsUseCase(
51
+ intent_parser=intent_parser or DeterministicPlaceChatIntentParser(),
52
  anchor_resolver=anchor_resolver or MockPlaceAnchorResolver(),
53
+ retriever=(
54
+ retriever
55
+ or HybridContentPlaceChatRetriever(
56
+ embedding_provider=embedding,
57
+ place_repository=MockPlaceVectorRepository(embedding),
58
+ minimum_content_score=0.20,
59
+ )
60
  ),
61
  llm_provider=MockLLMProvider(),
62
  output_guard=PlaceChatOutputGuard(),
63
  ranking_version="places-chat-v2",
64
  taxonomy_version="places-taxonomy-v1",
65
  llm_enabled=llm_enabled,
66
+ nearby_place_provider=nearby_place_provider,
67
  )
68
 
69
 
 
84
  assert len(clarification.options[0].message) <= 200
85
 
86
 
87
+ def test_open_category_buttons_use_safe_ids_and_preserve_raw_values() -> None:
88
+ from app.modules.places.domain.clarifications import new_category_clarification
89
+
90
+ pending = new_category_clarification(
91
+ ("donas artesanales", "café de especialidad"),
92
+ kind="intent_category",
93
+ )
94
+ clarification = to_public_clarification(pending)
95
+
96
+ assert [option.option_id for option in pending.options] == [
97
+ "donas_artesanales_1",
98
+ "cafe_de_especialidad_2",
99
+ ]
100
+ assert [option.value for option in pending.options] == [
101
+ "donas artesanales",
102
+ "café de especialidad",
103
+ ]
104
+ assert [option.option_id for option in clarification.options] == [
105
+ "donas_artesanales_1",
106
+ "cafe_de_especialidad_2",
107
+ ]
108
+
109
+
110
  @pytest.mark.asyncio
111
  async def test_llm_cannot_change_action_or_candidates() -> None:
112
  without_llm = await build_use_case(llm_enabled=False).execute(
 
134
  assert with_llm.guard_reason == "candidate_name_deferred_to_main_api"
135
 
136
 
137
+ class LowConfidenceIntentParser:
138
+ def __init__(self, alternatives: tuple[IntentAlternative, ...]) -> None:
139
+ self._alternatives = alternatives
140
+
141
+ def parse(
142
+ self,
143
+ message,
144
+ state,
145
+ has_user_location,
146
+ clarification_choice=None,
147
+ ):
148
+ del message, clarification_choice
149
+ parsed = DeterministicPlaceChatIntentParser().parse(
150
+ message="una panaderia tranquila",
151
+ state=state,
152
+ has_user_location=has_user_location,
153
+ )
154
+ return replace(
155
+ parsed,
156
+ semantic_query="algo dulce tranquilo",
157
+ confidence=0.45,
158
+ alternatives=self._alternatives,
159
+ category_source="semantic_activity",
160
+ )
161
+
162
+
163
+ class CategoryEvidenceRetriever:
164
+ def __init__(self) -> None:
165
+ self.calls = 0
166
+
167
+ async def retrieve(self, intent, limit):
168
+ del intent, limit
169
+ self.calls += 1
170
+ return [
171
+ PlaceChatCandidate(
172
+ place_id="bakery_1",
173
+ name="Panaderia Local",
174
+ category="bakery",
175
+ content_score=0.62,
176
+ semantic_score=0.64,
177
+ lexical_score=0.20,
178
+ match_level="broad",
179
+ matched_reasons=("algo dulce",),
180
+ )
181
+ ]
182
+
183
+
184
+ @pytest.mark.asyncio
185
+ async def test_low_confidence_uses_dynamic_category_hypotheses() -> None:
186
+ parser = LowConfidenceIntentParser(
187
+ alternatives=(
188
+ IntentAlternative(
189
+ key="bakery",
190
+ description="Panaderias artesanales",
191
+ confidence=0.68,
192
+ ),
193
+ IntentAlternative(
194
+ key="ice_cream",
195
+ description="Postres y heladerias",
196
+ confidence=0.64,
197
+ ),
198
+ )
199
+ )
200
+
201
+ result = await build_use_case(
202
+ llm_enabled=False,
203
+ intent_parser=parser,
204
+ ).execute(
205
+ message="quiero algo dulce y tranquilo",
206
+ state=ConversationState(),
207
+ user_latitude=16.7531,
208
+ user_longitude=-93.1156,
209
+ candidate_limit=5,
210
+ result_limit=3,
211
+ )
212
+
213
+ assert result.action == "clarification"
214
+ assert result.clarification is not None
215
+ assert [option.option_id for option in result.clarification.options] == [
216
+ "bakery",
217
+ "ice_cream",
218
+ ]
219
+ assert [option.label for option in result.clarification.options] == [
220
+ "Panaderias artesanales",
221
+ "Postres y heladerias",
222
+ ]
223
+ assert result.state_patch["target_category"] == "bakery"
224
+ assert result.state_patch["soft_preferences"] == ["tranquilo"]
225
+ assert [
226
+ option["id"]
227
+ for option in result.state_patch["pending_clarification"]["options"]
228
+ ] == ["bakery", "ice_cream"]
229
+
230
+
231
+ @pytest.mark.asyncio
232
+ async def test_low_confidence_without_top_k_uses_retrieval_evidence_not_fixed_menu() -> None:
233
+ retriever = CategoryEvidenceRetriever()
234
+ result = await build_use_case(
235
+ llm_enabled=False,
236
+ intent_parser=LowConfidenceIntentParser(alternatives=()),
237
+ retriever=retriever,
238
+ ).execute(
239
+ message="quiero algo dulce y tranquilo",
240
+ state=ConversationState(),
241
+ user_latitude=16.7531,
242
+ user_longitude=-93.1156,
243
+ candidate_limit=5,
244
+ result_limit=3,
245
+ )
246
+
247
+ assert retriever.calls == 1
248
+ assert result.action == "recommendations"
249
+ assert result.clarification is None
250
+ assert [candidate.place_id for candidate in result.candidates] == ["bakery_1"]
251
+ assert "intent_confidence" in result.unresolved
252
+ assert "restaurante" not in result.message.casefold()
253
+ assert "cafeteria" not in result.message.casefold()
254
+ assert "parque" not in result.message.casefold()
255
+ assert result.state_patch["soft_preferences"] == ["tranquilo"]
256
+
257
+
258
+ @pytest.mark.asyncio
259
+ async def test_low_quality_hypotheses_do_not_create_an_arbitrary_menu() -> None:
260
+ parser = LowConfidenceIntentParser(
261
+ alternatives=(
262
+ IntentAlternative("bakery", "Panaderias", 0.55),
263
+ IntentAlternative("outdoors", "Espacios abiertos", 0.52),
264
+ )
265
+ )
266
+
267
+ result = await build_use_case(
268
+ llm_enabled=False,
269
+ intent_parser=parser,
270
+ retriever=CategoryEvidenceRetriever(),
271
+ ).execute(
272
+ message="algo dificil de interpretar",
273
+ state=ConversationState(),
274
+ user_latitude=16.7531,
275
+ user_longitude=-93.1156,
276
+ candidate_limit=5,
277
+ result_limit=3,
278
+ )
279
+
280
+ assert result.action == "recommendations"
281
+ assert result.clarification is None
282
+ assert "intent_confidence" in result.unresolved
283
+
284
+
285
+ class WeakEvidenceRetriever:
286
+ async def retrieve(self, intent, limit):
287
+ del intent, limit
288
+ return [
289
+ PlaceChatCandidate(
290
+ place_id="weak_1",
291
+ name="Coincidencia tenue",
292
+ category="cafe",
293
+ content_score=0.0,
294
+ semantic_score=0.0,
295
+ lexical_score=0.0,
296
+ match_level="broad",
297
+ matched_reasons=(),
298
+ metadata={
299
+ "retrieval_diagnostics": {
300
+ "meets_minimum_content_score": False,
301
+ }
302
+ },
303
+ )
304
+ ]
305
+
306
+
307
+ @pytest.mark.asyncio
308
+ async def test_weak_candidates_are_returned_as_reviewable_not_confident() -> None:
309
+ result = await build_use_case(
310
+ llm_enabled=False,
311
+ retriever=WeakEvidenceRetriever(),
312
+ ).execute(
313
+ message="una cafeteria",
314
+ state=ConversationState(),
315
+ user_latitude=16.7531,
316
+ user_longitude=-93.1156,
317
+ candidate_limit=5,
318
+ result_limit=3,
319
+ )
320
+
321
+ assert result.action == "recommendations"
322
+ assert result.unresolved == ("retrieval_evidence",)
323
+ assert [candidate.place_id for candidate in result.candidates] == ["weak_1"]
324
+ assert "evidencia" in result.message.casefold()
325
+
326
+
327
+ class RecordingNearbyProvider:
328
+ def __init__(self) -> None:
329
+ self.call = None
330
+
331
+ async def get_nearby_place_ids(self, latitude, longitude, radius_meters):
332
+ self.call = (latitude, longitude, radius_meters)
333
+ return {"near_2", "near_1"}
334
+
335
+
336
+ class RecordingFilteredRetriever:
337
+ def __init__(self) -> None:
338
+ self.intent = None
339
+
340
+ async def retrieve(self, intent, limit):
341
+ del limit
342
+ self.intent = intent
343
+ return [
344
+ PlaceChatCandidate(
345
+ place_id="near_1",
346
+ name="Cercano",
347
+ category="cafe",
348
+ content_score=0.8,
349
+ semantic_score=0.8,
350
+ lexical_score=0.4,
351
+ match_level="exact",
352
+ matched_reasons=("cafe",),
353
+ metadata={
354
+ "retrieval_diagnostics": {
355
+ "meets_minimum_content_score": True,
356
+ }
357
+ },
358
+ )
359
+ ]
360
+
361
+
362
+ @pytest.mark.asyncio
363
+ async def test_current_location_ids_are_applied_before_content_retrieval() -> None:
364
+ nearby = RecordingNearbyProvider()
365
+ retriever = RecordingFilteredRetriever()
366
+
367
+ result = await build_use_case(
368
+ llm_enabled=False,
369
+ retriever=retriever,
370
+ nearby_place_provider=nearby,
371
+ ).execute(
372
+ message="una cafeteria",
373
+ state=ConversationState(),
374
+ user_latitude=16.7531,
375
+ user_longitude=-93.1156,
376
+ candidate_limit=5,
377
+ result_limit=3,
378
+ )
379
+
380
+ assert result.action == "recommendations"
381
+ assert nearby.call == (16.7531, -93.1156, 5_000)
382
+ assert retriever.intent.hard_filters["place_ids"] == ("near_1", "near_2")
383
+
384
+
385
+ class CoordinateAnchorResolver:
386
+ async def resolve(self, text, city, state, limit=3):
387
+ del text, city, state, limit
388
+ return [
389
+ ResolvedPlaceAnchor(
390
+ place_id="anchor_centro",
391
+ name="Centro",
392
+ latitude=16.75,
393
+ longitude=-93.12,
394
+ score=0.95,
395
+ )
396
+ ]
397
+
398
+
399
+ @pytest.mark.asyncio
400
+ async def test_explicit_anchor_coordinates_filter_before_retrieval() -> None:
401
+ nearby = RecordingNearbyProvider()
402
+ retriever = RecordingFilteredRetriever()
403
+
404
+ result = await build_use_case(
405
+ llm_enabled=False,
406
+ anchor_resolver=CoordinateAnchorResolver(),
407
+ retriever=retriever,
408
+ nearby_place_provider=nearby,
409
+ ).execute(
410
+ message="una cafeteria cerca del centro",
411
+ state=ConversationState(),
412
+ user_latitude=16.70,
413
+ user_longitude=-93.10,
414
+ candidate_limit=5,
415
+ result_limit=3,
416
+ )
417
+
418
+ assert result.action == "recommendations"
419
+ assert result.location_directive.source == "explicit_anchor"
420
+ assert nearby.call == (16.75, -93.12, 5_000)
421
+ assert retriever.intent.hard_filters["place_ids"] == ("near_1", "near_2")
422
+
423
+
424
+ @pytest.mark.asyncio
425
+ async def test_persisted_resolved_anchor_keeps_nearby_filter_on_followup() -> None:
426
+ nearby = RecordingNearbyProvider()
427
+ retriever = RecordingFilteredRetriever()
428
+
429
+ result = await build_use_case(
430
+ llm_enabled=False,
431
+ anchor_resolver=CoordinateAnchorResolver(),
432
+ retriever=retriever,
433
+ nearby_place_provider=nearby,
434
+ ).execute(
435
+ message="otra cafeteria tranquila",
436
+ state=ConversationState(
437
+ explicit_target_location=ExplicitTargetLocation(
438
+ anchor_text="Centro",
439
+ place_id="anchor_centro",
440
+ label="Centro",
441
+ radius_meters=3_000,
442
+ strict_radius=True,
443
+ )
444
+ ),
445
+ user_latitude=None,
446
+ user_longitude=None,
447
+ candidate_limit=5,
448
+ result_limit=3,
449
+ )
450
+
451
+ assert result.action == "recommendations"
452
+ assert result.location_directive.source == "state_anchor"
453
+ assert nearby.call == (16.75, -93.12, 3_000)
454
+ assert retriever.intent.hard_filters["place_ids"] == ("near_1", "near_2")
455
+
456
+
457
  class AmbiguousAnchorResolver:
458
  async def resolve(self, text, city, state, limit=3):
459
  del text, city, state, limit
 
667
  ("shopping", "shopping_exact"),
668
  ),
669
  )
670
+ async def test_short_category_queries_survive_sparse_scores_without_being_filtered(
671
  message: str,
672
  expected_id: str,
673
  ) -> None:
 
684
 
685
  candidates = await retriever.retrieve(intent=intent, limit=5)
686
 
687
+ candidate_ids = [candidate.place_id for candidate in candidates]
688
+ assert expected_id in candidate_ids
689
+ assert set(candidate_ids) == {
690
+ "park_exact",
691
+ "cinema_compatible",
692
+ "entertainment_noise",
693
+ "shopping_exact",
694
+ "family_noise",
695
+ }
696
+ assert all(
697
+ candidate.metadata["retrieval_diagnostics"]["content_quality"] == "weak"
698
+ for candidate in candidates
699
+ )
700
 
701
 
702
  @pytest.mark.asyncio
703
+ async def test_category_is_ranking_evidence_instead_of_a_hard_filter() -> None:
704
  embedding = MockEmbeddingProvider(dimension=16)
705
  retriever = HybridContentPlaceChatRetriever(
706
  embedding_provider=embedding,
 
715
 
716
  candidates = await retriever.retrieve(intent=intent, limit=10)
717
 
718
+ assert {candidate.place_id for candidate in candidates} == {
719
+ "park",
720
  "exact",
721
  "family",
722
  "broad",
723
+ }
724
+ diagnostics = {
725
+ candidate.place_id: candidate.metadata["retrieval_diagnostics"]
726
+ for candidate in candidates
727
+ }
728
+ assert diagnostics["park"]["category_match"] == "none"
729
+ assert diagnostics["exact"]["category_match"] == "exact"
730
 
731
 
732
  class ReferenceLocationResolver:
tests/test_place_embedding_configuration.py ADDED
@@ -0,0 +1,182 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ import json
2
+
3
+ import pytest
4
+ from pydantic import ValidationError
5
+
6
+ from app.modules.places.api import dependencies as place_dependencies
7
+ from app.modules.places.infrastructure.bert_intent_extractor import (
8
+ BertPlaceIntentExtractor,
9
+ )
10
+ from app.modules.places.infrastructure.place_category_catalog import (
11
+ load_place_category_concepts,
12
+ )
13
+ from app.shared.config.settings import Settings
14
+ from app.shared.nlp.embeddings.factory import create_place_embedding_provider
15
+ from app.shared.nlp.embeddings.mock import MockEmbeddingProvider
16
+ from app.shared.nlp.embeddings.sentence_transformer import (
17
+ SentenceTransformerEmbeddingProvider,
18
+ )
19
+
20
+
21
+ def test_places_embedding_provider_is_independent_from_global_dimension() -> None:
22
+ settings = Settings(
23
+ _env_file=None,
24
+ ENV="local",
25
+ EMBEDDING_PROVIDER="mock",
26
+ EMBEDDING_DIMENSION=16,
27
+ PLACES_EMBEDDING_PROVIDER="mock",
28
+ PLACES_EMBEDDING_DIMENSION=384,
29
+ PLACES_EMBEDDING_MODEL="places-test",
30
+ )
31
+
32
+ provider = create_place_embedding_provider(settings)
33
+
34
+ assert isinstance(provider, MockEmbeddingProvider)
35
+ assert provider.dimension == 384
36
+ assert settings.embedding_dimension == 16
37
+
38
+
39
+ def test_blank_optional_places_runtime_values_are_normalized_to_none() -> None:
40
+ settings = Settings(
41
+ _env_file=None,
42
+ PLACES_EMBEDDING_DEVICE=" ",
43
+ PLACES_CATEGORY_CATALOG_PATH="",
44
+ PLACES_PGVECTOR_HYBRID_FUNCTION=" ",
45
+ )
46
+
47
+ assert settings.places_embedding_device is None
48
+ assert settings.places_category_catalog_path is None
49
+ assert settings.places_pgvector_hybrid_function is None
50
+
51
+
52
+ def test_sentence_transformer_factory_configures_query_and_passage_prefixes() -> None:
53
+ settings = Settings(
54
+ _env_file=None,
55
+ ENV="local",
56
+ PLACES_EMBEDDING_PROVIDER="sentence_transformer",
57
+ PLACES_EMBEDDING_DIMENSION=768,
58
+ PLACES_EMBEDDING_MODEL="intfloat/multilingual-e5-base",
59
+ PLACES_EMBEDDING_QUERY_PREFIX="query: ",
60
+ PLACES_EMBEDDING_PASSAGE_PREFIX="passage: ",
61
+ )
62
+
63
+ query = create_place_embedding_provider(settings, text_role="query")
64
+ passage = create_place_embedding_provider(settings, text_role="passage")
65
+
66
+ assert isinstance(query, SentenceTransformerEmbeddingProvider)
67
+ assert isinstance(passage, SentenceTransformerEmbeddingProvider)
68
+ assert query.text_prefix == "query: "
69
+ assert passage.text_prefix == "passage: "
70
+ assert query.is_loaded is False
71
+ assert passage.is_loaded is False
72
+
73
+
74
+ def test_quoted_e5_prefixes_preserve_the_significant_space(tmp_path) -> None:
75
+ env_file = tmp_path / ".env"
76
+ env_file.write_text(
77
+ 'PLACES_EMBEDDING_QUERY_PREFIX="query: "\n'
78
+ 'PLACES_EMBEDDING_PASSAGE_PREFIX="passage: "\n',
79
+ encoding="utf-8",
80
+ )
81
+
82
+ settings = Settings(_env_file=env_file)
83
+
84
+ assert settings.places_embedding_query_prefix == "query: "
85
+ assert settings.places_embedding_passage_prefix == "passage: "
86
+
87
+
88
+ def test_places_embedding_provider_rejects_unknown_backend() -> None:
89
+ with pytest.raises(ValidationError, match="PLACES_EMBEDDING_PROVIDER"):
90
+ Settings(
91
+ _env_file=None,
92
+ ENV="local",
93
+ PLACES_EMBEDDING_PROVIDER="closed_taxonomy_magic",
94
+ )
95
+
96
+
97
+ def test_external_place_concept_catalog_is_data_driven(tmp_path) -> None:
98
+ path = tmp_path / "concepts.json"
99
+ path.write_text(
100
+ json.dumps(
101
+ {
102
+ "concepts": [
103
+ {
104
+ "id": "donut_shop",
105
+ "label": "Donas",
106
+ "description": "Lugar especializado en donas artesanales",
107
+ "examples": ["algo dulce glaseado"],
108
+ "storage_values": ["bakery", "dessert"],
109
+ }
110
+ ]
111
+ }
112
+ ),
113
+ encoding="utf-8",
114
+ )
115
+
116
+ concepts = load_place_category_concepts(str(path))
117
+
118
+ assert [concept.id for concept in concepts] == ["donut_shop"]
119
+ assert concepts[0].storage_values == ("bakery", "dessert")
120
+
121
+
122
+ def test_place_chat_intent_provider_defaults_to_deterministic() -> None:
123
+ settings = Settings(_env_file=None, ENV="local")
124
+
125
+ assert settings.places_chat_intent_provider == "deterministic"
126
+ assert settings.places_chat_bert_model_path is None
127
+
128
+
129
+ def test_place_chat_bert_provider_is_explicit_and_validated() -> None:
130
+ settings = Settings(
131
+ _env_file=None,
132
+ ENV="local",
133
+ PLACES_CHAT_INTENT_PROVIDER="BERT",
134
+ PLACES_CHAT_BERT_MODEL_PATH="models/places-intent",
135
+ PLACES_CHAT_BERT_MODEL_VERSION="intent-v3",
136
+ PLACES_CHAT_BERT_DEVICE="cpu",
137
+ PLACES_CHAT_BERT_MIN_TOKEN_CONFIDENCE=0.72,
138
+ )
139
+
140
+ assert settings.places_chat_intent_provider == "bert"
141
+ assert settings.places_chat_bert_model_path == "models/places-intent"
142
+ assert settings.places_chat_bert_model_version == "intent-v3"
143
+ assert settings.places_chat_bert_device == "cpu"
144
+ assert settings.places_chat_bert_min_token_confidence == pytest.approx(0.72)
145
+
146
+ with pytest.raises(ValidationError, match="PLACES_CHAT_BERT_MODEL_PATH"):
147
+ Settings(
148
+ _env_file=None,
149
+ ENV="local",
150
+ PLACES_CHAT_INTENT_PROVIDER="bert",
151
+ )
152
+
153
+
154
+ def test_place_chat_dependency_builds_bert_extractor_without_loading_it(
155
+ monkeypatch: pytest.MonkeyPatch,
156
+ ) -> None:
157
+ settings = Settings(
158
+ _env_file=None,
159
+ ENV="local",
160
+ PLACES_EMBEDDING_PROVIDER="mock",
161
+ PLACES_CHAT_INTENT_PROVIDER="bert",
162
+ PLACES_CHAT_BERT_MODEL_PATH="models/places-intent",
163
+ PLACES_CHAT_BERT_MODEL_VERSION="intent-v3",
164
+ PLACES_CHAT_BERT_DEVICE="cpu",
165
+ )
166
+ monkeypatch.setattr(place_dependencies, "get_settings", lambda: settings)
167
+ monkeypatch.setattr(
168
+ place_dependencies,
169
+ "load_place_category_concepts",
170
+ lambda *_args, **_kwargs: (),
171
+ )
172
+ place_dependencies.get_place_chat_intent_parser.cache_clear()
173
+ try:
174
+ parser = place_dependencies.get_place_chat_intent_parser()
175
+ extractor = parser._contextual_extractor
176
+
177
+ assert isinstance(extractor, BertPlaceIntentExtractor)
178
+ assert extractor.model_name == "models/places-intent"
179
+ assert extractor.model_version == "intent-v3"
180
+ assert extractor.is_loaded is False
181
+ finally:
182
+ place_dependencies.get_place_chat_intent_parser.cache_clear()
tests/test_places_use_cases.py CHANGED
@@ -11,7 +11,10 @@ from app.modules.places.infrastructure.mock_place_repository import MockPlaceVec
11
  from app.shared.nlp.embeddings.mock import MockEmbeddingProvider
12
  from app.shared.nlp.llm.base import LLMProvider, LLMResult, PlaceResponseMode
13
  from app.shared.nlp.llm.mock import MockLLMProvider
14
- from app.shared.nlp.llm.output_guard import DEFAULT_PLACE_CHAT_FALLBACK, PlaceChatOutputGuard
 
 
 
15
 
16
 
17
  @pytest.mark.asyncio
@@ -243,9 +246,10 @@ async def test_chat_places_falls_back_when_llm_fails() -> None:
243
  limit=3,
244
  )
245
 
246
- assert result.message == DEFAULT_PLACE_CHAT_FALLBACK
247
  assert result.places
248
  assert result.metadata["used_llm"] is False
 
249
 
250
 
251
  class FailingLLMProvider(LLMProvider):
 
11
  from app.shared.nlp.embeddings.mock import MockEmbeddingProvider
12
  from app.shared.nlp.llm.base import LLMProvider, LLMResult, PlaceResponseMode
13
  from app.shared.nlp.llm.mock import MockLLMProvider
14
+ from app.shared.nlp.llm.output_guard import (
15
+ LOW_CONFIDENCE_PLACE_CHAT_FALLBACK,
16
+ PlaceChatOutputGuard,
17
+ )
18
 
19
 
20
  @pytest.mark.asyncio
 
246
  limit=3,
247
  )
248
 
249
+ assert result.message == LOW_CONFIDENCE_PLACE_CHAT_FALLBACK
250
  assert result.places
251
  assert result.metadata["used_llm"] is False
252
+ assert result.metadata["response_mode"] == "low_confidence"
253
 
254
 
255
  class FailingLLMProvider(LLMProvider):
tests/test_semantic_activity_classifier.py CHANGED
@@ -1,6 +1,9 @@
1
  from app.modules.places.infrastructure.semantic_activity_classifier import (
2
  SemanticPlaceActivityClassifier,
3
  )
 
 
 
4
  from app.shared.nlp.embeddings.base import EmbeddingProvider
5
  from app.shared.nlp.preprocessing.text import prepare_for_embedding
6
 
@@ -45,9 +48,26 @@ class ControlledEmbeddingProvider(EmbeddingProvider):
45
  return [self.embed_text(text) for text in texts]
46
 
47
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
48
  def test_classifier_finds_activity_inside_a_long_message() -> None:
49
  classifier = SemanticPlaceActivityClassifier(
50
  embedding_provider=ControlledEmbeddingProvider(),
 
51
  )
52
 
53
  result = classifier.classify(
@@ -62,6 +82,7 @@ def test_classifier_finds_activity_inside_a_long_message() -> None:
62
  def test_classifier_abstains_when_the_best_categories_are_tied() -> None:
63
  classifier = SemanticPlaceActivityClassifier(
64
  embedding_provider=ControlledEmbeddingProvider(),
 
65
  )
66
 
67
  result = classifier.classify("mezcla")
@@ -72,6 +93,7 @@ def test_classifier_abstains_when_the_best_categories_are_tied() -> None:
72
  def test_classifier_abstains_for_a_generic_single_word_request() -> None:
73
  classifier = SemanticPlaceActivityClassifier(
74
  embedding_provider=ControlledEmbeddingProvider(),
 
75
  )
76
 
77
  assert classifier.classify("salir") is None
 
1
  from app.modules.places.infrastructure.semantic_activity_classifier import (
2
  SemanticPlaceActivityClassifier,
3
  )
4
+ from app.modules.places.infrastructure.open_vocabulary_category_classifier import (
5
+ PlaceCategoryConcept,
6
+ )
7
  from app.shared.nlp.embeddings.base import EmbeddingProvider
8
  from app.shared.nlp.preprocessing.text import prepare_for_embedding
9
 
 
48
  return [self.embed_text(text) for text in texts]
49
 
50
 
51
+ CONCEPTS = (
52
+ PlaceCategoryConcept(
53
+ id="restaurant",
54
+ label="comida restaurante",
55
+ description="hambre tacos pizza sushi antojo platillo cocina",
56
+ storage_values=("restaurant",),
57
+ ),
58
+ PlaceCategoryConcept(
59
+ id="sports",
60
+ label="ejercicio gimnasio deporte",
61
+ description="entrenar futbol cancha nadar fitness",
62
+ storage_values=("sports",),
63
+ ),
64
+ )
65
+
66
+
67
  def test_classifier_finds_activity_inside_a_long_message() -> None:
68
  classifier = SemanticPlaceActivityClassifier(
69
  embedding_provider=ControlledEmbeddingProvider(),
70
+ concepts=CONCEPTS,
71
  )
72
 
73
  result = classifier.classify(
 
82
  def test_classifier_abstains_when_the_best_categories_are_tied() -> None:
83
  classifier = SemanticPlaceActivityClassifier(
84
  embedding_provider=ControlledEmbeddingProvider(),
85
+ concepts=CONCEPTS,
86
  )
87
 
88
  result = classifier.classify("mezcla")
 
93
  def test_classifier_abstains_for_a_generic_single_word_request() -> None:
94
  classifier = SemanticPlaceActivityClassifier(
95
  embedding_provider=ControlledEmbeddingProvider(),
96
+ concepts=CONCEPTS,
97
  )
98
 
99
  assert classifier.classify("salir") is None
tests/test_sentence_transformer_embeddings.py ADDED
@@ -0,0 +1,223 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ import math
2
+
3
+ import pytest
4
+
5
+ from app.shared.nlp.embeddings.sentence_transformer import (
6
+ SentenceTransformerDimensionError,
7
+ SentenceTransformerEmbeddingProvider,
8
+ SentenceTransformerInferenceError,
9
+ SentenceTransformerModelLoadError,
10
+ )
11
+
12
+
13
+ class FakeSentenceTransformer:
14
+ def __init__(
15
+ self,
16
+ vectors: dict[str, list[float]],
17
+ dimension: int = 3,
18
+ ) -> None:
19
+ self.vectors = vectors
20
+ self.dimension = dimension
21
+ self.calls: list[tuple[list[str], dict[str, object]]] = []
22
+
23
+ def get_sentence_embedding_dimension(self) -> int:
24
+ return self.dimension
25
+
26
+ def encode(self, sentences: list[str], **kwargs: object) -> list[list[float]]:
27
+ self.calls.append((list(sentences), kwargs))
28
+ return [self.vectors[text] for text in sentences]
29
+
30
+
31
+ def test_provider_loads_lazily_and_batches_non_empty_texts() -> None:
32
+ model = FakeSentenceTransformer(
33
+ {
34
+ "query: donas": [3.0, 4.0, 0.0],
35
+ "query: cafecito": [0.0, 0.0, 2.0],
36
+ }
37
+ )
38
+ loader_calls: list[tuple[str, str | None]] = []
39
+
40
+ def loader(name: str, device: str | None) -> FakeSentenceTransformer:
41
+ loader_calls.append((name, device))
42
+ return model
43
+
44
+ provider = SentenceTransformerEmbeddingProvider(
45
+ "domain-model",
46
+ expected_dimension=3,
47
+ batch_size=8,
48
+ device="cpu",
49
+ text_prefix="query: ",
50
+ model_loader=loader,
51
+ )
52
+
53
+ assert provider.is_loaded is False
54
+ assert provider.dimension == 3
55
+ assert loader_calls == []
56
+
57
+ embeddings = provider.embed_batch([" donas ", " ", "cafecito"])
58
+
59
+ assert provider.is_loaded is True
60
+ assert loader_calls == [("domain-model", "cpu")]
61
+ assert embeddings[0] == pytest.approx([0.6, 0.8, 0.0])
62
+ assert embeddings[1] == [0.0, 0.0, 0.0]
63
+ assert embeddings[2] == pytest.approx([0.0, 0.0, 1.0])
64
+ assert model.calls == [
65
+ (
66
+ ["query: donas", "query: cafecito"],
67
+ {
68
+ "batch_size": 8,
69
+ "convert_to_numpy": True,
70
+ "normalize_embeddings": False,
71
+ "show_progress_bar": False,
72
+ },
73
+ )
74
+ ]
75
+
76
+
77
+ def test_empty_text_returns_zero_without_loading_model() -> None:
78
+ def loader(
79
+ _name: str, _device: str | None
80
+ ) -> FakeSentenceTransformer:
81
+ raise AssertionError("empty input must not load the model")
82
+
83
+ provider = SentenceTransformerEmbeddingProvider(
84
+ "domain-model",
85
+ expected_dimension=3,
86
+ model_loader=loader,
87
+ )
88
+
89
+ assert provider.embed_text("\t \n") == [0.0, 0.0, 0.0]
90
+ assert provider.embed_batch([]) == []
91
+ assert provider.is_loaded is False
92
+
93
+
94
+ def test_model_is_loaded_only_once_across_calls() -> None:
95
+ model = FakeSentenceTransformer({"first": [1.0, 0.0, 0.0], "second": [0.0, 1.0, 0.0]})
96
+ load_count = 0
97
+
98
+ def loader(
99
+ _name: str, _device: str | None
100
+ ) -> FakeSentenceTransformer:
101
+ nonlocal load_count
102
+ load_count += 1
103
+ return model
104
+
105
+ provider = SentenceTransformerEmbeddingProvider(
106
+ "domain-model",
107
+ expected_dimension=3,
108
+ model_loader=loader,
109
+ )
110
+
111
+ provider.embed_text("first")
112
+ provider.embed_text("second")
113
+
114
+ assert load_count == 1
115
+ assert len(model.calls) == 2
116
+
117
+
118
+ def test_provider_rejects_reported_model_dimension_mismatch() -> None:
119
+ model = FakeSentenceTransformer({}, dimension=4)
120
+ provider = SentenceTransformerEmbeddingProvider(
121
+ "wrong-model",
122
+ expected_dimension=3,
123
+ model_loader=lambda _name, _device: model,
124
+ )
125
+
126
+ with pytest.raises(
127
+ SentenceTransformerDimensionError,
128
+ match="model=4, configured=3",
129
+ ):
130
+ provider.embed_text("donas")
131
+
132
+ assert provider.is_loaded is False
133
+
134
+
135
+ def test_provider_rejects_bad_inference_output() -> None:
136
+ model = FakeSentenceTransformer({"donas": [1.0, 2.0]}, dimension=3)
137
+ provider = SentenceTransformerEmbeddingProvider(
138
+ "bad-output-model",
139
+ expected_dimension=3,
140
+ model_loader=lambda _name, _device: model,
141
+ )
142
+
143
+ with pytest.raises(
144
+ SentenceTransformerDimensionError,
145
+ match="returned=2, expected=3",
146
+ ):
147
+ provider.embed_text("donas")
148
+
149
+
150
+ def test_provider_wraps_loader_and_inference_failures_with_context() -> None:
151
+ def broken_loader(_name: str, _device: str | None) -> FakeSentenceTransformer:
152
+ raise OSError("model cache unavailable")
153
+
154
+ provider = SentenceTransformerEmbeddingProvider(
155
+ "missing-model",
156
+ expected_dimension=3,
157
+ model_loader=broken_loader,
158
+ )
159
+ with pytest.raises(
160
+ SentenceTransformerModelLoadError,
161
+ match="missing-model.*model cache unavailable",
162
+ ):
163
+ provider.embed_text("donas")
164
+
165
+ class BrokenModel(FakeSentenceTransformer):
166
+ def encode(
167
+ self, sentences: list[str], **kwargs: object
168
+ ) -> list[list[float]]:
169
+ raise RuntimeError("backend crashed")
170
+
171
+ broken_model = BrokenModel({}, dimension=3)
172
+ provider = SentenceTransformerEmbeddingProvider(
173
+ "broken-model",
174
+ expected_dimension=3,
175
+ model_loader=lambda _name, _device: broken_model,
176
+ )
177
+ with pytest.raises(
178
+ SentenceTransformerInferenceError,
179
+ match="broken-model.*backend crashed",
180
+ ):
181
+ provider.embed_text("donas")
182
+
183
+
184
+ def test_provider_rejects_zero_norm_and_non_finite_vectors() -> None:
185
+ zero_model = FakeSentenceTransformer({"zero": [0.0, 0.0, 0.0]})
186
+ provider = SentenceTransformerEmbeddingProvider(
187
+ "zero-model",
188
+ expected_dimension=3,
189
+ model_loader=lambda _name, _device: zero_model,
190
+ )
191
+ with pytest.raises(SentenceTransformerInferenceError, match="zero-norm"):
192
+ provider.embed_text("zero")
193
+
194
+ nan_model = FakeSentenceTransformer({"nan": [math.nan, 0.0, 1.0]})
195
+ provider = SentenceTransformerEmbeddingProvider(
196
+ "nan-model",
197
+ expected_dimension=3,
198
+ model_loader=lambda _name, _device: nan_model,
199
+ )
200
+ with pytest.raises(
201
+ SentenceTransformerInferenceError,
202
+ match="NaN or infinity",
203
+ ):
204
+ provider.embed_text("nan")
205
+
206
+
207
+ def test_provider_validates_constructor_and_input() -> None:
208
+ with pytest.raises(ValueError, match="model_name_or_path"):
209
+ SentenceTransformerEmbeddingProvider(" ", expected_dimension=3)
210
+ with pytest.raises(ValueError, match="expected_dimension"):
211
+ SentenceTransformerEmbeddingProvider("model", expected_dimension=0)
212
+ with pytest.raises(ValueError, match="batch_size"):
213
+ SentenceTransformerEmbeddingProvider(
214
+ "model", expected_dimension=3, batch_size=0
215
+ )
216
+
217
+ provider = SentenceTransformerEmbeddingProvider(
218
+ "model",
219
+ expected_dimension=3,
220
+ model_loader=lambda _name, _device: FakeSentenceTransformer({}),
221
+ )
222
+ with pytest.raises(TypeError, match=r"texts\[1\].*int"):
223
+ provider.embed_batch(["valid", 42]) # type: ignore[list-item]
tests/test_sql_contract.py CHANGED
@@ -103,3 +103,29 @@ def test_global_search_sql_does_not_directly_cast_untrusted_event_metadata() ->
103
  sql = path.read_text(encoding="utf-8")
104
  assert "NULLIF(e.metadata->>'start_time', '')::timestamptz" not in sql
105
  assert "(e.metadata->>'duration_minutes')::integer" not in sql
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
103
  sql = path.read_text(encoding="utf-8")
104
  assert "NULLIF(e.metadata->>'start_time', '')::timestamptz" not in sql
105
  assert "(e.metadata->>'duration_minutes')::integer" not in sql
106
+
107
+
108
+ def test_places_semantic_index_is_additive_and_hybrid() -> None:
109
+ migration = Path(
110
+ "sql/migrations/20260716_02_places_semantic_v1.sql"
111
+ ).read_text(encoding="utf-8")
112
+ upper = migration.upper()
113
+
114
+ assert "PLACE_EMBEDDINGS_SEMANTIC_V1" in upper
115
+ assert "VECTOR(768)" in upper
116
+ assert "SEARCH_PLACES_SEMANTIC_V1" in upper
117
+ assert "FULL OUTER JOIN" in upper
118
+ assert "WEBSEARCH_TO_TSQUERY" in upper
119
+ assert "USING HNSW" in upper
120
+ assert "AS MATERIALIZED" not in upper
121
+ assert "FROM PUBLIC.PLACE_EMBEDDINGS_SEMANTIC_V1 AS PLACE" in upper
122
+ assert "ALTER TABLE PUBLIC.PLACE_EMBEDDINGS" not in upper
123
+ assert "DROP TABLE" not in upper
124
+
125
+ verifier = Path("sql/verify_places_semantic_v1.sql")
126
+ assert verifier.exists()
127
+ verifier_sql = verifier.read_text(encoding="utf-8").upper()
128
+ assert "SET TRANSACTION READ ONLY" in verifier_sql
129
+ assert "VECTOR(768)" in verifier_sql
130
+ assert "USING HNSW" in verifier_sql
131
+ assert "EXPLAIN (ANALYZE, BUFFERS" in verifier_sql