added concept vector handling in study

routes/study.py

@@ -1,9 +1,9 @@
 # routes/study.py
+from services.study_service import StudyService
 from fastapi import APIRouter, HTTPException
 from loguru import logger
 from typing import List
 from pydantic import BaseModel
-from services.study_service import StudyService
 
 router = APIRouter(prefix="/api/study")
 
@@ -11,40 +11,79 @@ class NeighborhoodRequest(BaseModel):
     word: str
     n_neighbors: int = 20
 
+class VisualizationRequest(BaseModel):
+    words: List[str]
+
+
+
+class ConceptRequest(BaseModel):
+    positive_words: List[str]
+    negative_words: List[str] = []
+    n_results: int = 10
+
 class AnalogyRequest(BaseModel):
     word1: str
     word2: str
     word3: str
+    n_results: int = 10
 
-class VisualizationRequest(BaseModel):
+class SemanticFieldRequest(BaseModel):
     words: List[str]
+    n_neighbors: int = 5
 
 def init_router(study_service: StudyService):
-    @router.post("/neighborhood")
-    async def analyze_neighborhood(request: NeighborhoodRequest):
-        """Analyze word neighborhood with detailed semantic information"""
+    router = APIRouter(prefix="/api/study")
+
+    @router.post("/concept")
+    async def analyze_concept(request: ConceptRequest):
         try:
-            return await study_service.analyze_word_neighborhood(
-                request.word, 
-                request.n_neighbors
+            return await study_service.analyze_concept(
+                request.positive_words,
+                request.negative_words,
+                request.n_results
             )
         except Exception as e:
-            logger.error(f"Error analyzing word neighborhood: {str(e)}")
+            logger.error(f"Error analyzing concept: {str(e)}")
             raise HTTPException(status_code=500, detail="Internal server error")
 
     @router.post("/analogy")
     async def analyze_analogy(request: AnalogyRequest):
-        """Perform word analogy analysis"""
         try:
-            return await study_service.analyze_semantic_analogy(
+            return await study_service.analyze_analogy(
                 request.word1,
                 request.word2,
-                request.word3
+                request.word3,
+                request.n_results
             )
         except Exception as e:
             logger.error(f"Error analyzing analogy: {str(e)}")
             raise HTTPException(status_code=500, detail="Internal server error")
 
+    @router.post("/semantic-field")
+    async def analyze_semantic_field(request: SemanticFieldRequest):
+        try:
+            return await study_service.analyze_semantic_field(
+                request.words,
+                request.n_neighbors
+            )
+        except Exception as e:
+            logger.error(f"Error analyzing semantic field: {str(e)}")
+            raise HTTPException(status_code=500, detail="Internal server error")
+
+    @router.post("/neighborhood")
+    async def analyze_neighborhood(request: NeighborhoodRequest):
+        """Analyze word neighborhood with detailed semantic information"""
+        try:
+            return await study_service.analyze_word_neighborhood(
+                request.word, 
+                request.n_neighbors
+            )
+        except Exception as e:
+            logger.error(f"Error analyzing word neighborhood: {str(e)}")
+            raise HTTPException(status_code=500, detail="Internal server error")
+
+
+
     @router.post("/visualization")
     async def create_visualization(request: VisualizationRequest):
         """Create visualization data for a set of words"""

routes/word.py

@@ -1,51 +1,112 @@
-# routes/word.py
-from fastapi import APIRouter, HTTPException
+from fastapi import APIRouter, HTTPException, Query, Path
 from loguru import logger
 from typing import List, Optional
+from pydantic import BaseModel, Field, confloat, conint
 from services.word_service import WordEmbeddingService
 
 router = APIRouter(prefix="/api")
 
+# Request/Response Models
+class SimilarityRequest(BaseModel):
+    word1: str = Field(..., min_length=1, description="First word to compare")
+    word2: str = Field(..., min_length=1, description="Second word to compare")
+
+class WordsInRangeRequest(BaseModel):
+    word: str = Field(..., min_length=1, description="Target word")
+    min_sim: float = Field(..., ge=0.0, le=1.0, description="Minimum similarity threshold")
+    max_sim: float = Field(..., ge=0.0, le=1.0, description="Maximum similarity threshold")
+    n: int = Field(5, ge=1, le=100, description="Number of words to return")
+
+class WordResponse(BaseModel):
+    word: str
+    similarity: float
+
 def init_router(word_service: WordEmbeddingService):
     @router.post("/similarity")
-    async def calculate_similarity(word1: str, word2: str):
-        """Calculate semantic similarity between two words"""
+    async def calculate_similarity(request: SimilarityRequest):
+        """
+        Calculate semantic similarity between two words.
+        Returns a similarity score between 0 and 1.
+        """
         try:
-            similarity = await word_service.calculate_similarity(word1, word2)
+            similarity = await word_service.calculate_similarity(
+                request.word1, 
+                request.word2
+            )
             return {"similarity": similarity}
         except Exception as e:
             logger.error(f"Error calculating similarity: {str(e)}")
             raise HTTPException(status_code=500, detail="Internal server error")
 
     @router.get("/similar-words/{word}")
-    async def get_similar_words(word: str, n: int = 20):
-        """Get n most similar words to the input word"""
+    async def get_similar_words(
+        word: str = Path(..., min_length=1, description="Word to find similar words for"),
+        n: int = Query(20, ge=1, le=100, description="Number of similar words to return")
+    ):
+        """
+        Get n most similar words to the input word.
+        Returns a list of words with their similarity scores.
+        """
         try:
             words = await word_service.get_most_similar_words(word, n)
-            return {"words": words}
+            return {
+                "word": word,
+                "similar_words": words
+            }
         except Exception as e:
             logger.error(f"Error getting similar words: {str(e)}")
             raise HTTPException(status_code=500, detail="Internal server error")
 
     @router.post("/words-in-range")
-    async def get_words_in_range(word: str, min_sim: float, max_sim: float, n: int = 5):
-        """Get words within a specific similarity range"""
+    async def get_words_in_range(request: WordsInRangeRequest):
+        """
+        Get words within a specific similarity range.
+        Returns words with similarities between min_sim and max_sim.
+        """
         try:
-            words = await word_service.get_words_in_range(word, min_sim, max_sim, n)
+            if request.min_sim >= request.max_sim:
+                raise HTTPException(
+                    status_code=400, 
+                    detail="min_sim must be less than max_sim"
+                )
+
+            words = await word_service.get_words_in_range(
+                request.word,
+                request.min_sim,
+                request.max_sim,
+                request.n
+            )
             return {
+                "word": request.word,
+                "range": {
+                    "min": request.min_sim,
+                    "max": request.max_sim
+                },
                 "words": words,
                 "count": len(words)
             }
+        except HTTPException:
+            raise
         except Exception as e:
             logger.error(f"Error getting words in range: {str(e)}")
             raise HTTPException(status_code=500, detail="Internal server error")
 
     @router.get("/random-word")
     async def get_random_word():
-        """Get a random word from the vocabulary"""
+        """
+        Get a random word from the vocabulary.
+        Returns a single random word from the model's vocabulary.
+        """
         try:
             word = await word_service.get_random_word()
+            if not word:
+                raise HTTPException(
+                    status_code=500,
+                    detail="Could not generate random word"
+                )
             return {"word": word}
+        except HTTPException:
+            raise
         except Exception as e:
             logger.error(f"Error getting random word: {str(e)}")
             raise HTTPException(status_code=500, detail="Internal server error")

services/study_service.py

@@ -1,5 +1,5 @@
 from loguru import logger
-from typing import List, Dict
+from typing import List, Dict, Optional, Tuple
 import numpy as np
 from services.word_service import WordEmbeddingService
 
@@ -10,7 +10,6 @@ class StudyService:
     async def analyze_word_neighborhood(self, word: str, n_neighbors: int = 20) -> Dict:
         """Get detailed analysis of a word's semantic neighborhood"""
         try:
-            # Make sure to await the async calls
             vector = await self.word_service.get_vector(word)
             similar_words = await self.word_service.get_most_similar_words(word, n=n_neighbors)
             
@@ -29,61 +28,144 @@ class StudyService:
                 "vector_norm": None
             }
 
-    async def compare_words(self, words: List[str]) -> Dict:
-        """Compare multiple words to understand their relationships"""
-        results = []
-        similarity_matrix = []
+    async def analyze_concept(self, 
+                            positive_words: List[str], 
+                            negative_words: List[str] = None,
+                            n_results: int = 10) -> Dict:
+        """
+        Analyze a concept defined by positive and negative words
         
+        Example: "roi - homme + femme = reine"
+        """
         try:
-            for i, word1 in enumerate(words):
-                # Get vector for current word
-                vector = await self.word_service.get_vector(word1)
-                results.append({
-                    "word": word1,
-                    "in_vocabulary": vector is not None,
-                    "vector_norm": float(np.linalg.norm(vector)) if vector is not None else None
-                })
-                
-                # Calculate similarities
-                row = []
-                for j, word2 in enumerate(words):
-                    if i <= j:  # Only calculate upper triangle
-                        sim = await self.word_service.calculate_similarity(word1, word2)
-                        row.append(sim)
-                    else:  # Copy from upper triangle
-                        row.append(similarity_matrix[j][i])
-                similarity_matrix.append(row)
-
+            negative_words = negative_words or []
+            
+            # Get vectors for all words
+            concept_vec = np.zeros(300)  # FastText dimension
+            
+            # Add positive word vectors
+            for word in positive_words:
+                vector = await self.word_service.get_vector(word)
+                if vector is not None:
+                    concept_vec += vector
+            
+            # Subtract negative word vectors
+            for word in negative_words:
+                vector = await self.word_service.get_vector(word)
+                if vector is not None:
+                    concept_vec -= vector
+            
+            # Normalize the concept vector
+            concept_vec = concept_vec / np.linalg.norm(concept_vec)
+            
+            # Find similar words to the concept vector
+            similar_words = await self.word_service.get_similar_by_vector(concept_vec, n=n_results)
+            
             return {
-                "words": results,
-                "similarity_matrix": similarity_matrix
+                "concept": {
+                    "positive_words": positive_words,
+                    "negative_words": negative_words
+                },
+                "similar_words": similar_words,
+                "vector_norm": float(np.linalg.norm(concept_vec))
             }
         except Exception as e:
-            logger.exception(f"Error comparing words: {e}")
+            logger.exception(f"Error analyzing concept: {e}")
             return {
-                "words": [],
-                "similarity_matrix": []
+                "concept": {
+                    "positive_words": positive_words,
+                    "negative_words": negative_words
+                },
+                "similar_words": [],
+                "vector_norm": None
             }
 
-    async def get_similarity_ranges(self, word: str) -> Dict:
-        """Get words in different similarity ranges"""
-        ranges = [
-            (0.9, 1.0, "very_high"),
-            (0.7, 0.9, "high"),
-            (0.5, 0.7, "medium"),
-            (0.3, 0.5, "low"),
-            (0.1, 0.3, "very_low")
-        ]
+    async def analyze_analogy(self, 
+                            word1: str, 
+                            word2: str, 
+                            word3: str,
+                            n_results: int = 10) -> Dict:
+        """
+        Analyze word analogies (a:b :: c:?)
         
-        results = {}
+        Example: paris:france :: berlin:? (should find "allemagne")
+        """
         try:
-            for min_sim, max_sim, range_name in ranges:
-                words = await self.word_service.get_words_in_range(
-                    word, min_sim, max_sim, n=5
-                )
-                results[range_name] = words
-                
-            return results
+            # Get vectors
+            vec1 = await self.word_service.get_vector(word1)
+            vec2 = await self.word_service.get_vector(word2)
+            vec3 = await self.word_service.get_vector(word3)
+            
+            if None in (vec1, vec2, vec3):
+                return {
+                    "analogy": f"{word1}:{word2} :: {word3}:?",
+                    "similar_words": [],
+                    "error": "One or more words not found in vocabulary"
+                }
+            
+            # Calculate analogy vector (vec2 - vec1 + vec3)
+            analogy_vec = vec2 - vec1 + vec3
+            
+            # Normalize the vector
+            analogy_vec = analogy_vec / np.linalg.norm(analogy_vec)
+            
+            # Find similar words
+            similar_words = await self.word_service.get_similar_by_vector(analogy_vec, n=n_results)
+            
+            return {
+                "analogy": f"{word1}:{word2} :: {word3}:?",
+                "similar_words": similar_words
+            }
+        except Exception as e:
+            logger.exception(f"Error analyzing analogy: {e}")
+            return {
+                "analogy": f"{word1}:{word2} :: {word3}:?",
+                "similar_words": [],
+                "error": str(e)
+            }
+
+    async def analyze_semantic_field(self, 
+                                   words: List[str], 
+                                   n_neighbors: int = 5) -> Dict:
+        """
+        Analyze the semantic field created by a group of words
+        """
+        try:
+            results = []
+            center_vector = np.zeros(300)  # FastText dimension
+            valid_vectors = 0
+            
+            # Calculate center of the semantic field
+            for word in words:
+                vector = await self.word_service.get_vector(word)
+                if vector is not None:
+                    center_vector += vector
+                    valid_vectors += 1
+                    
+                    # Analyze each word
+                    similar = await self.word_service.get_most_similar_words(word, n=n_neighbors)
+                    results.append({
+                        "word": word,
+                        "similar_words": similar,
+                        "vector_norm": float(np.linalg.norm(vector))
+                    })
+            
+            if valid_vectors > 0:
+                center_vector = center_vector / valid_vectors
+                center_similar = await self.word_service.get_similar_by_vector(center_vector, n=n_neighbors)
+            else:
+                center_similar = []
+            
+            return {
+                "words": results,
+                "center_word_candidates": center_similar,
+                "valid_words_count": valid_vectors
+            }
         except Exception as e:
-            logger.exception(f"Error getting similarity ranges: {e}")
-            return {range_name: [] for _, _, range_name in ranges}
+            logger.exception(f"Error analyzing semantic field: {e}")
+            return {
+                "words": [],
+                "center_word_candidates": [],
+                "valid_words_count": 0,
+                "error": str(e)
+            }

services/word_service.py

@@ -172,4 +172,17 @@ class WordEmbeddingService:
             return random.choice(list(self.vocab_vectors.keys()))
         except Exception as e:
             logger.exception("Error getting random word")
-            return None
+            return None
+        
+    async def get_similar_by_vector(self, vector: np.ndarray, n: int = 10) -> List[Dict[str, float]]:
+        """Find words most similar to a given vector"""
+        await self._ensure_model_loaded()
+        try:
+            # Run the CPU-intensive operation in a thread pool
+            similar = await asyncio.to_thread(
+                lambda: WordEmbeddingService._model.similar_by_vector(vector, topn=n)
+            )
+            return [{'word': word, 'similarity': float(sim)} for word, sim in similar]
+        except Exception as e:
+            logger.exception(f"Error finding similar words by vector: {str(e)}")
+            return []