Spaces:
Sleeping
Sleeping
| import pandas as pd | |
| import numpy as np | |
| from sentence_transformers import SentenceTransformer | |
| import os | |
| # ============================================================================== | |
| # SCRIPT: embedding.py | |
| # DESCRIPTION: Converts text data into semantic vectors (embeddings). | |
| # ============================================================================== | |
| def generate_embeddings(input_path, output_metadata_path, output_vector_path): | |
| # 1. Data Loading | |
| if not os.path.exists(input_path): | |
| print(f"Error: {input_path} not found!") | |
| return | |
| print("Reading normalized POI data...") | |
| df = pd.read_json(input_path) | |
| # Validate that the necessary column exists for the model | |
| if 'enriched_description' not in df.columns: | |
| print("Error: 'enriched_description' column missing. Check the normalization step.") | |
| return | |
| # 2. Model Initialization (multilingual-e5-small) | |
| # This model is optimized for both Turkish and English semantic search. | |
| print("Loading model: multilingual-e5-small...") | |
| model = SentenceTransformer('intfloat/multilingual-e5-small') | |
| # 3. Text Preprocessing | |
| # The E5 model family requires the 'passage: ' prefix for documents to optimize retrieval. | |
| print("Preparing texts for vectorization...") | |
| texts = ["passage: " + str(text) for text in df['enriched_description']] | |
| # 4. Embedding Generation | |
| print(f"Computing embeddings for {len(texts)} locations... (This may take a moment)") | |
| # convert_to_numpy=True ensures the output is ready for binary storage and math operations. | |
| embeddings = model.encode(texts, show_progress_bar=True, convert_to_numpy=True) | |
| # 5. Modular Storage | |
| # A. Vector File (.npy): Stored in binary format for high-speed retrieval and low memory footprint. | |
| np.save(output_vector_path, embeddings) | |
| print(f"Vectors saved to: {output_vector_path}") | |
| # B. Metadata File (.json): Saves the original data without the heavy vector arrays. | |
| df_metadata = df.drop(columns=['embedding'], errors='ignore') | |
| df_metadata.to_json(output_metadata_path, orient="records", force_ascii=False, indent=4) | |
| print(f"Metadata saved to: {output_metadata_path}") | |
| print("\n Embedding process completed successfully!") | |
| if __name__ == "__main__": | |
| # Get the absolute path of the directory where this script is located | |
| script_dir = os.path.dirname(os.path.abspath(__file__)) | |
| # Navigate to the 'app' directory (one level up from 'scripts') | |
| app_dir = os.path.dirname(script_dir) | |
| # Define the data directory path | |
| data_dir = os.path.join(app_dir, "data") | |
| # Define file paths relative to the data directory | |
| INPUT_FILE = os.path.join(data_dir, "data_sightseeing_ready.json") | |
| META_OUTPUT = os.path.join(data_dir, "poi_metadata.json") | |
| VEC_OUTPUT = os.path.join(data_dir, "poi_vectors.npy") | |
| generate_embeddings(INPUT_FILE, META_OUTPUT, VEC_OUTPUT) | |