smart-search-by-notes.py

# source SMARTSEARCH/bin/activate
# pip install sentence-transformers faiss-gpu
# pip install --upgrade numpy==1.26

# Умный поиск по заметкам
# https://telegra.ph/Umnyj-poisk-po-zametkam-07-10

import os
from sentence_transformers import SentenceTransformer
import faiss
import numpy as np


# 📁 Путь к папке с заметками
NOTES_FOLDER = "notebooks"

# 📌 Модель для смысловых векторов
model = SentenceTransformer('sentence-transformers/all-MiniLM-L6-v2', token='hf_ZgVT...ULhFTGITiNVdUAG')

# 📥 Загрузка и индексация заметок
def load_notes(folder):
  notes = []
  filenames = []
  for file in os.listdir(folder):
    if file.endswith(".txt") or file.endswith(".md"):
      path = os.path.join(folder, file)
      with open(path, "r", encoding="utf-8") as f:
        content = f.read()
        notes.append(content)
        filenames.append(file)
  return notes, filenames

# 🔍 Поиск похожих заметок
def semantic_search(query, notes, vectors, filenames, top_k=6):
  query_vec = model.encode([query])
  D, I = index.search(query_vec, top_k)
  results = []
  for i, idx in enumerate(I[0]):
    snippet = notes[idx][:300].replace("\n", " ").strip()
    results.append((filenames[idx], D[0][i], snippet))
  return results

# ▶️ Основной блок
notes, filenames = load_notes(NOTES_FOLDER)
vectors = model.encode(notes)

# Построение индекса FAISS
dim = vectors.shape[1]
index = faiss.IndexFlatL2(dim)
index.add(np.array(vectors))

# 🔁 Цикл запросов
while True:
  q = input("\n🔎 Запрос (или 'exit'): ").strip() 
  if q.lower() in ['exit', 'quit']:
    break

  results = semantic_search(q, notes, vectors, filenames)
  print("\n📚 Результаты:")
  for fname, score, snippet in results:
    print(f"\n📝 {fname} (схожесть: {score:.2f})")
    print(f"  → {snippet[:150]}...")

#🔎 Запрос (или 'exit'): When was Amsterdam voted the best city to live in?

# 📚 Результаты:

# 📝 3.txt (схожесть: 0.65)
#   → In 2022, Amsterdam was ranked the ninth-best city to live in by the Economist Intelligence Unit[28] and 12th on quality of living for environment and ...

# 📝 12.txt (схожесть: 0.89)
#   → In 1300, Amsterdam's population was around 1,000 people.[89] While many towns in Holland experienced population decline during the 15th and 16th centu...

# 📝 10.txt (схожесть: 0.90)
#   → Amsterdam is located in the Western Netherlands, in the province of North Holland, the capital of which is not Amsterdam, but rather Haarlem. The rive...

# 📝 1.txt (схожесть: 0.91)
#   → Amsterdam (/ˈæmstərdæm/ AM-stər-dam, UK also /ˌæmstərˈdæm/ AM-stər-DAM;[12][13] Dutch: [ˌɑmstərˈdɑm] ⓘ; lit. 'Dam in the Amstel')[14] is the capital[a...

# 📝 7.txt (схожесть: 0.94)
#   → The end of the 19th century is sometimes called Amsterdam's second Golden Age.[57] New museums, a railway station, and the Concertgebouw were built; A...

# 📝 11.txt (схожесть: 0.96)
#   → Amsterdam has an oceanic climate (Köppen: Cfb)[81] strongly influenced by its proximity to the North Sea to the west, with prevailing westerly winds. ...


# 🔎 Запрос (или 'exit'): How many people live in Amsterdam?

# 📚 Результаты:

# 📝 3.txt (схожесть: 0.69)
#   → In 2022, Amsterdam was ranked the ninth-best city to live in by the Economist Intelligence Unit[28] and 12th on quality of living for environment and ...

# 📝 12.txt (схожесть: 0.70)
#   → In 1300, Amsterdam's population was around 1,000 people.[89] While many towns in Holland experienced population decline during the 15th and 16th centu...

# 📝 10.txt (схожесть: 0.82)
#   → Amsterdam is located in the Western Netherlands, in the province of North Holland, the capital of which is not Amsterdam, but rather Haarlem. The rive...

# 📝 1.txt (схожесть: 0.85)
#   → Amsterdam (/ˈæmstərdæm/ AM-stər-dam, UK also /ˌæmstərˈdæm/ AM-stər-DAM;[12][13] Dutch: [ˌɑmstərˈdɑm] ⓘ; lit. 'Dam in the Amstel')[14] is the capital[a...

# 📝 11.txt (схожесть: 1.01)
#   → Amsterdam has an oceanic climate (Köppen: Cfb)[81] strongly influenced by its proximity to the North Sea to the west, with prevailing westerly winds. ...

# 📝 2.txt (схожесть: 1.01)
#   → Amsterdam's main attractions include its historic canals; the Rijksmuseum, the state museum with Dutch Golden Age art; the Van Gogh Museum; the Dam Sq...