File size: 1,297 Bytes
64a1c56
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
"""Tests run with a fake embedder and a temp data dir: no model download, no Groq key needed."""
import os
import re
import tempfile
import zlib

_TMP = tempfile.mkdtemp(prefix="rag-test-")
os.environ["DATA_DIR"] = _TMP
os.environ["USE_RERANKER"] = "false"
os.environ["USE_QUERY_REWRITE"] = "false"
os.environ["GROQ_API_KEY"] = "test-key"

import numpy as np  # noqa: E402
import pytest  # noqa: E402

from app import store  # noqa: E402
from app.bm25 import bm25  # noqa: E402
from app.registry import get_registry  # noqa: E402


class FakeEmbedder:
    """Hashed bag-of-words vectors: deterministic, similar words -> similar vectors."""

    dim = 256

    def get_sentence_embedding_dimension(self):
        return self.dim

    def encode(self, texts, normalize_embeddings=True, **kw):
        out = np.zeros((len(texts), self.dim), dtype="float32")
        for i, t in enumerate(texts):
            for tok in re.findall(r"\w+", t.lower()):
                out[i, zlib.crc32(tok.encode()) % self.dim] += 1
        norms = np.linalg.norm(out, axis=1, keepdims=True)
        norms[norms == 0] = 1
        return out / norms


store.set_embedder(FakeEmbedder())


@pytest.fixture(autouse=True)
def clean_state():
    store.clear_all()
    get_registry().clear()
    bm25.mark_dirty()
    yield