Update README.md
Browse files
README.md
CHANGED
|
@@ -12,6 +12,9 @@ The model of SitEmb-v1.5-Qwen3.
|
|
| 12 |
import torch
|
| 13 |
|
| 14 |
from transformers import AutoTokenizer, AutoModel
|
|
|
|
|
|
|
|
|
|
| 15 |
|
| 16 |
residual = True
|
| 17 |
residual_factor = 0.5
|
|
@@ -122,8 +125,13 @@ query_hidden, _ = encode_query(
|
|
| 122 |
tokenizer, model, pooling_type="eos", queries=["Your query"],
|
| 123 |
batch_size=8, normalize=True, max_length=8192, residual=residual,
|
| 124 |
)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 125 |
candidate_hidden, candidate_hidden_residual = encode_passage(
|
| 126 |
-
tokenizer, model, pooling_type="eos", passages=["
|
| 127 |
batch_size=4, normalize=True, max_length=8192, residual=residual,
|
| 128 |
)
|
| 129 |
|
|
|
|
| 12 |
import torch
|
| 13 |
|
| 14 |
from transformers import AutoTokenizer, AutoModel
|
| 15 |
+
from tqdm import tqdm
|
| 16 |
+
from more_itertools import chunked
|
| 17 |
+
|
| 18 |
|
| 19 |
residual = True
|
| 20 |
residual_factor = 0.5
|
|
|
|
| 125 |
tokenizer, model, pooling_type="eos", queries=["Your query"],
|
| 126 |
batch_size=8, normalize=True, max_length=8192, residual=residual,
|
| 127 |
)
|
| 128 |
+
|
| 129 |
+
passage_affix = "The context in which the chunk is situated is given below. Please encode the chunk by being aware of the context. Context:\n"
|
| 130 |
+
your_chunk = "Your Chunk"
|
| 131 |
+
your_context = "Your Context"
|
| 132 |
+
|
| 133 |
candidate_hidden, candidate_hidden_residual = encode_passage(
|
| 134 |
+
tokenizer, model, pooling_type="eos", passages=[f"{your_chunk}<|endoftext|>{passage_affix}{your_context}"],
|
| 135 |
batch_size=4, normalize=True, max_length=8192, residual=residual,
|
| 136 |
)
|
| 137 |
|