KevinHuSh
commited on
Commit
·
5d16bca
1
Parent(s):
ba9251b
let index be batchly. (#733)
Browse files### What problem does this PR solve?
let index be batchly.
### Type of change
- [x] Refactoring
- rag/svr/task_executor.py +6 -1
rag/svr/task_executor.py
CHANGED
|
@@ -285,7 +285,12 @@ def main():
|
|
| 285 |
init_kb(r)
|
| 286 |
chunk_count = len(set([c["_id"] for c in cks]))
|
| 287 |
st = timer()
|
| 288 |
-
es_r =
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 289 |
cron_logger.info("Indexing elapsed({}): {}".format(r["name"], timer()-st))
|
| 290 |
if es_r:
|
| 291 |
callback(-1, "Index failure!")
|
|
|
|
| 285 |
init_kb(r)
|
| 286 |
chunk_count = len(set([c["_id"] for c in cks]))
|
| 287 |
st = timer()
|
| 288 |
+
es_r = ""
|
| 289 |
+
for b in range(0, len(cks), 32):
|
| 290 |
+
es_r = ELASTICSEARCH.bulk(cks[b:b+32], search.index_name(r["tenant_id"]))
|
| 291 |
+
if b % 128 == 0:
|
| 292 |
+
callback(prog=0.8 + 0.1 * (b + 1) / len(cks), msg="")
|
| 293 |
+
|
| 294 |
cron_logger.info("Indexing elapsed({}): {}".format(r["name"], timer()-st))
|
| 295 |
if es_r:
|
| 296 |
callback(-1, "Index failure!")
|