Spaces:
Sleeping
Sleeping
added ingestion functionality
Browse files
ingest.py
ADDED
|
@@ -0,0 +1,11 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
from llama_index import SimpleDirectoryReader, VectorStoreIndex, ServiceContext
|
| 2 |
+
from llama_index.text_splitter import SentenceSplitter
|
| 3 |
+
|
| 4 |
+
documents = SimpleDirectoryReader("./data").load_data()
|
| 5 |
+
|
| 6 |
+
text_splitter = SentenceSplitter(chunk_size=512, chunk_overlap=10)
|
| 7 |
+
service_context = ServiceContext.from_defaults(text_splitter=text_splitter)
|
| 8 |
+
|
| 9 |
+
index = VectorStoreIndex.from_documents(
|
| 10 |
+
documents, service_context=service_context
|
| 11 |
+
)
|