Spaces:
Runtime error
Runtime error
Create test_dataset
Browse files- test_dataset +17 -0
test_dataset
ADDED
|
@@ -0,0 +1,17 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
from chroma_datasets.utils import import_into_chroma
|
| 2 |
+
from chroma_datasets.utils import load_huggingface_dataset, import_chroma_exported_hf_dataset
|
| 3 |
+
import huggingface_hub
|
| 4 |
+
|
| 5 |
+
|
| 6 |
+
UB_paragraphs_dataset_name = "CenterFor/UB_paragraphs"
|
| 7 |
+
UB_paragraphs_dataset = load_huggingface_dataset(UB_paragraphs_dataset_name)
|
| 8 |
+
print(UB_paragraphs_dataset)
|
| 9 |
+
|
| 10 |
+
chroma_collection_name_from_HF = "entire_book_paragraph_content-from-HF"
|
| 11 |
+
paragraph_content_collection_from_HF = import_chroma_exported_hf_dataset(
|
| 12 |
+
chroma_client=chroma_client,
|
| 13 |
+
dataset=UB_paragraphs_dataset,
|
| 14 |
+
collection_name=chroma_collection_name_from_HF,
|
| 15 |
+
embedding_function=chroma_embedding_function
|
| 16 |
+
)
|
| 17 |
+
print(paragraph_content_collection_from_HF.count())
|