CenterFor commited on
Commit
bf86d68
·
1 Parent(s): 5c24ec3

Create test_dataset

Browse files
Files changed (1) hide show
  1. test_dataset +17 -0
test_dataset ADDED
@@ -0,0 +1,17 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ from chroma_datasets.utils import import_into_chroma
2
+ from chroma_datasets.utils import load_huggingface_dataset, import_chroma_exported_hf_dataset
3
+ import huggingface_hub
4
+
5
+
6
+ UB_paragraphs_dataset_name = "CenterFor/UB_paragraphs"
7
+ UB_paragraphs_dataset = load_huggingface_dataset(UB_paragraphs_dataset_name)
8
+ print(UB_paragraphs_dataset)
9
+
10
+ chroma_collection_name_from_HF = "entire_book_paragraph_content-from-HF"
11
+ paragraph_content_collection_from_HF = import_chroma_exported_hf_dataset(
12
+ chroma_client=chroma_client,
13
+ dataset=UB_paragraphs_dataset,
14
+ collection_name=chroma_collection_name_from_HF,
15
+ embedding_function=chroma_embedding_function
16
+ )
17
+ print(paragraph_content_collection_from_HF.count())