Update app.py
Browse files
app.py
CHANGED
|
@@ -15,10 +15,9 @@ loading and combining all the iterables together.
|
|
| 15 |
|
| 16 |
"""
|
| 17 |
|
| 18 |
-
from huggingface_hub import hf_hub_url, cached_download
|
| 19 |
-
import json
|
| 20 |
|
| 21 |
-
|
|
|
|
| 22 |
import json
|
| 23 |
from itertools import chain
|
| 24 |
|
|
@@ -27,7 +26,7 @@ target_subreddits = ["askscience", "gaming", "technology", "todayilearned", "pro
|
|
| 27 |
def load_reddit_split(subreddit_name):
|
| 28 |
"""Stream Reddit comments from a specific subreddit split"""
|
| 29 |
file_url = hf_hub_url(repo_id="HuggingFaceGECLM/REDDIT_comments", filename=f"{subreddit_name}.jsonl")
|
| 30 |
-
file_path =
|
| 31 |
with open(file_path, "r") as f:
|
| 32 |
for line in f:
|
| 33 |
yield json.loads(line)
|
|
|
|
| 15 |
|
| 16 |
"""
|
| 17 |
|
|
|
|
|
|
|
| 18 |
|
| 19 |
+
|
| 20 |
+
from huggingface_hub import hf_hub_download
|
| 21 |
import json
|
| 22 |
from itertools import chain
|
| 23 |
|
|
|
|
| 26 |
def load_reddit_split(subreddit_name):
|
| 27 |
"""Stream Reddit comments from a specific subreddit split"""
|
| 28 |
file_url = hf_hub_url(repo_id="HuggingFaceGECLM/REDDIT_comments", filename=f"{subreddit_name}.jsonl")
|
| 29 |
+
file_path = hf_hub_download(repo_id="HuggingFaceGECLM/REDDIT_comments", filename=f"{subreddit_name}.jsonl")
|
| 30 |
with open(file_path, "r") as f:
|
| 31 |
for line in f:
|
| 32 |
yield json.loads(line)
|