gdown

app.py

@@ -13,6 +13,9 @@ import io
 
 from nltk.tokenize import sent_tokenize
 
+import gdown
+
+
 import nltk
 nltk.download('punkt')
 
@@ -48,9 +51,14 @@ def sent_cross_load():
 
 @st.cache
 def load_data():
-    #dfs = [pd.read_json(f'passages_{i}.jsonl', lines=True) for i in range(0,5)]
-    #df = pd.concat(dfs)
-    df = pd.read_json('https://www.dropbox.com/s/82lwbaym3b1o6uq/passages.jsonl?raw=1', lines=True)
+    #df = pd.read_json('https://www.dropbox.com/s/82lwbaym3b1o6uq/passages.jsonl?raw=1', lines=True)
+    
+    url = "https://drive.google.com/uc?export=download&id=1nIBS9is8YCeiPBqA7MifVC5xeaKWH8uL"
+    output = "passages.jsonl"
+    gdown.download(url, output, quiet=False)
+
+    df = pd.read_json(output, lines=True)
+
     df.reset_index(inplace=True, drop=True)
     return df
 
@@ -63,8 +71,14 @@ with st.spinner(text="Loading data..."):
 def load_embeddings():
     #efs = [np.load(f'embeddings_{i}.pt.npy')  for i in range(0,5)]
     #corpus_embeddings = np.concatenate(efs)
-    response = requests.get("https://www.dropbox.com/s/px8kjdd3p5mzw6j/corpus_embeddings.pt.npy?raw=1")
-    corpus_embeddings = np.load(io.BytesIO(response.content))  
+    
+    url = "https://drive.google.com/uc?export=download&id=1z9eoBI07p_YtrdK1ZWZeCRT5T5mu5nhV"
+    output = "embeddings.npy"
+    gdown.download(url, output, quiet=False)
+
+    corpus_embeddings = np.load(output) 
+    #response = requests.get("https://www.dropbox.com/s/px8kjdd3p5mzw6j/corpus_embeddings.pt.npy?raw=1")
+    #corpus_embeddings = np.load(io.BytesIO(response.content))  
 
     
     return corpus_embeddings