Update fn.py

fn.py

@@ -14,8 +14,11 @@ import numpy as np
 from scipy.spatial.distance import cdist
 from duckduckgo_search import DDGS
 from bs4 import BeautifulSoup
+from markdownify import markdownify
 
-model_name = "intfloat/multilingual-e5-large"
+model_name = "cl-nagoya/ruri-large"
+# "mixedbread-ai/mxbai-embed-large-v1"
+# "intfloat/multilingual-e5-large"
 input_dir = 'input'
 vectors_dir = 'vectors'
 
@@ -54,6 +57,13 @@ def bs4(url):
 
     return text
 
+def md(url):
+    html = requests.get(url).text.replace("\r", '')
+    mdtxt = markdownify(html)
+    mdtxt = re.sub("\n+", "\n", mdtxt, flags=(re.MULTILINE | re.DOTALL))
+
+    return mdtxt
+
 def upload(name, filename, content):
     os.makedirs(f"{input_dir}/{name}", exist_ok=True)
     srcpath = f"{input_dir}/{name}/{filename}"