finish

app.py

@@ -5,19 +5,15 @@ import shutil
 import os
 import tqdm
 from huggingface_hub import snapshot_download
-from huggingface_hub import HfApi, login
+from huggingface_hub import HfApi
 import tempfile
 import re
-import pdfminer
+from pdfminer.high_level import extract_text
 import time
 
-print("pdfminer", pdfminer.__version__)
-print("pandoc", pypandoc.__version__)
-
 HF_TOKEN = os.environ.get("HF_TOKEN")
 
 api = HfApi()
-login(HF_TOKEN)
 
 
 #from docx import Document
@@ -50,7 +46,7 @@ def retrieve_lines(filename):
     extension = filename.split(".")[-1]
 
     if extension == "pdf":
-        text = pdfminer.high_level.extract_text(filename)
+        text = extract_text(filename)
         lines = text.split("\n")
     elif extension in ["docx", "doc"]:
         with tempfile.TemporaryDirectory() as tmpdirname:
@@ -98,7 +94,7 @@ def main(filename, codewords_mapping):
                 out += 25 * "="
                 out += "\n\n"
     
-            out += f"## Source: {filename}\n"
+            out += f"## Source: {'/'.join(filename.split('/')[-2:])}\n"
             out += 25 * "-"
             out += "\n"
             out += "\n".join([f'-{v}' for k,v in match.items()])
@@ -123,31 +119,31 @@ def convert(*keywords):
     os.makedirs(RESULTS_FOLDER)
 
     result_files = []
-    for folder in tqdm.tqdm(glob.glob(os.path.join(DOC_FOLDER, "*"))):
-        all_files = tqdm.tqdm(glob.glob(f"./{folder}/*"))
+    folders = glob.glob(os.path.join(DOC_FOLDER, "*"))
+
+
+    for folder in tqdm.tqdm(folders):
+        all_files = tqdm.tqdm(glob.glob(f"{folder}/*"))
         num_files += len(all_files)
 
         for filename in all_files:
             try:
-                result_files += main(filename)
+                result_files += main(filename, codewords_mapping)
             except Exception as e:
                 print(f"{filename} not working because \n {e}")
 
-        break
-
-    print(f"Len {result_files}")
-
     result_files = list(set(result_files))
 
-    for file in result_files:
-        api.upload_file(
-            path_or_fileobj=file,
-            path_in_repo=file,
-            repo_id="patrickvonplaten/atlas",
-            repo_type="dataset",
-        )
+    api.upload_folder(
+        repo_id="patrickvonplaten/atlas",
+        folder_path=RESULTS_FOLDER,
+        path_in_repo=f"results_{time.time()}",
+        repo_type="dataset",
+        token=HF_TOKEN,
+    )
+
+    return f"Done. Processed {len(result_files)} files."
 
-    return f"Done: {len(result_files)}"
 
 inputs = [gr.Textbox(label=f"Enter your keywords for {k}", max_lines=2, placeholder=CAT_TO_CODEWORDS[k], value=",".join(CAT_TO_CODEWORDS[k])) for k in CATEGORIES]
 

requirements.txt

@@ -1,2 +1,2 @@
 pypandoc
-pdfminer
+pdfminer.six