add pdf crawl

app.py

@@ -5,6 +5,35 @@ from reference_string_parsing import *
 from controlled_summarization import *
 from dataset_extraction import *
 
+import requests
+def download_pdf(url, dest_folder):
+   
+    """
+    Download a PDF from a given URL and save it to a specified destination folder.
+    Parameters:
+        url (str): URL of the PDF
+        dest_folder (str): Destination folder to save the downloaded PDF
+    """
+    
+    if not os.path.exists(dest_folder):
+        os.makedirs(dest_folder)
+
+    response = requests.get(url, stream=True)
+    filename = os.path.join(dest_folder, url.split("/")[-1])
+
+    with open(filename, 'wb') as file:
+        for chunk in response.iter_content(chunk_size=1024):
+            if chunk:
+                file.write(chunk)
+    #print(f"Downloaded {url} to {filename}")
+    return filename
+
+# Example Usage
+#url = "https://arxiv.org/pdf/2305.14996.pdf"
+#dest_folder = "./examples/"
+#download_pdf(url, dest_folder)
+
+
 with gr.Blocks(css="#htext span {white-space: pre-line}") as demo:
     gr.Markdown("# Gradio Demo for SciAssist")
     with gr.Tabs():
@@ -16,7 +45,8 @@ with gr.Blocks(css="#htext span {white-space: pre-line}") as demo:
                 gr.Markdown(ctrlsum_file_md)
                 with gr.Row():
                     with gr.Column():
-                        ctrlsum_file = gr.File(label="Input File")
+                        ctrlsum_url = gr.TextArea(label="PDF URL", max_lines=1)
+                        ctrlsum_file = gr.File(label="Input File", max_lines=2)
                         ctrlsum_str = gr.TextArea(label="Input String", max_lines=5)
                         with gr.Column():
                             gr.Markdown("* Length 0 will exert no control over length.")
@@ -33,6 +63,9 @@ with gr.Blocks(css="#htext span {white-space: pre-line}") as demo:
                 ctrlsum_file_examples = gr.Examples(examples=[["examples/H01-1042_body.txt", 50, "automatic evaluation technique"],["examples/H01-1042.pdf", 0, "automatic evaluation technique"]],
                                                 inputs=[ctrlsum_file, ctrlsum_file_length, ctrlsum_file_keywords])
 
+        if ctrlsum_url is not None and len(ctrlsum_url) > 4:
+            ctrlsum_file = download_pdf(ctrlsum_url, './examples/')
+
         ctrlsum_file_btn.click(
             fn=ctrlsum_for_file,
             inputs=[ctrlsum_file, ctrlsum_file_length, ctrlsum_file_keywords, ctrlsum_str],
@@ -143,4 +176,4 @@ with gr.Blocks(css="#htext span {white-space: pre-line}") as demo:
         )
 
 
-demo.launch(share=False)
+demo.launch(share=False)