Update app.py

app.py

@@ -1,9 +1,20 @@
+#Choose the extractor. Both extractnet & dragnet have dependency conflicts with bertopic
+EXTRACTOR_NET = 'trafilatura'
+
 import numpy as np
 import onnxruntime
 import onnx
 import gradio as gr
 import requests
 import json
+
+if(EXTRACTOR_NET == 'extractnet'):
+  from extractnet import Extractor
+elif(EXTRACTOR_NET == 'dragnet'):
+  from dragnet import extract_content
+elif(EXTRACTOR_NET == 'trafilatura'):
+  import trafilatura
+
 from extractnet import Extractor
 import math
 from transformers import AutoTokenizer
@@ -275,8 +286,15 @@ def inference(input_batch,isurl,use_archive,limit_companies=10):
                 if archive['archived']:
                     url = archive['url']
             #Extract the data from url
-            extracted = Extractor().extract(requests.get(url).text)
-            input_batch_content.append(extracted['content'])
+            if(EXTRACTOR_NET == 'extractnet'):
+              extracted = Extractor().extract(requests.get(url).text)
+              input_batch_content.append(extracted['content'])
+            elif(EXTRACTOR_NET == 'dragnet'):
+              extracted = extract_content(requests.get(url).content)
+              input_batch_content.append(extracted)
+            elif(EXTRACTOR_NET == 'trafilatura'):
+              extracted = trafilatura.extract(trafilatura.fetch_url(url), include_comments=False)
+              input_batch_content.append(extracted)
     else:
         print("[i] Data is news contents")
         if isinstance(input_batch_r[0], list):