Update app.py

app.py

@@ -306,7 +306,16 @@ def inference(input_batch,isurl,use_archive,filt_companies_topic,limit_companies
               extracted = extract_content(requests.get(url).content)
               input_batch_content.append(extracted)
             elif(EXTRACTOR_NET == 'trafilatura'):
-              extracted = trafilatura.extract(trafilatura.fetch_url(url), include_comments=False, config=trafilatura_config, include_tables=False)
+              try:
+                  extracted = trafilatura.extract(trafilatura.fetch_url(url), include_comments=False, config=trafilatura_config, include_tables=False)
+              except:
+                    archive = is_in_archive(url)
+                    if archive['archived']:
+                        print("[W] Using archive.org version of",url)
+                        url = archive['url']
+                        extracted = trafilatura.extract(trafilatura.fetch_url(url), include_comments=False, config=trafilatura_config, include_tables=False)
+                    else:
+                        print("[E] URL=",url,"not found")
               input_batch_content.append(extracted)
     else:
         print("[i] Data is news contents")
@@ -349,7 +358,7 @@ def inference(input_batch,isurl,use_archive,filt_companies_topic,limit_companies
       if ner_labels[idx]: #not empty
         for ner in ner_labels[idx]:
           if filt_companies_topic:
-              if news_sectors[idx] != ner[1]:
+              if news_sectors[idx][0] not in ner[1]:
                   continue
           dfo = pd.concat( [dfo, df.loc[[idx]].assign(company=ner[0], sector=ner[1], symbol=ner[2])], join='outer', ignore_index=True) #axis=0
     print("[i] Pandas output shape:",dfo.shape)