Switched to simple check for domain extension.

functions/helper_functions.py

@@ -20,6 +20,7 @@ def get_url(company_name: str) -> str:
     '''
 
     logger = logging.getLogger(__name__ + '.get_url')
+    logger.info('Getting website URL for %s', company_name)
 
     query = f'{company_name} official website'
 
@@ -45,10 +46,6 @@ def get_feed(website_url: str) -> str:
 
     feeds = feed_search(website_url)
 
-    logger.info('Feeds search result is: %s', type(feeds))
-    logger.info('Feeds search results: %s', len(feeds))
-    logger.info('Feeds results: %s', list(feeds))
-
     if len(feeds) > 0:
         return str(feeds[0].url)
 

functions/tools.py

@@ -2,11 +2,11 @@
 
 import logging
 from urllib.parse import urlparse
-import validators
 import functions.helper_functions as helper_funcs
 
 FEED_URIS = {}
 RSS_EXTENSIONS = ['xml', 'rss', 'atom']
+COMMON_EXTENSIONS = ['com', 'net', 'org', 'edu', 'gov', 'co', 'us']
 
 
 def get_content(website: str) -> list:
@@ -39,7 +39,7 @@ def get_content(website: str) -> list:
     # If neither of those get it - try feedparse if it looks like a url
     # or else just google it
     else:
-        if validators.url(website):
+        if website.split('.')[-1] in COMMON_EXTENSIONS:
             website_url = website
             logger.info('%s looks like a website URL', website)