Create utils.py

utils.py

@@ -0,0 +1,28 @@
+import os 
+import random
+import requests
+
+user_agents_url = 'https://raw.githubusercontent.com/danielmiessler/SecLists/master/Fuzzing/User-Agents/UserAgents-IE.txt'
+
+os.makedirs('./tmp', exist_ok=True)
+download_path = './tmp/user_agents.txt'
+
+# download user-agents
+if not os.path.exists(download_path):
+    urlretrieve(user_agents_url, download_path)
+
+# reading user-agents file
+with open(download_path, 'r') as f:
+    user_agents = f.readlines()
+    user_agents = list(map(lambda x: x.strip('\n'), user_agents))
+
+def return_user_agent():
+    """
+    this function returns different user agent randomly 
+    """
+    ua = random.choice(user_agents)
+    # using this header to pretend as regular user so that we are not blocked by website
+    headers = {
+        'User-Agent': ua
+    }
+    return headers