add comments

data_crawler/Tiki_Crawl.py

@@ -64,6 +64,19 @@ params = {
 }
 
 def comment_parser(json):
+    """
+    A function to extract information from json produced by Tiki API
+
+    Parameters
+    ----------
+    json : dictionary
+            The json format produced by Tiki API.
+
+    Returns
+    ----------
+    dict
+        a dictionary containing relevant information from the reviews
+    """
     d = dict()
     d['id'] = json.get(id)
     d['title'] = json.get('title')
@@ -78,11 +91,27 @@ def comment_parser(json):
 
 
 def crawl_tiki(link, pages = 1): 
-  pid = re.search(r"p(\d+)", link).group(1)
-  result = []
-  params['product_id'] = pid
-  print('Crawl comment for product {}'.format(pid))
-  for i in range(pages):
+    """
+    A function to request and retrieve information from Tiki API
+    
+    Parameters
+    ----------
+    link : string
+        Product link from Tiki website
+    
+    pages: int
+        Number of pages from the product page to crawl
+    
+    Returns
+    ----------
+    dict
+    a DataFrame containing relevant information from the reviews
+    """
+    pid = re.search(r"p(\d+)", link).group(1)
+    result = []
+    params['product_id'] = pid
+    print('Crawl comment for product {}'.format(pid))
+    for i in range(pages):
       params['page'] = i
       response = requests.get('https://tiki.vn/api/v2/reviews', headers=headers, params=params, cookies=cookies)
       if response.status_code == 200:
@@ -91,4 +120,4 @@ def crawl_tiki(link, pages = 1):
               result.append(comment_parser(comment))
       else:
         break
-  return pd.DataFrame(result)
+    return pd.DataFrame(result)