Update TextProcessor.py

TextProcessor.py

@@ -12,7 +12,7 @@ def paragraphs(folderpath):
     pg_indices = df['Pg Ind'].tolist()
 
     x1s, y1s, x2s, y2s, line_texts, line_inds, pg_inds, baselines, rights = [], [], [], [], [], [], [], {}, {}
-    paras, types = [], []
+    paras = []
     for (i, pg_ind) in enumerate(pg_indices):
         lines = eval(df[df['Pg Ind'] == i]['Lines'].tolist()[0])
         pg_x1s, pg_x2s = [], []
@@ -48,8 +48,8 @@ def paragraphs(folderpath):
 
             current_tabbed = x1s[j] - baselines[pg_inds[j]] > 7
             prior_tabbed = x1s[j - 1] - baselines[pg_inds[j - 1]] > 7
-            prior_supertabbed = x1s[j - 1] - baselines[pg_inds[j - 1]] >= 11
-            current_supertabbed = x1s[j] - baselines[pg_inds[j]] >= 11
+            prior_supertabbed = x1s[j - 1] - baselines[pg_inds[j - 1]] >= 14
+            current_supertabbed = x1s[j] - baselines[pg_inds[j]] >= 14
             prior_more_left = (x1s[j] - baselines[pg_inds[j]]) - (x1s[j - 1] - baselines[pg_inds[j - 1]]) > 7
             prior_right_margin = x1s[j - 1] > prior_median
 
@@ -58,24 +58,21 @@ def paragraphs(folderpath):
             is_start_blockquote = (prior_endswith_period and current_supertabbed and prior_more_left and not is_inblock)
             is_after_blockquote = (prior_endswith_period and not current_supertabbed and is_inblock)
             is_after_disposition = (prior_right_margin and current_tabbed)
-            # is_after_oneline_paragraph = (prior_tabbed and current_tabbed and not prior_supertabbed and not current_supertabbed and not is_inblock)
 
             if is_start_blockquote and not is_section_header:
                 is_inblock = True
-            if is_after_blockquote or prior_is_date: # This date hack helps ensure that slip opinion headers do not get caught
+                # print("START BLOCK")
+            if is_after_blockquote or prior_is_date:  # This date hack helps ensure that slip opinion headers do not get caught
                 is_inblock = False
 
             if is_section_header or is_the_classic or is_start_blockquote or is_after_blockquote or is_after_disposition:
                 paras.append(para)
-                types.append(is_inblock)
                 para = []
             #     print('\n')
-            # print(str([j, pg_inds[j]]) + ':\t' + str(is_the_classic) + '\t' + str(is_start_blockquote)  + '\t' + str(is_after_blockquote)+ '\t' + str(is_after_disposition) + '\t' + str(is_after_oneline_paragraph) + '\t' + line_text)
-            para.append((pg_inds[j], line_inds[j], line_text))
-
+            # print(str([j, pg_inds[j]]) + ':\t' + str(is_the_classic) + '\t' + str(is_start_blockquote) + '\t' + str(is_after_blockquote)+ '\t' + str(is_after_disposition) + '\t' + str(is_section_header) + '\t' + line_text)
+            para.append((pg_inds[j], line_inds[j], is_inblock, line_text))
     paras.append(para)
-    types.append(is_inblock)
-    paras_df = pd.DataFrame({'Lines': paras, 'Block Quote': types})
+    paras_df = pd.DataFrame({'Lines': paras})
     return paras_df
 
 def process_file(folderpath):
@@ -86,7 +83,7 @@ def process_file(folderpath):
     indents = []
     for (i, para_lines) in enumerate(paras_lines):
         para = []
-        para_start_pg_ind, para_start_line_ind, para_first_line = para_lines[0]
+        para_start_pg_ind, para_start_line_ind, _, para_first_line = para_lines[0]
         page_df = data_df[data_df['Pg Ind'] == para_start_pg_ind]
         pg_lines = eval(page_df['Lines'].tolist()[0])
         x1, y1, x2, y2, text = pg_lines[para_start_line_ind]
@@ -97,5 +94,3 @@ def process_file(folderpath):
         image = cv2.imread(folderpath + '/' + str(pg_ind) + '-processed.png')
         cv2.circle(image, (x1 - 15, int(0.5 * (y1 + y2))), radius=1, color=(240, 32, 160), thickness=2)
         cv2.imwrite(folderpath + '/' + str(pg_ind) + '-processed.png', image)
-
-process_file('PDF Cases/333_178')