Update TextProcessor.py

TextProcessor.py

@@ -35,7 +35,10 @@ def paragraphs(folderpath):
             continue
 
         if len(line_texts[j]) > 0:
-            prior_endswith_period = re.search('\.([^A-z]{0,2})$', line_texts[j - 1].strip()) is not None
+            prior_median = (baselines[pg_inds[j - 1]] + rights[pg_inds[j - 1]]) / 2
+            current_median = (baselines[pg_inds[j]] + rights[pg_inds[j]]) / 2
+
+            prior_endswith_period = re.search('[:\.]([^A-z]{0,2})$',line_texts[j - 1].strip()) is not None  # Include colon?
             prior_is_section_header = re.search('^([ABCDEIVX]+)$', line_texts[j - 1].strip()) is not None
             current_is_section_header = re.search('^([ABCDEIVX]+)$', line_texts[j].strip()) is not None
             prior_is_asterisk = re.search('^([\s\*]+)$', line_texts[j - 1].strip()) is not None
@@ -46,18 +49,23 @@ def paragraphs(folderpath):
             prior_tabbed = x1s[j - 1] - baselines[pg_inds[j - 1]] > 7
             prior_supertabbed = x1s[j - 1] - baselines[pg_inds[j - 1]] > 18
             current_supertabbed = x1s[j] - baselines[pg_inds[j]] > 18
-            prior_more_left = x1s[j] - x1s[j - 1] > 7
-            prior_right_margin = abs(x2s[j - 1] - rights[pg_inds[j - 1]]) < 10
+            prior_more_left = (x1s[j] - baselines[pg_inds[j]]) - (x1s[j - 1] - baselines[pg_inds[j - 1]]) > 7
+            prior_right_margin = x1s[j - 1] > prior_median
 
             is_section_header = (prior_is_section_header or current_is_section_header or prior_is_asterisk or current_is_asterisk or prior_is_date)
-            prior_period_current_tabbed = (prior_endswith_period and current_tabbed and (prior_more_left or prior_right_margin or (not prior_supertabbed and not current_supertabbed)))
+            is_the_classic = (prior_endswith_period and current_tabbed and prior_more_left and not prior_supertabbed)  # Note: Supertabbing oofs stuff
+            is_start_blockquote = (prior_endswith_period and current_supertabbed and prior_more_left)
+            is_after_blockquote = (prior_endswith_period and not current_supertabbed and prior_supertabbed)
+            is_after_disposition = (prior_right_margin and current_tabbed)
+            is_after_oneline_paragraph = (prior_tabbed and current_tabbed and not prior_supertabbed and not current_supertabbed)
 
-            if is_section_header or prior_period_current_tabbed:
+            if is_section_header or is_the_classic or is_after_oneline_paragraph or is_start_blockquote or is_after_blockquote or is_after_disposition:
                 paras.append(para)
                 para = []
             #     print('\n')
-            # print(str(j) + ':\t' + str(prior_endswith_period) + '\t' + str(current_tabbed)  + '\t' + str(prior_more_left)+ '\t' + str(prior_supertabbed) + '\t' + str(prior_right_margin)+ '\t' + str(current_supertabbed) +'\t' + line_text)
+            # print(str([j, pg_inds[j]]) + ':\t' + str(is_the_classic) + '\t' + str(is_start_blockquote)  + '\t' + str(is_after_blockquote)+ '\t' + str(is_after_disposition) + '\t' + line_text)
             para.append((pg_inds[j], line_inds[j], line_text))
+
     paras.append(para)
     paras_df = pd.DataFrame({'Lines': paras})
     return paras_df
@@ -80,4 +88,6 @@ def process_file(folderpath):
         x1, y1, x2, y2, para_first_line, pg_ind = indent
         image = cv2.imread(folderpath + '/' + str(pg_ind) + '-processed.png')
         cv2.circle(image, (x1 - 15, int(0.5 * (y1 + y2))), radius=1, color=(240, 32, 160), thickness=2)
-        cv2.imwrite(folderpath + '/' + str(pg_ind) + '-processed.png', image)
+        cv2.imwrite(folderpath + '/' + str(pg_ind) + '-processed.png', image)
+
+process_file('PDF Cases/462_122')