Update ImageProcessor.py

Added paragraph functionality

ImageProcessor.py

@@ -22,17 +22,14 @@ def is_leftmost(image, x, y_top, y_bot):
     return np.sum(left_portion) == 0
 
 def get_indents(filename, body_bbox, page):
-    # doc = fitz.open(filename)
     indented_lines = []
-
-    # for (pg_ind, page) in enumerate(doc):
     image = cv2.imread(filename)
     body_rect = fitz.Rect(body_bbox)
     pg_dict = page.get_text('dict', clip=body_rect)
     all_lines = [(int(line['bbox'][0]), int(line['bbox'][1]), int(line['bbox'][2]), int(line['bbox'][3]), line)for block in pg_dict['blocks'] for line in block['lines']]
     body_text = page.get_text("text", clip=body_rect).strip()
     baseline = min([l[0] for l in all_lines])
-    indented_inds = [i for (i,l) in enumerate(all_lines) if (l[0]-baseline > 9 and l[0]-baseline < 30 and is_leftmost(image, l[0]-10, l[1], l[3]))]
+    indented_inds = [i for (i,l) in enumerate(all_lines) if (l[0]-baseline > 9 and is_leftmost(image, l[0]-12, l[1], l[3]))]# and l[0]-baseline < 30
     for i in indented_inds:
         indented_lines.append((i, all_lines[i][0], all_lines[i][1], all_lines[i][2], all_lines[i][3]))
     return indented_lines
@@ -144,6 +141,68 @@ def get_page_elements(filename, page):
 
     return page_bbox, header_bbox, fn_bbox, body_bbox, case_separator_bbox, indent_lines, image
 
+def paragraphs(folderpath):
+    doc = fitz.open(folderpath + '/opinion.pdf')
+    df = pd.read_csv(folderpath + '/data.csv')
+    df = df.replace({np.nan: None})
+    nl_inds = df['Indent Lines'].tolist()
+    nl_inds = [eval(nli) for nli in nl_inds]
+    nl_indents = [nli[1] for page_nli in nl_inds for nli in page_nli]
+    nl_inds = [(i, nli[0]) for (i, page_nli) in zip(df['Pg Ind'].tolist(), nl_inds) for nli in page_nli]
+    paras = [([], 0, 0, 0)]  # Text, indent amount, start pg ind, end pg ind
+    para_lines = []
+    for (i, page) in enumerate(doc):
+        ind = df.index[df['Pg Ind'] == i].tolist()[0]
+        body_bbox = [df.iloc[ind]['Body X1'], df.iloc[ind]['Body Y1'], df.iloc[ind]['Body X2'], df.iloc[ind]['Body Y2']]
+        case_separator = df.iloc[ind]['Case Separator Y']
+        if case_separator is not None:
+            body_bbox[-1] = case_separator
+        body_rect = fitz.Rect(body_bbox)
+        pg_dict = page.get_text('dict', clip=body_rect)
+        all_lines = [get_line_text(line) for block in pg_dict['blocks'] for line in block['lines']]
+        for (j, line) in enumerate(all_lines):
+            if line == "":
+                continue
+            if (i, j) in nl_inds:
+                indent_amt = nl_indents[nl_inds.index((i, j))]  # This is for the starting one
+                paras.append(([], indent_amt, i, i))
+            paras[-1] = list(paras[-1])
+            paras[-1][0].append(line.strip())
+            paras[-1][-1] = i  # Update the page ind
+            paras[-1] = tuple(paras[-1])
+    paras = block_quotes(paras)
+    paras_df = pd.DataFrame(data=paras, index=None, columns=['Text', 'Indent Amount', 'Start Pg Ind', 'End Pg Ind'])
+    return paras_df
+
+def get_line_text(line):
+    words = []
+    for s in line['spans']:
+        text = s['text'].strip()
+        if text != "":
+            words.append(text)
+    words = " ".join(words)
+    return words
+
+def block_quotes(paras):
+    modified_paras = []
+    start_para, end_para, end_quote_passed = None, None, None
+    for (i, (para, ind_amt, start_pg_ind, end_pg_ind)) in enumerate(paras):
+        if i == len(paras) - 1:
+            break
+        if len(para) == 1 and "“" == para[0][0] and start_para is None:
+            start_para = i
+        if len(para) == 1 and "”" == para[0][-1] and start_para is not None:
+            end_quote_passed = True
+        if len(para) == 1 and (
+                paras[i + 1][1] - ind_amt) < -5 and end_para is None and start_para is not None and end_quote_passed:
+            end_para = i
+        if start_para is not None and end_para is not None:
+            para = [p[0][0] for p in paras[start_para:end_para + 1]]
+            start_para, end_para, end_quote_passed = None, None, False
+        if start_para is None and end_para is None:
+            modified_paras.append((para, ind_amt, start_pg_ind, end_pg_ind))
+    return modified_paras
+
 def process_file(folderpath):
     pdf2png(folderpath)
     doc = fitz.open(folderpath + '/opinion.pdf')
@@ -173,4 +232,10 @@ def process_file(folderpath):
         data_df = pd.concat([data_df, row_df], ignore_index=True)
         cv2.imwrite(folderpath + '/' + str(ind) + '-processed.png', image)
     data_df['Pg Ind'] = data_df['Pg Ind'].astype('int')
-    data_df.to_csv(folderpath +'/data.csv', index=False)
+    data_df.to_csv(folderpath +'/data.csv', index=False)
+    paras_df = paragraphs(folderpath)
+    paras_df.to_csv(folderpath + '/paragraphs.csv', index=False)
+
+
+
+process_file('PDF Cases/19-896_2135')