File size: 13,680 Bytes
8b0c83a be66014 8b0c83a |
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 193 194 195 196 197 198 199 200 201 202 203 204 205 206 207 208 209 210 211 212 213 214 215 216 217 218 219 220 221 222 223 224 225 226 227 228 229 230 231 232 233 234 235 236 237 238 239 240 241 242 243 244 245 246 247 248 249 250 251 |
# import streamlit as st
# from doctr.io import DocumentFile
# from doctr.models import ocr_predictor
# import json
# from doctr.models import ocr_predictor
# import openai
# import re
# import os
# model = ocr_predictor(det_arch='db_resnet50', reco_arch='crnn_vgg16_bn', pretrained=True)
# # import the OpenAI Python library for calling the OpenAI API
# from openai import OpenAI
# import os
# # Function to extract text from PDF using OCR
# def extract_text_from_pdf(pdf_file):
# doc = DocumentFile.from_pdf(pdf_file)
# result = model(doc) # Assuming 'model' is your OCR predictor
# return result
# # Function to clean extracted text
# def clean_extracted_text(result):
# extracted_text = []
# for page in result.pages:
# for block in page.blocks:
# for line in block.lines:
# line_text = ' '.join(word.value for word in line.words)
# extracted_text.append(line_text)
# return '\n'.join(extracted_text)
# # Function to extract critical information using a language model
# def extract_critical_information(text):
# # Here you would integrate your OpenAI language model (LLM) or similar
# # For demonstration, let's assume it returns a mock JSON
# MODEL = "gpt-3.5-turbo"
# #response = client.chat.completions.create(
# response = client.chat.completions.create(
# model=MODEL,
# messages=[
# {"role": "system", "content": "You are a property deed expert for the US."},
# {"role": "user", "content": f"Given the raw information extracted using OCR from a PDF, you should extract the most important parts of the deed such as the owner's name, property number, address, and any other important factors. The OCR results are stored in the variable `result`.\n\nOCR Result:\n{text}\n\nPlease provide the extracted information in JSON format.\n\nExample JSON output:\n{{\n \"owner_name\": \"\",\n \"property_address\": \"\",\n \"legal_description\": \"\",\n \"grantor_name\": \"\",\n \"grantee_name\": \"\",\n \"deed_type\": \"\",\n \"liens_and_encumbrances\": \"\",\n \"signatures\": \"\",\n \"notarization_details\": \"\",\n \"recording_information\": \"\",\n \"consideration\": \"\",\n \"habendum_clause\": \"\",\n \"warranty_clauses\": \"\",\n \"tax_information\": \"\",\n \"title_insurance_details\": \"\"\n}}Extracted Information JSON:, Warning: Do not make up fake information"}],
# temperature=0,
# )
# extracted_info= (response.choices[0].message.content)
# return extracted_info
# def clean_and_convert_to_json(input_string):
# # Remove the markdown code block indicators and any leading/trailing whitespace
# cleaned_string = input_string.strip()
# cleaned_string = re.sub(r'^```json\s*|\s*```$', '', cleaned_string, flags=re.MULTILINE)
# # Remove any non-printable characters except newlines
# cleaned_string = ''.join(char for char in cleaned_string if char.isprintable() or char in '\n\r')
# # Ensure the string starts with { and ends with }
# cleaned_string = cleaned_string.strip()
# if not cleaned_string.startswith('{'):
# cleaned_string = '{' + cleaned_string
# if not cleaned_string.endswith('}'):
# cleaned_string = cleaned_string + '}'
# try:
# # Parse the string as JSON
# data = json.loads(cleaned_string)
# # Convert back to a JSON string with proper formatting
# cleaned_json = json.dumps(data, indent=2)
# return cleaned_json
# except json.JSONDecodeError as e:
# print(f"JSON Decode Error: {e}")
# print(f"Problematic string:\n{cleaned_string}")
# return None
# def main():
# st.title("Property Deed Information Extraction App")
# uploaded_file = st.file_uploader("Upload a PDF file", type="pdf")
# if uploaded_file is not None:
# st.write("File uploaded successfully!")
# # Process the uploaded PDF file
# result = extract_text_from_pdf(uploaded_file)
# cleaned_text = clean_extracted_text(result)
# extracted_info = extract_critical_information(cleaned_text)
# extracted_info = clean_and_convert_to_json(extracted_info)
# # Parse the JSON result
# try:
# extracted_info_json = json.loads(extracted_info)
# except json.JSONDecodeError as e:
# st.error(f"An error occurred while parsing the JSON response: {e}")
# extracted_info_json = {}
# # Display extracted information
# st.subheader("Extracted Information:")
# st.text_input("Owner Name", value=extracted_info_json.get("owner_name", ""))
# st.text_input("Property Address", value=extracted_info_json.get("property_address", ""))
# st.text_input("Legal Description", value=extracted_info_json.get("legal_description", ""))
# st.text_input("Grantor Name", value=extracted_info_json.get("grantor_name", ""))
# st.text_input("Grantee Name", value=extracted_info_json.get("grantee_name", ""))
# st.text_input("Deed Type", value=extracted_info_json.get("deed_type", ""))
# st.text_input("Liens and Encumbrances", value=extracted_info_json.get("liens_and_encumbrances", ""))
# st.text_input("Signatures", value=extracted_info_json.get("signatures", ""))
# st.text_input("Notarization Details", value=extracted_info_json.get("notarization_details", ""))
# st.text_input("Recording Information", value=extracted_info_json.get("recording_information", ""))
# st.text_input("Consideration", value=extracted_info_json.get("consideration", ""))
# st.text_input("Habendum Clause", value=extracted_info_json.get("habendum_clause", ""))
# st.text_input("Warranty Clauses", value=extracted_info_json.get("warranty_clauses", ""))
# st.text_input("Tax Information", value=extracted_info_json.get("tax_information", ""))
# st.text_input("Title Insurance Details", value=extracted_info_json.get("title_insurance_details", ""))
# if __name__ == "__main__":
# main()
import streamlit as st
from doctr.io import DocumentFile
from doctr.models import ocr_predictor
import json
import openai
import re
import os
from openai import OpenAI
model = ocr_predictor(det_arch='db_resnet50', reco_arch='crnn_vgg16_bn', pretrained=True).to('cpu')
# Set up OpenAI client
api_key= os.environ.get("OPENAI_API_KEY","default_value_if_not_found")
# if not api_key:
# st.error("API Key is missing. Please set the OPENAI_API_KEY environment variable.")
# else:
# client = openai.OpenAI(api_key=api_key)
# st.write(f"API Key Loaded: {api_key[:4]}...")
# Function to extract text from PDF using OCR
def extract_text_from_pdf(pdf_file):
doc = DocumentFile.from_pdf(pdf_file)
result = model(doc) # Assuming 'model' is your OCR predictor
return result
# Function to clean extracted text
def clean_extracted_text(result):
extracted_text = []
for page in result.pages:
for block in page.blocks:
for line in block.lines:
line_text = ' '.join(word.value for word in line.words)
extracted_text.append(line_text)
return '\n'.join(extracted_text)
# Function to extract critical information using a language model
# def extract_critical_information(text):
# MODEL = "gpt-3.5-turbo"
# try:
# response = client.ChatCompletion.create(
# model=MODEL,
# messages=[
# {"role": "system", "content": "You are a property deed expert for the US."},
# {"role": "user", "content": f"Given the raw information extracted using OCR from a PDF, you should extract the most important parts of the deed such as the owner's name, property number, address, and any other important factors. The OCR results are stored in the variable `result`.\n\nOCR Result:\n{text}\n\nPlease provide the extracted information in JSON format.\n\nExample JSON output:\n{{\n \"owner_name\": \"\",\n \"property_address\": \"\",\n \"legal_description\": \"\",\n \"grantor_name\": \"\",\n \"grantee_name\": \"\",\n \"deed_type\": \"\",\n \"liens_and_encumbrances\": \"\",\n \"signatures\": \"\",\n \"notarization_details\": \"\",\n \"recording_information\": \"\",\n \"consideration\": \"\",\n \"habendum_clause\": \"\",\n \"warranty_clauses\": \"\",\n \"tax_information\": \"\",\n \"title_insurance_details\": \"\"\n}}Extracted Information JSON:, Warning: Do not make up fake information"}],
# temperature=0,
# )
# extracted_info = response['choices'][0]['message']['content']
# return extracted_info
# except openai.error.OpenAIError as e:
# st.error(f"OpenAI API Error: {e}")
# return None
def extract_critical_information(text):
# Here you would integrate your OpenAI language model (LLM) or similar
# For demonstration, let's assume it returns a mock JSON
MODEL = "gpt-4o-mini"
#response = client.chat.completions.create(
response = client.chat.completions.create(
model=MODEL,
messages=[
{"role": "system", "content": "You are a property deed expert for the US."},
{"role": "user", "content": f"Given the raw information extracted using OCR from a PDF, you should extract the most important parts of the deed such as the owner's name, property number, address, and any other important factors. The OCR results are stored in the variable `result`.\n\nOCR Result:\n{text}\n\nPlease provide the extracted information in JSON format.\n\nExample JSON output:\n{{\n \"owner_name\": \"\",\n \"property_address\": \"\",\n \"legal_description\": \"\",\n \"grantor_name\": \"\",\n \"grantee_name\": \"\",\n \"deed_type\": \"\",\n \"liens_and_encumbrances\": \"\",\n \"signatures\": \"\",\n \"notarization_details\": \"\",\n \"recording_information\": \"\",\n \"consideration\": \"\",\n \"habendum_clause\": \"\",\n \"warranty_clauses\": \"\",\n \"tax_information\": \"\",\n \"title_insurance_details\": \"\"\n}}Extracted Information JSON:, Warning: Do not make up fake information"}],
temperature=0,
)
extracted_info= (response.choices[0].message.content)
return extracted_info
# Function to clean and convert text to JSON
def clean_and_convert_to_json(input_string):
cleaned_string = input_string.strip()
cleaned_string = re.sub(r'^```json\s*|\s*```$', '', cleaned_string, flags=re.MULTILINE)
cleaned_string = ''.join(char for char in cleaned_string if char.isprintable() or char in '\n\r')
cleaned_string = cleaned_string.strip()
if not cleaned_string.startswith('{'):
cleaned_string = '{' + cleaned_string
if not cleaned_string.endswith('}'):
cleaned_string = cleaned_string + '}'
try:
data = json.loads(cleaned_string)
cleaned_json = json.dumps(data, indent=2)
return cleaned_json
except json.JSONDecodeError as e:
st.error(f"JSON Decode Error: {e}")
return None
# Streamlit app
def main():
st.title("Property Deed Information Extraction App")
uploaded_file = st.file_uploader("Upload a PDF file", type="pdf")
if uploaded_file is not None:
st.write("File uploaded successfully!")
result = extract_text_from_pdf(uploaded_file)
cleaned_text = clean_extracted_text(result)
extracted_info = extract_critical_information(cleaned_text)
if extracted_info:
extracted_info = clean_and_convert_to_json(extracted_info)
if extracted_info:
try:
extracted_info_json = json.loads(extracted_info)
except json.JSONDecodeError as e:
st.error(f"An error occurred while parsing the JSON response: {e}")
extracted_info_json = {}
st.subheader("Extracted Information:")
st.text_input("Owner Name", value=extracted_info_json.get("owner_name", ""))
st.text_input("Property Address", value=extracted_info_json.get("property_address", ""))
st.text_input("Legal Description", value=extracted_info_json.get("legal_description", ""))
st.text_input("Grantor Name", value=extracted_info_json.get("grantor_name", ""))
st.text_input("Grantee Name", value=extracted_info_json.get("grantee_name", ""))
st.text_input("Deed Type", value=extracted_info_json.get("deed_type", ""))
st.text_input("Liens and Encumbrances", value=extracted_info_json.get("liens_and_encumbrances", ""))
st.text_input("Signatures", value=extracted_info_json.get("signatures", ""))
st.text_input("Notarization Details", value=extracted_info_json.get("notarization_details", ""))
st.text_input("Recording Information", value=extracted_info_json.get("recording_information", ""))
st.text_input("Consideration", value=extracted_info_json.get("consideration", ""))
st.text_input("Habendum Clause", value=extracted_info_json.get("habendum_clause", ""))
st.text_input("Warranty Clauses", value=extracted_info_json.get("warranty_clauses", ""))
st.text_input("Tax Information", value=extracted_info_json.get("tax_information", ""))
st.text_input("Title Insurance Details", value=extracted_info_json.get("title_insurance_details", ""))
else:
st.error("Failed to clean and convert extracted information to JSON.")
else:
st.error("Failed to extract critical information from text.")
if __name__ == "__main__":
main()
|