Spaces:
Runtime error
Runtime error
File size: 5,755 Bytes
121d037 53e11fa 121d037 ce86dab 121d037 53e11fa 7ffe108 53e11fa 7ffe108 53e11fa 7ffe108 121d037 ce86dab 121d037 ce86dab 45cfcf7 ce86dab 121d037 |
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 |
import streamlit as st
import os
import shutil
import fitz
import pandas as pd
import easyocr
from openai import OpenAI
from dotenv import load_dotenv
import ast
load_dotenv()
@st.cache_data
def convert_df(df):
# IMPORTANT: Cache the conversion to prevent computation on every rerun
return df.to_csv().encode('utf-8')
def list_files(directory):
for root, dirs, files in os.walk(directory):
for name in files:
yield os.path.join(root, name)
def correct_list(client, list_str):
base_prompt = '''Above python list has syntax error.
Correct the syntax without changing the values. Output should only be the corrected list.
'''
prompt = list_str + base_prompt
chat_completion = client.chat.completions.create(
messages=[
{
"role": "user",
"content": prompt,
}
],
model="gpt-3.5-turbo",
)
list_str_correct = chat_completion.choices[0].message.content
return list_str_correct
if os.path.exists('prediction') and os.path.isdir('prediction'):
shutil.rmtree('prediction')
if os.path.exists('temp_pdf') and os.path.isdir('temp_pdf'):
shutil.rmtree('temp_pdf')
# Check if the directory exists
if not os.path.exists('temp_pdf'):
# If it does not exist, create it
os.makedirs('temp_pdf')
print('not_found')
else:
print('found')
temp_file_path = 'temp//temp.pdf'
reader = easyocr.Reader(['en'])
def main():
# Set the title of the app
st.title("Transcript parser")
credential = st.text_input('Credential')
if credential is not '':
st.markdown(credential)
# credential = os.environ.get("OPENAI_API_KEY")
# Create a file uploader to upload PDF files
uploaded_file = st.file_uploader("Choose a PDF file", type="pdf")
if uploaded_file is not None:
# Create a temporary directory
with open(temp_file_path, 'wb') as f:
f.write(uploaded_file.getbuffer())
image_paths=[]
input_path = "temp//temp.pdf"
# st.markdown('### Images of detected tables')
with st.spinner('Performing OCR...'):
doc = fitz.open(input_path)
zoom = 4
mat = fitz.Matrix(zoom, zoom)
count = 0
context = ''
for p in doc:
count += 1
if count>4:
count=4
st.error('Page limit exceeded. processing first 4 images')
for i in range(count):
st.markdown(f"Processing page {i+1}...")
val = f"image_{i}.png"
val = os.path.join('temp_pdf', val)
page = doc.load_page(i)
pix = page.get_pixmap(matrix=mat)
pix.save(val)
image_paths.append(val)
text = reader.readtext(val, detail=0)
context = context + ' '.join(text)
doc.close()
print(context)
st.success('OCR completed')
# context = ''
# with st.spinner('Performing OCR on tables to extract images...'):
# for image in image_paths:
# text = reader.readtext(image, detail=0)
# # print(text)
# context = context + ' '.join(text)
# print(context)
# pass
with st.spinner('Parsing extracted text...'):
st.markdown('### Extracted data from transcripts')
base_prompt='''Above is the OCR extracted transcript.
Extract student's grade along with subject. Output should only be a lists of dict with course and grade as its keys.
'''
base_prompt='''Above is the OCR extracted transcript.
Extract student's points/scores along with subject. Output should only be a lists of dict with course and points/scores as its keys.
'''
client = OpenAI(
# This is the default and can be omitted
api_key=credential,
)
prompt = context + base_prompt
chat_completion = client.chat.completions.create(
messages=[
{
"role": "user",
"content": prompt,
}
],
model="gpt-3.5-turbo",
)
list_str = chat_completion.choices[0].message.content
print(list_str)
try:
actual_list = ast.literal_eval(list_str)
except:
list_str_correct = correct_list(client, list_str)
actual_list = ast.literal_eval(list_str_correct)
df = pd.DataFrame(columns=['Courses', 'Grade'])
# Saving the keys in a variable (as a list)
keys_list = list(actual_list[0].keys())
print(keys_list)
# for subject in actual_list:
# df.loc[len(df)] = [subject['course'], subject['grade']]
for subject in actual_list:
df.loc[len(df)] = [subject[keys_list[0]], subject[keys_list[1]]]
st.dataframe(df)
csv = convert_df(df)
st.download_button(
label="Download Parsed transcript",
data=csv,
file_name='transcript.csv',
mime='text/csv',
)
st.success('Transcript Processing Completed!')
# Run the app
if __name__ == "__main__":
main() |