import streamlit as st import base64 import cv2 import numpy as np from openai import OpenAI # Image Preprocessing Function def preprocess_image(image_file): # Convert the file to an OpenCV image file_bytes = np.asarray(bytearray(image_file.read()), dtype=np.uint8) image = cv2.imdecode(file_bytes, cv2.IMREAD_COLOR) # Resize the image, if necessary # image = cv2.resize(image, (desired_width, desired_height)) # Convert to grayscale gray_image = cv2.cvtColor(image, cv2.COLOR_BGR2GRAY) # Apply Gaussian blur to reduce noise and improve OCR accuracy blurred_image = cv2.GaussianBlur(gray_image, (5, 5), 0) # Adaptive Histogram Equalization clahe = cv2.createCLAHE(clipLimit=2.0, tileGridSize=(8, 8)) equalized_image = clahe.apply(blurred_image) # Return the preprocessed image return image_file # Function to encode the image to base64 def encode_image(image_file): return base64.b64encode(image_file.getvalue()).decode("utf-8") st.set_page_config(page_title="Scientific and Engineering Image Analyst", layout="centered", initial_sidebar_state="collapsed") # Streamlit page setup st.title("Scientific and Engineering Image Analyst X Omnisys") # Text input for the user to enter their OpenAI API Key api_key = st.text_input("Enter your OpenAI API Key:", type="password") # Initialize the OpenAI client with the API key client = OpenAI(api_key=api_key) # File uploader allows user to add their own image uploaded_file = st.file_uploader("Upload an image", type=["jpg", "png", "jpeg"]) # Checkbox to add an additional prompt add_prompt = st.checkbox("Add additional prompt instructions") # Initialize a variable for additional prompt text additional_prompt_text = "" # Conditional text input for additional prompt if add_prompt: additional_prompt_text = st.text_area("Enter additional prompt instructions:") if uploaded_file: # Display the uploaded image with st.expander("Image", expanded=True): st.image(uploaded_file, caption=uploaded_file.name, use_column_width=True) # Toggle for showing additional details input show_details = st.checkbox("Add details about the image", value=False) if show_details: # Text input for additional details about the image, shown only if toggle is True additional_details = st.text_area( "Add any additional details or context about the image here:", disabled=not show_details ) # Button to trigger the analysis analyze_button = st.button("Analyse the Image") # Check if an image has been uploaded, if the API key is available, and if the button has been pressed if uploaded_file is not None and api_key and analyze_button: with st.spinner("Analysing the image ..."): processed_image = preprocess_image(uploaded_file) # Display the preprocessed image with st.expander("Processed Image", expanded=True): st.image(processed_image, caption="Processed Image", use_column_width=True) # Encode the image base64_image = encode_image(uploaded_file) # Standard prompt for image analysis prompt_text = ( "As an expert in scientific and engineering diagram analysis, your keen eye for detail is crucial. " # "Your primary task is to conduct a meticulous examination of the provided image. " # "Focus on identifying every numerical value visible in the diagram, such as dimensions, tolerances, and material properties. " "Offer a detailed, fact-based, and technically precise explanation of the diagram, with an emphasis on the scientific or engineering principles it illustrates. " # "Highlight the significance of each numerical value, explaining how they affect the diagram's functionality and design. " "Structure your analysis in a clear, markdown format, targeting an audience with a background in science or engineering. " "Incorporate appropriate scientific or engineering terminology to provide a thorough understanding of the numerical details. " # "Conclude with a bold, concise caption summarizing the key aspects and numerical details of the image, and their relevance in the diagram's context." "Analyze the uploaded scientific or engineering diagram with a detailed focus. Extract all visible numerical values, such as dimensions, tolerances, and material properties. Crosscheck and verify their accuracy against common standards or logical expectations in the field. Highlight any discrepancies or unusual values and provide explanations. Conclude with a bold, concise caption summarizing the key aspects and numerical details of the image, and their relevance in the diagram's context, ensuring the accuracy of each value is validated." ) # Append additional prompt text if provided if additional_prompt_text: prompt_text += f"\n\nAdditional Prompt Instructions:\n{additional_prompt_text}" # Append additional details if provided if show_details and additional_details: prompt_text += f"\n\nAdditional Context Provided by the User:\n{additional_details}" # Create the payload for the completion request messages = [ { "role": "user", "content": [ {"type": "text", "text": prompt_text}, { "type": "image_url", "image_url": f"data:image/jpeg;base64,{base64_image}", }, ], } ] # Make the request to the OpenAI API try: # Stream the response full_response = "" message_placeholder = st.empty() for completion in client.chat.completions.create( model="gpt-4-vision-preview", messages=messages, max_tokens=1200, stream=True ): # Check if there is content to display if completion.choices[0].delta.content is not None: full_response += completion.choices[0].delta.content message_placeholder.markdown(full_response + "▌") # Final update to placeholder after the stream ends message_placeholder.markdown(full_response) except Exception as e: st.error(f"An error occurred: {e}") else: # Warnings for user action required if not uploaded_file and analyze_button: st.warning("Please upload an image.") if not api_key: st.warning("Please enter your OpenAI API key.")