import os import cv2 import re import base64 import numpy as np import pandas as pd import gradio as gr from roboflow import Roboflow from openai import OpenAI from openpyxl import load_workbook # ====================================================== # CONFIG # ====================================================== ROBOFLOW_API_KEY = "uP19IAi98TqwLvHmNB8V" ROBOFLOW_PROJECT = "braker3" ROBOFLOW_VERSION = 6 CONF_THRESHOLD = 0.35 IOU_THRESHOLD = 0.4 PAD_PIXELS = 20 EXCEL_PATH = "List.xlsm" # ====================================================== # OPENAI # ====================================================== api_key = os.getenv("OPENAI_API_KEY") if not api_key: raise RuntimeError("OPENAI_API_KEY not found") client = OpenAI(api_key=api_key) # ====================================================== # ROBOFLOW # ====================================================== rf = Roboflow(api_key=ROBOFLOW_API_KEY) project = rf.workspace().project(ROBOFLOW_PROJECT) model = project.version(ROBOFLOW_VERSION).model # ====================================================== # CONSTANTS # ====================================================== KNOWN_MANUFACTURERS = [ "MITSUBISHI ELECTRIC","SIEMENS","SCHNEIDER ELECTRIC", "ABB","LS ELECTRIC","HITACHI","FUJI ELECTRIC","EATON" ] IGNORED_LABELS = { "NO-FUSE BREAKER","NO FUSE BREAKER","NO-FUSE","FUSE BREAKER" } SPEC_JAPANESE = { "Manufacture Name": "メーカー", "Circuit Name": "回路番号", "Load Name": "負荷名称", "Breaking Capacity": "遮断容量", "AT": "トリップ(AT)", "AF": "フレーム(AF)" } # ====================================================== # IMAGE HELPERS # ====================================================== def resize_for_roboflow(img, max_side=1280): h, w = img.shape[:2] scale = min(max_side / max(h, w), 1.0) if scale < 1: img = cv2.resize(img, (int(w*scale), int(h*scale))) return img def img_to_base64(img): ok, buf = cv2.imencode(".jpg", img) return base64.b64encode(buf).decode() if ok else None def crop_with_padding(img, x1, y1, x2, y2, pad=20): h, w = img.shape[:2] return img[max(0,y1-pad):min(h,y2+pad), max(0,x1-pad):min(w,x2+pad)] def expand_box_directional(img, x1, y1, x2, y2): h, w = img.shape[:2] return img[max(0,y1-20):min(h,y2+20), max(0,x1-10):min(w,x2+100)] def expand_circuit_crop(img, x1, y1, x2, y2): h, w = img.shape[:2] return img[max(0,y1-20):min(h,y2+20), max(0,x1-40):min(w,x2+40)] def expand_manufacturer_crop(img, x1, y1, x2, y2): h, w = img.shape[:2] return img[max(0,y1-40):min(h,y2+40), max(0,x1-20):min(w,x2+120)] def rotate_image(img, angle): return cv2.rotate(img, cv2.ROTATE_90_CLOCKWISE) if angle == 90 else img def upscale_and_clahe(img, scale=3): gray = cv2.cvtColor(img, cv2.COLOR_BGR2GRAY) gray = cv2.resize(gray, None, fx=scale, fy=scale, interpolation=cv2.INTER_CUBIC) clahe = cv2.createCLAHE(2.0,(8,8)) return cv2.cvtColor(clahe.apply(gray), cv2.COLOR_GRAY2BGR) # ====================================================== # NORMALIZATION & MATCH HELPERS # ====================================================== def normalize_text(s): s = str(s).upper().strip() s = s.replace(" ", "").replace("_", "-") s = re.sub(r"-+", "-", s) return s def extract_digits(s): nums = re.findall(r"\d+", normalize_text(s)) return nums[0] if nums else "" def extract_ka(s): nums = re.findall(r"(\d+)\s*KA", normalize_text(s)) return nums[0] if nums else "" def extract_code_prefix(s): m = re.match(r"^[A-Z0-9]+(?:-[A-Z0-9]+)*", normalize_text(s)) return m.group(0) if m else "" def is_bad_expression(s): return bool(re.search(r"\d+\s*[Xx×]\s*\d+", str(s))) def tokenize_company(s): tokens = re.sub(r"[^A-Z0-9]", " ", normalize_text(s)).split() stop = {"ELECTRIC","CO","LTD","LIMITED","CORP","CORPORATION","INC"} return {t for t in tokens if t not in stop and len(t) >= 3} # ====================================================== # CLEANERS # ====================================================== def clean_manufacturer_exact(text): text = text.upper() for b in KNOWN_MANUFACTURERS: if b in text: return b return "" def clean_code_exact(text): text = re.sub(r"\s+","",text.upper()) text = text.replace("_","-") return re.sub(r"[^A-Z0-9\-]","",text) def extract_breaking_capacity_strict(text): digits = re.findall(r"\d+", text) for d in digits: if d in ["3","8","36","85"]: return "85" return "85" # ====================================================== # MATCH LOGIC (YOURS) # ====================================================== def match_value(spec, d_raw, e_raw): d = normalize_text(d_raw) e = normalize_text(e_raw) if e == "" or e.lower() == "nan": return False if spec == "Manufacture Name": if "MITSUBISHI" in d and "三菱" in str(e_raw): return True if len(str(e_raw).strip()) <= 2: return False if d == e: return True return len(tokenize_company(d_raw) & tokenize_company(e_raw)) >= 1 if spec in ["AT", "AF"]: if is_bad_expression(d_raw) or is_bad_expression(e_raw): return False return extract_digits(d_raw) == extract_digits(e_raw) if spec == "Breaking Capacity": if "/" in str(d_raw) or "/" in str(e_raw): return False dk, ek = extract_ka(d_raw), extract_ka(e_raw) if dk and ek: return dk == ek return extract_digits(d_raw) == extract_digits(e_raw) if spec in ["Circuit Name", "Load Name"]: return extract_code_prefix(d_raw) == extract_code_prefix(e_raw) return d == e # ====================================================== # GPT OCR # ====================================================== def gpt_ocr(label, crop): label_l = label.lower() crop = upscale_and_clahe(crop, 3) angles = [0,90] if label_l == "manufacture name" else [0] rule = { "manufacture name": "Return ONLY the manufacturer brand name.", "breaking capacity": "Return ONLY the number.", "load name": "Return ONLY the code exactly as printed.", "circuit name": "Read the text exactly as printed." }.get(label_l, "Return ONLY the numeric value.") outputs = [] for a in angles: img_try = rotate_image(crop, a) b64 = img_to_base64(img_try) if not b64: continue resp = client.chat.completions.create( model="gpt-5.2", messages=[ {"role":"system","content":"You are a strict OCR engine."}, {"role":"user","content":[ {"type":"text","text":rule}, {"type":"image_url", "image_url":{"url":f"data:image/jpeg;base64,{b64}"}} ]} ], temperature=0 ) txt = resp.choices[0].message.content.strip() if txt: outputs.append(txt) if not outputs: return "" text = max(outputs, key=len) if label_l == "manufacture name": return clean_manufacturer_exact(text) if label_l == "breaking capacity": return extract_breaking_capacity_strict(text) if label.upper() in ["AT","AF"]: return extract_digits(text) if label_l == "load name": return clean_code_exact(text) return text # ====================================================== # VERIFY # ====================================================== def verify_mcb(excel_path, detected_specs): wb = load_workbook(excel_path, data_only=True) if "MCB" not in wb.sheetnames: return pd.DataFrame( [["MCB sheet not found","","NO"]], columns=["仕様","検出値","Excelに存在?"] ) ws = wb["MCB"] df = pd.DataFrame([list(r) for r in ws.iter_rows(values_only=True)]) df.dropna(how="all", inplace=True) results = [] for spec, det_val in detected_specs.items(): found = False for col in df.columns: for excel_val in df[col].dropna(): if match_value(spec, det_val, excel_val): found = True break if found: break results.append([ SPEC_JAPANESE.get(spec, spec), det_val, "YES" if found else "NO" ]) return pd.DataFrame(results, columns=["仕様","検出値","Excelに存在?"]) # ====================================================== # MAIN PIPELINE # ====================================================== def run_pipeline(image): image = resize_for_roboflow( cv2.cvtColor(np.array(image), cv2.COLOR_RGB2BGR) ) preds = model.predict( image, confidence=int(CONF_THRESHOLD*100), overlap=int(IOU_THRESHOLD*100) ).json()["predictions"] best = {} vis = image.copy() for p in preds: label = p["class"] conf = p["confidence"] x,y,w,h = map(int,[p["x"],p["y"],p["width"],p["height"]]) x1,y1,x2,y2 = x-w//2,y-h//2,x+w//2,y+h//2 cv2.rectangle(vis,(x1,y1),(x2,y2),(0,255,0),2) cv2.putText(vis,label,(x1,max(y1-10,20)), cv2.FONT_HERSHEY_SIMPLEX,0.6,(0,0,255),2) if label.lower() == "manufacture name": crop = expand_manufacturer_crop(image,x1,y1,x2,y2) elif label.lower() == "circuit name": crop = expand_circuit_crop(image,x1,y1,x2,y2) elif label.lower() == "load name": crop = expand_box_directional(image,x1,y1,x2,y2) else: crop = crop_with_padding(image,x1,y1,x2,y2) if label not in best or conf > best[label][0]: best[label] = (conf, crop) extracted_rows = [] detected_specs = {} for label,(_,crop) in best.items(): if label.upper() in IGNORED_LABELS: continue val = gpt_ocr(label, crop) if val: detected_specs[label] = val extracted_rows.append([label, val]) extracted_df = pd.DataFrame( extracted_rows, columns=["Field", "Extracted Text"] ) verification_df = verify_mcb(EXCEL_PATH, detected_specs) output_path = "verification_result.xlsx" verification_df.to_excel(output_path, index=False) vis = cv2.cvtColor(vis, cv2.COLOR_BGR2RGB) return vis, extracted_df, verification_df, output_path # ====================================================== # GRADIO UI # ====================================================== with gr.Blocks(theme=gr.themes.Soft()) as demo: gr.Markdown(""" # AI-Based Visual Inspection of Breaker Panel Specifications """) # ============================== # STEP 1: INPUT + DETECTION (SIDE BY SIDE) # ============================== with gr.Row(): image_input = gr.Image( type="pil", label="📷 Upload Breaker Image" ) detected_image = gr.Image( label="🟢 Detected Image" ) run_btn = gr.Button("🚀 Run Verification", variant="primary") # ============================== # STEP 2: OCR EXTRACTION # ============================== gr.Markdown("## 🟡 Extracted Text") extracted_table = gr.Dataframe( label="Extracted OCR Text", interactive=False ) # ============================== # STEP 3: VERIFICATION # ============================== gr.Markdown("## 🔵 Verification Result") verification_table = gr.Dataframe( label="Load List Verification Result", interactive=False ) download_file = gr.File( label="⬇️ Download Verification Excel" ) # ============================== # BUTTON ACTION # ============================== run_btn.click( fn=run_pipeline, inputs=image_input, outputs=[ detected_image, extracted_table, verification_table, download_file ] ) demo.launch()