Spaces:
Sleeping
Sleeping
Claude commited on
Commit ·
20968a7
1
Parent(s): 548f2eb
fix: ocr unpack 3 vals, no duplicate rename, detected font in review, Reviewer label
Browse files- smoke_signal_tab.py +31 -28
smoke_signal_tab.py
CHANGED
|
@@ -1194,11 +1194,8 @@ def ingest_pdfs(files, rights_class: str, notes: str, book_code_hint: str = "",
|
|
| 1194 |
existing_book_id = df.loc[df["sha256"] == file_hash, "book_id"].values[0]
|
| 1195 |
existing_status = df.loc[df["sha256"] == file_hash, "status"].values[0]
|
| 1196 |
existing_rights = df.loc[df["sha256"] == file_hash, "rights_class"].values[0]
|
| 1197 |
-
if
|
| 1198 |
-
|
| 1199 |
-
df.loc[df["sha256"] == file_hash, "book_id"] = desired_book_id
|
| 1200 |
-
existing_book_id = desired_book_id
|
| 1201 |
-
log.append(log_line(f"↻ Renamed duplicate book ID → {desired_book_id}"))
|
| 1202 |
|
| 1203 |
if rights_class != "unknown" and existing_rights != rights_class:
|
| 1204 |
df.loc[df["sha256"] == file_hash, "rights_class"] = rights_class
|
|
@@ -1950,7 +1947,7 @@ def run_full_pipeline(
|
|
| 1950 |
|
| 1951 |
try:
|
| 1952 |
ocr_gen = run_ocr(scope_include, scope_exclude, replace_book_queue=True)
|
| 1953 |
-
for ocr_status_html, ocr_log in ocr_gen:
|
| 1954 |
for line in str(ocr_log).splitlines():
|
| 1955 |
if line.strip() and line not in log:
|
| 1956 |
log.append(line)
|
|
@@ -2533,14 +2530,14 @@ def get_review_item(idx: int) -> tuple:
|
|
| 2533 |
q_df = load_queue_df()
|
| 2534 |
d_df = load_decisions_df()
|
| 2535 |
if q_df.empty:
|
| 2536 |
-
return None, "", "", 0, 0
|
| 2537 |
|
| 2538 |
decided_ids = set(d_df["region_id"].tolist()) if not d_df.empty else set()
|
| 2539 |
pending = q_df[~q_df["region_id"].isin(decided_ids)]
|
| 2540 |
if not pending.empty and {"book_id", "page"}.issubset(pending.columns):
|
| 2541 |
pending = pending.sort_values(["book_id", "page"], ascending=[True, True], kind="stable")
|
| 2542 |
if pending.empty:
|
| 2543 |
-
return None, "All items reviewed!", "", len(q_df), len(q_df)
|
| 2544 |
|
| 2545 |
idx = idx % len(pending)
|
| 2546 |
item = pending.iloc[idx]
|
|
@@ -2581,7 +2578,8 @@ def get_review_item(idx: int) -> tuple:
|
|
| 2581 |
f"conf: <b style='color:{'var(--ss-red)' if float(item.get('confidence',0)) < 0.6 else 'var(--ss-gold)'}'>"
|
| 2582 |
f"{float(item.get('confidence',0)):.0%}</b></div>")
|
| 2583 |
|
| 2584 |
-
|
|
|
|
| 2585 |
|
| 2586 |
|
| 2587 |
def save_review_decision(idx: int, final_text: str, action: str, reviewer: str, reason: str, conf_override: bool = False) -> tuple:
|
|
@@ -3081,7 +3079,12 @@ def smoke_signal_tab():
|
|
| 3081 |
lines=8,
|
| 3082 |
interactive=True,
|
| 3083 |
)
|
| 3084 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 3085 |
reason_code = gr.Dropdown(
|
| 3086 |
label="Reason code",
|
| 3087 |
choices=["","OCR_MISS","OCR_WRONG_WORD","DECORATIVE_FONT",
|
|
@@ -3108,14 +3111,14 @@ def smoke_signal_tab():
|
|
| 3108 |
current_idx = gr.State(0)
|
| 3109 |
|
| 3110 |
def load_review():
|
| 3111 |
-
img, raw, info, done, total = get_review_item(0)
|
| 3112 |
status = _review_status_html()
|
| 3113 |
-
return 0, status, img, raw, raw, info
|
| 3114 |
|
| 3115 |
def next_item(idx):
|
| 3116 |
new_idx = idx + 1
|
| 3117 |
-
img, raw, info, done, total = get_review_item(new_idx)
|
| 3118 |
-
return new_idx, img, raw, raw, info
|
| 3119 |
|
| 3120 |
def do_accept(idx, final, reviewer, reason, conf_ov=False):
|
| 3121 |
action = "edited" if final.strip() != "" else "accepted"
|
|
@@ -3124,27 +3127,27 @@ def smoke_signal_tab():
|
|
| 3124 |
)
|
| 3125 |
status = _review_status_html()
|
| 3126 |
new_idx = idx + 1
|
| 3127 |
-
img2, raw2, info2, _, _ = get_review_item(new_idx)
|
| 3128 |
-
return fb, status, new_idx, img2, raw2, raw2, info2
|
| 3129 |
|
| 3130 |
def do_reject(idx, final, reviewer, reason):
|
| 3131 |
fb, img, raw, info, _, _ = save_review_decision(idx, final, "rejected", reviewer, reason)
|
| 3132 |
status = _review_status_html()
|
| 3133 |
new_idx = idx + 1
|
| 3134 |
-
img2, raw2, info2, _, _ = get_review_item(new_idx)
|
| 3135 |
-
return fb, status, new_idx, img2, raw2, raw2, info2
|
| 3136 |
|
| 3137 |
def do_quarantine(idx, final, reviewer, reason):
|
| 3138 |
fb, img, raw, info, _, _ = save_review_decision(idx, final, "quarantined", reviewer, reason)
|
| 3139 |
status = _review_status_html()
|
| 3140 |
new_idx = idx + 1
|
| 3141 |
-
img2, raw2, info2, _, _ = get_review_item(new_idx)
|
| 3142 |
-
return fb, status, new_idx, img2, raw2, raw2, info2
|
| 3143 |
|
| 3144 |
action_outputs = [training_feedback, review_status, current_idx,
|
| 3145 |
-
review_image, raw_text_box, final_text_box, item_info]
|
| 3146 |
|
| 3147 |
-
review_load_outputs = [current_idx, review_status, review_image, raw_text_box, final_text_box, item_info]
|
| 3148 |
|
| 3149 |
load_review_btn.click(
|
| 3150 |
load_review,
|
|
@@ -3161,18 +3164,18 @@ def smoke_signal_tab():
|
|
| 3161 |
|
| 3162 |
def _next(idx):
|
| 3163 |
new_idx = idx + 1
|
| 3164 |
-
img, raw, info, _, _ = get_review_item(new_idx)
|
| 3165 |
-
return new_idx, img, raw, raw, info
|
| 3166 |
|
| 3167 |
def _prev(idx):
|
| 3168 |
new_idx = max(0, idx - 1)
|
| 3169 |
-
img, raw, info, _, _ = get_review_item(new_idx)
|
| 3170 |
-
return new_idx, img, raw, raw, info
|
| 3171 |
|
| 3172 |
prev_btn.click(_prev, inputs=[current_idx],
|
| 3173 |
-
outputs=[current_idx, review_image, raw_text_box, final_text_box, item_info])
|
| 3174 |
next_btn.click(_next, inputs=[current_idx],
|
| 3175 |
-
outputs=[current_idx, review_image, raw_text_box, final_text_box, item_info])
|
| 3176 |
|
| 3177 |
# Review now auto-loads on tab select and after OCR run completion.
|
| 3178 |
|
|
|
|
| 1194 |
existing_book_id = df.loc[df["sha256"] == file_hash, "book_id"].values[0]
|
| 1195 |
existing_status = df.loc[df["sha256"] == file_hash, "status"].values[0]
|
| 1196 |
existing_rights = df.loc[df["sha256"] == file_hash, "rights_class"].values[0]
|
| 1197 |
+
if False: # disabled: do not rename on duplicate re-registration
|
| 1198 |
+
pass
|
|
|
|
|
|
|
|
|
|
| 1199 |
|
| 1200 |
if rights_class != "unknown" and existing_rights != rights_class:
|
| 1201 |
df.loc[df["sha256"] == file_hash, "rights_class"] = rights_class
|
|
|
|
| 1947 |
|
| 1948 |
try:
|
| 1949 |
ocr_gen = run_ocr(scope_include, scope_exclude, replace_book_queue=True)
|
| 1950 |
+
for ocr_status_html, ocr_log, _ocr_done in ocr_gen:
|
| 1951 |
for line in str(ocr_log).splitlines():
|
| 1952 |
if line.strip() and line not in log:
|
| 1953 |
log.append(line)
|
|
|
|
| 2530 |
q_df = load_queue_df()
|
| 2531 |
d_df = load_decisions_df()
|
| 2532 |
if q_df.empty:
|
| 2533 |
+
return None, "", "", 0, 0, ""
|
| 2534 |
|
| 2535 |
decided_ids = set(d_df["region_id"].tolist()) if not d_df.empty else set()
|
| 2536 |
pending = q_df[~q_df["region_id"].isin(decided_ids)]
|
| 2537 |
if not pending.empty and {"book_id", "page"}.issubset(pending.columns):
|
| 2538 |
pending = pending.sort_values(["book_id", "page"], ascending=[True, True], kind="stable")
|
| 2539 |
if pending.empty:
|
| 2540 |
+
return None, "All items reviewed!", "", len(q_df), len(q_df), ""
|
| 2541 |
|
| 2542 |
idx = idx % len(pending)
|
| 2543 |
item = pending.iloc[idx]
|
|
|
|
| 2578 |
f"conf: <b style='color:{'var(--ss-red)' if float(item.get('confidence',0)) < 0.6 else 'var(--ss-gold)'}'>"
|
| 2579 |
f"{float(item.get('confidence',0)):.0%}</b></div>")
|
| 2580 |
|
| 2581 |
+
font_name_val = str(item.get("font_name", "") or "").strip()
|
| 2582 |
+
return img_path, item.get("raw_ocr",""), info, len(q_df) - len(pending), len(q_df), font_name_val
|
| 2583 |
|
| 2584 |
|
| 2585 |
def save_review_decision(idx: int, final_text: str, action: str, reviewer: str, reason: str, conf_override: bool = False) -> tuple:
|
|
|
|
| 3079 |
lines=8,
|
| 3080 |
interactive=True,
|
| 3081 |
)
|
| 3082 |
+
detected_font_display = gr.Textbox(
|
| 3083 |
+
label="Detected Font",
|
| 3084 |
+
interactive=False,
|
| 3085 |
+
scale=1,
|
| 3086 |
+
)
|
| 3087 |
+
reviewer_name = gr.Textbox(label="Reviewer", placeholder="e.g. jamal", scale=1)
|
| 3088 |
reason_code = gr.Dropdown(
|
| 3089 |
label="Reason code",
|
| 3090 |
choices=["","OCR_MISS","OCR_WRONG_WORD","DECORATIVE_FONT",
|
|
|
|
| 3111 |
current_idx = gr.State(0)
|
| 3112 |
|
| 3113 |
def load_review():
|
| 3114 |
+
img, raw, info, done, total, font = get_review_item(0)
|
| 3115 |
status = _review_status_html()
|
| 3116 |
+
return 0, status, img, raw, raw, info, font
|
| 3117 |
|
| 3118 |
def next_item(idx):
|
| 3119 |
new_idx = idx + 1
|
| 3120 |
+
img, raw, info, done, total, font = get_review_item(new_idx)
|
| 3121 |
+
return new_idx, img, raw, raw, info, font
|
| 3122 |
|
| 3123 |
def do_accept(idx, final, reviewer, reason, conf_ov=False):
|
| 3124 |
action = "edited" if final.strip() != "" else "accepted"
|
|
|
|
| 3127 |
)
|
| 3128 |
status = _review_status_html()
|
| 3129 |
new_idx = idx + 1
|
| 3130 |
+
img2, raw2, info2, _, _, font2 = get_review_item(new_idx)
|
| 3131 |
+
return fb, status, new_idx, img2, raw2, raw2, info2, font2
|
| 3132 |
|
| 3133 |
def do_reject(idx, final, reviewer, reason):
|
| 3134 |
fb, img, raw, info, _, _ = save_review_decision(idx, final, "rejected", reviewer, reason)
|
| 3135 |
status = _review_status_html()
|
| 3136 |
new_idx = idx + 1
|
| 3137 |
+
img2, raw2, info2, _, _, font2 = get_review_item(new_idx)
|
| 3138 |
+
return fb, status, new_idx, img2, raw2, raw2, info2, font2
|
| 3139 |
|
| 3140 |
def do_quarantine(idx, final, reviewer, reason):
|
| 3141 |
fb, img, raw, info, _, _ = save_review_decision(idx, final, "quarantined", reviewer, reason)
|
| 3142 |
status = _review_status_html()
|
| 3143 |
new_idx = idx + 1
|
| 3144 |
+
img2, raw2, info2, _, _, font2 = get_review_item(new_idx)
|
| 3145 |
+
return fb, status, new_idx, img2, raw2, raw2, info2, font2
|
| 3146 |
|
| 3147 |
action_outputs = [training_feedback, review_status, current_idx,
|
| 3148 |
+
review_image, raw_text_box, final_text_box, item_info, detected_font_display]
|
| 3149 |
|
| 3150 |
+
review_load_outputs = [current_idx, review_status, review_image, raw_text_box, final_text_box, item_info, detected_font_display]
|
| 3151 |
|
| 3152 |
load_review_btn.click(
|
| 3153 |
load_review,
|
|
|
|
| 3164 |
|
| 3165 |
def _next(idx):
|
| 3166 |
new_idx = idx + 1
|
| 3167 |
+
img, raw, info, _, _, font = get_review_item(new_idx)
|
| 3168 |
+
return new_idx, img, raw, raw, info, font
|
| 3169 |
|
| 3170 |
def _prev(idx):
|
| 3171 |
new_idx = max(0, idx - 1)
|
| 3172 |
+
img, raw, info, _, _, font = get_review_item(new_idx)
|
| 3173 |
+
return new_idx, img, raw, raw, info, font
|
| 3174 |
|
| 3175 |
prev_btn.click(_prev, inputs=[current_idx],
|
| 3176 |
+
outputs=[current_idx, review_image, raw_text_box, final_text_box, item_info, detected_font_display])
|
| 3177 |
next_btn.click(_next, inputs=[current_idx],
|
| 3178 |
+
outputs=[current_idx, review_image, raw_text_box, final_text_box, item_info, detected_font_display])
|
| 3179 |
|
| 3180 |
# Review now auto-loads on tab select and after OCR run completion.
|
| 3181 |
|