Update page_files/categorized/Backend/category_push.py

page_files/categorized/Backend/category_push.py

@@ -52,7 +52,7 @@ EXTRAS_COLUMNS = [
     "confirmed_by_gpt", "confirmed_by_claude", "verified_by", "match_layer",
     "doi_url", "mapped_figure_id", "mapped_figure_caption", "mapped_figure_number",
     "mapped_figure_page", "mapped_subplot", "map_score", "map_signals",
-    "plot_image_path",
+    "plot_image_path", "plot_local_path",
 ]
 
 _CLASS_OF = {"Composites_materials": "composite", "Fibers": "fiber", "Polymers": "polymer"}
@@ -204,7 +204,8 @@ def conform_to_category(
 
         # resolve -> S3 (preferred) or local copy -> optionally embed
         img_bytes = None
-        stored_path = ""              # what extras.plot_image_path records
+        stored_path = ""              # primary path extras.plot_image_path records
+        local_path = ""               # local relative copy (always kept when possible)
         rel = _s(row.get("plot_image_path"))
         if rel and _is_url(rel):
             # mapper already uploaded this crop to S3 - reuse the URL as-is
@@ -217,6 +218,15 @@ def conform_to_category(
                 if key:
                     try:
                         img_bytes = _s3_get(key)
+                        # also drop a local copy alongside so a local path exists too
+                        try:
+                            fn = os.path.basename(key)
+                            os.makedirs(dest_dir, exist_ok=True)
+                            with open(os.path.join(dest_dir, fn), "wb") as _fh:
+                                _fh.write(img_bytes)
+                            local_path = os.path.join(category_table, stem, fn)
+                        except Exception as e:
+                            log.warning(f"category_push: local copy from S3 failed for {rel} ({e}).")
                     except Exception as e:
                         log.warning(f"category_push: S3 fetch failed for {rel} ({e})")
         elif rel:
@@ -227,15 +237,25 @@ def conform_to_category(
                     data = fh.read()
                 if embed_image:
                     img_bytes = data
+                local_rel = ""
+                s3_url = ""
+                # ALWAYS keep a local copy under plots_root/<category>/<stem>/
+                try:
+                    os.makedirs(dest_dir, exist_ok=True)
+                    shutil.copyfile(abs_src, os.path.join(dest_dir, fn))
+                    local_rel = os.path.join(category_table, stem, fn)
+                except Exception as e:
+                    log.warning(f"category_push: local copy failed for {fn} ({e}).")
+                # AND upload to S3 when configured
                 if _s3_ready():
                     try:
-                        stored_path = _s3_put(data, _s3_key(category_table, stem, fn))
+                        s3_url = _s3_put(data, _s3_key(category_table, stem, fn))
                     except Exception as e:
-                        log.warning(f"category_push: S3 upload failed for {fn} ({e}); copying locally.")
-                if not stored_path:            # no S3, or upload failed -> local copy
-                    os.makedirs(dest_dir, exist_ok=True)
-                    shutil.copyfile(abs_src, os.path.join(dest_dir, fn))
-                    stored_path = os.path.join(category_table, stem, fn)
+                        log.warning(f"category_push: S3 upload failed for {fn} ({e}); keeping local path.")
+                # primary path recorded in extras: prefer the S3 URL, else local
+                stored_path = s3_url or local_rel
+                # keep both resolvable references on the row
+                local_path = local_rel
                 if fn not in copied:
                     copied.add(fn)
                     n_plots += 1
@@ -301,6 +321,7 @@ def conform_to_category(
             "map_score":             _s(row.get("map_score")),
             "map_signals":           _s(row.get("map_signals")),
             "plot_image_path":       stored_path,
+            "plot_local_path":       local_path,
         })
 
     main_df = pd.DataFrame(main_rows, columns=TARGET_COLUMNS)