init project

app.py

@@ -37,6 +37,8 @@ from modules.mobilesamv2.utils.transforms import ResizeLongestSide
 from modules.pe3r.models import Models
 import torchvision.transforms as tvf
 
+from transformers import AutoTokenizer, AutoModel, AutoProcessor, SamModel
+
 silent = False
 # device = 'cuda' if torch.cuda.is_available() else 'cpu'
 pe3r = Models('cpu') #
@@ -304,6 +306,10 @@ def get_cog_feats(images):
 
     device = 'cuda' if torch.cuda.is_available() else 'cpu'
     pe3r.sam2.to(device)
+
+    siglip = AutoModel.from_pretrained("google/siglip-large-patch16-256", device_map=device)
+    siglip_processor = AutoProcessor.from_pretrained("google/siglip-large-patch16-256", device_map=device)
+
     # pe3r.siglip_processor.to(device)
     # pe3r.siglip.to(device)
 
@@ -399,10 +405,10 @@ def get_cog_feats(images):
         seg_imgs = np.stack(seg_img_list, axis=0) # b,H,W,3
         seg_imgs = torch.from_numpy(seg_imgs).permute(0,3,1,2) # / 255.0
         
-        inputs = pe3r.siglip_processor(images=seg_imgs, return_tensors="pt")
+        inputs = siglip_processor(images=seg_imgs, return_tensors="pt")
         inputs = {key: value.to(device) for key, value in inputs.items()}
         
-        image_features = pe3r.siglip.get_image_features(**inputs)
+        image_features = siglip.get_image_features(**inputs)
         image_features = image_features / image_features.norm(dim=-1, keepdim=True)
         image_features = image_features.detach().cpu()
 
@@ -519,13 +525,14 @@ def get_3D_object_from_scene(outdir, text, threshold, scene, min_conf_thr, as_po
     
     device = 'cuda' if torch.cuda.is_available() else 'cpu'
     # pe3r.siglip_tokenizer.to(device)
-
+    siglip = AutoModel.from_pretrained("google/siglip-large-patch16-256", device_map=device)
+    siglip_tokenizer = AutoTokenizer.from_pretrained("google/siglip-large-patch16-256", device_map=device)
 
     texts = [text]
-    inputs = pe3r.siglip_tokenizer(text=texts, padding="max_length", return_tensors="pt")
+    inputs = siglip_tokenizer(text=texts, padding="max_length", return_tensors="pt")
     inputs = {key: value.to(device) for key, value in inputs.items()}
     with torch.no_grad():
-        text_feats =pe3r.siglip.get_text_features(**inputs)
+        text_feats =siglip.get_text_features(**inputs)
         text_feats = text_feats / text_feats.norm(dim=-1, keepdim=True)
     scene.render_image(text_feats, threshold)
     scene.ori_imgs = scene.rendered_imgs

modules/pe3r/models.py

@@ -47,6 +47,6 @@ class Models:
         self.yolov8 = ObjectAwareModel(YOLO8_CKP)
 
         # -- siglip --
-        self.siglip = AutoModel.from_pretrained("google/siglip-large-patch16-256", device_map=device)
-        self.siglip_tokenizer = AutoTokenizer.from_pretrained("google/siglip-large-patch16-256", device_map=device)
-        self.siglip_processor = AutoProcessor.from_pretrained("google/siglip-large-patch16-256", device_map=device)
+        # self.siglip = AutoModel.from_pretrained("google/siglip-large-patch16-256", device_map=device)
+        # self.siglip_tokenizer = AutoTokenizer.from_pretrained("google/siglip-large-patch16-256", device_map=device)
+        # self.siglip_processor = AutoProcessor.from_pretrained("google/siglip-large-patch16-256", device_map=device)