f lisa

app.py

@@ -20,10 +20,14 @@ import gradio as gr
 
 import torch
 import torch.nn.functional as F
+import transformers
 from PIL import Image
 import numpy as np
 import time
 import threading
+import subprocess
+import sys
+import importlib
 
 from ncut_pytorch.backbone import extract_features, load_model
 from ncut_pytorch.backbone import MODEL_DICT, LAYER_DICT, RES_DICT
@@ -541,6 +545,48 @@ def run_fn(
     images = [transform_image(image, resolution=resolution) for image in images]
     images = torch.stack(images)
     
+    
+    if is_lisa:
+        # LISA and Llava is not compatible with the current version of transformers
+        # please contact the author for update
+        # this is a dirty patch for the LISA model
+        
+        # pre-import the SD3 pipeline
+        from diffusers import StableDiffusion3Pipeline
+        
+        # unloading the current transformers
+        for module in list(sys.modules.keys()):
+            if "transformers" in module:
+                del sys.modules[module]
+            
+
+        def install_transformers_version(version, target_dir):
+            """Install a specific version of transformers to a target directory."""
+            if not os.path.exists(target_dir):
+                os.makedirs(target_dir)
+            
+            # Use subprocess to run the pip command
+            subprocess.check_call([sys.executable, '-m', 'pip', 'install', f'transformers=={version}', '-t', target_dir])
+
+        target_dir = '/tmp/lisa_transformers_v433'
+        if not os.path.exists(target_dir):
+            install_transformers_version('4.33.0', target_dir)
+
+        # Add the new version path to sys.path
+        sys.path.insert(0, target_dir)
+        
+        transformers = importlib.import_module("transformers")
+
+    if not is_lisa:
+        # remove the LISA model from the sys.path
+            
+        if "/tmp/lisa_transformers_v433" in sys.path:
+            sys.path.remove("/tmp/lisa_transformers_v433")
+        
+        transformers = importlib.import_module("transformers")
+        
+    
+    
     if "AlignedThreeModelAttnNodes" == model_name:
         # dirty patch for the alignedcut paper
         model = load_alignedthreemodel()
@@ -1244,6 +1290,10 @@ with demo:
         with gr.Column():
             gr.Markdown("###### Running out of GPU? Try [Demo](https://ncut-pytorch.readthedocs.io/en/latest/demo/) hosted at UPenn")
 
+# for local development
+if os.path.exists("/hf_token.txt"):
+    os.environ["HF_ACCESS_TOKEN"] = open("/hf_token.txt").read().strip()
+
 if DOWNLOAD_ALL_MODELS_DATASETS:
     from ncut_pytorch.backbone import download_all_models
     threading.Thread(target=download_all_models).start()

requirements.txt

@@ -12,7 +12,7 @@ pillow==9.4.0
 SAM-2 @ git+https://github.com/huzeyann/segment-anything-2.git
 segment-anything @ git+https://github.com/facebookresearch/segment-anything.git@6fdee8f
 mobile-sam @ git+https://github.com/ChaoningZhang/MobileSAM.git@c12dd83
-lisa @ git+https://github.com/huzeyann/LISA.git
+lisa @ git+https://github.com/huzeyann/LISA/tree/old-transformers-working.git
 timm==0.9.2
 open-clip-torch==2.20.0
 ncut-pytorch>=1.3.13