修正

LLM_package.py

@@ -38,7 +38,7 @@ class MoondreamInference:
                     obj["y_max"], obj["x_max"]
                 ]
             })
-            print(parsed)
+           
         return parsed
 
 
@@ -56,22 +56,19 @@ class GeminiInference:
         client = genai.Client(api_key=self.api_key_source)
         my_file = client.files.upload(file=file_path)
         response = client.models.generate_content(
-            model="gemini-2.0-flash",
+            model="gemini-2.5-pro",
             contents=[my_file, prompt],
         )
         return response.text
     def get_response_text(self,prompt):
         client = genai.Client(api_key=self.api_key_source)
         response = client.models.generate_content(
-            model="gemini-2.0-flash",
+            model="gemini-2.5-pro",
             contents=[prompt],
         )
         text = response.text
         return text
     def parse(self, text):
-        """
-        レスポンス JSON をパース。'label' と 'box_2d'([0-1000]正規化) を取り出し、[0,1]正規化に変換して返すリスト。
-        """
         json_str = text
         if '```json' in text:
             json_str = text[text.find('```json') + len('```json'):]
@@ -82,7 +79,9 @@ class GeminiInference:
         """
         レスポンス JSON をパース。'label' と 'box_2d'([0-1000]正規化) を取り出し、[0,1]正規化に変換して返すリスト。
         """
-        print(text)
+        print("GeminiInference.parse_response:", text)
+        if not text:
+            return {'state': 'empty'}
         json_str = text
         if '```json' in text:
             json_str = text[text.find('```json') + len('```json'):]
@@ -101,3 +100,41 @@ class GeminiInference:
                 norm = [c / 1000.0 for c in coords]
                 parsed.append({'label': obj['label'], 'box_2d': norm})
         return parsed
+class ObjectDetector:
+    def __init__(self, API_KEY=None):
+        self.model  = GeminiInference(API_KEY) 
+        self.prompt_objects=None
+        self.text=None
+        
+    def detect_objects(self, image_path):
+        self.prompt= f"""
+    Detect all {self.prompt_objects} in the image. The box_2d should be [ymin, xmin, ymax, xmax] normalized to 0-1000.
+    Please provide the response as a JSON array of objects, where each object has a 'label' and 'box_2d' field.
+    Example:
+    [
+        {{"label": "face", "box_2d": [100, 200, 300, 400]}},
+        {{"label": "license_plate", "box_2d": [500, 600, 700, 800]}}
+    ]
+    """
+        print(self.prompt)
+        detected_objects_norm_0_1= self.model.parse_response(self.model.get_response(image_path, self.prompt))
+        return detected_objects_norm_0_1
+    """
+    Detects the danger level of the image.
+    """
+    def detect_danger_level(self, image_path):
+        analysis_prompt = f"""
+            画像の個人情報漏洩リスクを分析し、厳密にJSON形式で返答してください。なおこの時、資料があれば、資料を参考にしてください：
+                {{
+                    "risk_level": "high|medium|low",
+                    "risk_reason": "リスクの具体的理由",
+                    "objects_to_remove": ["消去すべきオブジェクトリスト(英語で、例: 'face', 'license_plate')"]
+                }}
+                <資料>
+                {self.text if self.text else "なし"}
+                </資料>
+                """
+        
+        response = self.model.parse(self.model.get_response(image_path, analysis_prompt))
+        print(f"Response: {response}")
+        return response

app.py

@@ -17,6 +17,7 @@ import supervision as sv
 from PIL import Image, ImageFilter
 import numpy as np
 import cv2
+from LLM_package import ObjectDetector,GeminiInference
 import pycocotools.mask as mask_util
 import insightface
 from fastapi import FastAPI, File, UploadFile, Form
@@ -227,10 +228,39 @@ import easyocr
 
 
 
-def llm_to_process_image(risk_level, image_path, point1, point2, thresholds=None):
+def llm_to_process_image_simple(risk_level, image_path, point1, point2, thresholds=None):
+    print(risk_level, image_path, point1, point2, thresholds)
     print('point1,point2', point1, point2)
+    GEMINI_API_KEY=os.getenv('GEMINI_API_KEY')
     # 画像処理のロジックをここに追加
-    pass
+    Objectdetector = ObjectDetector(API_KEY=GEMINI_API_KEY)
+    debug_image_path='/test_llm.jpg'
+    Objectdetector.prompt_objects={'text', 'poster', 'Name tag', 'License plate', 'Digital screens',
+        'signboard', 'sign', 'logo', 'manhole', 'electricity pole', 'cardboard'}
+    # 画像の読み込みとRGB変換
+    
+    image = cv2.imread(image_path)
+    image_rgb = cv2.cvtColor(image, cv2.COLOR_BGR2RGB) 
+    mask_llm = np.zeros(image.shape[:2], dtype=np.uint8)
+    llm_results = Objectdetector.detect_objects(image_path)
+    for result in llm_results:
+            bbox=result['box_2d']
+            x1, y1 = int(bbox[1]* image.shape[1]), int(bbox[0]* image.shape[0])
+            x2, y2 = int(bbox[3]* image.shape[1]), int(bbox[2]* image.shape[0])
+            mask_llm[y1:y2, x1:x2] = 255  # テキスト領域をマスク
+    p1_x, p1_y = int(point1[0] * image.shape[1]), int(point1[1] * image.shape[0])
+    p2_x, p2_y = int(point2[0] * image.shape[1]), int(point2[1] * image.shape[0])
+    x_min, y_min = max(0, min(p1_x, p2_x)), max(0, min(p1_y, p2_y))
+    x_max, y_max = min(image.shape[1], max(p1_x, p2_x)), min(image.shape[0], max(p1_y, p2_y))
+    mask_llm[y_min:y_max, x_min:x_max] = 0  # 範囲を黒に設定
+    save_dir = "./saved_images"
+    os.makedirs(save_dir, exist_ok=True)
+    debug_image_pil = Image.fromarray(mask_llm)
+    debug_image_pil.save(save_dir + debug_image_path)
+    return save_dir + debug_image_path
+
+
+
 
 # 特殊な処理を行う関数
 def special_process_image_yolo(risk_level, image_path, point1, point2, thresholds=None):
@@ -838,6 +868,91 @@ async def mosaic_face(file: UploadFile = File(...)):
     # 一時ファイルをレスポンスとして返す
     return FileResponse(path=temp_file_path, media_type="image/jpeg", filename="mosaic_image.jpg")
 
+
+
+
+
+
+
+
+@app.post("/create-mask-and-inpaint-sum-llm-simple")
+async def create_mask_sum(image: UploadFile = File(...), risk_level: int = Form(...),
+    x1: float = Form(...),
+    y1: float = Form(...),
+    x2: float = Form(...),
+    y2: float = Form(...),):
+    default_x = 0.001
+    default_y = 0.001
+
+
+    point1 = [default_x if math.isnan(x1) else x1, default_y if math.isnan(y1) else y1]
+
+    point2 = [default_x if math.isnan(x2) else x2, default_y if math.isnan(y2) else y2]
+    timestamp = datetime.now().strftime("%Y%m%d_%H%M%S")
+    # 一意な識別子を生成
+    unique_id = uuid.uuid4().hex
+    input_path = save_image(image.file, f"./input_{timestamp}_{unique_id}.jpg")
+    mask_path = llm_to_process_image_simple(risk_level, input_path, point1, point2,thresholds=thresholds)
+    output_path = f"./output_simple_lama_{timestamp}_{unique_id}.jpg"
+    print('point1,point2',point1,point2)#消去したくない範囲のこと
+    # OpenCVでインペイント
+    inpaint_image_with_mask1(input_path, mask_path, output_path)
+
+    return FileResponse(output_path)
+
+# カスケードファイルの読み込み (顔検出)
+face_cascade = cv2.CascadeClassifier(cv2.data.haarcascades + 'haarcascade_frontalface_default.xml')
+
+def apply_mosaic(image, x, y, w, h, mosaic_level=15):
+    """ 指定範囲にモザイク処理を適用 """
+    face = image[y:y+h, x:x+w]
+    face = cv2.resize(face, (w // mosaic_level, h // mosaic_level))
+    face = cv2.resize(face, (w, h), interpolation=cv2.INTER_NEAREST)
+    image[y:y+h, x:x+w] = face
+    return image
+
+@app.post("/mosaic_face")
+async def mosaic_face(file: UploadFile = File(...)):
+    # 画像ファイルを読み込み
+    image_data = await file.read()
+    np_array = np.frombuffer(image_data, np.uint8)
+    img = cv2.imdecode(np_array, cv2.IMREAD_COLOR)
+
+    # グレースケール変換と顔検出
+    gray = cv2.cvtColor(img, cv2.COLOR_BGR2GRAY)
+    faces = face_cascade.detectMultiScale(gray, scaleFactor=1.1, minNeighbors=4, minSize=(30, 30))
+
+    # 検出した顔にモザイクを適用
+    for (x, y, w, h) in faces:
+        img = apply_mosaic(img, x, y, w, h)
+
+    # 一時ファイルに保存
+    with tempfile.NamedTemporaryFile(delete=False, suffix=".jpg") as temp_file:
+        temp_file_path = Path(temp_file.name)
+        cv2.imwrite(str(temp_file_path), img)
+
+    # 一時ファイルをレスポンスとして返す
+    return FileResponse(path=temp_file_path, media_type="image/jpeg", filename="mosaic_image.jpg")
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
 
 # Helper function to read image file
 def read_image(file: UploadFile):

detector.py

@@ -16,21 +16,24 @@ class ObjectDetector:
     ]
     """
     def detect_objects(self, image_path):
-        detected_objects_norm_0_1= self.model.parse_response(self.model.get_response(image_path, self.prompt))        return detected_objects_norm_0_1
+        detected_objects_norm_0_1= self.model.parse_response(self.model.get_response(image_path, self.prompt))
+        return detected_objects_norm_0_1
+    """
+    Detects the danger level of the image.
+    """
     def detect_danger_level(self, image_path):
-        """
-        Detects the danger level of the image.
-        """
         analysis_prompt = f"""
             画像の個人情報漏洩リスクを分析し、厳密にJSON形式で返答してください。なおこの時、資料があれば、資料を参考にしてください：
-            {{
-                "risk_level": "high|medium|low",
-                "risk_reason": "リスクの具体的理由",
-                "objects_to_remove": ["消去すべきオブジェクトリスト(英語で、例: 'face', 'license_plate')"]
-            }}
-            <資料>
-            {self.text if self.text else "なし"}
-            </資料>
-            """
-        response = json.loads(self.model.get_response_text(image_path, analysis_prompt))
+                {{
+                    "risk_level": "high|medium|low",
+                    "risk_reason": "リスクの具体的理由",
+                    "objects_to_remove": ["消去すべきオブジェクトリスト(英語で、例: 'face', 'license_plate')"]
+                }}
+                <資料>
+                {self.text if self.text else "なし"}
+                </資料>
+                """
+        
+        response = self.model.parse(self.model.get_response(image_path, analysis_prompt))
+        print(f"Response: {response}")
         return response

requirements.txt

@@ -72,4 +72,5 @@ uvicorn==0.32.0
 zipp==3.20.2
 supervision
 onnxruntime
-insightface
+
+dotenv 

test.py

@@ -0,0 +1,39 @@
+from LLM_package import ObjectDetector
+import os
+from dotenv import load_dotenv
+import numpy as np
+import cv2
+from PIL import Image
+load_dotenv(dotenv_path='../.env')
+def llm_to_process_image(risk_level, image_path, point1, point2, thresholds=None):
+    print(risk_level, image_path, point1, point2, thresholds)
+    print('point1,point2', point1, point2)
+    GEMINI_API_KEY=os.getenv('GEMINI_API_KEY')
+    # 画像処理のロジックをここに追加
+    Objectdetector = ObjectDetector(API_KEY=GEMINI_API_KEY)
+    debug_image_path='/test_llm.jpg'
+    Objectdetector.prompt_objects={'face', 'poster', 'Name tag', 'License plate', 'Digital screens',
+        'signboard', 'sign', 'logo', 'manhole', 'electricity pole', 'cardboard'}
+    # 画像の読み込みとRGB変換
+    
+    image = cv2.imread(image_path)
+    image_rgb = cv2.cvtColor(image, cv2.COLOR_BGR2RGB) 
+    mask_llm = np.zeros(image.shape[:2], dtype=np.uint8)
+    llm_results = Objectdetector.detect_objects(image_path)
+    for result in llm_results:
+            bbox=result['box_2d']
+            x1, y1 = int(bbox[1]* image.shape[1]), int(bbox[0]* image.shape[0])
+            x2, y2 = int(bbox[3]* image.shape[1]), int(bbox[2]* image.shape[0])
+            mask_llm[y1:y2, x1:x2] = 255  # テキスト領域をマスク
+    p1_x, p1_y = int(point1[0] * image.shape[1]), int(point1[1] * image.shape[0])
+    p2_x, p2_y = int(point2[0] * image.shape[1]), int(point2[1] * image.shape[0])
+    x_min, y_min = max(0, min(p1_x, p2_x)), max(0, min(p1_y, p2_y))
+    x_max, y_max = min(image.shape[1], max(p1_x, p2_x)), min(image.shape[0], max(p1_y, p2_y))
+    mask_llm[y_min:y_max, x_min:x_max] = 0  # 範囲を黒に設定
+    save_dir = "./saved_images"
+    os.makedirs(save_dir, exist_ok=True)
+    debug_image_pil = Image.fromarray(mask_llm)
+    debug_image_pil.save(save_dir + debug_image_path)
+
+llm_to_process_image(50, "../../16508.jpg", (0, 0), (0, 0), thresholds=None)
+