tonyliu404 commited on
Commit
532b220
·
1 Parent(s): 0e97134

fixed docker model issues

Browse files
services/OCR_glm_service.py CHANGED
@@ -1,14 +1,20 @@
1
  from transformers import AutoModelForImageTextToText, AutoProcessor
2
  import torch
 
3
  from pathlib import Path
4
  from helpers import get_project_root
5
 
6
 
7
  class OCR_Glm_Service:
8
  def __init__(self, ocr_path=None, device=None):
 
 
 
9
  if not ocr_path:
10
- ROOT = get_project_root()
11
- ocr_path = ROOT / "backend" / "models" / "GlmOcr"
 
 
12
  processor_path = ocr_path / "processor"
13
  model_path = ocr_path / "model"
14
 
@@ -49,13 +55,12 @@ class OCR_Glm_Service:
49
  return output_text
50
 
51
  def load_model(self):
52
- ROOT = get_project_root()
53
- GLMOCR_MODEL_DIR = ROOT / "backend" / "models" / "GlmOcr"
54
- MODEL_PATH = "zai-org/GLM-OCR"
55
 
56
- model = AutoModelForImageTextToText.from_pretrained(MODEL_PATH)
57
  model.save_pretrained(GLMOCR_MODEL_DIR / "model")
58
- processor = AutoProcessor.from_pretrained( MODEL_PATH)
59
  processor.save_pretrained(GLMOCR_MODEL_DIR / "processor")
60
 
61
  print(f"Downloaded GLM OCR to: {GLMOCR_MODEL_DIR}")
 
1
  from transformers import AutoModelForImageTextToText, AutoProcessor
2
  import torch
3
+ import os
4
  from pathlib import Path
5
  from helpers import get_project_root
6
 
7
 
8
  class OCR_Glm_Service:
9
  def __init__(self, ocr_path=None, device=None):
10
+ ROOT = get_project_root()
11
+ self.base_model_path = Path(os.getenv("MODEL_PATH", ROOT / "backend" / "models"))
12
+
13
  if not ocr_path:
14
+ ocr_path = self.base_model_path / "GlmOcr"
15
+ else:
16
+ ocr_path = Path(ocr_path)
17
+
18
  processor_path = ocr_path / "processor"
19
  model_path = ocr_path / "model"
20
 
 
55
  return output_text
56
 
57
  def load_model(self):
58
+ GLMOCR_MODEL_DIR = self.base_model_path / "GlmOcr"
59
+ DOWNLOAD_MODEL = "zai-org/GLM-OCR"
 
60
 
61
+ model = AutoModelForImageTextToText.from_pretrained(DOWNLOAD_MODEL)
62
  model.save_pretrained(GLMOCR_MODEL_DIR / "model")
63
+ processor = AutoProcessor.from_pretrained( DOWNLOAD_MODEL)
64
  processor.save_pretrained(GLMOCR_MODEL_DIR / "processor")
65
 
66
  print(f"Downloaded GLM OCR to: {GLMOCR_MODEL_DIR}")
services/OCR_japanese_service.py CHANGED
@@ -2,14 +2,19 @@ from transformers import AutoTokenizer, AutoImageProcessor, VisionEncoderDecoder
2
  from PIL import Image
3
  from pathlib import Path
4
  from helpers import get_project_root
 
5
 
6
  ROOT = get_project_root()
7
 
8
  class OCR_Japanese_Service:
9
  def __init__(self, ocr_path=None, device=None):
 
 
 
10
  if not ocr_path:
11
- ROOT = get_project_root()
12
- ocr_path = ROOT / "backend" / "models" / "Kha-white"
 
13
 
14
  processor_path = ocr_path / "processor"
15
  model_path = ocr_path / "model"
@@ -43,13 +48,12 @@ class OCR_Japanese_Service:
43
  return generated_text
44
 
45
  def load_model(self):
46
- ROOT = get_project_root()
47
- MODEL_PATH = "kha-white/manga-ocr-base"
48
- JAPANESE_OCR_DIR = ROOT / "backend" / "models" / "Kha-white"
49
 
50
- tokenizer = AutoTokenizer.from_pretrained(MODEL_PATH)
51
- model = VisionEncoderDecoderModel.from_pretrained(MODEL_PATH)
52
- processor = AutoImageProcessor.from_pretrained(MODEL_PATH)
53
 
54
  tokenizer.save_pretrained(JAPANESE_OCR_DIR / "tokenizer")
55
  model.save_pretrained(JAPANESE_OCR_DIR / "model")
 
2
  from PIL import Image
3
  from pathlib import Path
4
  from helpers import get_project_root
5
+ import os
6
 
7
  ROOT = get_project_root()
8
 
9
  class OCR_Japanese_Service:
10
  def __init__(self, ocr_path=None, device=None):
11
+ ROOT = get_project_root()
12
+ self.base_model_path = Path(os.getenv("MODEL_PATH", ROOT / "backend" / "models"))
13
+
14
  if not ocr_path:
15
+ ocr_path = self.base_model_path / "Kha-white"
16
+ else:
17
+ ocr_path = Path(ocr_path)
18
 
19
  processor_path = ocr_path / "processor"
20
  model_path = ocr_path / "model"
 
48
  return generated_text
49
 
50
  def load_model(self):
51
+ DOWNLOAD_MODEL = "kha-white/manga-ocr-base"
52
+ JAPANESE_OCR_DIR = self.base_model_path / "Kha-white"
 
53
 
54
+ tokenizer = AutoTokenizer.from_pretrained(DOWNLOAD_MODEL)
55
+ model = VisionEncoderDecoderModel.from_pretrained(DOWNLOAD_MODEL)
56
+ processor = AutoImageProcessor.from_pretrained(DOWNLOAD_MODEL)
57
 
58
  tokenizer.save_pretrained(JAPANESE_OCR_DIR / "tokenizer")
59
  model.save_pretrained(JAPANESE_OCR_DIR / "model")
services/bubble_detector_kiuyha_service.py CHANGED
@@ -3,24 +3,27 @@ from PIL import Image
3
  from helpers import get_project_root
4
  from huggingface_hub import hf_hub_download
5
  from pathlib import Path
 
6
 
7
  class Bubble_Detector_Kiuyha_Service:
8
  def __init__(self, path=None):
 
 
 
9
  if not path:
10
- ROOT = get_project_root()
11
- path = ROOT / "backend" / "models"
 
12
 
13
- model_path = path / "kiuyha.pt"
14
-
15
- if not model_path.exists():
16
- print(f"Kiuyha model not found at {model_path}. Attempting to download")
17
  self.load_model()
18
 
19
- if model_path.exists():
20
- self.model = YOLO(model_path)
21
  print("Loaded Bubble Detector Kiuyha")
22
  else:
23
- raise FileNotFoundError(f"Error: Could not find or retrieve {model_path}")
24
 
25
  def predict(self, img_path, conf=0.2, iou=0.4, show_labels=True, show_conf=True, imgsz=640):
26
  results = self.model.predict(
@@ -62,9 +65,7 @@ class Bubble_Detector_Kiuyha_Service:
62
  return sorted_boxes
63
 
64
  def load_model(self):
65
- ROOT = get_project_root()
66
- model_dir = ROOT / "backend" / "models"
67
- target_path = model_dir / "kiuyha.pt"
68
 
69
  if target_path.exists():
70
  print(f"Kiuya Model already exists at {target_path}")
@@ -73,7 +74,7 @@ class Bubble_Detector_Kiuyha_Service:
73
  downloaded_path = hf_hub_download(
74
  repo_id="Kiuyha/Manga-Bubble-YOLO",
75
  filename="model.pt",
76
- local_dir=model_dir
77
  )
78
 
79
  final_path = Path(downloaded_path).rename(target_path)
 
3
  from helpers import get_project_root
4
  from huggingface_hub import hf_hub_download
5
  from pathlib import Path
6
+ import os
7
 
8
  class Bubble_Detector_Kiuyha_Service:
9
  def __init__(self, path=None):
10
+ ROOT = get_project_root()
11
+ self.base_model_path = Path(os.getenv("MODEL_PATH", ROOT / "backend" / "models"))
12
+
13
  if not path:
14
+ path = self.base_model_path / "GlmOcr"
15
+ else:
16
+ path = Path(path)
17
 
18
+ if not self.base_model_path.exists():
19
+ print(f"Kiuyha model not found at {self.base_model_path}. Attempting to download")
 
 
20
  self.load_model()
21
 
22
+ if self.base_model_path.exists():
23
+ self.model = YOLO(self.base_model_path)
24
  print("Loaded Bubble Detector Kiuyha")
25
  else:
26
+ raise FileNotFoundError(f"Error: Could not find or retrieve {self.base_model_path}")
27
 
28
  def predict(self, img_path, conf=0.2, iou=0.4, show_labels=True, show_conf=True, imgsz=640):
29
  results = self.model.predict(
 
65
  return sorted_boxes
66
 
67
  def load_model(self):
68
+ target_path = self.base_model_path / "kiuyha.pt"
 
 
69
 
70
  if target_path.exists():
71
  print(f"Kiuya Model already exists at {target_path}")
 
74
  downloaded_path = hf_hub_download(
75
  repo_id="Kiuyha/Manga-Bubble-YOLO",
76
  filename="model.pt",
77
+ local_dir=self.base_model_path
78
  )
79
 
80
  final_path = Path(downloaded_path).rename(target_path)
services/translate_qwen_service.py CHANGED
@@ -1,14 +1,16 @@
1
  from transformers import AutoTokenizer, AutoModelForCausalLM
2
  import json
3
  import torch
 
 
4
  from helpers import get_project_root
5
 
6
  class Translate_Qwen_Service:
7
  def __init__(self, path=None, device=None):
8
- if not path:
9
- ROOT = get_project_root()
10
- path = ROOT / "backend" / "models" / "Qwen"
11
-
12
  tokenizer_path = path / "tokenizer"
13
  model_path = path / "model"
14
 
@@ -74,12 +76,11 @@ class Translate_Qwen_Service:
74
  return {"error": "Invalid JSON", "raw": output_text}
75
 
76
  def load_model(self):
77
- ROOT = get_project_root()
78
- MODEL_PATH = "Qwen/Qwen2.5-7B-Instruct"
79
- QWEN_DIR = ROOT / "backend" / "models" / "Qwen"
80
 
81
- tokenizer = AutoTokenizer.from_pretrained(MODEL_PATH)
82
- model = AutoModelForCausalLM.from_pretrained(MODEL_PATH)
83
  tokenizer.save_pretrained(QWEN_DIR / "tokenizer")
84
  model.save_pretrained(QWEN_DIR / "model")
85
 
 
1
  from transformers import AutoTokenizer, AutoModelForCausalLM
2
  import json
3
  import torch
4
+ import os
5
+ from pathlib import Path
6
  from helpers import get_project_root
7
 
8
  class Translate_Qwen_Service:
9
  def __init__(self, path=None, device=None):
10
+ ROOT = get_project_root()
11
+ self.base_model_path = Path(os.getenv("MODEL_PATH", ROOT / "backend" / "models"))
12
+
13
+ path = self.base_model_path / "Qwen"
14
  tokenizer_path = path / "tokenizer"
15
  model_path = path / "model"
16
 
 
76
  return {"error": "Invalid JSON", "raw": output_text}
77
 
78
  def load_model(self):
79
+ QWEN_DIR = self.base_model_path / "Qwen"
80
+ DOWNLOAD_MODEL = "Qwen/Qwen2.5-7B-Instruct"
 
81
 
82
+ tokenizer = AutoTokenizer.from_pretrained(DOWNLOAD_MODEL)
83
+ model = AutoModelForCausalLM.from_pretrained(DOWNLOAD_MODEL)
84
  tokenizer.save_pretrained(QWEN_DIR / "tokenizer")
85
  model.save_pretrained(QWEN_DIR / "model")
86