Spaces:
Runtime error
Runtime error
| from django_thread import Thread | |
| from time import sleep | |
| from backend.module.utils import date_utils | |
| from django.db import connections | |
| from backend.models.model_cache import SocketRequestChapterQueueCache | |
| from backend.models.model_1 import ComicStorageCache | |
| from core.settings import BASE_DIR | |
| from backend.module import web_scrap | |
| from backend.module.utils import manage_image | |
| from backend.module.utils.directory_info import GetDirectorySize | |
| from channels.layers import get_channel_layer | |
| from asgiref.sync import async_to_sync | |
| from django.db.models import Count | |
| import requests, environ, os, subprocess, shutil, zipfile, uuid, sys | |
| env = environ.Env() | |
| # os.environ["KMP_DUPLICATE_LIB_OK"] = "TRUE" | |
| STORAGE_DIR = os.path.join(BASE_DIR,"storage") | |
| if not os.path.exists(STORAGE_DIR): os.makedirs(STORAGE_DIR) | |
| COMIC_STORAGE_DIR = os.path.join(STORAGE_DIR,"comics") | |
| if not os.path.exists(COMIC_STORAGE_DIR): os.makedirs(COMIC_STORAGE_DIR) | |
| LOG_DIR = os.path.join(BASE_DIR, "log") | |
| if not os.path.exists(LOG_DIR): os.makedirs(LOG_DIR) | |
| MAX_STORAGE_SIZE = 20 * 1024**3 | |
| class Job(Thread): | |
| def run(self): | |
| while True: | |
| input_dir = "" | |
| managed_output_dir = "" | |
| try: | |
| query_result = SocketRequestChapterQueueCache.objects.order_by("datetime").first() | |
| while True: | |
| if (GetDirectorySize(COMIC_STORAGE_DIR) >= MAX_STORAGE_SIZE): | |
| query_result_2 = ComicStorageCache.objects.order_by("datetime").first() | |
| if (query_result_2): | |
| file_path = query_result_2.file_path | |
| if os.path.exists(file_path): os.remove(file_path) | |
| ComicStorageCache.objects.filter(id=query_result_2.id).delete() | |
| else: break | |
| if (query_result): | |
| socket_id = query_result.socket_id | |
| channel_name = query_result.channel_name | |
| source = query_result.source | |
| comic_id = query_result.comic_id | |
| chapter_id = query_result.chapter_id | |
| chapter_idx = query_result.chapter_idx | |
| options = query_result.options | |
| if (options.get("translate").get("state")): | |
| target_lang = options.get("translate").get("target") | |
| else: target_lang = "" | |
| stored = ComicStorageCache.objects.filter( | |
| source=source, | |
| comic_id=comic_id, | |
| chapter_id=chapter_id, | |
| chapter_idx=chapter_idx, | |
| colorize= options.get("colorize"), | |
| translate= options.get("translate").get("state"), | |
| target_lang= target_lang, | |
| ).first() | |
| if (stored): | |
| SocketRequestChapterQueueCache.objects.filter(id=query_result.id).delete() | |
| connections['cache'].close() | |
| connections['DB1'].close() | |
| else: | |
| connections['cache'].close() | |
| connections['DB1'].close() | |
| if (options.get("colorize") or options.get("translate").get("state")): | |
| script = [] | |
| input_dir = os.path.join(COMIC_STORAGE_DIR,source,comic_id,str(chapter_idx),"temp") | |
| merge_output_dir = os.path.join(COMIC_STORAGE_DIR,source,comic_id,str(chapter_idx),"merged") | |
| if (options.get("translate").get("state") and options.get("colorize")): | |
| managed_output_dir = os.path.join(COMIC_STORAGE_DIR,source,comic_id,str(chapter_idx),f"{options.get("translate").get("target")}_translated_colorized") | |
| script = [ | |
| "python", "-m", "manga_translator", "-v", "--overwrite", "--attempts=3", "--ocr=mocr", "--no-text-lang-skip", "--det-auto-rotate", "--det-gamma-correct", "--colorize=mc2", "--translator=m2m100_big", | |
| "-l", f"{options.get("translate").get("target")}", "-i", f"{merge_output_dir}", "-o", f"{managed_output_dir}" | |
| ] | |
| elif (options.get("translate").get("state") and not options.get("colorize")): | |
| managed_output_dir = os.path.join(COMIC_STORAGE_DIR,source,comic_id,str(chapter_idx),f"{options.get("translate").get("target")}_translated") | |
| script = [ | |
| "python", "-m", "manga_translator", "-v", "--overwrite", "--attempts=3", "--ocr=mocr", "--no-text-lang-skip", "--det-auto-rotate", "--det-gamma-correct", "--translator=m2m100_big", | |
| "-l", f"{options.get("translate").get("target")}", "-i", f"{merge_output_dir}", "-o", f"{managed_output_dir}" | |
| ] | |
| elif (options.get("colorize") and not options.get("translate").get("state")): | |
| managed_output_dir = os.path.join(COMIC_STORAGE_DIR,source,comic_id,str(chapter_idx),"colorized") | |
| script = [ | |
| "python", "-m", "manga_translator", "-v", "--overwrite", "--attempts=3", "--detector=none", "--translator=original", "--colorize=mc2", "--colorization-size=-1", | |
| "-i", f"{merge_output_dir}", "-o", f"{managed_output_dir}" | |
| ] | |
| if target_lang == "ENG": script.append("--manga2eng") | |
| if os.path.exists(input_dir): shutil.rmtree(input_dir) | |
| if os.path.exists(merge_output_dir): shutil.rmtree(merge_output_dir) | |
| if os.path.exists(managed_output_dir): shutil.rmtree(managed_output_dir) | |
| job = web_scrap.source_control[source].get_chapter.scrap(comic_id=comic_id,chapter_id=chapter_id,output_dir=input_dir) | |
| if job.get("status") == "success": | |
| manage_image.merge_images_vertically(input_dir, merge_output_dir, max_height=1800) | |
| shutil.rmtree(input_dir) | |
| with open(os.path.join(LOG_DIR,"image_translator_output.log"), "w") as file: | |
| result = subprocess.run( | |
| script, | |
| cwd=os.path.join(BASE_DIR, "backend", "module", "utils", "image_translator"), | |
| check=True, | |
| shell=False if sys.platform.startswith('linux') else True, | |
| stdout=file, | |
| stderr=file, | |
| text=True, | |
| ) | |
| if result.returncode != 0: raise Exception("Image Translator Execution error!") | |
| os.makedirs(managed_output_dir,exist_ok=True) | |
| shutil.rmtree(merge_output_dir) | |
| with zipfile.ZipFile(managed_output_dir + '.zip', 'w') as zipf: | |
| for foldername, subfolders, filenames in os.walk(managed_output_dir): | |
| for filename in filenames: | |
| if filename.endswith(('.png', '.jpg', '.jpeg')): | |
| file_path = os.path.join(foldername, filename) | |
| zipf.write(file_path, os.path.basename(file_path)) | |
| shutil.rmtree(managed_output_dir) | |
| ComicStorageCache( | |
| source = source, | |
| comic_id = comic_id, | |
| chapter_id = chapter_id, | |
| chapter_idx = chapter_idx, | |
| file_path = managed_output_dir + ".zip", | |
| colorize = options.get("colorize"), | |
| translate = options.get("translate").get("state"), | |
| target_lang = target_lang, | |
| ).save() | |
| query_result_3 = SocketRequestChapterQueueCache.objects.filter(id=query_result.id).first() | |
| channel_name = query_result_3.channel_name if query_result_3 else "" | |
| SocketRequestChapterQueueCache.objects.filter(id=query_result.id).delete() | |
| if channel_name: | |
| channel_layer = get_channel_layer() | |
| async_to_sync(channel_layer.send)(channel_name, { | |
| 'type': 'event_send', | |
| 'data': { | |
| "type": "chapter_ready_to_download", | |
| "data": { | |
| "source": source, | |
| "comic_id": comic_id, | |
| "chapter_id": chapter_id, | |
| "chapter_idx": chapter_idx | |
| } | |
| } | |
| }) | |
| connections['cache'].close() | |
| connections['DB1'].close() | |
| else: | |
| connections['cache'].close() | |
| connections['DB1'].close() | |
| raise Exception("#1 Dowload chapter error!") | |
| else: | |
| input_dir = os.path.join(COMIC_STORAGE_DIR,source,comic_id,str(chapter_idx),"original") | |
| merge_output_dir = os.path.join(COMIC_STORAGE_DIR,source,comic_id,str(chapter_idx),"origin-merged") | |
| if os.path.exists(input_dir): shutil.rmtree(input_dir) | |
| if os.path.exists(merge_output_dir): shutil.rmtree(merge_output_dir) | |
| job = web_scrap.source_control["colamanga"].get_chapter.scrap(comic_id=comic_id,chapter_id=chapter_id,output_dir=input_dir) | |
| if job.get("status") == "success": | |
| manage_image.merge_images_vertically(input_dir, merge_output_dir, max_height=1800) | |
| if os.path.exists(input_dir): shutil.rmtree(input_dir) | |
| with zipfile.ZipFile(input_dir + '.zip', 'w') as zipf: | |
| for foldername, subfolders, filenames in os.walk(merge_output_dir): | |
| for filename in filenames: | |
| if filename.endswith(('.png', '.jpg', '.jpeg')): | |
| file_path = os.path.join(foldername, filename) | |
| zipf.write(file_path, os.path.basename(file_path)) | |
| shutil.rmtree(merge_output_dir) | |
| ComicStorageCache( | |
| source = source, | |
| comic_id = comic_id, | |
| chapter_id = chapter_id, | |
| chapter_idx = chapter_idx, | |
| file_path = input_dir + '.zip', | |
| colorize = False, | |
| translate = False, | |
| target_lang = "", | |
| ).save() | |
| query_result_3 = SocketRequestChapterQueueCache.objects.filter(id=query_result.id).first() | |
| channel_name = query_result_3.channel_name if query_result_3 else "" | |
| SocketRequestChapterQueueCache.objects.filter(id=query_result.id).delete() | |
| if channel_name: | |
| channel_layer = get_channel_layer() | |
| async_to_sync(channel_layer.send)(channel_name, { | |
| 'type': 'event_send', | |
| 'data': { | |
| "type": "chapter_ready_to_download", | |
| "data": { | |
| "source": source, | |
| "comic_id": comic_id, | |
| "chapter_id": chapter_id, | |
| "chapter_idx": chapter_idx | |
| } | |
| } | |
| }) | |
| else: | |
| connections['cache'].close() | |
| connections['DB1'].close() | |
| raise Exception("#2 Dowload chapter error!") | |
| connections['cache'].close() | |
| connections['DB1'].close() | |
| else: | |
| connections['cache'].close() | |
| connections['DB1'].close() | |
| sleep(5) | |
| except Exception as e: | |
| print("[Error] Chapter Queue Socket:", e) | |
| if (input_dir): | |
| if os.path.exists(input_dir): shutil.rmtree(input_dir) | |
| if (managed_output_dir): | |
| if os.path.exists(managed_output_dir): shutil.rmtree(managed_output_dir) | |
| query_result_3 = SocketRequestChapterQueueCache.objects.filter(id=query_result.id).first() | |
| channel_name = query_result_3.channel_name if query_result_3 else "" | |
| SocketRequestChapterQueueCache.objects.filter(id=query_result.id).delete() | |
| if (channel_name): | |
| channel_layer = get_channel_layer() | |
| async_to_sync(channel_layer.send)(channel_name, { | |
| 'type': 'event_send', | |
| 'data': { | |
| "type": "chapter_ready_to_download", | |
| "data": {"state":"error"} | |
| } | |
| }) | |
| connections['cache'].close() | |
| connections['DB1'].close() | |
| sleep(10) | |
| thread = Job() | |
| thread.daemon = True | |
| thread.start() | |
| class UpdateSocketQueue(Thread): | |
| def run(self): | |
| while True: | |
| try: | |
| queue = 0 | |
| MAX_QUEUE = SocketRequestChapterQueueCache.objects.count() | |
| if (MAX_QUEUE): | |
| query_result = list(set(SocketRequestChapterQueueCache.objects.order_by("datetime").values_list('socket_id', flat=True).distinct())) | |
| for socket_id in query_result: | |
| object = {} | |
| query_result_2 = SocketRequestChapterQueueCache.objects.filter(socket_id=socket_id).order_by("datetime").values("source","comic_id","chapter_idx") | |
| for item in query_result_2: | |
| source = item.get("source") | |
| comic_id = item.get("comic_id") | |
| chapter_idx = item.get("chapter_idx") | |
| object[f"{source}-{comic_id}-{chapter_idx}"] = queue | |
| queue += 1 | |
| query_result_3 = SocketRequestChapterQueueCache.objects.filter(socket_id=socket_id).first() | |
| if (query_result_3): | |
| channel_layer = get_channel_layer() | |
| async_to_sync(channel_layer.send)(query_result_3.channel_name, { | |
| 'type': 'event_send', | |
| 'data': { | |
| "type": "chapter_queue_info", | |
| "chapter_queue": { | |
| "queue": object, | |
| "max_queue": MAX_QUEUE, | |
| } | |
| } | |
| }) | |
| else: | |
| pass | |
| except Exception as e: | |
| print(e) | |
| connections['cache'].close() | |
| connections['DB1'].close() | |
| sleep(10) | |
| thread = UpdateSocketQueue() | |
| thread.daemon = True | |
| thread.start() |