fc7e96b182b21f5be42d5060b4cc04b1c2e517067a0c471c97505b29f93f45e5

.gitattributes

@@ -55,3 +55,12 @@ SD-CN-Animation/examples/gold_1.gif filter=lfs diff=lfs merge=lfs -text
 SD-CN-Animation/examples/macaroni_1.gif filter=lfs diff=lfs merge=lfs -text
 SD-CN-Animation/examples/tree_2.gif filter=lfs diff=lfs merge=lfs -text
 SD-CN-Animation/examples/tree_2.mp4 filter=lfs diff=lfs merge=lfs -text
+ebsynth_utility/imgs/sample1.mp4 filter=lfs diff=lfs merge=lfs -text
+ebsynth_utility/imgs/sample2.mp4 filter=lfs diff=lfs merge=lfs -text
+ebsynth_utility/imgs/sample3.mp4 filter=lfs diff=lfs merge=lfs -text
+ebsynth_utility/imgs/sample4.mp4 filter=lfs diff=lfs merge=lfs -text
+ebsynth_utility/imgs/sample5.mp4 filter=lfs diff=lfs merge=lfs -text
+ebsynth_utility/imgs/sample6.mp4 filter=lfs diff=lfs merge=lfs -text
+ebsynth_utility/imgs/sample_anyaheh.mp4 filter=lfs diff=lfs merge=lfs -text
+ebsynth_utility/imgs/sample_autotag.mp4 filter=lfs diff=lfs merge=lfs -text
+ebsynth_utility/imgs/sample_clipseg.mp4 filter=lfs diff=lfs merge=lfs -text

ebsynth_utility/calculator.py

@@ -0,0 +1,237 @@
+# https://www.mycompiler.io/view/3TFZagC
+
+class ParseError(Exception):
+    def __init__(self, pos, msg, *args):
+        self.pos = pos
+        self.msg = msg
+        self.args = args
+
+    def __str__(self):
+        return '%s at position %s' % (self.msg % self.args, self.pos)
+
+class Parser:
+    def __init__(self):
+        self.cache = {}
+
+    def parse(self, text):
+        self.text = text
+        self.pos = -1
+        self.len = len(text) - 1
+        rv = self.start()
+        self.assert_end()
+        return rv
+
+    def assert_end(self):
+        if self.pos < self.len:
+            raise ParseError(
+                self.pos + 1,
+                'Expected end of string but got %s',
+                self.text[self.pos + 1]
+            )
+
+    def eat_whitespace(self):
+        while self.pos < self.len and self.text[self.pos + 1] in " \f\v\r\t\n":
+            self.pos += 1
+
+    def split_char_ranges(self, chars):
+        try:
+            return self.cache[chars]
+        except KeyError:
+            pass
+
+        rv = []
+        index = 0
+        length = len(chars)
+
+        while index < length:
+            if index + 2 < length and chars[index + 1] == '-':
+                if chars[index] >= chars[index + 2]:
+                    raise ValueError('Bad character range')
+
+                rv.append(chars[index:index + 3])
+                index += 3
+            else:
+                rv.append(chars[index])
+                index += 1
+
+        self.cache[chars] = rv
+        return rv
+
+    def char(self, chars=None):
+        if self.pos >= self.len:
+            raise ParseError(
+                self.pos + 1,
+                'Expected %s but got end of string',
+                'character' if chars is None else '[%s]' % chars
+            )
+
+        next_char = self.text[self.pos + 1]
+        if chars == None:
+            self.pos += 1
+            return next_char
+
+        for char_range in self.split_char_ranges(chars):
+            if len(char_range) == 1:
+                if next_char == char_range:
+                    self.pos += 1
+                    return next_char
+            elif char_range[0] <= next_char <= char_range[2]:
+                self.pos += 1
+                return next_char
+
+        raise ParseError(
+            self.pos + 1,
+            'Expected %s but got %s',
+            'character' if chars is None else '[%s]' % chars,
+            next_char
+        )
+
+    def keyword(self, *keywords):
+        self.eat_whitespace()
+        if self.pos >= self.len:
+            raise ParseError(
+                self.pos + 1,
+                'Expected %s but got end of string',
+                ','.join(keywords)
+            )
+
+        for keyword in keywords:
+            low = self.pos + 1
+            high = low + len(keyword)
+
+            if self.text[low:high] == keyword:
+                self.pos += len(keyword)
+                self.eat_whitespace()
+                return keyword
+
+        raise ParseError(
+            self.pos + 1,
+            'Expected %s but got %s',
+            ','.join(keywords),
+            self.text[self.pos + 1],
+        )
+
+    def match(self, *rules):
+        self.eat_whitespace()
+        last_error_pos = -1
+        last_exception = None
+        last_error_rules = []
+
+        for rule in rules:
+            initial_pos = self.pos
+            try:
+                rv = getattr(self, rule)()
+                self.eat_whitespace()
+                return rv
+            except ParseError as e:
+                self.pos = initial_pos
+
+                if e.pos > last_error_pos:
+                    last_exception = e
+                    last_error_pos = e.pos
+                    last_error_rules.clear()
+                    last_error_rules.append(rule)
+                elif e.pos == last_error_pos:
+                    last_error_rules.append(rule)
+
+        if len(last_error_rules) == 1:
+            raise last_exception
+        else:
+            raise ParseError(
+                last_error_pos,
+                'Expected %s but got %s',
+                ','.join(last_error_rules),
+                self.text[last_error_pos]
+            )
+
+    def maybe_char(self, chars=None):
+        try:
+            return self.char(chars)
+        except ParseError:
+            return None
+
+    def maybe_match(self, *rules):
+        try:
+            return self.match(*rules)
+        except ParseError:
+            return None
+
+    def maybe_keyword(self, *keywords):
+        try:
+            return self.keyword(*keywords)
+        except ParseError:
+            return None
+
+class CalcParser(Parser):
+    def start(self):
+        return self.expression()
+
+    def expression(self):
+        rv = self.match('term')
+        while True:
+            op = self.maybe_keyword('+', '-')
+            if op is None:
+                break
+
+            term = self.match('term')
+            if op == '+':
+                rv += term
+            else:
+                rv -= term
+
+        return rv
+
+    def term(self):
+        rv = self.match('factor')
+        while True:
+            op = self.maybe_keyword('*', '/')
+            if op is None:
+                break
+
+            term = self.match('factor')
+            if op == '*':
+                rv *= term
+            else:
+                rv /= term
+
+        return rv
+
+    def factor(self):
+        if self.maybe_keyword('('):
+            rv = self.match('expression')
+            self.keyword(')')
+
+            return rv
+
+        return self.match('number')
+
+    def number(self):
+        chars = []
+
+        sign = self.maybe_keyword('+', '-')
+        if sign is not None:
+            chars.append(sign)
+
+        chars.append(self.char('0-9'))
+
+        while True:
+            char = self.maybe_char('0-9')
+            if char is None:
+                break
+
+            chars.append(char)
+
+        if self.maybe_char('.'):
+            chars.append('.')
+            chars.append(self.char('0-9'))
+
+            while True:
+                char = self.maybe_char('0-9')
+                if char is None:
+                    break
+
+                chars.append(char)
+
+        rv = float(''.join(chars))
+        return rv
+

ebsynth_utility/ebsynth_utility.py

@@ -0,0 +1,185 @@
+import os
+
+from modules.ui import plaintext_to_html
+
+import cv2
+import glob
+from PIL import Image
+
+from extensions.ebsynth_utility.stage1 import ebsynth_utility_stage1,ebsynth_utility_stage1_invert
+from extensions.ebsynth_utility.stage2 import ebsynth_utility_stage2
+from extensions.ebsynth_utility.stage5 import ebsynth_utility_stage5
+from extensions.ebsynth_utility.stage7 import ebsynth_utility_stage7
+from extensions.ebsynth_utility.stage8 import ebsynth_utility_stage8
+from extensions.ebsynth_utility.stage3_5 import ebsynth_utility_stage3_5
+
+
+def x_ceiling(value, step):
+  return -(-value // step) * step
+
+def dump_dict(string, d:dict):
+    for key in d.keys():
+        string += ( key + " : " + str(d[key]) + "\n")
+    return string
+
+class debug_string:
+    txt = ""
+    def print(self, comment):
+        print(comment)
+        self.txt += comment + '\n'
+    def to_string(self):
+        return self.txt
+
+def ebsynth_utility_process(stage_index: int, project_dir:str, original_movie_path:str, frame_width:int, frame_height:int, st1_masking_method_index:int, st1_mask_threshold:float, tb_use_fast_mode:bool, tb_use_jit:bool, clipseg_mask_prompt:str, clipseg_exclude_prompt:str, clipseg_mask_threshold:int, clipseg_mask_blur_size:int, clipseg_mask_blur_size2:int, key_min_gap:int, key_max_gap:int, key_th:float, key_add_last_frame:bool, color_matcher_method:str, st3_5_use_mask:bool, st3_5_use_mask_ref:bool, st3_5_use_mask_org:bool, color_matcher_ref_type:int, color_matcher_ref_image:Image, blend_rate:float, export_type:str, bg_src:str, bg_type:str, mask_blur_size:int, mask_threshold:float, fg_transparency:float, mask_mode:str):
+    args = locals()
+    info = ""
+    info = dump_dict(info, args)
+    dbg = debug_string()
+
+
+    def process_end(dbg, info):
+        return plaintext_to_html(dbg.to_string()), plaintext_to_html(info)
+
+
+    if not os.path.isdir(project_dir):
+        dbg.print("{0} project_dir not found".format(project_dir))
+        return process_end( dbg, info )
+
+    if not os.path.isfile(original_movie_path):
+        dbg.print("{0} original_movie_path not found".format(original_movie_path))
+        return process_end( dbg, info )
+    
+    is_invert_mask = False
+    if mask_mode == "Invert":
+        is_invert_mask = True
+
+    frame_path = os.path.join(project_dir , "video_frame")
+    frame_mask_path = os.path.join(project_dir, "video_mask")
+
+    if is_invert_mask:
+        inv_path = os.path.join(project_dir, "inv")
+        os.makedirs(inv_path, exist_ok=True)
+
+        org_key_path = os.path.join(inv_path, "video_key")
+        img2img_key_path = os.path.join(inv_path, "img2img_key")
+        img2img_upscale_key_path = os.path.join(inv_path, "img2img_upscale_key")
+    else:
+        org_key_path = os.path.join(project_dir, "video_key")
+        img2img_key_path = os.path.join(project_dir, "img2img_key")
+        img2img_upscale_key_path = os.path.join(project_dir, "img2img_upscale_key")
+
+    if mask_mode == "None":
+        frame_mask_path = ""
+    
+
+    project_args = [project_dir, original_movie_path, frame_path, frame_mask_path, org_key_path, img2img_key_path, img2img_upscale_key_path]
+
+
+    if stage_index == 0:
+        ebsynth_utility_stage1(dbg, project_args, frame_width, frame_height, st1_masking_method_index, st1_mask_threshold, tb_use_fast_mode, tb_use_jit, clipseg_mask_prompt, clipseg_exclude_prompt, clipseg_mask_threshold, clipseg_mask_blur_size, clipseg_mask_blur_size2, is_invert_mask)
+        if is_invert_mask:
+            inv_mask_path = os.path.join(inv_path, "inv_video_mask")
+            ebsynth_utility_stage1_invert(dbg, frame_mask_path, inv_mask_path)
+
+    elif stage_index == 1:
+        ebsynth_utility_stage2(dbg, project_args, key_min_gap, key_max_gap, key_th, key_add_last_frame, is_invert_mask)
+    elif stage_index == 2:
+
+        sample_image = glob.glob( os.path.join(frame_path , "*.png" ) )[0]
+        img_height, img_width, _ = cv2.imread(sample_image).shape
+        if img_width < img_height:
+            re_w = 512
+            re_h = int(x_ceiling( (512 / img_width) * img_height , 64))
+        else:
+            re_w = int(x_ceiling( (512 / img_height) * img_width , 64))
+            re_h = 512
+        img_width = re_w
+        img_height = re_h
+
+        dbg.print("stage 3")
+        dbg.print("")
+        dbg.print("!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!")
+        dbg.print("1. Go to img2img tab")
+        dbg.print("2. Select [ebsynth utility] in the script combo box")
+        dbg.print("3. Fill in the \"Project directory\" field with [" + project_dir + "]" )
+        dbg.print("4. Select in the \"Mask Mode(Override img2img Mask mode)\" field with [" + ("Invert" if is_invert_mask else "Normal") + "]" )
+        dbg.print("5. I recommend to fill in the \"Width\" field with [" + str(img_width) + "]" )
+        dbg.print("6. I recommend to fill in the \"Height\" field with [" + str(img_height) + "]" )
+        dbg.print("7. I recommend to fill in the \"Denoising strength\" field with lower than 0.35" )
+        dbg.print("   (When using controlnet together, you can put in large values (even 1.0 is possible).)")
+        dbg.print("8. Fill in the remaining configuration fields of img2img. No image and mask settings are required.")
+        dbg.print("9. Drop any image onto the img2img main screen. This is necessary to avoid errors, but does not affect the results of img2img.")
+        dbg.print("10. Generate")
+        dbg.print("(Images are output to [" + img2img_key_path + "])")
+        dbg.print("!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!")
+        return process_end( dbg, "" )
+    
+    elif stage_index == 3:
+        ebsynth_utility_stage3_5(dbg, project_args, color_matcher_method, st3_5_use_mask, st3_5_use_mask_ref, st3_5_use_mask_org, color_matcher_ref_type, color_matcher_ref_image)
+
+    elif stage_index == 4:
+        sample_image = glob.glob( os.path.join(frame_path , "*.png" ) )[0]
+        img_height, img_width, _ = cv2.imread(sample_image).shape
+
+        sample_img2img_key = glob.glob( os.path.join(img2img_key_path , "*.png" ) )[0]
+        img_height_key, img_width_key, _ = cv2.imread(sample_img2img_key).shape
+
+        if is_invert_mask:
+            project_dir = inv_path
+
+        dbg.print("stage 4")
+        dbg.print("")
+
+        if img_height == img_height_key and img_width == img_width_key:
+            dbg.print("!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!")
+            dbg.print("!! The size of frame and img2img_key matched.")
+            dbg.print("!! You can skip this stage.")
+
+        dbg.print("!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!")
+        dbg.print("0. Enable the following item")
+        dbg.print("Settings ->")
+        dbg.print("  Saving images/grids ->")
+        dbg.print("    Use original name for output filename during batch process in extras tab")
+        dbg.print("!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!")
+        dbg.print("1. If \"img2img_upscale_key\" directory already exists in the %s, delete it manually before executing."%(project_dir))
+        dbg.print("2. Go to Extras tab")
+        dbg.print("3. Go to Batch from Directory tab")
+        dbg.print("4. Fill in the \"Input directory\" field with [" + img2img_key_path + "]" )
+        dbg.print("5. Fill in the \"Output directory\" field with [" + img2img_upscale_key_path + "]" )
+        dbg.print("6. Go to Scale to tab")
+        dbg.print("7. Fill in the \"Width\" field with [" + str(img_width) + "]" )
+        dbg.print("8. Fill in the \"Height\" field with [" + str(img_height) + "]" )
+        dbg.print("9. Fill in the remaining configuration fields of Upscaler.")
+        dbg.print("10. Generate")
+        dbg.print("!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!")
+        return process_end( dbg, "" )
+    elif stage_index == 5:
+        ebsynth_utility_stage5(dbg, project_args, is_invert_mask)
+    elif stage_index == 6:
+
+        if is_invert_mask:
+            project_dir = inv_path
+
+        dbg.print("stage 6")
+        dbg.print("")
+        dbg.print("!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!")
+        dbg.print("Running ebsynth.(on your self)")
+        dbg.print("Open the generated .ebs under %s and press [Run All] button."%(project_dir))
+        dbg.print("If ""out-*"" directory already exists in the %s, delete it manually before executing."%(project_dir))
+        dbg.print("If multiple .ebs files are generated, run them all.")
+        dbg.print("(I recommend associating the .ebs file with EbSynth.exe.)")
+        dbg.print("!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!")
+        return process_end( dbg, "" )
+    elif stage_index == 7:
+        ebsynth_utility_stage7(dbg, project_args, blend_rate, export_type, is_invert_mask)
+    elif stage_index == 8:
+        if mask_mode != "Normal":
+            dbg.print("!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!")
+            dbg.print("Please reset [configuration]->[etc]->[Mask Mode] to Normal.")
+            dbg.print("!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!")
+            return process_end( dbg, "" )
+        ebsynth_utility_stage8(dbg, project_args, bg_src, bg_type, mask_blur_size, mask_threshold, fg_transparency, export_type)
+    else:
+        pass
+
+    return process_end( dbg, info )

ebsynth_utility/imgs/sample1.mp4

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:9c5458eea82a691a584af26f51d1db728092069a3409c6f9eb2dd14fd2b71173
+size 4824162

ebsynth_utility/imgs/sample2.mp4

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:537b8331b74d8ea49ee580aed138d460735ef897ab31ca694031d7a56d99ff72
+size 2920523

ebsynth_utility/imgs/sample3.mp4

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:c49739c2ef1f2ecaf14453f463e46b1a05de1688ce22b50200563a03b1758ddf
+size 5161880

ebsynth_utility/imgs/sample4.mp4

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:62ed25263b6e5328a49460714c0b6e6ac46759921c87c91850f749f5bf068cfa
+size 5617838

ebsynth_utility/imgs/sample5.mp4

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:4e429cfef8b3ed7829ce3219895fcfdbbe94e1494a2e4dcd87988e03509c8d50
+size 4190467

ebsynth_utility/imgs/sample6.mp4

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:fb9a9ea8662ef1b7fc7151b987006eef8dd3598e320242bd87a2838ac8733df6
+size 6890883

ebsynth_utility/imgs/sample_anyaheh.mp4

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:4de4e9b9758cefe28c430f909da9dfc086e5b3510e9d0aa7becab7b4be355447
+size 12159686

ebsynth_utility/imgs/sample_autotag.mp4

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:4e904d68fc0fd9ac09ce153a9d54e9f1ce9f8db7cf5e96109c496f7e64924c92
+size 7058129

ebsynth_utility/imgs/sample_clipseg.mp4

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:17701c5c3a376d3c4cf8ce0acfb991033830d56670ca3178eedd6e671e096af3
+size 10249706

ebsynth_utility/install.py

@@ -0,0 +1,24 @@
+import launch
+
+def update_transparent_background():
+    from importlib.metadata import version as meta_version
+    from packaging import version
+    v = meta_version("transparent-background")
+    print("current transparent-background " + v)
+    if version.parse(v) < version.parse('1.2.3'):
+        launch.run_pip("install -U transparent-background", "update transparent-background version for Ebsynth Utility")
+
+if not launch.is_installed("transparent_background"):
+    launch.run_pip("install transparent-background", "requirements for Ebsynth Utility")
+
+update_transparent_background()
+
+if not launch.is_installed("IPython"):
+    launch.run_pip("install ipython", "requirements for Ebsynth Utility")
+
+if not launch.is_installed("seaborn"):
+    launch.run_pip("install ""seaborn>=0.11.0""", "requirements for Ebsynth Utility")
+
+if not launch.is_installed("color_matcher"):
+    launch.run_pip("install color-matcher", "requirements for Ebsynth Utility")
+

ebsynth_utility/sample/add_token.txt

@@ -0,0 +1,54 @@
+[
+    {
+        "target":"smile",
+        "min_score":0.5,
+        "token": ["lottalewds_v0", "1.2"],
+        "type":"lora"
+    },
+    {
+        "target":"smile",
+        "min_score":0.5,
+        "token": ["anyahehface", "score*1.2"],
+        "type":"normal"
+    },
+    {
+        "target":"smile",
+        "min_score":0.5,
+        "token": ["wicked smug", "score*1.2"],
+        "type":"normal"
+    },
+    {
+        "target":"smile",
+        "min_score":0.5,
+        "token": ["half closed eyes", "0.2 + score*0.3"],
+        "type":"normal"
+    },
+    
+    
+    
+    {
+        "target":"test_token",
+        "min_score":0.8,
+        "token": ["lora_name_A", "0.5"],
+        "type":"lora"
+    },
+    {
+        "target":"test_token",
+        "min_score":0.5,
+        "token": ["bbbb", "score - 0.1"],
+        "type":"normal"
+    },
+    {
+        "target":"test_token2",
+        "min_score":0.8,
+        "token": ["hypernet_name_A", "score"],
+        "type":"hypernet"
+    },
+    {
+        "target":"test_token3",
+        "min_score":0.0,
+        "token": ["dddd", "score"],
+        "type":"normal"
+    }
+]
+

ebsynth_utility/sample/blacklist.txt

@@ -0,0 +1,10 @@
+motion_blur
+blurry
+realistic
+depth_of_field
+mountain
+tree
+water
+underwater
+tongue
+tongue_out

ebsynth_utility/scripts/custom_script.py

@@ -0,0 +1,1012 @@
+import modules.scripts as scripts
+import gradio as gr
+import os
+import torch
+import random
+import time
+import pprint
+import shutil
+
+from modules.processing import process_images,Processed
+from modules.paths import models_path
+from modules.textual_inversion import autocrop
+import modules.images
+from modules import shared,deepbooru,masking
+import cv2
+import copy
+import numpy as np
+from PIL import Image,ImageOps
+import glob
+import requests
+import json
+import re
+from extensions.ebsynth_utility.calculator import CalcParser,ParseError
+
+def get_my_dir():
+    if os.path.isdir("extensions/ebsynth_utility"):
+        return "extensions/ebsynth_utility"
+    return scripts.basedir()
+
+def x_ceiling(value, step):
+    return -(-value // step) * step
+
+def remove_pngs_in_dir(path):
+    if not os.path.isdir(path):
+        return
+    pngs = glob.glob( os.path.join(path, "*.png") )
+    for png in pngs:
+        os.remove(png)
+
+def resize_img(img, w, h):
+    if img.shape[0] + img.shape[1] < h + w:
+        interpolation = interpolation=cv2.INTER_CUBIC
+    else:
+        interpolation = interpolation=cv2.INTER_AREA
+
+    return cv2.resize(img, (w, h), interpolation=interpolation)
+
+def download_and_cache_models(dirname):
+    download_url = 'https://github.com/zymk9/yolov5_anime/blob/8b50add22dbd8224904221be3173390f56046794/weights/yolov5s_anime.pt?raw=true'
+    model_file_name = 'yolov5s_anime.pt'
+
+    if not os.path.exists(dirname):
+        os.makedirs(dirname)
+
+    cache_file = os.path.join(dirname, model_file_name)
+    if not os.path.exists(cache_file):
+        print(f"downloading face detection model from '{download_url}' to '{cache_file}'")
+        response = requests.get(download_url)
+        with open(cache_file, "wb") as f:
+            f.write(response.content)
+
+    if os.path.exists(cache_file):
+        return cache_file
+    return None
+
+class Script(scripts.Script):
+    anime_face_detector = None
+    face_detector = None
+    face_merge_mask_filename = "face_crop_img2img_mask.png"
+    face_merge_mask_image = None
+    prompts_dir = ""
+    calc_parser = None
+    is_invert_mask = False
+    controlnet_weight = 0.5
+    controlnet_weight_for_face = 0.5
+    add_tag_replace_underscore = False
+
+
+# The title of the script. This is what will be displayed in the dropdown menu.
+    def title(self):
+        return "ebsynth utility"
+
+# Determines when the script should be shown in the dropdown menu via the 
+# returned value. As an example:
+# is_img2img is True if the current tab is img2img, and False if it is txt2img.
+# Thus, return is_img2img to only show the script on the img2img tab.
+
+    def show(self, is_img2img):
+        return is_img2img
+
+# How the script's is displayed in the UI. See https://gradio.app/docs/#components
+# for the different UI components you can use and how to create them.
+# Most UI components can return a value, such as a boolean for a checkbox.
+# The returned values are passed to the run method as parameters.
+
+    def ui(self, is_img2img):
+        with gr.Column(variant='panel'):
+            with gr.Column():
+                project_dir = gr.Textbox(label='Project directory', lines=1)
+                generation_test = gr.Checkbox(False, label="Generation TEST!!(Ignore Project directory and use the image and mask specified in the main UI)")
+
+            with gr.Accordion("Mask option"):
+                mask_mode = gr.Dropdown(choices=["Normal","Invert","None","Don't Override"], value="Normal" ,label="Mask Mode(Override img2img Mask mode)")
+                inpaint_area = gr.Dropdown(choices=["Whole picture","Only masked","Don't Override"], type = "index", value="Only masked" ,label="Inpaint Area(Override img2img Inpaint area)")
+                use_depth = gr.Checkbox(True, label="Use Depth Map If exists in /video_key_depth")
+                gr.HTML(value="<p style='margin-bottom: 0.7em'>\
+                        See \
+                        <font color=\"blue\"><a href=\"https://github.com/thygate/stable-diffusion-webui-depthmap-script\">[here]</a></font> for depth map.\
+                        </p>")
+
+            with gr.Accordion("ControlNet option"):
+                controlnet_weight = gr.Slider(minimum=0.0, maximum=2.0, step=0.01, value=0.5, label="Control Net Weight")
+                controlnet_weight_for_face = gr.Slider(minimum=0.0, maximum=2.0, step=0.01, value=0.5, label="Control Net Weight For Face")
+                use_preprocess_img = gr.Checkbox(True, label="Use Preprocess image If exists in /controlnet_preprocess")
+                gr.HTML(value="<p style='margin-bottom: 0.7em'>\
+                        Please enable the following settings to use controlnet from this script.<br>\
+                        <font color=\"red\">\
+                        Settings->ControlNet->Allow other script to control this extension\
+                        </font>\
+                        </p>")
+            
+            with gr.Accordion("Loopback option"):
+                img2img_repeat_count = gr.Slider(minimum=1, maximum=30, step=1, value=1, label="Img2Img Repeat Count (Loop Back)")
+                inc_seed = gr.Slider(minimum=0, maximum=9999999, step=1, value=1, label="Add N to seed when repeating ")
+
+            with gr.Accordion("Auto Tagging option"):
+                auto_tag_mode = gr.Dropdown(choices=["None","DeepDanbooru","CLIP"], value="None" ,label="Auto Tagging")
+                add_tag_to_head = gr.Checkbox(False, label="Add additional prompts to the head")
+                add_tag_replace_underscore = gr.Checkbox(False, label="Replace '_' with ' '(Does not affect the function to add tokens using add_token.txt.)")
+                gr.HTML(value="<p style='margin-bottom: 0.7em'>\
+                        The results are stored in timestamp_prompts.txt.<br>\
+                        If you want to use the same tagging results the next time you run img2img, rename the file to prompts.txt<br>\
+                        Recommend enabling the following settings.<br>\
+                        <font color=\"red\">\
+                        Settings->Interrogate Option->Interrogate: include ranks of model tags matches in results\
+                        </font>\
+                        </p>")
+
+            with gr.Accordion("Face Crop option"):
+                is_facecrop = gr.Checkbox(False, label="use Face Crop img2img")
+
+                with gr.Row():
+                    face_detection_method = gr.Dropdown(choices=["YuNet","Yolov5_anime"], value="YuNet" ,label="Face Detection Method")
+                    gr.HTML(value="<p style='margin-bottom: 0.7em'>\
+                            If loading of the Yolov5_anime model fails, check\
+                            <font color=\"blue\"><a href=\"https://github.com/AUTOMATIC1111/stable-diffusion-webui/issues/2235\">[this]</a></font> solution.\
+                            </p>")
+                face_crop_resolution = gr.Slider(minimum=128, maximum=2048, step=1, value=512, label="Face Crop Resolution")
+                max_crop_size = gr.Slider(minimum=0, maximum=2048, step=1, value=1024, label="Max Crop Size")
+                face_denoising_strength = gr.Slider(minimum=0.00, maximum=1.00, step=0.01, value=0.5, label="Face Denoising Strength")
+                face_area_magnification = gr.Slider(minimum=1.00, maximum=10.00, step=0.01, value=1.5, label="Face Area Magnification ")
+                disable_facecrop_lpbk_last_time = gr.Checkbox(False, label="Disable at the last loopback time")
+                
+                with gr.Column():
+                    enable_face_prompt = gr.Checkbox(False, label="Enable Face Prompt")
+                    face_prompt = gr.Textbox(label="Face Prompt", show_label=False, lines=2,
+                        placeholder="Prompt for Face",
+                        value = "face close up,"
+                    )
+
+        return [project_dir, generation_test, mask_mode, inpaint_area, use_depth, img2img_repeat_count, inc_seed, auto_tag_mode, add_tag_to_head, add_tag_replace_underscore, is_facecrop, face_detection_method, face_crop_resolution, max_crop_size, face_denoising_strength, face_area_magnification, enable_face_prompt, face_prompt, controlnet_weight, controlnet_weight_for_face, disable_facecrop_lpbk_last_time,use_preprocess_img]
+
+
+    def detect_face_from_img(self, img_array):
+        if not self.face_detector:
+            dnn_model_path = autocrop.download_and_cache_models(os.path.join(models_path, "opencv"))
+            self.face_detector = cv2.FaceDetectorYN.create(dnn_model_path, "", (0, 0))
+        
+        self.face_detector.setInputSize((img_array.shape[1], img_array.shape[0]))
+        _, result = self.face_detector.detect(img_array)
+        return result
+
+    def detect_anime_face_from_img(self, img_array):
+        import sys
+
+        if not self.anime_face_detector:
+            if 'models' in sys.modules:
+                del sys.modules['models']
+
+            anime_model_path = download_and_cache_models(os.path.join(models_path, "yolov5_anime"))
+
+            if not os.path.isfile(anime_model_path):
+                print( "WARNING!! " + anime_model_path + " not found.")
+                print( "use YuNet instead.")
+                return self.detect_face_from_img(img_array)
+
+            self.anime_face_detector = torch.hub.load('ultralytics/yolov5', 'custom', path=anime_model_path)
+
+            # warmup
+            test = np.zeros([512,512,3],dtype=np.uint8)
+            _ = self.anime_face_detector(test)
+
+        result = self.anime_face_detector(img_array)
+        #models.common.Detections
+        faces = []
+        for x_c, y_c, w, h, _, _ in result.xywh[0].tolist():
+            faces.append( [ x_c - w/2 , y_c - h/2, w, h ] )
+
+        return faces
+
+    def detect_face(self, img, mask, face_detection_method, max_crop_size):
+        img_array = np.array(img)
+
+        # image without alpha
+        if img_array.shape[2] == 4:
+            img_array = img_array[:,:,:3]
+
+        if mask is not None:
+            if self.is_invert_mask:
+                mask = ImageOps.invert(mask)
+            mask_array = np.array(mask)/255
+            if mask_array.ndim == 2:
+                mask_array = mask_array[:, :, np.newaxis]
+
+            if mask_array.shape[2] == 4:
+                mask_array = mask_array[:,:,:3]
+            
+            img_array = mask_array * img_array
+            img_array = img_array.astype(np.uint8)
+
+        if face_detection_method == "YuNet":
+            faces = self.detect_face_from_img(img_array)
+        elif face_detection_method == "Yolov5_anime":
+            faces = self.detect_anime_face_from_img(img_array)
+        else:
+            faces = self.detect_face_from_img(img_array)
+        
+        if faces is None or len(faces) == 0:
+            return []
+        
+        face_coords = []
+        for face in faces:
+            x = int(face[0])
+            y = int(face[1])
+            w = int(face[2])
+            h = int(face[3])
+            if max(w,h) > max_crop_size:
+                print("ignore big face")
+                continue
+            if w == 0 or h == 0:
+                print("ignore w,h = 0 face")
+                continue
+
+            face_coords.append( [ x/img_array.shape[1],y/img_array.shape[0],w/img_array.shape[1],h/img_array.shape[0]] )
+
+        return face_coords
+
+    def get_mask(self):
+        def create_mask( output, x_rate, y_rate, k_size ):
+            img = np.zeros((512, 512, 3))
+            img = cv2.ellipse(img, ((256, 256), (int(512 * x_rate), int(512 * y_rate)), 0), (255, 255, 255), thickness=-1)
+            img = cv2.GaussianBlur(img, (k_size, k_size), 0)
+            cv2.imwrite(output, img)
+        
+        if self.face_merge_mask_image is None:
+            mask_file_path = os.path.join( get_my_dir() , self.face_merge_mask_filename)
+            if not os.path.isfile(mask_file_path):
+                create_mask( mask_file_path, 0.9, 0.9, 91)
+
+            m = cv2.imread( mask_file_path )[:,:,0]
+            m = m[:, :, np.newaxis]
+            self.face_merge_mask_image = m / 255
+
+        return self.face_merge_mask_image
+
+    def face_img_crop(self, img, face_coords,face_area_magnification):
+        img_array = np.array(img)
+        face_imgs =[]
+        new_coords = []
+
+        for face in face_coords:
+            x = int(face[0] * img_array.shape[1])
+            y = int(face[1] * img_array.shape[0])
+            w = int(face[2] * img_array.shape[1])
+            h = int(face[3] * img_array.shape[0])
+            print([x,y,w,h])
+
+            cx = x + int(w/2)
+            cy = y + int(h/2)
+
+            x = cx - int(w*face_area_magnification / 2)
+            x = x if x > 0 else 0
+            w = cx + int(w*face_area_magnification / 2) - x
+            w = w if x+w < img.width else img.width - x
+
+            y = cy - int(h*face_area_magnification / 2)
+            y = y if y > 0 else 0
+            h = cy + int(h*face_area_magnification / 2) - y
+            h = h if y+h < img.height else img.height - y
+
+            print([x,y,w,h])
+
+            face_imgs.append( img_array[y: y+h, x: x+w] )
+            new_coords.append( [x,y,w,h] )
+        
+        resized = []
+        for face_img in face_imgs:
+            if face_img.shape[1] < face_img.shape[0]:
+                re_w = self.face_crop_resolution
+                re_h = int(x_ceiling( (self.face_crop_resolution / face_img.shape[1]) * face_img.shape[0] , 64))
+            else:
+                re_w = int(x_ceiling( (self.face_crop_resolution / face_img.shape[0]) * face_img.shape[1] , 64))
+                re_h = self.face_crop_resolution
+            
+            face_img = resize_img(face_img, re_w, re_h)
+            resized.append( Image.fromarray(face_img))
+
+        return resized, new_coords
+
+    def face_crop_img2img(self, p, face_coords, face_denoising_strength, face_area_magnification, enable_face_prompt, face_prompt, controlnet_input_img, controlnet_input_face_imgs, preprocess_img_exist):
+
+        def merge_face(img, face_img, face_coord, base_img_size, mask):
+            x_rate = img.width / base_img_size[0]
+            y_rate = img.height / base_img_size[1]
+
+            img_array = np.array(img)
+            x = int(face_coord[0] * x_rate)
+            y = int(face_coord[1] * y_rate)
+            w = int(face_coord[2] * x_rate)
+            h = int(face_coord[3] * y_rate)
+
+            face_array = np.array(face_img)
+            face_array = resize_img(face_array, w, h)
+            mask = resize_img(mask, w, h)
+            if mask.ndim == 2:
+                mask = mask[:, :, np.newaxis]
+            
+            bg = img_array[y: y+h, x: x+w]
+            img_array[y: y+h, x: x+w] = mask * face_array + (1-mask)*bg
+
+            return Image.fromarray(img_array)
+
+        base_img = p.init_images[0]
+
+        base_img_size = (base_img.width, base_img.height)
+
+        if face_coords is None or len(face_coords) == 0:
+            print("no face detected")
+            return process_images(p)
+
+        print(face_coords)
+        face_imgs, new_coords = self.face_img_crop(base_img, face_coords, face_area_magnification)
+
+        if not face_imgs:
+            return process_images(p)
+
+        face_p = copy.copy(p)
+
+        ### img2img base img
+        proc = self.process_images(p, controlnet_input_img, self.controlnet_weight, preprocess_img_exist)
+        print(proc.seed)
+
+        ### img2img for each face
+        face_img2img_results = []
+
+        for face, coord, controlnet_input_face in zip(face_imgs, new_coords, controlnet_input_face_imgs):
+            # cv2.imwrite("scripts/face.png", np.array(face)[:, :, ::-1])
+            face_p.init_images = [face]
+            face_p.width = face.width
+            face_p.height = face.height
+            face_p.denoising_strength = face_denoising_strength
+            
+            if enable_face_prompt:
+                face_p.prompt = face_prompt
+            else:
+                face_p.prompt = "close-up face ," + face_p.prompt
+
+            if p.image_mask is not None:
+                x,y,w,h = coord
+                cropped_face_mask = Image.fromarray(np.array(p.image_mask)[y: y+h, x: x+w])
+                face_p.image_mask = modules.images.resize_image(0, cropped_face_mask, face.width, face.height)
+            
+            face_proc = self.process_images(face_p, controlnet_input_face, self.controlnet_weight_for_face, preprocess_img_exist)
+            print(face_proc.seed)
+
+            face_img2img_results.append((face_proc.images[0], coord))
+        
+        ### merge faces
+        bg = proc.images[0]
+        mask = self.get_mask()
+
+        for face_img, coord in face_img2img_results:
+            bg = merge_face(bg, face_img, coord, base_img_size, mask)
+        
+        proc.images[0] = bg
+
+        return proc
+
+    def get_depth_map(self, mask, depth_path ,img_basename, is_invert_mask):
+        depth_img_path = os.path.join( depth_path , img_basename )
+
+        depth = None
+
+        if os.path.isfile( depth_img_path ):
+            depth = Image.open(depth_img_path)
+        else:
+            # try 00001-0000.png
+            os.path.splitext(img_basename)[0]
+            depth_img_path = os.path.join( depth_path , os.path.splitext(img_basename)[0] + "-0000.png" )
+            if os.path.isfile( depth_img_path ):
+                depth = Image.open(depth_img_path)
+        
+        if depth:
+            if mask:
+                mask_array = np.array(mask)
+                depth_array = np.array(depth)
+
+                if is_invert_mask == False:
+                    depth_array[mask_array[:,:,0] == 0] = 0
+                else:
+                    depth_array[mask_array[:,:,0] != 0] = 0
+
+                depth = Image.fromarray(depth_array)
+
+                tmp_path = os.path.join( depth_path , "tmp" )
+                os.makedirs(tmp_path, exist_ok=True)
+                tmp_path = os.path.join( tmp_path , img_basename )
+                depth_array = depth_array.astype(np.uint16)
+                cv2.imwrite(tmp_path, depth_array)
+
+            mask = depth
+        
+        return depth!=None, mask
+    
+### auto tagging
+    debug_count = 0
+
+    def get_masked_image(self, image, mask_image):
+
+        if mask_image == None:
+            return image.convert("RGB")
+        
+        mask = mask_image.convert('L')
+        if self.is_invert_mask:
+            mask = ImageOps.invert(mask)
+        crop_region = masking.get_crop_region(np.array(mask), 0)
+#        crop_region = masking.expand_crop_region(crop_region, self.width, self.height, mask.width, mask.height)
+#        x1, y1, x2, y2 = crop_region
+        image = image.crop(crop_region).convert("RGB")
+        mask = mask.crop(crop_region)
+
+        base_img = Image.new("RGB", image.size, (255, 190, 200))
+
+        image = Image.composite( image, base_img, mask )
+
+#        image.save("scripts/get_masked_image_test_"+ str(self.debug_count) + ".png")
+#        self.debug_count += 1
+
+        return image
+    
+    def interrogate_deepdanbooru(self, imgs, masks):
+        prompts_dict = {}
+        cause_err = False
+
+        try:
+            deepbooru.model.start()
+
+            for img,mask in zip(imgs,masks):
+                key = os.path.basename(img)
+                print(key + " interrogate deepdanbooru")
+
+                image = Image.open(img)
+                mask_image = Image.open(mask) if mask else None
+                image = self.get_masked_image(image, mask_image)
+
+                prompt = deepbooru.model.tag_multi(image)
+
+                prompts_dict[key] = prompt
+        except Exception as e:
+            import traceback
+            traceback.print_exc()
+            print(e)
+            cause_err = True
+        finally:
+            deepbooru.model.stop()
+            if cause_err:
+                print("Exception occurred during auto-tagging(deepdanbooru)")
+                return Processed()
+
+        return prompts_dict
+
+
+    def interrogate_clip(self, imgs, masks):
+        from modules import devices, shared, lowvram, paths
+        import importlib
+        import models
+
+        caption_list = []
+        prompts_dict = {}
+        cause_err = False
+
+        try:
+            if shared.cmd_opts.lowvram or shared.cmd_opts.medvram:
+                lowvram.send_everything_to_cpu()
+                devices.torch_gc()
+
+            with paths.Prioritize("BLIP"):
+                importlib.reload(models)
+                shared.interrogator.load()
+
+            for img,mask in zip(imgs,masks):
+                key = os.path.basename(img)
+                print(key + " generate caption")
+
+                image = Image.open(img)
+                mask_image = Image.open(mask) if mask else None
+                image = self.get_masked_image(image, mask_image)
+
+                caption = shared.interrogator.generate_caption(image)
+                caption_list.append(caption)
+
+            shared.interrogator.send_blip_to_ram()
+            devices.torch_gc()
+
+            for img,mask,caption in zip(imgs,masks,caption_list):
+                key = os.path.basename(img)
+                print(key + " interrogate clip")
+
+                image = Image.open(img)
+                mask_image = Image.open(mask) if mask else None
+                image = self.get_masked_image(image, mask_image)
+
+                clip_image = shared.interrogator.clip_preprocess(image).unsqueeze(0).type(shared.interrogator.dtype).to(devices.device_interrogate)
+
+                res = ""
+
+                with torch.no_grad(), devices.autocast():
+                    image_features = shared.interrogator.clip_model.encode_image(clip_image).type(shared.interrogator.dtype)
+                    image_features /= image_features.norm(dim=-1, keepdim=True)
+
+                    for name, topn, items in shared.interrogator.categories():
+                        matches = shared.interrogator.rank(image_features, items, top_count=topn)
+                        for match, score in matches:
+                            if shared.opts.interrogate_return_ranks:
+                                res += f", ({match}:{score/100:.3f})"
+                            else:
+                                res += ", " + match
+
+                prompts_dict[key] = (caption + res)
+
+        except Exception as e:
+            import traceback
+            traceback.print_exc()
+            print(e)
+            cause_err = True
+        finally:
+            shared.interrogator.unload()
+            if cause_err:
+                print("Exception occurred during auto-tagging(blip/clip)")
+                return Processed()
+        
+        return prompts_dict
+
+
+    def remove_reserved_token(self, token_list):
+        reserved_list = ["pink_background","simple_background","pink","pink_theme"]
+
+        result_list = []
+
+        head_token = token_list[0]
+
+        if head_token[2] == "normal":
+            head_token_str = head_token[0].replace('pink background', '')
+            token_list[0] = (head_token_str, head_token[1], head_token[2])
+
+        for token in token_list:
+            if token[0] in reserved_list:
+                continue
+            result_list.append(token)
+
+        return result_list
+
+    def remove_blacklisted_token(self, token_list):
+        black_list_path = os.path.join(self.prompts_dir, "blacklist.txt") 
+        if not os.path.isfile(black_list_path):
+            print(black_list_path + " not found.")
+            return token_list
+
+        with open(black_list_path) as f:
+            black_list = [s.strip() for s in f.readlines()]
+
+            result_list = []
+
+            for token in token_list:
+                if token[0] in black_list:
+                    continue
+                result_list.append(token)
+            
+            token_list = result_list
+
+        return token_list
+
+    def add_token(self, token_list):
+        add_list_path = os.path.join(self.prompts_dir, "add_token.txt") 
+        if not os.path.isfile(add_list_path):
+            print(add_list_path + " not found.")
+
+            if self.add_tag_replace_underscore:
+                token_list = [ (x[0].replace("_"," "), x[1], x[2]) for x in token_list ]
+
+            return token_list
+        
+        if not self.calc_parser:
+            self.calc_parser = CalcParser()
+
+        with open(add_list_path) as f:
+            add_list = json.load(f)
+            '''
+            [
+                {
+                    "target":"test_token",
+                    "min_score":0.8,
+                    "token": ["lora_name_A", "0.5"],
+                    "type":"lora"
+                },
+                {
+                    "target":"test_token",
+                    "min_score":0.5,
+                    "token": ["bbbb", "score - 0.1"],
+                    "type":"normal"
+                },
+                {
+                    "target":"test_token2",
+                    "min_score":0.8,
+                    "token": ["hypernet_name_A", "score"],
+                    "type":"hypernet"
+                },
+                {
+                    "target":"test_token3",
+                    "min_score":0.0,
+                    "token": ["dddd", "score"],
+                    "type":"normal"
+                }
+            ]
+            '''
+            result_list = []
+
+            for token in token_list:
+                for add_item in add_list:
+                    if token[0] == add_item["target"]:
+                        if token[1] > add_item["min_score"]:
+                            # hit
+                            formula = str(add_item["token"][1])
+                            formula = formula.replace("score",str(token[1]))
+                            print('Input: %s' % str(add_item["token"][1]))
+
+                            try:
+                                score = self.calc_parser.parse(formula)
+                                score = round(score, 3)
+                            except (ParseError, ZeroDivisionError) as e:
+                                print('Input: %s' % str(add_item["token"][1]))
+                                print('Error: %s' % e)
+                                print("ignore this token")
+                                continue
+
+                            print("score = " + str(score))
+                            result_list.append( ( add_item["token"][0], score, add_item["type"] ) )
+            
+            if self.add_tag_replace_underscore:
+                token_list = [ (x[0].replace("_"," "), x[1], x[2]) for x in token_list ]
+
+            token_list = token_list + result_list
+
+        return token_list
+
+    def create_prompts_dict(self, imgs, masks, auto_tag_mode):
+        prompts_dict = {}
+
+        if auto_tag_mode == "DeepDanbooru":
+            raw_dict = self.interrogate_deepdanbooru(imgs, masks)
+        elif auto_tag_mode == "CLIP":
+            raw_dict = self.interrogate_clip(imgs, masks)
+        
+        repatter = re.compile(r'\((.+)\:([0-9\.]+)\)')
+
+        for key, value_str in raw_dict.items():
+            value_list = [x.strip() for x in value_str.split(',')]
+
+            value = []
+            for v in value_list:
+                m = repatter.fullmatch(v)
+                if m:
+                    value.append((m.group(1), float(m.group(2)), "normal"))
+                else:
+                    value.append((v, 1, "no_score"))
+            
+#            print(value)
+            value = self.remove_reserved_token(value)
+#            print(value)
+            value = self.remove_blacklisted_token(value)
+#            print(value)
+            value = self.add_token(value)
+#            print(value)
+
+            def create_token_str(x):
+                print(x)
+                if x[2] == "no_score":
+                    return x[0]
+                elif x[2] == "lora":
+                    return "<lora:" + x[0] + ":" + str(x[1]) + ">"
+                elif x[2] == "hypernet":
+                    return "<hypernet:" + x[0] + ":" + str(x[1]) + ">"
+                else:
+                    return "(" + x[0] + ":" + str(x[1]) + ")"
+
+            value_list = [create_token_str(x) for x in value]
+            value = ",".join(value_list)
+
+            prompts_dict[key] = value
+
+        return prompts_dict
+
+    def load_prompts_dict(self, imgs, default_token):
+        prompts_path = os.path.join(self.prompts_dir, "prompts.txt") 
+        if not os.path.isfile(prompts_path):
+            print(prompts_path + " not found.")
+            return {}
+        
+        prompts_dict = {}
+
+        print(prompts_path + " found!!")
+        print("skip auto tagging.")
+        
+        with open(prompts_path) as f:
+            raw_dict = json.load(f)
+            prev_value = default_token
+            for img in imgs:
+                key = os.path.basename(img)
+
+                if key in raw_dict:
+                    prompts_dict[key] = raw_dict[key]
+                    prev_value = raw_dict[key]
+                else:
+                    prompts_dict[key] = prev_value
+
+        return prompts_dict
+    
+    def process_images(self, p, input_img, controlnet_weight, input_img_is_preprocessed):
+        p.control_net_input_image = input_img
+        p.control_net_weight = controlnet_weight
+        if input_img_is_preprocessed:
+            p.control_net_module = "none"
+        return process_images(p)
+
+# This is where the additional processing is implemented. The parameters include
+# self, the model object "p" (a StableDiffusionProcessing class, see
+# processing.py), and the parameters returned by the ui method.
+# Custom functions can be defined here, and additional libraries can be imported 
+# to be used in processing. The return value should be a Processed object, which is
+# what is returned by the process_images method.
+    def run(self, p, project_dir, generation_test, mask_mode, inpaint_area, use_depth, img2img_repeat_count, inc_seed, auto_tag_mode, add_tag_to_head, add_tag_replace_underscore, is_facecrop, face_detection_method, face_crop_resolution, max_crop_size, face_denoising_strength, face_area_magnification, enable_face_prompt, face_prompt, controlnet_weight, controlnet_weight_for_face, disable_facecrop_lpbk_last_time, use_preprocess_img):
+        args = locals()
+
+        if generation_test:
+            print("generation_test")
+            test_proj_dir = os.path.join( get_my_dir() , "generation_test_proj")
+            os.makedirs(test_proj_dir, exist_ok=True)
+            test_video_key_path = os.path.join( test_proj_dir , "video_key")
+            os.makedirs(test_video_key_path, exist_ok=True)
+            test_video_mask_path = os.path.join( test_proj_dir , "video_mask")
+            os.makedirs(test_video_mask_path, exist_ok=True)
+
+            controlnet_input_path = os.path.join(test_proj_dir, "controlnet_input")
+            if os.path.isdir(controlnet_input_path):
+                shutil.rmtree(controlnet_input_path)
+
+            remove_pngs_in_dir(test_video_key_path)
+            remove_pngs_in_dir(test_video_mask_path)
+
+            test_base_img = p.init_images[0]
+            test_mask = p.image_mask
+
+            if test_base_img:
+                test_base_img.save( os.path.join( test_video_key_path , "00001.png") )
+            if test_mask:
+                test_mask.save( os.path.join( test_video_mask_path , "00001.png") )
+            
+            project_dir = test_proj_dir
+        else:
+            if not os.path.isdir(project_dir):
+                print("project_dir not found")
+                return Processed()
+        
+        self.controlnet_weight = controlnet_weight
+        self.controlnet_weight_for_face = controlnet_weight_for_face
+
+        self.add_tag_replace_underscore = add_tag_replace_underscore
+        self.face_crop_resolution = face_crop_resolution
+        
+        if p.seed == -1:
+            p.seed = int(random.randrange(4294967294))
+
+        if mask_mode == "Normal":
+            p.inpainting_mask_invert = 0
+        elif mask_mode == "Invert":
+            p.inpainting_mask_invert = 1
+        
+        if inpaint_area in (0,1):  #"Whole picture","Only masked"
+            p.inpaint_full_res = inpaint_area
+
+        is_invert_mask = False
+        if mask_mode == "Invert":
+            is_invert_mask = True
+
+            inv_path = os.path.join(project_dir, "inv")
+            if not os.path.isdir(inv_path):
+                print("project_dir/inv not found")
+                return Processed()
+            
+            org_key_path = os.path.join(inv_path, "video_key")
+            img2img_key_path = os.path.join(inv_path, "img2img_key")
+            depth_path = os.path.join(inv_path, "video_key_depth")
+
+            preprocess_path = os.path.join(inv_path, "controlnet_preprocess")
+
+            controlnet_input_path = os.path.join(inv_path, "controlnet_input")
+
+            self.prompts_dir = inv_path
+            self.is_invert_mask = True
+        else:
+            org_key_path = os.path.join(project_dir, "video_key")
+            img2img_key_path = os.path.join(project_dir, "img2img_key")
+            depth_path = os.path.join(project_dir, "video_key_depth")
+
+            preprocess_path = os.path.join(project_dir, "controlnet_preprocess")
+
+            controlnet_input_path = os.path.join(project_dir, "controlnet_input")
+
+            self.prompts_dir = project_dir
+            self.is_invert_mask = False
+
+        frame_mask_path = os.path.join(project_dir, "video_mask")
+
+        if not use_depth:
+            depth_path = None
+
+        if not os.path.isdir(org_key_path):
+            print(org_key_path + " not found")
+            print("Generate key frames first." if is_invert_mask == False else \
+                    "Generate key frames first.(with [Ebsynth Utility] Tab -> [configuration] -> [etc]-> [Mask Mode] = Invert setting)")
+            return Processed()
+
+        if not os.path.isdir(controlnet_input_path):
+            print(controlnet_input_path + " not found")
+            print("copy {0} -> {1}".format(org_key_path,controlnet_input_path))
+
+            os.makedirs(controlnet_input_path, exist_ok=True)
+
+            imgs = glob.glob( os.path.join(org_key_path ,"*.png") )
+            for img in imgs:
+                img_basename = os.path.basename(img)
+                shutil.copy( img , os.path.join(controlnet_input_path, img_basename) )
+
+        remove_pngs_in_dir(img2img_key_path)
+        os.makedirs(img2img_key_path, exist_ok=True)
+
+
+        def get_mask_of_img(img):
+            img_basename = os.path.basename(img)
+            
+            if mask_mode != "None":
+                mask_path = os.path.join( frame_mask_path , img_basename )
+                if os.path.isfile( mask_path ):
+                    return mask_path
+            return ""
+        
+        def get_pair_of_img(img, target_dir):
+            img_basename = os.path.basename(img)
+            
+            pair_path = os.path.join( target_dir , img_basename )
+            if os.path.isfile( pair_path ):
+                return pair_path
+            print("!!! pair of "+ img + " not in " + target_dir)
+            return ""
+
+        def get_controlnet_input_img(img):
+            pair_img = get_pair_of_img(img, controlnet_input_path)
+            if not pair_img:
+                pair_img = get_pair_of_img(img, org_key_path)
+            return pair_img
+        
+        imgs = glob.glob( os.path.join(org_key_path ,"*.png") )
+        masks = [ get_mask_of_img(i) for i in imgs ]
+        controlnet_input_imgs = [ get_controlnet_input_img(i) for i in imgs ]
+
+        for mask in masks:
+            m = cv2.imread(mask) if mask else None
+            if m is not None:
+                if m.max() == 0:
+                    print("{0} blank mask found".format(mask))
+                    if m.ndim == 2:
+                        m[0,0] = 255
+                    else:
+                        m = m[:,:,:3]
+                        m[0,0,0:3] = 255
+                    cv2.imwrite(mask, m)
+
+        ######################
+        # face crop
+        face_coords_dict={}
+        for img,mask in zip(imgs,masks):
+            face_detected = False
+            if is_facecrop:
+                image = Image.open(img)
+                mask_image = Image.open(mask) if mask else None
+                face_coords = self.detect_face(image, mask_image, face_detection_method, max_crop_size)
+                if face_coords is None or len(face_coords) == 0:
+                    print("no face detected")
+                else:
+                    print("face detected")
+                    face_detected = True
+            
+            key = os.path.basename(img)
+            face_coords_dict[key] = face_coords if face_detected else []
+
+        with open( os.path.join( project_dir if is_invert_mask == False else inv_path,"faces.txt" ), "w") as f:
+            f.write(json.dumps(face_coords_dict,indent=4))
+
+        ######################
+        # prompts
+        prompts_dict = self.load_prompts_dict(imgs, p.prompt)
+
+        if not prompts_dict:
+            if auto_tag_mode != "None":
+                prompts_dict = self.create_prompts_dict(imgs, masks, auto_tag_mode)
+
+                for key, value in prompts_dict.items():
+                    prompts_dict[key] = (value + "," + p.prompt) if add_tag_to_head else (p.prompt + "," + value)
+
+            else:
+                for img in imgs:
+                    key = os.path.basename(img)
+                    prompts_dict[key] = p.prompt
+            
+        with open( os.path.join( project_dir if is_invert_mask == False else inv_path, time.strftime("%Y%m%d-%H%M%S_") + "prompts.txt" ), "w") as f:
+            f.write(json.dumps(prompts_dict,indent=4))
+
+
+        ######################
+        # img2img
+        for img, mask, controlnet_input_img, face_coords, prompts in zip(imgs, masks, controlnet_input_imgs, face_coords_dict.values(), prompts_dict.values()):
+
+            # Generation cancelled.
+            if shared.state.interrupted:
+                print("Generation cancelled.")
+                break
+
+            image = Image.open(img)
+            mask_image = Image.open(mask) if mask else None
+
+            img_basename = os.path.basename(img)
+            
+            _p = copy.copy(p)
+            
+            _p.init_images=[image]
+            _p.image_mask = mask_image
+            _p.prompt = prompts
+            resized_mask = None
+
+            repeat_count = img2img_repeat_count
+            
+            if mask_mode != "None" or use_depth:
+                if use_depth:
+                    depth_found, _p.image_mask = self.get_depth_map( mask_image, depth_path ,img_basename, is_invert_mask )
+                    mask_image = _p.image_mask
+                    if depth_found:
+                        _p.inpainting_mask_invert = 0
+            
+            preprocess_img_exist = False
+            controlnet_input_base_img = Image.open(controlnet_input_img) if controlnet_input_img else None
+
+            if use_preprocess_img:
+                preprocess_img = os.path.join(preprocess_path, img_basename)
+                if os.path.isfile( preprocess_img ):
+                    controlnet_input_base_img = Image.open(preprocess_img)
+                    preprocess_img_exist = True
+
+            if face_coords:
+                controlnet_input_face_imgs, _ = self.face_img_crop(controlnet_input_base_img, face_coords, face_area_magnification)
+
+            while repeat_count > 0:
+
+                if disable_facecrop_lpbk_last_time:
+                    if img2img_repeat_count > 1:
+                        if repeat_count == 1:
+                            face_coords = None
+
+                if face_coords:
+                    proc = self.face_crop_img2img(_p, face_coords, face_denoising_strength, face_area_magnification, enable_face_prompt, face_prompt, controlnet_input_base_img, controlnet_input_face_imgs, preprocess_img_exist)
+                else:
+                    proc = self.process_images(_p, controlnet_input_base_img, self.controlnet_weight, preprocess_img_exist)
+                    print(proc.seed)
+                
+                repeat_count -= 1
+
+                if repeat_count > 0:
+                    _p.init_images=[proc.images[0]]
+
+                    if mask_image is not None and resized_mask is None:
+                        resized_mask = resize_img(np.array(mask_image) , proc.images[0].width, proc.images[0].height)
+                        resized_mask = Image.fromarray(resized_mask)
+                    _p.image_mask = resized_mask
+                    _p.seed += inc_seed
+
+            proc.images[0].save( os.path.join( img2img_key_path , img_basename ) )
+
+        with open( os.path.join( project_dir if is_invert_mask == False else inv_path,"param.txt" ), "w") as f:
+            f.write(pprint.pformat(proc.info))
+        with open( os.path.join( project_dir if is_invert_mask == False else inv_path ,"args.txt" ), "w") as f:
+            f.write(pprint.pformat(args))
+
+        return proc

ebsynth_utility/scripts/ui.py

@@ -0,0 +1,199 @@
+
+import gradio as gr
+
+from ebsynth_utility import ebsynth_utility_process
+from modules import script_callbacks
+from modules.call_queue import wrap_gradio_gpu_call
+
+def on_ui_tabs():
+
+    with gr.Blocks(analytics_enabled=False) as ebs_interface:
+        with gr.Row().style(equal_height=False):
+            with gr.Column(variant='panel'):
+
+                with gr.Row():
+                    with gr.Tabs(elem_id="ebs_settings"):
+                        with gr.TabItem('project setting', elem_id='ebs_project_setting'):
+                            project_dir = gr.Textbox(label='Project directory', lines=1)
+                            original_movie_path = gr.Textbox(label='Original Movie Path', lines=1)
+
+                            org_video = gr.Video(interactive=True, mirror_webcam=False)
+                            def fn_upload_org_video(video):
+                                return video
+                            org_video.upload(fn_upload_org_video, org_video, original_movie_path)
+                            gr.HTML(value="<p style='margin-bottom: 1.2em'>\
+                                    If you have trouble entering the video path manually, you can also use drag and drop.For large videos, please enter the path manually. \
+                                    </p>")
+
+                        with gr.TabItem('configuration', elem_id='ebs_configuration'):
+                            with gr.Tabs(elem_id="ebs_configuration_tab"):
+                                with gr.TabItem(label="stage 1",elem_id='ebs_configuration_tab1'):
+                                    with gr.Row():
+                                        frame_width = gr.Number(value=-1, label="Frame Width", precision=0, interactive=True)
+                                        frame_height = gr.Number(value=-1, label="Frame Height", precision=0, interactive=True)
+                                    gr.HTML(value="<p style='margin-bottom: 1.2em'>\
+                                            -1 means that it is calculated automatically. If both are -1, the size will be the same as the source size. \
+                                            </p>")
+
+                                    st1_masking_method_index = gr.Radio(label='Masking Method', choices=["transparent-background","clipseg","transparent-background AND clipseg"], value="transparent-background", type="index")
+
+                                    with gr.Accordion(label="transparent-background options"):
+                                        st1_mask_threshold = gr.Slider(minimum=0.0, maximum=1.0, step=0.01, label='Mask Threshold', value=0.0)
+
+                                        # https://pypi.org/project/transparent-background/
+                                        gr.HTML(value="<p style='margin-bottom: 0.7em'>\
+                                                configuration for \
+                                                <font color=\"blue\"><a href=\"https://pypi.org/project/transparent-background\">[transparent-background]</a></font>\
+                                                </p>")
+                                        tb_use_fast_mode = gr.Checkbox(label="Use Fast Mode(It will be faster, but the quality of the mask will be lower.)", value=False)
+                                        tb_use_jit = gr.Checkbox(label="Use Jit", value=False)
+
+                                    with gr.Accordion(label="clipseg options"):
+                                        clipseg_mask_prompt = gr.Textbox(label='Mask Target (e.g., girl, cats)', lines=1)
+                                        clipseg_exclude_prompt = gr.Textbox(label='Exclude Target (e.g., finger, book)', lines=1)
+                                        clipseg_mask_threshold = gr.Slider(minimum=0.0, maximum=1.0, step=0.01, label='Mask Threshold', value=0.4)
+                                        clipseg_mask_blur_size = gr.Slider(minimum=0, maximum=150, step=1, label='Mask Blur Kernel Size(MedianBlur)', value=11)
+                                        clipseg_mask_blur_size2 = gr.Slider(minimum=0, maximum=150, step=1, label='Mask Blur Kernel Size(GaussianBlur)', value=11)
+
+                                with gr.TabItem(label="stage 2", elem_id='ebs_configuration_tab2'):
+                                    key_min_gap = gr.Slider(minimum=0, maximum=500, step=1, label='Minimum keyframe gap', value=10)
+                                    key_max_gap = gr.Slider(minimum=0, maximum=1000, step=1, label='Maximum keyframe gap', value=300)
+                                    key_th = gr.Slider(minimum=0.0, maximum=100.0, step=0.1, label='Threshold of delta frame edge', value=8.5)
+                                    key_add_last_frame = gr.Checkbox(label="Add last frame to keyframes", value=True)
+
+                                with gr.TabItem(label="stage 3.5", elem_id='ebs_configuration_tab3_5'):
+                                    gr.HTML(value="<p style='margin-bottom: 0.7em'>\
+                                            <font color=\"blue\"><a href=\"https://github.com/hahnec/color-matcher\">[color-matcher]</a></font>\
+                                            </p>")
+                                    
+                                    color_matcher_method = gr.Radio(label='Color Transfer Method', choices=['default', 'hm', 'reinhard', 'mvgd', 'mkl', 'hm-mvgd-hm', 'hm-mkl-hm'], value="hm-mkl-hm", type="value")
+                                    color_matcher_ref_type = gr.Radio(label='Color Matcher Ref Image Type', choices=['original video frame', 'first frame of img2img result'], value="original video frame", type="index")
+                                    gr.HTML(value="<p style='margin-bottom: 0.7em'>\
+                                            <font color=\"red\">If an image is specified below, it will be used with highest priority.</font>\
+                                            </p>")
+                                    color_matcher_ref_image = gr.Image(label="Color Matcher Ref Image", source='upload', mirror_webcam=False, type='pil')
+                                    st3_5_use_mask = gr.Checkbox(label="Apply mask to the result", value=True)
+                                    st3_5_use_mask_ref = gr.Checkbox(label="Apply mask to the Ref Image", value=False)
+                                    st3_5_use_mask_org = gr.Checkbox(label="Apply mask to original image", value=False)
+                                    #st3_5_number_of_itr = gr.Slider(minimum=1, maximum=10, step=1, label='Number of iterations', value=1)
+
+                                with gr.TabItem(label="stage 7", elem_id='ebs_configuration_tab7'):
+                                    blend_rate = gr.Slider(minimum=0.0, maximum=1.0, step=0.01, label='Crossfade blend rate', value=1.0)
+                                    export_type = gr.Dropdown(choices=["mp4","webm","gif","rawvideo"], value="mp4" ,label="Export type")
+
+                                with gr.TabItem(label="stage 8", elem_id='ebs_configuration_tab8'):
+                                    bg_src = gr.Textbox(label='Background source(mp4 or directory containing images)', lines=1)
+                                    bg_type = gr.Dropdown(choices=["Fit video length","Loop"], value="Fit video length" ,label="Background type")
+                                    mask_blur_size = gr.Slider(minimum=0, maximum=150, step=1, label='Mask Blur Kernel Size', value=5)
+                                    mask_threshold = gr.Slider(minimum=0.0, maximum=1.0, step=0.01, label='Mask Threshold', value=0.0)
+                                    #is_transparent = gr.Checkbox(label="Is Transparent", value=True, visible = False)
+                                    fg_transparency = gr.Slider(minimum=0.0, maximum=1.0, step=0.01, label='Foreground Transparency', value=0.0)
+
+                                with gr.TabItem(label="etc", elem_id='ebs_configuration_tab_etc'):
+                                    mask_mode = gr.Dropdown(choices=["Normal","Invert","None"], value="Normal" ,label="Mask Mode")
+
+                    with gr.Column(variant='panel'):
+                        with gr.Column(scale=1):
+                            with gr.Group():
+                                debug_info = gr.HTML(elem_id="ebs_info_area", value=".")
+
+                            with gr.Column(scale=2):
+                                stage_index = gr.Radio(label='Process Stage', choices=["stage 1","stage 2","stage 3","stage 3.5","stage 4","stage 5","stage 6","stage 7","stage 8"], value="stage 1", type="index")
+                                gr.HTML(value="<p style='margin-bottom: 0.7em'>\
+                                                The process of creating a video can be divided into the following stages.<br>\
+                                                (Stage 3, 4, and 6 only show a guide and do nothing actual processing.)<br><br>\
+                                                <b>stage 1</b> <br>\
+                                                    Extract frames from the original video. <br>\
+                                                    Generate a mask image. <br><br>\
+                                                <b>stage 2</b> <br>\
+                                                    Select keyframes to be given to ebsynth.<br><br>\
+                                                <b>stage 3</b> <br>\
+                                                    img2img keyframes.<br><br>\
+                                                <b>stage 3.5</b> <br>\
+                                                    (this is optional. Perform color correction on the img2img results and expect flickering to decrease. Or, you can simply change the color tone from the generated result.)<br><br>\
+                                                <b>stage 4</b> <br>\
+                                                    and upscale to the size of the original video.<br><br>\
+                                                <b>stage 5</b> <br>\
+                                                    Rename keyframes.<br>\
+                                                    Generate .ebs file.(ebsynth project file)<br><br>\
+                                                <b>stage 6</b> <br>\
+                                                    Running ebsynth.(on your self)<br>\
+                                                    Open the generated .ebs under project directory and press [Run All] button. <br>\
+                                                    If ""out-*"" directory already exists in the Project directory, delete it manually before executing.<br>\
+                                                    If multiple .ebs files are generated, run them all.<br><br>\
+                                                <b>stage 7</b> <br>\
+                                                    Concatenate each frame while crossfading.<br>\
+                                                    Composite audio files extracted from the original video onto the concatenated video.<br><br>\
+                                                <b>stage 8</b> <br>\
+                                                    This is an extra stage.<br>\
+                                                    You can put any image or images or video you like in the background.<br>\
+                                                    You can specify in this field -> [Ebsynth Utility]->[configuration]->[stage 8]->[Background source]<br>\
+                                                    If you have already created a background video in Invert Mask Mode([Ebsynth Utility]->[configuration]->[etc]->[Mask Mode]),<br>\
+                                                    You can specify \"path_to_project_dir/inv/crossfade_tmp\".<br>\
+                                                </p>")
+                            
+                            with gr.Row():
+                                generate_btn = gr.Button('Generate', elem_id="ebs_generate_btn", variant='primary')
+                            
+                            with gr.Group():
+                                html_info = gr.HTML()
+
+
+            ebs_args = dict(
+                fn=wrap_gradio_gpu_call(ebsynth_utility_process),
+                inputs=[
+                    stage_index,
+
+                    project_dir,
+                    original_movie_path,
+
+                    frame_width,
+                    frame_height,
+                    st1_masking_method_index,
+                    st1_mask_threshold,
+                    tb_use_fast_mode,
+                    tb_use_jit,
+                    clipseg_mask_prompt,
+                    clipseg_exclude_prompt,
+                    clipseg_mask_threshold,
+                    clipseg_mask_blur_size,
+                    clipseg_mask_blur_size2,
+
+                    key_min_gap,
+                    key_max_gap,
+                    key_th,
+                    key_add_last_frame,
+
+                    color_matcher_method,
+                    st3_5_use_mask,
+                    st3_5_use_mask_ref,
+                    st3_5_use_mask_org,
+                    color_matcher_ref_type,
+                    color_matcher_ref_image,
+
+                    blend_rate,
+                    export_type,
+
+                    bg_src,
+                    bg_type,
+                    mask_blur_size,
+                    mask_threshold,
+                    fg_transparency,
+
+                    mask_mode,
+
+                ],
+                outputs=[
+                    debug_info,
+                    html_info,
+                ],
+                show_progress=False,
+            )
+            generate_btn.click(**ebs_args)
+           
+    return (ebs_interface, "Ebsynth Utility", "ebs_interface"),
+
+
+
+script_callbacks.on_ui_tabs(on_ui_tabs)
+

ebsynth_utility/stage1.py

@@ -0,0 +1,258 @@
+import os
+import subprocess
+import glob
+import cv2
+import re
+
+from transformers import AutoProcessor, CLIPSegForImageSegmentation
+from PIL import Image
+import torch
+import numpy as np
+
+
+def resize_img(img, w, h):
+    if img.shape[0] + img.shape[1] < h + w:
+        interpolation = interpolation=cv2.INTER_CUBIC
+    else:
+        interpolation = interpolation=cv2.INTER_AREA
+
+    return cv2.resize(img, (w, h), interpolation=interpolation)
+
+def resize_all_img(path, frame_width, frame_height):
+    if not os.path.isdir(path):
+        return
+    
+    pngs = glob.glob( os.path.join(path, "*.png") )
+    img = cv2.imread(pngs[0])
+    org_h,org_w = img.shape[0],img.shape[1]
+
+    if frame_width == -1 and frame_height == -1:
+        return
+    elif frame_width == -1 and frame_height != -1:
+        frame_width = int(frame_height * org_w / org_h)
+    elif frame_width != -1 and frame_height == -1:
+        frame_height = int(frame_width * org_h / org_w)
+    else:
+        pass
+    print("({0},{1}) resize to ({2},{3})".format(org_w, org_h, frame_width, frame_height))
+
+    for png in pngs:
+        img = cv2.imread(png)
+        img = resize_img(img, frame_width, frame_height)
+        cv2.imwrite(png, img)
+
+def remove_pngs_in_dir(path):
+    if not os.path.isdir(path):
+        return
+    
+    pngs = glob.glob( os.path.join(path, "*.png") )
+    for png in pngs:
+        os.remove(png)
+
+def create_and_mask(mask_dir1, mask_dir2, output_dir):
+    masks = glob.glob( os.path.join(mask_dir1, "*.png") )
+
+    for mask1 in masks:
+        base_name = os.path.basename(mask1)
+        print("combine {0}".format(base_name))
+        
+        mask2 = os.path.join(mask_dir2, base_name)
+        if not os.path.isfile(mask2):
+            print("{0} not found!!! -> skip".format(mask2))
+            continue
+
+        img_1 = cv2.imread(mask1)
+        img_2 = cv2.imread(mask2)
+        img_1 = np.minimum(img_1,img_2)
+
+        out_path = os.path.join(output_dir, base_name)
+        cv2.imwrite(out_path, img_1)
+
+
+def create_mask_clipseg(input_dir, output_dir, clipseg_mask_prompt, clipseg_exclude_prompt, clipseg_mask_threshold, mask_blur_size, mask_blur_size2):
+    from modules import devices
+
+    devices.torch_gc()
+
+    device = devices.get_optimal_device_name()
+
+    processor = AutoProcessor.from_pretrained("CIDAS/clipseg-rd64-refined")
+    model = CLIPSegForImageSegmentation.from_pretrained("CIDAS/clipseg-rd64-refined")
+    model.to(device)
+
+    imgs = glob.glob( os.path.join(input_dir, "*.png") )
+    texts = [x.strip() for x in clipseg_mask_prompt.split(',')]
+    exclude_texts = [x.strip() for x in clipseg_exclude_prompt.split(',')] if clipseg_exclude_prompt else None
+    
+    if exclude_texts:
+        all_texts = texts + exclude_texts
+    else:
+        all_texts = texts
+
+
+    for img_count,img in enumerate(imgs):
+        image = Image.open(img)
+        base_name = os.path.basename(img)
+
+        inputs = processor(text=all_texts, images=[image] * len(all_texts), padding="max_length", return_tensors="pt")
+        inputs = inputs.to(device)
+
+        with torch.no_grad(), devices.autocast():
+            outputs = model(**inputs)
+        
+        if len(all_texts) == 1:
+            preds = outputs.logits.unsqueeze(0)
+        else:
+            preds = outputs.logits
+
+        mask_img = None
+
+        for i in range(len(all_texts)):
+            x = torch.sigmoid(preds[i])
+            x = x.to('cpu').detach().numpy()
+
+#            x[x < clipseg_mask_threshold] = 0
+            x = x > clipseg_mask_threshold
+
+            if i < len(texts):
+                if mask_img is None:
+                    mask_img = x
+                else:
+                    mask_img = np.maximum(mask_img,x)
+            else:
+                mask_img[x > 0] = 0
+
+        mask_img = mask_img*255
+        mask_img = mask_img.astype(np.uint8)
+        
+        if mask_blur_size > 0:
+            mask_blur_size = mask_blur_size//2 * 2 + 1
+            mask_img = cv2.medianBlur(mask_img, mask_blur_size)
+
+        if mask_blur_size2 > 0:
+            mask_blur_size2 = mask_blur_size2//2 * 2 + 1
+            mask_img = cv2.GaussianBlur(mask_img, (mask_blur_size2, mask_blur_size2), 0)
+
+        mask_img = resize_img(mask_img, image.width, image.height)
+
+        mask_img = cv2.cvtColor(mask_img, cv2.COLOR_GRAY2RGB)
+        save_path = os.path.join(output_dir, base_name)
+        cv2.imwrite(save_path, mask_img)
+
+        print("{0} / {1}".format( img_count+1,len(imgs) ))
+    
+    devices.torch_gc()
+
+
+def create_mask_transparent_background(input_dir, output_dir, tb_use_fast_mode, tb_use_jit, st1_mask_threshold):
+    fast_str = " --fast" if tb_use_fast_mode else ""
+    jit_str = " --jit" if tb_use_jit else ""
+    venv = "venv"
+    if 'VIRTUAL_ENV' in os.environ:
+        venv = os.environ['VIRTUAL_ENV']
+    bin_path = os.path.join(venv, "Scripts")
+    bin_path = os.path.join(bin_path, "transparent-background")
+
+    if os.path.isfile(bin_path) or os.path.isfile(bin_path + ".exe"):
+        subprocess.call(bin_path + " --source " + input_dir + " --dest " + output_dir + " --type map" + fast_str + jit_str, shell=True)
+    else:
+        subprocess.call("transparent-background --source " + input_dir + " --dest " + output_dir + " --type map" + fast_str + jit_str, shell=True)
+
+    mask_imgs = glob.glob( os.path.join(output_dir, "*.png") )
+    
+    for m in mask_imgs:
+        img = cv2.imread(m)
+        img[img < int( 255 * st1_mask_threshold )] = 0
+        cv2.imwrite(m, img)
+
+    p = re.compile(r'([0-9]+)_[a-z]*\.png')
+
+    for mask in mask_imgs:
+        base_name = os.path.basename(mask)
+        m = p.fullmatch(base_name)
+        if m:
+            os.rename(mask, os.path.join(output_dir, m.group(1) + ".png"))
+
+def ebsynth_utility_stage1(dbg, project_args, frame_width, frame_height, st1_masking_method_index, st1_mask_threshold, tb_use_fast_mode, tb_use_jit, clipseg_mask_prompt, clipseg_exclude_prompt, clipseg_mask_threshold, clipseg_mask_blur_size, clipseg_mask_blur_size2, is_invert_mask):
+    dbg.print("stage1")
+    dbg.print("")
+
+    if st1_masking_method_index == 1 and (not clipseg_mask_prompt):
+        dbg.print("Error: clipseg_mask_prompt is Empty")
+        return
+
+    project_dir, original_movie_path, frame_path, frame_mask_path, _, _, _ = project_args
+
+    if is_invert_mask:
+        if os.path.isdir( frame_path ) and os.path.isdir( frame_mask_path ):
+            dbg.print("Skip as it appears that the frame and normal masks have already been generated.")
+            return
+
+    # remove_pngs_in_dir(frame_path)
+
+    if frame_mask_path:
+        remove_pngs_in_dir(frame_mask_path)
+    
+    if frame_mask_path:
+        os.makedirs(frame_mask_path, exist_ok=True)
+
+    if os.path.isdir( frame_path ):
+        dbg.print("Skip frame extraction")
+    else:
+        os.makedirs(frame_path, exist_ok=True)
+
+        png_path = os.path.join(frame_path , "%05d.png")
+        # ffmpeg.exe -ss 00:00:00  -y -i %1 -qscale 0 -f image2 -c:v png "%05d.png"
+        subprocess.call("ffmpeg -ss 00:00:00  -y -i " + original_movie_path + " -qscale 0 -f image2 -c:v png " + png_path, shell=True)
+
+        dbg.print("frame extracted")
+
+        frame_width = max(frame_width,-1)
+        frame_height = max(frame_height,-1)
+
+        if frame_width != -1 or frame_height != -1:
+            resize_all_img(frame_path, frame_width, frame_height)
+
+    if frame_mask_path:
+        if st1_masking_method_index == 0:
+            create_mask_transparent_background(frame_path, frame_mask_path, tb_use_fast_mode, tb_use_jit, st1_mask_threshold)
+        elif st1_masking_method_index == 1:
+            create_mask_clipseg(frame_path, frame_mask_path, clipseg_mask_prompt, clipseg_exclude_prompt, clipseg_mask_threshold, clipseg_mask_blur_size, clipseg_mask_blur_size2)
+        elif st1_masking_method_index == 2:
+            tb_tmp_path = os.path.join(project_dir , "tb_mask_tmp")
+            if not os.path.isdir( tb_tmp_path ):
+                os.makedirs(tb_tmp_path, exist_ok=True)
+                create_mask_transparent_background(frame_path, tb_tmp_path, tb_use_fast_mode, tb_use_jit, st1_mask_threshold)
+            create_mask_clipseg(frame_path, frame_mask_path, clipseg_mask_prompt, clipseg_exclude_prompt, clipseg_mask_threshold, clipseg_mask_blur_size, clipseg_mask_blur_size2)
+            create_and_mask(tb_tmp_path,frame_mask_path,frame_mask_path)
+
+
+        dbg.print("mask created")
+    
+    dbg.print("")
+    dbg.print("completed.")
+
+
+def ebsynth_utility_stage1_invert(dbg, frame_mask_path, inv_mask_path):
+    dbg.print("stage 1 create_invert_mask")
+    dbg.print("")
+
+    if not os.path.isdir( frame_mask_path ):
+        dbg.print( frame_mask_path + " not found")
+        dbg.print("Normal masks must be generated previously.")
+        dbg.print("Do stage 1 with [Ebsynth Utility] Tab -> [configuration] -> [etc]-> [Mask Mode] = Normal setting first")
+        return
+
+    os.makedirs(inv_mask_path, exist_ok=True)
+
+    mask_imgs = glob.glob( os.path.join(frame_mask_path, "*.png") )
+    
+    for m in mask_imgs:
+        img = cv2.imread(m)
+        inv = cv2.bitwise_not(img)
+
+        base_name = os.path.basename(m)
+        cv2.imwrite(os.path.join(inv_mask_path,base_name), inv)
+
+    dbg.print("")
+    dbg.print("completed.")

ebsynth_utility/stage2.py

@@ -0,0 +1,173 @@
+import cv2
+import os
+import glob
+import shutil
+import numpy as np
+import math
+
+#---------------------------------
+# Copied from PySceneDetect
+def mean_pixel_distance(left: np.ndarray, right: np.ndarray) -> float:
+    """Return the mean average distance in pixel values between `left` and `right`.
+    Both `left and `right` should be 2 dimensional 8-bit images of the same shape.
+    """
+    assert len(left.shape) == 2 and len(right.shape) == 2
+    assert left.shape == right.shape
+    num_pixels: float = float(left.shape[0] * left.shape[1])
+    return (np.sum(np.abs(left.astype(np.int32) - right.astype(np.int32))) / num_pixels)
+
+
+def estimated_kernel_size(frame_width: int, frame_height: int) -> int:
+    """Estimate kernel size based on video resolution."""
+    size: int = 4 + round(math.sqrt(frame_width * frame_height) / 192)
+    if size % 2 == 0:
+        size += 1
+    return size
+
+_kernel = None
+
+def _detect_edges(lum: np.ndarray) -> np.ndarray:
+    global _kernel
+    """Detect edges using the luma channel of a frame.
+    Arguments:
+        lum: 2D 8-bit image representing the luma channel of a frame.
+    Returns:
+        2D 8-bit image of the same size as the input, where pixels with values of 255
+        represent edges, and all other pixels are 0.
+    """
+    # Initialize kernel.
+    if _kernel is None:
+        kernel_size = estimated_kernel_size(lum.shape[1], lum.shape[0])
+        _kernel = np.ones((kernel_size, kernel_size), np.uint8)
+
+    # Estimate levels for thresholding.
+    sigma: float = 1.0 / 3.0
+    median = np.median(lum)
+    low = int(max(0, (1.0 - sigma) * median))
+    high = int(min(255, (1.0 + sigma) * median))
+
+    # Calculate edges using Canny algorithm, and reduce noise by dilating the edges.
+    # This increases edge overlap leading to improved robustness against noise and slow
+    # camera movement. Note that very large kernel sizes can negatively affect accuracy.
+    edges = cv2.Canny(lum, low, high)
+    return cv2.dilate(edges, _kernel)
+
+#---------------------------------
+
+def detect_edges(img_path, mask_path, is_invert_mask):
+    im = cv2.imread(img_path)
+    if mask_path:
+        mask = cv2.imread(mask_path)[:,:,0]
+        mask = mask[:, :, np.newaxis]
+        im = im * ( (mask == 0) if is_invert_mask else (mask > 0) )
+#        im = im * (mask/255)
+#        im = im.astype(np.uint8)
+#        cv2.imwrite( os.path.join( os.path.dirname(mask_path) , "tmp.png" ) , im)
+
+    hue, sat, lum = cv2.split(cv2.cvtColor( im , cv2.COLOR_BGR2HSV))
+    return _detect_edges(lum)
+
+def get_mask_path_of_img(img_path, mask_dir):
+    img_basename = os.path.basename(img_path)
+    mask_path = os.path.join( mask_dir , img_basename )
+    return mask_path if os.path.isfile( mask_path ) else None
+
+def analyze_key_frames(png_dir, mask_dir, th, min_gap, max_gap, add_last_frame, is_invert_mask):
+    keys = []
+    
+    frames = sorted(glob.glob( os.path.join(png_dir, "[0-9]*.png") ))
+    
+    key_frame = frames[0]
+    keys.append( int(os.path.splitext(os.path.basename(key_frame))[0]) )
+    key_edges = detect_edges( key_frame, get_mask_path_of_img( key_frame, mask_dir ), is_invert_mask )
+    gap = 0
+    
+    for frame in frames:
+        gap += 1
+        if gap < min_gap:
+            continue
+        
+        edges = detect_edges( frame, get_mask_path_of_img( frame, mask_dir ), is_invert_mask )
+        
+        delta = mean_pixel_distance( edges, key_edges )
+        
+        _th = th * (max_gap - gap)/max_gap
+        
+        if _th < delta:
+            basename_without_ext = os.path.splitext(os.path.basename(frame))[0]
+            keys.append( int(basename_without_ext) )
+            key_frame = frame
+            key_edges = edges
+            gap = 0
+    
+    if add_last_frame:
+        basename_without_ext = os.path.splitext(os.path.basename(frames[-1]))[0]
+        last_frame = int(basename_without_ext)
+        if not last_frame in keys:
+            keys.append( last_frame )
+
+    return keys
+
+def remove_pngs_in_dir(path):
+    if not os.path.isdir(path):
+        return
+    
+    pngs = glob.glob( os.path.join(path, "*.png") )
+    for png in pngs:
+        os.remove(png)
+
+def ebsynth_utility_stage2(dbg, project_args, key_min_gap, key_max_gap, key_th, key_add_last_frame, is_invert_mask):
+    dbg.print("stage2")
+    dbg.print("")
+
+    _, original_movie_path, frame_path, frame_mask_path, org_key_path, _, _ = project_args
+
+    remove_pngs_in_dir(org_key_path)
+    os.makedirs(org_key_path, exist_ok=True)
+
+    fps = 30
+    clip = cv2.VideoCapture(original_movie_path)
+    if clip:
+        fps = clip.get(cv2.CAP_PROP_FPS)
+        clip.release()
+
+    if key_min_gap == -1:
+        key_min_gap = int(10 * fps/30)
+    else:
+        key_min_gap = max(1, key_min_gap)
+        key_min_gap = int(key_min_gap * fps/30)
+        
+    if key_max_gap == -1:
+        key_max_gap = int(300 * fps/30)
+    else:
+        key_max_gap = max(10, key_max_gap)
+        key_max_gap = int(key_max_gap * fps/30)
+    
+    key_min_gap,key_max_gap = (key_min_gap,key_max_gap) if key_min_gap < key_max_gap else (key_max_gap,key_min_gap)
+    
+    dbg.print("fps: {}".format(fps))
+    dbg.print("key_min_gap: {}".format(key_min_gap))
+    dbg.print("key_max_gap: {}".format(key_max_gap))
+    dbg.print("key_th: {}".format(key_th))
+
+    keys = analyze_key_frames(frame_path, frame_mask_path, key_th, key_min_gap, key_max_gap, key_add_last_frame, is_invert_mask)
+
+    dbg.print("keys : " + str(keys))
+    
+    for k in keys:
+        filename = str(k).zfill(5) + ".png"
+        shutil.copy( os.path.join( frame_path , filename) , os.path.join(org_key_path, filename) )
+
+
+    dbg.print("")
+    dbg.print("Keyframes are output to [" + org_key_path + "]")
+    dbg.print("")
+    dbg.print("[Ebsynth Utility]->[configuration]->[stage 2]->[Threshold of delta frame edge]")
+    dbg.print("The smaller this value, the narrower the keyframe spacing, and if set to 0, the keyframes will be equally spaced at the value of [Minimum keyframe gap].")
+    dbg.print("")
+    dbg.print("If you do not like the selection, you can modify it manually.")
+    dbg.print("(Delete keyframe, or Add keyframe from ["+frame_path+"])")
+
+    dbg.print("")
+    dbg.print("completed.")
+

ebsynth_utility/stage3_5.py

@@ -0,0 +1,178 @@
+import cv2
+import os
+import glob
+import shutil
+import numpy as np
+from PIL import Image
+
+from color_matcher import ColorMatcher
+from color_matcher.normalizer import Normalizer
+
+def resize_img(img, w, h):
+    if img.shape[0] + img.shape[1] < h + w:
+        interpolation = interpolation=cv2.INTER_CUBIC
+    else:
+        interpolation = interpolation=cv2.INTER_AREA
+
+    return cv2.resize(img, (w, h), interpolation=interpolation)
+
+def get_pair_of_img(img_path, target_dir):
+    img_basename = os.path.basename(img_path)
+    target_path = os.path.join( target_dir , img_basename )
+    return target_path if os.path.isfile( target_path ) else None
+
+def remove_pngs_in_dir(path):
+    if not os.path.isdir(path):
+        return
+    
+    pngs = glob.glob( os.path.join(path, "*.png") )
+    for png in pngs:
+        os.remove(png)
+
+def get_pair_of_img(img, target_dir):
+    img_basename = os.path.basename(img)
+    
+    pair_path = os.path.join( target_dir , img_basename )
+    if os.path.isfile( pair_path ):
+        return pair_path
+    print("!!! pair of "+ img + " not in " + target_dir)
+    return ""
+
+def get_mask_array(mask_path):
+    if not mask_path:
+        return None
+    mask_array = np.asarray(Image.open( mask_path ))
+    if mask_array.ndim == 2:
+        mask_array = mask_array[:, :, np.newaxis]
+    mask_array = mask_array[:,:,:1]
+    mask_array = mask_array/255
+    return mask_array
+
+def color_match(imgs, ref_image, color_matcher_method, dst_path):
+    cm = ColorMatcher(method=color_matcher_method)
+
+    i = 0
+    total = len(imgs)
+
+    for fname in imgs:
+
+        img_src = Image.open(fname)
+        img_src = Normalizer(np.asarray(img_src)).type_norm()
+
+        img_src = cm.transfer(src=img_src, ref=ref_image, method=color_matcher_method)
+
+        img_src = Normalizer(img_src).uint8_norm()
+        Image.fromarray(img_src).save(os.path.join(dst_path, os.path.basename(fname)))
+
+        i += 1
+        print("{0}/{1}".format(i, total))
+
+    imgs = sorted( glob.glob( os.path.join(dst_path, "*.png") ) )
+
+
+def ebsynth_utility_stage3_5(dbg, project_args, color_matcher_method, st3_5_use_mask, st3_5_use_mask_ref, st3_5_use_mask_org, color_matcher_ref_type, color_matcher_ref_image):
+    dbg.print("stage3.5")
+    dbg.print("")
+
+    _, _, frame_path, frame_mask_path, org_key_path, img2img_key_path, _ = project_args
+
+    backup_path = os.path.join( os.path.join( img2img_key_path, "..") , "st3_5_backup_img2img_key")
+    backup_path = os.path.normpath(backup_path)
+    
+    if not os.path.isdir( backup_path ):
+        dbg.print("{0} not found -> create backup.".format(backup_path))
+        os.makedirs(backup_path, exist_ok=True)
+
+        imgs = glob.glob( os.path.join(img2img_key_path, "*.png") )
+
+        for img in imgs:
+            img_basename = os.path.basename(img)
+            pair_path = os.path.join( backup_path , img_basename )            
+            shutil.copy( img , pair_path)
+    
+    else:
+        dbg.print("{0} found -> Treat the images here as originals.".format(backup_path))
+    
+    org_imgs = sorted( glob.glob( os.path.join(backup_path, "*.png") ) )
+    head_of_keyframe = org_imgs[0]
+
+    # open ref img
+    ref_image = color_matcher_ref_image
+    if not ref_image:
+        dbg.print("color_matcher_ref_image not set")
+
+        if color_matcher_ref_type == 0:
+            #'original video frame'
+            dbg.print("select -> original video frame")
+            ref_image = Image.open( get_pair_of_img(head_of_keyframe, frame_path) )
+        else:
+            #'first frame of img2img result'
+            dbg.print("select -> first frame of img2img result")
+            ref_image = Image.open( get_pair_of_img(head_of_keyframe, backup_path) )
+
+        ref_image = np.asarray(ref_image)
+
+        if st3_5_use_mask_ref:
+            mask = get_pair_of_img(head_of_keyframe, frame_mask_path)
+            if mask:
+                mask_array = get_mask_array( mask )
+                ref_image = ref_image * mask_array
+                ref_image = ref_image.astype(np.uint8)
+
+    else:
+        dbg.print("select -> color_matcher_ref_image")
+        ref_image = np.asarray(ref_image)
+    
+
+    if color_matcher_method in ('mvgd', 'hm-mvgd-hm'):
+        sample_img = Image.open(head_of_keyframe)
+        ref_image = resize_img( ref_image, sample_img.width, sample_img.height )
+    
+    ref_image = Normalizer(ref_image).type_norm()
+    
+
+    if st3_5_use_mask_org:
+        tmp_path = os.path.join( os.path.join( img2img_key_path, "..") , "st3_5_tmp")
+        tmp_path = os.path.normpath(tmp_path)
+        dbg.print("create {0} for masked original image".format(tmp_path))
+
+        remove_pngs_in_dir(tmp_path)
+        os.makedirs(tmp_path, exist_ok=True)
+
+        for org_img in org_imgs:
+            image_basename = os.path.basename(org_img)
+
+            org_image = np.asarray(Image.open(org_img))
+
+            mask = get_pair_of_img(org_img, frame_mask_path)
+            if mask:
+                mask_array = get_mask_array( mask )
+                org_image = org_image * mask_array
+                org_image = org_image.astype(np.uint8)
+
+            Image.fromarray(org_image).save( os.path.join( tmp_path, image_basename ) )
+        
+        org_imgs = sorted( glob.glob( os.path.join(tmp_path, "*.png") ) )
+    
+
+    color_match(org_imgs, ref_image, color_matcher_method, img2img_key_path)
+
+
+    if st3_5_use_mask or st3_5_use_mask_org:
+        imgs = sorted( glob.glob( os.path.join(img2img_key_path, "*.png") ) )
+        for img in imgs:
+            mask = get_pair_of_img(img, frame_mask_path)
+            if mask:
+                mask_array = get_mask_array( mask )
+                bg = get_pair_of_img(img, frame_path)
+                bg_image = np.asarray(Image.open( bg ))
+                fg_image = np.asarray(Image.open( img ))
+
+                final_img = fg_image * mask_array + bg_image * (1-mask_array)
+                final_img = final_img.astype(np.uint8)
+
+                Image.fromarray(final_img).save(img)
+
+    dbg.print("")
+    dbg.print("completed.")
+

ebsynth_utility/stage5.py

@@ -0,0 +1,279 @@
+import cv2
+import re
+import os
+import glob
+import time
+
+from sys import byteorder
+import binascii
+import numpy as np
+
+SYNTHS_PER_PROJECT = 15
+
+def to_float_bytes(f):
+    if byteorder == 'little':
+        return np.array([ float(f) ], dtype='<f4').tobytes()
+    else:
+        return np.array([ float(f) ], dtype='>f4').tobytes()
+
+def path2framenum(path):
+    return int( os.path.splitext(os.path.basename( path ))[0] )
+
+def search_key_dir(key_dir):
+    frames = glob.glob( os.path.join(key_dir ,"[0-9]*.png"), recursive=False)
+    
+    frames = sorted(frames)
+    
+    basename = os.path.splitext(os.path.basename( frames[0] ))[0]
+    
+    key_list = [ path2framenum(key) for key in frames ]
+    
+    print("digits = " + str(len(basename)))
+    print("keys = " + str(key_list))
+    
+    return len(basename), key_list
+
+
+def search_video_dir(video_dir):
+    frames = glob.glob( os.path.join(video_dir, "[0-9]*.png"), recursive=False)
+    
+    frames = sorted(frames)
+    
+    first = path2framenum( frames[0] )
+    last = path2framenum( frames[-1] )
+    
+    return first, last
+
+def export_project( project, proj_filename ):
+    
+    proj_path = os.path.join( project["proj_dir"] , proj_filename + ".ebs")
+    
+    with open(proj_path, 'wb') as f:
+        # header
+        f.write( binascii.unhexlify('45') )
+        f.write( binascii.unhexlify('42') )
+        f.write( binascii.unhexlify('53') )
+        f.write( binascii.unhexlify('30') )
+        f.write( binascii.unhexlify('35') )
+        f.write( binascii.unhexlify('00') )
+        
+        # video
+        f.write( len( project["video_dir"] + project["file_name"]).to_bytes(4, byteorder) )
+        f.write( (project["video_dir"] + project["file_name"]).encode() )
+        
+        # mask
+        if project["mask_dir"]:
+            f.write( len( project["mask_dir"] + project["file_name"]).to_bytes(4, byteorder) )
+            f.write( (project["mask_dir"] + project["file_name"]).encode() )
+        else:
+            f.write( int(0).to_bytes(4, byteorder) )
+            
+        # key
+        f.write( len( project["key_dir"] + project["file_name"]).to_bytes(4, byteorder) )
+        f.write( (project["key_dir"] + project["file_name"]).encode() )
+        
+        # mask on flag
+        if project["mask_dir"]:
+            f.write( int(1).to_bytes(1, byteorder) )
+        else:
+            f.write( int(0).to_bytes(1, byteorder) )
+        
+        
+        # keyframe weight
+        f.write( to_float_bytes( project["key_weight"] ) )
+        
+        # video weight
+        f.write( to_float_bytes( project["video_weight"] ) )
+        
+        # mask weight
+        f.write( to_float_bytes( project["mask_weight"] ) )
+        
+        # mapping
+        f.write( to_float_bytes( project["adv_mapping"] ) )
+        
+        # de-flicker
+        f.write( to_float_bytes( project["adv_de-flicker"] ) )
+        
+        # diversity
+        f.write( to_float_bytes( project["adv_diversity"] ) )
+        
+        
+        # num of synths
+        f.write( len( project["synth_list"] ).to_bytes(4, byteorder) )
+        
+        # synth
+        for synth in project["synth_list"]:
+            # key frame
+            f.write( int( synth["key"] ).to_bytes(4, byteorder) )
+            # is start frame exist
+            f.write( int(1).to_bytes(1, byteorder) )
+            # is end frame exist
+            f.write( int(1).to_bytes(1, byteorder) )
+            # start frame
+            f.write( int( synth["prev_key"] ).to_bytes(4, byteorder) )
+            # end frame
+            f.write( int( synth["next_key"] ).to_bytes(4, byteorder) )
+            
+            # out path
+            path =  "out-" + str(synth["key"]).zfill( project["number_of_digits"] ) + project["file_name"]
+            f.write( len(path).to_bytes(4, byteorder) )
+            f.write( path.encode() )
+            
+        # ?
+        f.write( binascii.unhexlify('56') )
+        f.write( binascii.unhexlify('30') )
+        f.write( binascii.unhexlify('32') )
+        f.write( binascii.unhexlify('00') )
+        
+        # synthesis detail
+        f.write( int( project["adv_detail"] ).to_bytes(1, byteorder) )
+        
+        # padding
+        f.write( binascii.unhexlify('00') )
+        f.write( binascii.unhexlify('00') )
+        f.write( binascii.unhexlify('00') )
+        
+        # use gpu
+        f.write( int( project["adv_gpu"] ).to_bytes(1, byteorder) )
+        
+        # ?
+        f.write( binascii.unhexlify('00') )
+        f.write( binascii.unhexlify('00') )
+        f.write( binascii.unhexlify('F0') )
+        f.write( binascii.unhexlify('41') )
+        f.write( binascii.unhexlify('C0') )
+        f.write( binascii.unhexlify('02') )
+        f.write( binascii.unhexlify('00') )
+        f.write( binascii.unhexlify('00') )
+
+
+def rename_keys(key_dir):
+    imgs = glob.glob(os.path.join(key_dir, "*.png"), recursive=False)
+    
+    if not imgs:
+        print('no files in %s' % key_dir)
+        return
+    
+    p = re.compile(r'([0-9]+).*\.png')
+    
+    for img in imgs:
+        
+        filename = os.path.basename(img)
+        
+        m = p.fullmatch(filename)
+        
+        if m:
+            f = m.group(1) + ".png"
+            dirname = os.path.dirname(img)
+            os.rename(img, os.path.join(dirname, f))
+
+def ebsynth_utility_stage5(dbg, project_args, is_invert_mask):
+    dbg.print("stage5")
+    dbg.print("")
+    
+    project_dir, _, frame_path, frame_mask_path, _, img2img_key_path, img2img_upscale_key_path = project_args
+
+    if not os.path.isdir(project_dir):
+        dbg.print('project_dir : no such dir %s' % project_dir)
+        return
+    if not os.path.isdir(frame_path):
+        dbg.print('frame_path : no such dir %s' % frame_path)
+        return
+
+    no_upscale = False
+
+    if not os.path.isdir(img2img_upscale_key_path):
+        dbg.print('img2img_upscale_key_path : no such dir %s' % img2img_upscale_key_path)
+        if not os.path.isdir(img2img_key_path):
+            return
+        
+        sample_img2img_key = glob.glob( os.path.join(img2img_key_path , "*.png" ) )[0]
+        img_height1, img_width1, _ = cv2.imread(sample_img2img_key).shape
+        sample_frame = glob.glob( os.path.join(frame_path , "*.png" ) )[0]
+        img_height2, img_width2, _ = cv2.imread(sample_frame).shape
+
+        if img_height1 != img_height2 or img_width1 != img_width2:
+            return
+        
+        dbg.print('The size of frame and img2img_key matched. use %s instead' % img2img_key_path)
+        img2img_upscale_key_path = img2img_key_path
+        no_upscale = True
+
+    else:
+        rename_keys(img2img_upscale_key_path)
+    
+    number_of_digits, keys = search_key_dir( img2img_upscale_key_path )
+    
+    if number_of_digits == -1:
+        dbg.print('no key frame')
+        return
+    
+    first_frame, last_frame = search_video_dir( frame_path )
+    
+    ### add next key
+    synth_list = []
+    next_key = last_frame
+    
+    for key in keys[::-1]:
+        synth_list.append( { "next_key": next_key })
+        next_key = key
+        
+    synth_list = synth_list[::-1]
+    prev_key = first_frame
+    
+    ### add key / prev key
+    for i, key in enumerate(keys):
+        synth_list[i]["key"] = key
+        synth_list[i]["prev_key"] = prev_key
+        prev_key = key
+    
+    project = {
+        "proj_dir" : project_dir if is_invert_mask == False else os.path.join(project_dir, "inv"),
+        "file_name" : "/[" + "#" *  number_of_digits + "].png",
+        "number_of_digits" : number_of_digits,
+        
+        "key_dir" : "img2img_upscale_key" if no_upscale == False else "img2img_key",
+        "video_dir" : "video_frame" if is_invert_mask == False else "../video_frame",
+        "mask_dir" : "video_mask" if is_invert_mask == False else "inv_video_mask",
+        "key_weight" : 1.0,
+        "video_weight" : 4.0,
+        "mask_weight" : 1.0,
+        "adv_mapping" : 10.0,
+        "adv_de-flicker" : 1.0,
+        "adv_diversity" : 3500.0,
+        "adv_detail" : 1,   # high
+        "adv_gpu" : 1,      # use gpu
+    }
+
+    if not frame_mask_path:
+        # no mask
+        project["mask_dir"] = ""
+
+    proj_base_name = time.strftime("%Y%m%d-%H%M%S")
+    if is_invert_mask:
+        proj_base_name = "inv_" + proj_base_name
+
+    tmp=[]
+    proj_index = 0
+    for i, synth in enumerate(synth_list):
+        tmp.append(synth)
+        if (i % SYNTHS_PER_PROJECT == SYNTHS_PER_PROJECT-1):
+            project["synth_list"] = tmp
+            proj_file_name = proj_base_name + "_" + str(proj_index).zfill(5)
+            export_project( project, proj_file_name )
+            proj_index += 1
+            tmp = []
+            dbg.print("exported : " + proj_file_name + ".ebs" )
+    
+    if tmp:
+        project["synth_list"] = tmp
+        proj_file_name = proj_base_name + "_" + str(proj_index).zfill(5)
+        export_project( project, proj_file_name )
+        proj_index += 1
+        dbg.print("exported : " + proj_file_name + ".ebs" )
+    
+    dbg.print("")
+    dbg.print("completed.")
+
+
+

ebsynth_utility/stage7.py

@@ -0,0 +1,234 @@
+import os
+import re
+import subprocess
+import glob
+import shutil
+import time
+import cv2
+import numpy as np
+
+
+def clamp(n, smallest, largest):
+    return sorted([smallest, n, largest])[1]
+
+
+def create_movie_from_frames( dir, start, end, number_of_digits, fps, output_path, export_type):
+    def get_export_str(export_type):
+        if export_type == "mp4":
+            return " -vcodec libx264 -pix_fmt yuv420p "
+        elif export_type == "webm":
+#            return " -vcodec vp9 -crf 10 -b:v 0 "
+            return " -crf 40 -b:v 0 -threads 4 "
+        elif export_type == "gif":
+            return " "
+        elif export_type == "rawvideo":
+            return " -vcodec rawvideo -pix_fmt bgr24 "
+
+    vframes = end - start + 1
+    path = os.path.join(dir , '%0' + str(number_of_digits) + 'd.png')
+    
+    # ffmpeg -r 10 -start_number n -i snapshot_%03d.png -vframes 50 example.gif
+    subprocess.call("ffmpeg -framerate " + str(fps) + " -r " + str(fps) +
+                        " -start_number " + str(start) +
+                        " -i " + path + 
+                        " -vframes " + str( vframes ) +
+                        get_export_str(export_type) +
+                        output_path, shell=True)
+
+
+def search_out_dirs(proj_dir, blend_rate):
+    ### create out_dirs
+    p = re.compile(r'.*[\\\/]out\-([0-9]+)[\\\/]')
+
+    number_of_digits = -1
+    
+    out_dirs=[]
+    for d in glob.glob( os.path.join(proj_dir ,"out-*/"), recursive=False):
+        m = p.fullmatch(d)
+        if m:
+            if number_of_digits == -1:
+                number_of_digits = len(m.group(1))
+            out_dirs.append({ 'keyframe':int(m.group(1)), 'path':d })
+    
+    out_dirs = sorted(out_dirs, key=lambda x: x['keyframe'], reverse=True)
+    
+    print(number_of_digits)
+    
+    prev_key = -1
+    for out_d in out_dirs:
+        out_d['next_keyframe'] = prev_key
+        prev_key = out_d['keyframe']
+    
+    out_dirs = sorted(out_dirs, key=lambda x: x['keyframe'])
+    
+    
+    ### search start/end frame
+    prev_key = 0
+    for out_d in out_dirs:
+        imgs = sorted(glob.glob(  os.path.join( out_d['path'], '[0-9]'*number_of_digits + '.png') ))
+        
+        first_img = imgs[0]
+        print(first_img)
+        basename_without_ext = os.path.splitext(os.path.basename(first_img))[0]
+        blend_timing = (prev_key - out_d['keyframe'])*blend_rate + out_d['keyframe']
+        blend_timing = round(blend_timing)
+        start_frame = max( blend_timing, int(basename_without_ext) )
+        out_d['startframe'] = start_frame
+        
+        last_img = imgs[-1]
+        print(last_img)
+        basename_without_ext = os.path.splitext(os.path.basename(last_img))[0]
+        end_frame = min( out_d['next_keyframe'], int(basename_without_ext) )
+        if end_frame == -1:
+            end_frame = int(basename_without_ext)
+        out_d['endframe'] = end_frame
+        prev_key = out_d['keyframe']
+    
+    return number_of_digits, out_dirs
+
+def get_ext(export_type):
+    if export_type in ("mp4","webm","gif"):
+        return "." + export_type
+    else:
+        return ".avi"
+
+def trying_to_add_audio(original_movie_path, no_snd_movie_path, output_path, tmp_dir ):
+    if os.path.isfile(original_movie_path):
+        sound_path = os.path.join(tmp_dir , 'sound.mp4')
+        subprocess.call("ffmpeg -i " + original_movie_path + " -vn -acodec copy " + sound_path, shell=True)
+        
+        if os.path.isfile(sound_path):
+            # ffmpeg -i video.mp4 -i audio.wav -c:v copy -c:a aac -map 0:v:0 -map 1:a:0 output.mp4
+
+            subprocess.call("ffmpeg -i " + no_snd_movie_path + " -i " + sound_path + " -c:v copy -c:a aac -map 0:v:0 -map 1:a:0 " + output_path, shell=True)
+            return True
+    
+    return False
+
+def ebsynth_utility_stage7(dbg, project_args, blend_rate,export_type,is_invert_mask):
+    dbg.print("stage7")
+    dbg.print("")
+
+    project_dir, original_movie_path, _, _, _, _, _ = project_args
+
+    fps = 30
+    clip = cv2.VideoCapture(original_movie_path)
+    if clip:
+        fps = clip.get(cv2.CAP_PROP_FPS)
+        clip.release()
+    
+    blend_rate = clamp(blend_rate, 0.0, 1.0)
+
+    dbg.print("blend_rate: {}".format(blend_rate))
+    dbg.print("export_type: {}".format(export_type))
+    dbg.print("fps: {}".format(fps))
+    
+    if is_invert_mask:
+        project_dir = os.path.join( project_dir , "inv")
+
+    tmp_dir = os.path.join( project_dir , "crossfade_tmp")
+
+    
+    if os.path.isdir(tmp_dir):
+        shutil.rmtree(tmp_dir)
+    os.mkdir(tmp_dir)
+    
+    number_of_digits, out_dirs = search_out_dirs( project_dir, blend_rate )
+    
+    if number_of_digits == -1:
+        dbg.print('no out dir')
+        return
+    
+    ### create frame imgs
+    
+    start = out_dirs[0]['startframe']
+    end = out_dirs[-1]['endframe']
+    
+    cur_clip = 0
+    next_clip = cur_clip+1 if len(out_dirs) > cur_clip+1 else -1
+    
+    current_frame = 0
+    
+    print(str(start) + " -> " + str(end+1))
+    
+    black_img = np.zeros_like( cv2.imread( os.path.join(out_dirs[cur_clip]['path'], str(start).zfill(number_of_digits) + ".png") ) )
+    
+    for i in range(start, end+1):
+        
+        print(str(i) + " / " + str(end))
+
+        if next_clip == -1:
+            break
+        
+        if i in range( out_dirs[cur_clip]['startframe'], out_dirs[cur_clip]['endframe'] +1):
+            pass
+        elif i in range( out_dirs[next_clip]['startframe'], out_dirs[next_clip]['endframe'] +1):
+            cur_clip = next_clip
+            next_clip = cur_clip+1 if len(out_dirs) > cur_clip+1 else -1
+            if next_clip == -1:
+                break
+        else:
+            ### black
+            # front ... none
+            # back ... none
+            cv2.imwrite( os.path.join(tmp_dir, filename) , black_img)
+            current_frame = i
+            continue
+        
+        filename = str(i).zfill(number_of_digits) + ".png"
+        
+        # front ... cur_clip
+        # back ... next_clip or none
+        
+        if i in range( out_dirs[next_clip]['startframe'], out_dirs[next_clip]['endframe'] +1):
+            # front ... cur_clip
+            # back ... next_clip
+            img_f = cv2.imread( os.path.join(out_dirs[cur_clip]['path'] , filename) )
+            img_b = cv2.imread( os.path.join(out_dirs[next_clip]['path'] , filename) )
+            
+            back_rate = (i - out_dirs[next_clip]['startframe'])/ max( 1 , (out_dirs[cur_clip]['endframe'] - out_dirs[next_clip]['startframe']) )
+            
+            img = cv2.addWeighted(img_f, 1.0 - back_rate, img_b, back_rate, 0)
+            
+            cv2.imwrite( os.path.join(tmp_dir , filename) , img)
+        else:
+            # front ... cur_clip
+            # back ... none
+            filename = str(i).zfill(number_of_digits) + ".png"
+            shutil.copy( os.path.join(out_dirs[cur_clip]['path'] , filename) , os.path.join(tmp_dir , filename) )
+        
+        current_frame = i
+    
+    
+    start2 = current_frame+1
+    
+    print(str(start2) + " -> " + str(end+1))
+    
+    for i in range(start2, end+1):
+        filename = str(i).zfill(number_of_digits) + ".png"
+        shutil.copy( os.path.join(out_dirs[cur_clip]['path'] , filename) , os.path.join(tmp_dir , filename) )
+    
+    ### create movie
+    movie_base_name = time.strftime("%Y%m%d-%H%M%S")
+    if is_invert_mask:
+        movie_base_name = "inv_" + movie_base_name
+    
+    nosnd_path = os.path.join(project_dir , movie_base_name + get_ext(export_type))
+    
+    start = out_dirs[0]['startframe']
+    end = out_dirs[-1]['endframe']
+
+    create_movie_from_frames( tmp_dir, start, end, number_of_digits, fps, nosnd_path, export_type)
+
+    dbg.print("exported : " + nosnd_path)
+    
+    if export_type == "mp4":
+
+        with_snd_path = os.path.join(project_dir , movie_base_name + '_with_snd.mp4')
+
+        if trying_to_add_audio(original_movie_path, nosnd_path, with_snd_path, tmp_dir):
+            dbg.print("exported : " + with_snd_path)
+    
+    dbg.print("")
+    dbg.print("completed.")
+

ebsynth_utility/stage8.py

@@ -0,0 +1,146 @@
+import os
+import re
+import subprocess
+import glob
+import shutil
+import time
+import cv2
+import numpy as np
+import itertools
+from extensions.ebsynth_utility.stage7 import create_movie_from_frames, get_ext, trying_to_add_audio
+
+def clamp(n, smallest, largest):
+    return sorted([smallest, n, largest])[1]
+
+def resize_img(img, w, h):
+    if img.shape[0] + img.shape[1] < h + w:
+        interpolation = interpolation=cv2.INTER_CUBIC
+    else:
+        interpolation = interpolation=cv2.INTER_AREA
+
+    return cv2.resize(img, (w, h), interpolation=interpolation)
+
+def merge_bg_src(base_frame_dir, bg_dir, frame_mask_path, tmp_dir, bg_type, mask_blur_size, mask_threshold, fg_transparency):
+
+    base_frames = sorted(glob.glob( os.path.join(base_frame_dir, "[0-9]*.png"), recursive=False) )
+
+    bg_frames = sorted(glob.glob( os.path.join(bg_dir, "*.png"), recursive=False) )
+    
+    def bg_frame(total_frames):
+        bg_len = len(bg_frames)
+
+        if bg_type == "Loop":
+            itr = itertools.cycle(bg_frames)
+            while True:
+                yield next(itr)
+        else:
+            for i in range(total_frames):
+                yield bg_frames[ int(bg_len * (i/total_frames))]
+    
+    bg_itr = bg_frame(len(base_frames))
+
+    for base_frame in base_frames:
+        im = cv2.imread(base_frame)
+        bg = cv2.imread( next(bg_itr) )
+        bg = resize_img(bg, im.shape[1], im.shape[0] )
+
+        basename = os.path.basename(base_frame)
+        mask_path = os.path.join(frame_mask_path, basename)
+        mask = cv2.imread(mask_path)[:,:,0]
+
+        mask[mask < int( 255 * mask_threshold )] = 0
+
+        if mask_blur_size > 0:
+            mask_blur_size = mask_blur_size//2 * 2 + 1
+            mask = cv2.GaussianBlur(mask, (mask_blur_size, mask_blur_size), 0)
+        mask = mask[:, :, np.newaxis]
+
+        fore_rate = (mask/255) * (1 - fg_transparency)
+
+        im = im * fore_rate + bg * (1- fore_rate)
+        im = im.astype(np.uint8)
+        cv2.imwrite( os.path.join( tmp_dir , basename ) , im)
+
+def extract_frames(movie_path , output_dir, fps):
+    png_path = os.path.join(output_dir , "%05d.png")
+    # ffmpeg.exe -ss 00:00:00  -y -i %1 -qscale 0 -f image2 -c:v png "%05d.png"
+    subprocess.call("ffmpeg -ss 00:00:00  -y -i " + movie_path + " -vf fps=" + str( round(fps, 2)) + " -qscale 0 -f image2 -c:v png " + png_path, shell=True)
+
+def ebsynth_utility_stage8(dbg, project_args, bg_src, bg_type, mask_blur_size, mask_threshold, fg_transparency, export_type):
+    dbg.print("stage8")
+    dbg.print("")
+
+    if not bg_src:
+        dbg.print("Fill [configuration] -> [stage 8] -> [Background source]")
+        return
+
+    project_dir, original_movie_path, _, frame_mask_path, _, _, _ = project_args
+
+    fps = 30
+    clip = cv2.VideoCapture(original_movie_path)
+    if clip:
+        fps = clip.get(cv2.CAP_PROP_FPS)
+        clip.release()
+
+    dbg.print("bg_src: {}".format(bg_src))
+    dbg.print("bg_type: {}".format(bg_type))
+    dbg.print("mask_blur_size: {}".format(mask_blur_size))
+    dbg.print("export_type: {}".format(export_type))
+    dbg.print("fps: {}".format(fps))
+    
+    base_frame_dir = os.path.join( project_dir , "crossfade_tmp")
+    
+    if not os.path.isdir(base_frame_dir):
+        dbg.print(base_frame_dir + " base frame not found")
+        return
+    
+    tmp_dir = os.path.join( project_dir , "bg_merge_tmp")
+    if os.path.isdir(tmp_dir):
+        shutil.rmtree(tmp_dir)
+    os.mkdir(tmp_dir)
+
+    ### create frame imgs
+    if os.path.isfile(bg_src):
+        bg_ext = os.path.splitext(os.path.basename(bg_src))[1]
+        if bg_ext == ".mp4":
+            bg_tmp_dir = os.path.join( project_dir , "bg_extract_tmp")
+            if os.path.isdir(bg_tmp_dir):
+                shutil.rmtree(bg_tmp_dir)
+            os.mkdir(bg_tmp_dir)
+
+            extract_frames(bg_src, bg_tmp_dir, fps)
+
+            bg_src = bg_tmp_dir
+        else:
+            dbg.print(bg_src + " must be mp4 or directory")
+            return
+    elif not os.path.isdir(bg_src):
+        dbg.print(bg_src + " must be mp4 or directory")
+        return
+
+    merge_bg_src(base_frame_dir, bg_src, frame_mask_path, tmp_dir, bg_type, mask_blur_size, mask_threshold, fg_transparency)
+    
+    ### create movie
+    movie_base_name = time.strftime("%Y%m%d-%H%M%S")
+    movie_base_name = "merge_" + movie_base_name
+    
+    nosnd_path = os.path.join(project_dir , movie_base_name + get_ext(export_type))
+
+    merged_frames = sorted(glob.glob( os.path.join(tmp_dir, "[0-9]*.png"), recursive=False) )
+    start = int(os.path.splitext(os.path.basename(merged_frames[0]))[0])
+    end = int(os.path.splitext(os.path.basename(merged_frames[-1]))[0])
+
+    create_movie_from_frames(tmp_dir,start,end,5,fps,nosnd_path,export_type)
+
+    dbg.print("exported : " + nosnd_path)
+    
+    if export_type == "mp4":
+
+        with_snd_path = os.path.join(project_dir , movie_base_name + '_with_snd.mp4')
+
+        if trying_to_add_audio(original_movie_path, nosnd_path, with_snd_path, tmp_dir):
+            dbg.print("exported : " + with_snd_path)
+    
+    dbg.print("")
+    dbg.print("completed.")
+

ebsynth_utility/style.css

@@ -0,0 +1,39 @@
+#ebs_info_area {
+	border: black 2px solid;
+    border-radius: 5px;
+    font-size: 15px;
+    margin: 10px;
+    padding: 10px;
+}
+
+#ebs_configuration_tab1>div{
+    margin: 5px;
+    padding: 5px;
+}
+
+#ebs_configuration_tab2>div{
+    margin: 5px;
+    padding: 5px;
+}
+
+#ebs_configuration_tab3_5>div{
+    margin: 5px;
+    padding: 5px;
+}
+
+#ebs_configuration_tab7>div{
+    margin: 5px;
+    padding: 5px;
+}
+
+#ebs_configuration_tab8>div{
+    margin: 5px;
+    padding: 5px;
+}
+
+#ebs_configuration_tab_etc>div{
+    margin: 5px;
+    padding: 5px;
+}
+
+