Upload 77 files

.gitattributes

@@ -33,3 +33,6 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
 *.zip filter=lfs diff=lfs merge=lfs -text
 *.zst filter=lfs diff=lfs merge=lfs -text
 *tfevents* filter=lfs diff=lfs merge=lfs -text
+imgs/example/a_dcp.png filter=lfs diff=lfs merge=lfs -text
+imgs/example/b_dcp.png filter=lfs diff=lfs merge=lfs -text
+imgs/hand.gif filter=lfs diff=lfs merge=lfs -text

cores/__init__.py

@@ -0,0 +1 @@
+from .options import *

cores/add.py

@@ -0,0 +1,130 @@
+import os
+from queue import Queue
+from threading import Thread
+import time
+import numpy as np
+import cv2
+from models import runmodel
+from util import mosaic,util,ffmpeg,filt
+from util import image_processing as impro
+from .init import video_init
+
+
+'''
+---------------------Add Mosaic---------------------
+'''
+def addmosaic_img(opt,netS):
+    path = opt.media_path
+    print('Add Mosaic:',path)
+    img = impro.imread(path)
+    mask = runmodel.get_ROI_position(img,netS,opt)[0]
+    img = mosaic.addmosaic(img,mask,opt)
+    impro.imwrite(os.path.join(opt.result_dir,os.path.splitext(os.path.basename(path))[0]+'_add.jpg'),img)
+
+def get_roi_positions(opt,netS,imagepaths,savemask=True):
+    # resume
+    continue_flag = False
+    if os.path.isfile(os.path.join(opt.temp_dir,'step.json')):
+        step = util.loadjson(os.path.join(opt.temp_dir,'step.json'))
+        resume_frame = int(step['frame'])
+        if int(step['step'])>2:
+            mask_index = np.load(os.path.join(opt.temp_dir,'mask_index.npy'))
+            return mask_index
+        if int(step['step'])>=2 and resume_frame>0:
+            pre_positions = np.load(os.path.join(opt.temp_dir,'roi_positions.npy'))
+            continue_flag = True
+            imagepaths = imagepaths[resume_frame:]
+            
+    positions = []
+    t1 = time.time()
+    if not opt.no_preview:
+        cv2.namedWindow('mask', cv2.WINDOW_NORMAL)
+    print('Step:2/4 -- Find mosaic location')
+
+    img_read_pool = Queue(4)
+    def loader(imagepaths):
+        for imagepath in imagepaths:
+            img_origin = impro.imread(os.path.join(opt.temp_dir+'/video2image',imagepath))
+            img_read_pool.put(img_origin)
+    t = Thread(target=loader,args=(imagepaths,))
+    t.daemon = True
+    t.start()
+
+    for i,imagepath in enumerate(imagepaths,1):
+        img_origin = img_read_pool.get()
+        mask,x,y,size,area = runmodel.get_ROI_position(img_origin,netS,opt)
+        positions.append([x,y,area])  
+        if savemask:
+            t = Thread(target=cv2.imwrite,args=(os.path.join(opt.temp_dir+'/ROI_mask',imagepath), mask,))
+            t.start()
+        if i%1000==0:
+            save_positions = np.array(positions)
+            if continue_flag:
+                save_positions = np.concatenate((pre_positions,save_positions),axis=0)
+            np.save(os.path.join(opt.temp_dir,'roi_positions.npy'),save_positions)
+            step = {'step':2,'frame':i+resume_frame}
+            util.savejson(os.path.join(opt.temp_dir,'step.json'),step)
+
+        #preview result and print
+        if not opt.no_preview:
+            cv2.imshow('mask',mask)
+            cv2.waitKey(1) & 0xFF
+        t2 = time.time()
+        print('\r',str(i)+'/'+str(len(imagepaths)),util.get_bar(100*i/len(imagepaths),num=35),util.counttime(t1,t2,i,len(imagepaths)),end='')
+    
+    if not opt.no_preview:
+        cv2.destroyAllWindows()
+
+    print('\nOptimize ROI locations...')
+    if continue_flag:
+        positions = np.concatenate((pre_positions,positions),axis=0)
+    mask_index = filt.position_medfilt(np.array(positions), 7)
+    step = {'step':3,'frame':0}
+    util.savejson(os.path.join(opt.temp_dir,'step.json'),step)
+    np.save(os.path.join(opt.temp_dir,'roi_positions.npy'),positions)
+    np.save(os.path.join(opt.temp_dir,'mask_index.npy'),np.array(mask_index))
+
+    return mask_index
+
+def addmosaic_video(opt,netS):
+    path = opt.media_path
+    fps,imagepaths = video_init(opt,path)[:2]
+    length = len(imagepaths)
+    start_frame = int(imagepaths[0][7:13])
+    mask_index = get_roi_positions(opt,netS,imagepaths)[(start_frame-1):]
+
+    t1 = time.time()
+    if not opt.no_preview:
+        cv2.namedWindow('preview', cv2.WINDOW_NORMAL)
+
+    # add mosaic
+    print('Step:3/4 -- Add Mosaic:')
+    t1 = time.time()
+    # print(mask_index)
+    for i,imagepath in enumerate(imagepaths,1):
+        mask = impro.imread(os.path.join(opt.temp_dir+'/ROI_mask',imagepaths[np.clip(mask_index[i-1]-start_frame,0,1000000)]),'gray')
+        img = impro.imread(os.path.join(opt.temp_dir+'/video2image',imagepath))
+        if impro.mask_area(mask)>100: 
+            try:#Avoid unknown errors
+                img = mosaic.addmosaic(img, mask, opt)
+            except Exception as e:
+                   print('Warning:',e)
+        t = Thread(target=cv2.imwrite,args=(os.path.join(opt.temp_dir+'/addmosaic_image',imagepath),img))
+        t.start()
+        os.remove(os.path.join(opt.temp_dir+'/video2image',imagepath))
+        
+        #preview result and print
+        if not opt.no_preview:
+            cv2.imshow('preview',img)
+            cv2.waitKey(1) & 0xFF
+        t2 = time.time()
+        print('\r',str(i)+'/'+str(length),util.get_bar(100*i/length,num=35),util.counttime(t1,t2,i,length),end='')
+    
+    print()
+    if not opt.no_preview:
+        cv2.destroyAllWindows()
+    print('Step:4/4 -- Convert images to video')
+    ffmpeg.image2video( fps,
+                        opt.temp_dir+'/addmosaic_image/output_%06d.'+opt.tempimage_type,
+                        opt.temp_dir+'/voice_tmp.mp3',
+                         os.path.join(opt.result_dir,os.path.splitext(os.path.basename(path))[0]+'_add.mp4'))

cores/clean.py

@@ -0,0 +1,249 @@
+import os
+import time
+import numpy as np
+import cv2
+import torch
+from models import runmodel
+from util import data,util,ffmpeg,filt
+from util import image_processing as impro
+from .init import video_init
+from multiprocessing import Queue, Process
+from threading import Thread
+
+'''
+---------------------Clean Mosaic---------------------
+'''
+def get_mosaic_positions(opt,netM,imagepaths,savemask=True):
+    # resume
+    continue_flag = False
+    if os.path.isfile(os.path.join(opt.temp_dir,'step.json')):
+        step = util.loadjson(os.path.join(opt.temp_dir,'step.json'))
+        resume_frame = int(step['frame'])
+        if int(step['step'])>2:
+            pre_positions = np.load(os.path.join(opt.temp_dir,'mosaic_positions.npy'))
+            return pre_positions
+        if int(step['step'])>=2 and resume_frame>0:
+            pre_positions = np.load(os.path.join(opt.temp_dir,'mosaic_positions.npy'))
+            continue_flag = True
+            imagepaths = imagepaths[resume_frame:]
+            
+    positions = []
+    t1 = time.time()
+    if not opt.no_preview:
+        cv2.namedWindow('mosaic mask', cv2.WINDOW_NORMAL)
+    print('Step:2/4 -- Find mosaic location')
+
+    img_read_pool = Queue(4)
+    def loader(imagepaths):
+        for imagepath in imagepaths:
+            img_origin = impro.imread(os.path.join(opt.temp_dir+'/video2image',imagepath))
+            img_read_pool.put(img_origin)
+    t = Thread(target=loader,args=(imagepaths,))
+    t.setDaemon(True)
+    t.start()
+
+    for i,imagepath in enumerate(imagepaths,1):
+        img_origin = img_read_pool.get()
+        x,y,size,mask = runmodel.get_mosaic_position(img_origin,netM,opt)
+        positions.append([x,y,size])
+        if savemask:
+            t = Thread(target=cv2.imwrite,args=(os.path.join(opt.temp_dir+'/mosaic_mask',imagepath), mask,))
+            t.start()
+        if i%1000==0:
+            save_positions = np.array(positions)
+            if continue_flag:
+                save_positions = np.concatenate((pre_positions,save_positions),axis=0)
+            np.save(os.path.join(opt.temp_dir,'mosaic_positions.npy'),save_positions)
+            step = {'step':2,'frame':i+resume_frame}
+            util.savejson(os.path.join(opt.temp_dir,'step.json'),step)
+
+        #preview result and print
+        if not opt.no_preview:
+            cv2.imshow('mosaic mask',mask)
+            cv2.waitKey(1) & 0xFF
+        t2 = time.time()
+        print('\r',str(i)+'/'+str(len(imagepaths)),util.get_bar(100*i/len(imagepaths),num=35),util.counttime(t1,t2,i,len(imagepaths)),end='')
+    
+    if not opt.no_preview:
+        cv2.destroyAllWindows()
+    print('\nOptimize mosaic locations...')
+    positions =np.array(positions)
+    if continue_flag:
+        positions = np.concatenate((pre_positions,positions),axis=0)
+    for i in range(3):positions[:,i] = filt.medfilt(positions[:,i],opt.medfilt_num)
+    step = {'step':3,'frame':0}
+    util.savejson(os.path.join(opt.temp_dir,'step.json'),step)
+    np.save(os.path.join(opt.temp_dir,'mosaic_positions.npy'),positions)
+
+    return positions
+
+def cleanmosaic_img(opt,netG,netM):
+
+    path = opt.media_path
+    print('Clean Mosaic:',path)
+    img_origin = impro.imread(path)
+    x,y,size,mask = runmodel.get_mosaic_position(img_origin,netM,opt)
+    #cv2.imwrite('./mask/'+os.path.basename(path), mask)
+    img_result = img_origin.copy()
+    if size > 100 :
+        img_mosaic = img_origin[y-size:y+size,x-size:x+size]
+        if opt.traditional:
+            img_fake = runmodel.traditional_cleaner(img_mosaic,opt)
+        else:
+            img_fake = runmodel.run_pix2pix(img_mosaic,netG,opt)
+        img_result = impro.replace_mosaic(img_origin,img_fake,mask,x,y,size,opt.no_feather)
+    else:
+        print('Do not find mosaic')
+    impro.imwrite(os.path.join(opt.result_dir,os.path.splitext(os.path.basename(path))[0]+'_clean.jpg'),img_result)
+
+def cleanmosaic_img_server(opt,img_origin,netG,netM):
+    x,y,size,mask = runmodel.get_mosaic_position(img_origin,netM,opt)
+    img_result = img_origin.copy()
+    if size > 100 :
+        img_mosaic = img_origin[y-size:y+size,x-size:x+size]
+        if opt.traditional:
+            img_fake = runmodel.traditional_cleaner(img_mosaic,opt)
+        else:
+            img_fake = runmodel.run_pix2pix(img_mosaic,netG,opt)
+        img_result = impro.replace_mosaic(img_origin,img_fake,mask,x,y,size,opt.no_feather)
+    return img_result
+
+def cleanmosaic_video_byframe(opt,netG,netM):
+    path = opt.media_path
+    fps,imagepaths,height,width = video_init(opt,path)
+    start_frame = int(imagepaths[0][7:13])
+    positions = get_mosaic_positions(opt,netM,imagepaths,savemask=True)[(start_frame-1):]
+
+    t1 = time.time()
+    if not opt.no_preview:
+        cv2.namedWindow('clean', cv2.WINDOW_NORMAL)
+
+    # clean mosaic
+    print('Step:3/4 -- Clean Mosaic:')
+    length = len(imagepaths)
+    for i,imagepath in enumerate(imagepaths,0):
+        x,y,size = positions[i][0],positions[i][1],positions[i][2]
+        img_origin = impro.imread(os.path.join(opt.temp_dir+'/video2image',imagepath))
+        img_result = img_origin.copy()
+        if size > 100:
+            try:#Avoid unknown errors
+                img_mosaic = img_origin[y-size:y+size,x-size:x+size]
+                if opt.traditional:
+                    img_fake = runmodel.traditional_cleaner(img_mosaic,opt)
+                else:
+                    img_fake = runmodel.run_pix2pix(img_mosaic,netG,opt)
+                mask = cv2.imread(os.path.join(opt.temp_dir+'/mosaic_mask',imagepath),0)
+                img_result = impro.replace_mosaic(img_origin,img_fake,mask,x,y,size,opt.no_feather)
+            except Exception as e:
+                print('Warning:',e)
+        t = Thread(target=cv2.imwrite,args=(os.path.join(opt.temp_dir+'/replace_mosaic',imagepath), img_result,))
+        t.start()
+        os.remove(os.path.join(opt.temp_dir+'/video2image',imagepath))
+        
+        #preview result and print
+        if not opt.no_preview:
+            cv2.imshow('clean',img_result)
+            cv2.waitKey(1) & 0xFF
+        t2 = time.time()
+        print('\r',str(i+1)+'/'+str(length),util.get_bar(100*i/length,num=35),util.counttime(t1,t2,i+1,len(imagepaths)),end='')
+    print()
+    if not opt.no_preview:
+        cv2.destroyAllWindows()
+    print('Step:4/4 -- Convert images to video')
+    ffmpeg.image2video( fps,
+                opt.temp_dir+'/replace_mosaic/output_%06d.'+opt.tempimage_type,
+                opt.temp_dir+'/voice_tmp.mp3',
+                 os.path.join(opt.result_dir,os.path.splitext(os.path.basename(path))[0]+'_clean.mp4'))  
+
+def cleanmosaic_video_fusion(opt,netG,netM):
+    path = opt.media_path
+    N,T,S = 2,5,3
+    LEFT_FRAME = (N*S)
+    POOL_NUM = LEFT_FRAME*2+1
+    INPUT_SIZE = 256
+    FRAME_POS = np.linspace(0, (T-1)*S,T,dtype=np.int64)
+    img_pool = []
+    previous_frame = None
+    init_flag = True
+    
+    fps,imagepaths,height,width = video_init(opt,path)
+    start_frame = int(imagepaths[0][7:13])
+    positions = get_mosaic_positions(opt,netM,imagepaths,savemask=True)[(start_frame-1):]
+    t1 = time.time()
+    if not opt.no_preview:
+        cv2.namedWindow('clean', cv2.WINDOW_NORMAL)
+    
+    # clean mosaic
+    print('Step:3/4 -- Clean Mosaic:')
+    length = len(imagepaths)
+    write_pool = Queue(4)
+    show_pool = Queue(4)
+    def write_result():
+        while True:
+            save_ori,imagepath,img_origin,img_fake,x,y,size = write_pool.get()
+            if save_ori:
+                img_result = img_origin
+            else:
+                mask = cv2.imread(os.path.join(opt.temp_dir+'/mosaic_mask',imagepath),0)
+                img_result = impro.replace_mosaic(img_origin,img_fake,mask,x,y,size,opt.no_feather)
+            if not opt.no_preview:
+                show_pool.put(img_result.copy())
+            cv2.imwrite(os.path.join(opt.temp_dir+'/replace_mosaic',imagepath),img_result)
+            os.remove(os.path.join(opt.temp_dir+'/video2image',imagepath))
+    t = Thread(target=write_result,args=())
+    t.setDaemon(True)
+    t.start()
+
+    for i,imagepath in enumerate(imagepaths,0):
+        x,y,size = positions[i][0],positions[i][1],positions[i][2]
+        input_stream = []
+        # image read stream
+        if i==0 :# init
+            for j in range(POOL_NUM):
+                img_pool.append(impro.imread(os.path.join(opt.temp_dir+'/video2image',imagepaths[np.clip(i+j-LEFT_FRAME,0,len(imagepaths)-1)])))
+        else: # load next frame
+            img_pool.pop(0)
+            img_pool.append(impro.imread(os.path.join(opt.temp_dir+'/video2image',imagepaths[np.clip(i+LEFT_FRAME,0,len(imagepaths)-1)])))
+        img_origin = img_pool[LEFT_FRAME]
+
+        # preview result and print
+        if not opt.no_preview:
+            if show_pool.qsize()>3:   
+                cv2.imshow('clean',show_pool.get())
+                cv2.waitKey(1) & 0xFF
+
+        if size>50:
+            try:#Avoid unknown errors
+                for pos in FRAME_POS:
+                    input_stream.append(impro.resize(img_pool[pos][y-size:y+size,x-size:x+size], INPUT_SIZE,interpolation=cv2.INTER_CUBIC)[:,:,::-1])
+                if init_flag:
+                    init_flag = False
+                    previous_frame = input_stream[N]
+                    previous_frame = data.im2tensor(previous_frame,bgr2rgb=True,gpu_id=opt.gpu_id)
+                
+                input_stream = np.array(input_stream).reshape(1,T,INPUT_SIZE,INPUT_SIZE,3).transpose((0,4,1,2,3))
+                input_stream = data.to_tensor(data.normalize(input_stream),gpu_id=opt.gpu_id)
+                with torch.no_grad():
+                    unmosaic_pred = netG(input_stream,previous_frame)
+                img_fake = data.tensor2im(unmosaic_pred,rgb2bgr = True)
+                previous_frame = unmosaic_pred
+                write_pool.put([False,imagepath,img_origin.copy(),img_fake.copy(),x,y,size])
+            except Exception as e:
+                init_flag = True
+                print('Error:',e)
+        else:
+            write_pool.put([True,imagepath,img_origin.copy(),-1,-1,-1,-1])
+            init_flag = True
+        
+        t2 = time.time()
+        print('\r',str(i+1)+'/'+str(length),util.get_bar(100*i/length,num=35),util.counttime(t1,t2,i+1,len(imagepaths)),end='')
+    print()
+    write_pool.close()
+    show_pool.close()
+    if not opt.no_preview:
+        cv2.destroyAllWindows()
+    print('Step:4/4 -- Convert images to video')
+    ffmpeg.image2video( fps,
+                opt.temp_dir+'/replace_mosaic/output_%06d.'+opt.tempimage_type,
+                opt.temp_dir+'/voice_tmp.mp3',
+                 os.path.join(opt.result_dir,os.path.splitext(os.path.basename(path))[0]+'_clean.mp4')) 

cores/init.py

@@ -0,0 +1,31 @@
+import os
+from util import util,ffmpeg
+
+'''
+---------------------Video Init---------------------
+'''
+def video_init(opt,path):
+    fps,endtime,height,width = ffmpeg.get_video_infos(path)
+    if opt.fps !=0:
+        fps = opt.fps
+
+    # resume
+    if os.path.isfile(os.path.join(opt.temp_dir,'step.json')):
+        step = util.loadjson(os.path.join(opt.temp_dir,'step.json'))
+        if int(step['step'])>=1:
+            choose = input('There is an unfinished video. Continue it? [y/n] ')
+            if choose.lower() =='yes' or choose.lower() == 'y':
+                imagepaths = os.listdir(opt.temp_dir+'/video2image')
+                imagepaths.sort()
+                return fps,imagepaths,height,width
+    
+    print('Step:1/4 -- Convert video to images')
+    util.file_init(opt)
+    ffmpeg.video2voice(path,opt.temp_dir+'/voice_tmp.mp3',opt.start_time,opt.last_time)
+    ffmpeg.video2image(path,opt.temp_dir+'/video2image/output_%06d.'+opt.tempimage_type,fps,opt.start_time,opt.last_time)
+    imagepaths = os.listdir(opt.temp_dir+'/video2image')
+    imagepaths.sort()
+    step = {'step':2,'frame':0}
+    util.savejson(os.path.join(opt.temp_dir,'step.json'),step)
+
+    return fps,imagepaths,height,width

cores/options.py

@@ -0,0 +1,130 @@
+import argparse
+import os
+import sys
+
+
+class Options():
+    def __init__(self):
+        self.parser = argparse.ArgumentParser(formatter_class=argparse.ArgumentDefaultsHelpFormatter)
+        self.initialized = False
+
+    def initialize(self):
+
+        #base
+        self.parser.add_argument('--debug', action='store_true', help='if specified, start debug mode')
+        self.parser.add_argument('--gpu_id', type=str,default='0', help='if -1, use cpu')
+        self.parser.add_argument('--media_path', type=str, default='./imgs/ruoruo.jpg',help='your videos or images path')
+        self.parser.add_argument('-ss', '--start_time', type=str, default='00:00:00',help='start position of video, default is the beginning of video')
+        self.parser.add_argument('-t', '--last_time', type=str, default='00:00:00',help='duration of the video, default is the entire video')
+        self.parser.add_argument('--mode', type=str, default='auto',help='Program running mode. auto | add | clean | style')
+        self.parser.add_argument('--model_path', type=str, default='./pretrained_models/mosaic/add_face.pth',help='pretrained model path')
+        self.parser.add_argument('--result_dir', type=str, default='./result',help='output media will be saved here')
+        self.parser.add_argument('--temp_dir', type=str, default='./tmp', help='Temporary files will go here')
+        self.parser.add_argument('--tempimage_type', type=str, default='jpg',help='type of temp image, png | jpg, png is better but occupy more storage space')
+        self.parser.add_argument('--netG', type=str, default='auto',
+            help='select model to use for netG(Clean mosaic and Transfer style) -> auto | unet_128 | unet_256 | resnet_9blocks | HD | video')
+        self.parser.add_argument('--fps', type=int, default=0,help='read and output fps, if 0-> origin')
+        self.parser.add_argument('--no_preview', action='store_true', help='if specified,do not preview images when processing video. eg.(when run it on server)')
+        self.parser.add_argument('--output_size', type=int, default=0,help='size of output media, if 0 -> origin')
+        self.parser.add_argument('--mask_threshold', type=int, default=64,help='Mosaic detection threshold (0~255). The smaller is it, the more likely judged as a mosaic area.')
+
+        #AddMosaic
+        self.parser.add_argument('--mosaic_mod', type=str, default='squa_avg',help='type of mosaic -> squa_avg | squa_random | squa_avg_circle_edge | rect_avg | random')
+        self.parser.add_argument('--mosaic_size', type=int, default=0,help='mosaic size,if 0 auto size')
+        self.parser.add_argument('--mask_extend', type=int, default=10,help='extend mosaic area')
+        
+        #CleanMosaic     
+        self.parser.add_argument('--mosaic_position_model_path', type=str, default='auto',help='name of model use to find mosaic position')
+        self.parser.add_argument('--traditional', action='store_true', help='if specified, use traditional image processing methods to clean mosaic')
+        self.parser.add_argument('--tr_blur', type=int, default=10, help='ksize of blur when using traditional method, it will affect final quality')
+        self.parser.add_argument('--tr_down', type=int, default=10, help='downsample when using traditional method,it will affect final quality')
+        self.parser.add_argument('--no_feather', action='store_true', help='if specified, no edge feather and color correction, but run faster')
+        self.parser.add_argument('--all_mosaic_area', action='store_true', help='if specified, find all mosaic area, else only find the largest area')
+        self.parser.add_argument('--medfilt_num', type=int, default=11,help='medfilt window of mosaic movement in the video')
+        self.parser.add_argument('--ex_mult', type=str, default='auto',help='mosaic area expansion')
+        
+        #StyleTransfer
+        self.parser.add_argument('--preprocess', type=str, default='resize', help='resize and cropping of images at load time [ resize | resize_scale_width | edges | gray] or resize,edges(use comma to split)')
+        self.parser.add_argument('--edges', action='store_true', help='if specified, use edges to generate pictures,(input_nc = 1)')  
+        self.parser.add_argument('--canny', type=int, default=150,help='threshold of canny')
+        self.parser.add_argument('--only_edges', action='store_true', help='if specified, output media will be edges')
+
+        self.initialized = True
+
+
+    def getparse(self, test_flag = False):
+        if not self.initialized:
+            self.initialize()
+        self.opt = self.parser.parse_args()
+        
+        model_name = os.path.basename(self.opt.model_path)
+        self.opt.temp_dir = os.path.join(self.opt.temp_dir, 'DeepMosaics_temp')
+
+        if self.opt.gpu_id != '-1':
+            os.environ["CUDA_VISIBLE_DEVICES"] = str(self.opt.gpu_id)
+            import torch
+            if not torch.cuda.is_available():
+                self.opt.gpu_id = '-1'
+        # else:
+        #     self.opt.gpu_id = '-1'
+
+        if test_flag:
+            if not os.path.exists(self.opt.media_path):
+                print('Error: Media does not exist!')
+                input('Please press any key to exit.\n')
+                sys.exit(0)
+            if not os.path.exists(self.opt.model_path):
+                print('Error: Model does not exist!')
+                input('Please press any key to exit.\n')
+                sys.exit(0)
+
+            if self.opt.mode == 'auto':
+                if 'clean' in model_name or self.opt.traditional:
+                    self.opt.mode = 'clean'
+                elif 'add' in model_name:
+                    self.opt.mode = 'add'
+                elif 'style' in model_name or 'edges' in model_name:
+                    self.opt.mode = 'style'
+                else:
+                    print('Please check model_path!')
+                    input('Please press any key to exit.\n')
+                    sys.exit(0)
+
+            if self.opt.output_size == 0 and self.opt.mode == 'style':
+                self.opt.output_size = 512
+
+            if 'edges' in model_name or 'edges' in self.opt.preprocess:
+                self.opt.edges = True
+
+            if self.opt.netG == 'auto' and self.opt.mode =='clean':
+                if 'unet_128' in model_name:
+                    self.opt.netG = 'unet_128'
+                elif 'resnet_9blocks' in model_name:
+                    self.opt.netG = 'resnet_9blocks'
+                elif 'HD' in model_name and 'video' not in model_name:
+                    self.opt.netG = 'HD'
+                elif 'video' in model_name:
+                    self.opt.netG = 'video'
+                else:
+                    print('Type of Generator error!')
+                    input('Please press any key to exit.\n')
+                    sys.exit(0)
+
+            if self.opt.ex_mult == 'auto':
+                if 'face' in model_name:
+                    self.opt.ex_mult = 1.1
+                else:
+                    self.opt.ex_mult = 1.5
+            else:
+                self.opt.ex_mult = float(self.opt.ex_mult)
+
+            if self.opt.mosaic_position_model_path == 'auto' and self.opt.mode == 'clean':
+                _path = os.path.join(os.path.split(self.opt.model_path)[0],'mosaic_position.pth')
+                if os.path.isfile(_path):
+                    self.opt.mosaic_position_model_path = _path
+                else:
+                    input('Please check mosaic_position_model_path!')
+                    input('Please press any key to exit.\n')
+                    sys.exit(0)
+
+        return self.opt

cores/style.py

@@ -0,0 +1,50 @@
+import os
+import time
+import numpy as np
+import cv2
+from models import runmodel
+from util import mosaic,util,ffmpeg,filt
+from util import image_processing as impro
+from .init import video_init
+
+'''
+---------------------Style Transfer---------------------
+'''
+def styletransfer_img(opt,netG):
+    print('Style Transfer_img:',opt.media_path)
+    img = impro.imread(opt.media_path)
+    img = runmodel.run_styletransfer(opt, netG, img)
+    suffix = os.path.basename(opt.model_path).replace('.pth','').replace('style_','')
+    impro.imwrite(os.path.join(opt.result_dir,os.path.splitext(os.path.basename(opt.media_path))[0]+'_'+suffix+'.jpg'),img)
+
+def styletransfer_video(opt,netG):
+    path = opt.media_path
+    fps,imagepaths = video_init(opt,path)[:2]
+    print('Step:2/4 -- Transfer')
+    t1 = time.time()
+    if not opt.no_preview:
+        cv2.namedWindow('preview', cv2.WINDOW_NORMAL)
+    length = len(imagepaths)
+
+    for i,imagepath in enumerate(imagepaths,1):
+        img = impro.imread(os.path.join(opt.temp_dir+'/video2image',imagepath))
+        img = runmodel.run_styletransfer(opt, netG, img)
+        cv2.imwrite(os.path.join(opt.temp_dir+'/style_transfer',imagepath),img)
+        os.remove(os.path.join(opt.temp_dir+'/video2image',imagepath))
+        
+        #preview result and print
+        if not opt.no_preview:
+            cv2.imshow('preview',img)
+            cv2.waitKey(1) & 0xFF
+        t2 = time.time()
+        print('\r',str(i)+'/'+str(length),util.get_bar(100*i/length,num=35),util.counttime(t1,t2,i,len(imagepaths)),end='')
+    
+    print()
+    if not opt.no_preview:
+        cv2.destroyAllWindows()
+    suffix = os.path.basename(opt.model_path).replace('.pth','').replace('style_','')
+    print('Step:4/4 -- Convert images to video')
+    ffmpeg.image2video( fps,
+                opt.temp_dir+'/style_transfer/output_%06d.'+opt.tempimage_type,
+                opt.temp_dir+'/voice_tmp.mp3',
+                 os.path.join(opt.result_dir,os.path.splitext(os.path.basename(path))[0]+'_'+suffix+'.mp4')) 

cpp/CMakeLists.txt

@@ -0,0 +1,38 @@
+cmake_minimum_required(VERSION 3.0 FATAL_ERROR)
+set(CMAKE_CXX_STANDARD 14)
+
+project(DeepMosaics)
+set(LIBRARY_OUTPUT_PATH ${PROJECT_SOURCE_DIR}/lib) #链接库路径
+
+set(Torch_DIR /home/hypo/libtorch/share/cmake/Torch)
+find_package(Torch REQUIRED)
+
+set(OpenCV_DIR /home/hypo/opencv-4.4.0)
+find_package(OpenCV REQUIRED)
+
+# Add sub directories
+add_subdirectory(example)
+add_subdirectory(utils)
+
+# set_property(TARGET ${PROJECT_NAME} PROPERTY CXX_STANDARD 14)
+# cmake_minimum_required(VERSION 3.0 FATAL_ERROR)
+# project(main)
+# set(LIBRARY_OUTPUT_PATH ${PROJECT_SOURCE_DIR}/lib) #链接库路径
+
+# set(Torch_DIR /home/hypo/libtorch/share/cmake/Torch)
+# find_package(Torch REQUIRED)
+
+# set(OpenCV_DIR /home/hypo/opencv-4.4.0)
+# find_package(OpenCV REQUIRED)
+
+# # 查找当前目录下的所有源文件
+# # 并将名称保存到 DIR_SRCS 变量
+# # aux_source_directory(. DIR_SRCS)
+# add_subdirectory(utils)
+
+# add_executable(main main.cpp)
+# # target_link_libraries(main )
+# # include_directories( "${OpenCV_INCLUDE_DIRS}" )
+# target_link_libraries( main "${TORCH_LIBRARIES}" "${OpenCV_LIBS}" utils)
+
+# set_property(TARGET main PROPERTY CXX_STANDARD 14)

cpp/README.md

@@ -0,0 +1,3 @@
+### C++ version for DeepMosaics
+* I am learning c++ through this project...
+* It is under development...

cpp/example/CMakeLists.txt

@@ -0,0 +1,17 @@
+# project(example)
+# add_executable("${PROJECT_NAME}" deepmosaic.cpp)
+# target_link_libraries(  "${PROJECT_NAME}" 
+#                         "${TORCH_LIBRARIES}" 
+#                         "${OpenCV_LIBS}"
+#                         utils)
+
+file(GLOB_RECURSE srcs RELATIVE "${CMAKE_CURRENT_SOURCE_DIR}" "${CMAKE_CURRENT_SOURCE_DIR}/*.cpp")
+foreach(sourcefile IN LISTS srcs)
+    string( REPLACE ".cpp" "" binname ${sourcefile})
+    add_executable( ${binname} ${sourcefile} )
+    target_link_libraries(  ${binname} 
+                            "${TORCH_LIBRARIES}" 
+                            "${OpenCV_LIBS}"
+                            utils)
+    # set_property(TARGET ${binname} PROPERTY CXX_STANDARD 14)
+endforeach()

cpp/example/deepmosaic.cpp

@@ -0,0 +1,52 @@
+#include <stdio.h>
+#include <string.h>
+#include <unistd.h>
+#include <iostream>
+#include <list>
+#include <vector>
+
+#include <torch/script.h>
+#include <torch/torch.h>
+#include <opencv2/opencv.hpp>
+
+#include "data.hpp"
+#include "util.hpp"
+
+int main() {
+    std::string path = util::current_path();
+
+    std::string net_path = "../res/models/mosaic_position.pth";
+    std::string img_path = "../res/test_media/face/d.jpg";
+
+    cv::Mat img = cv::imread(img_path);
+    cv::resize(img, img, cv::Size(360, 360), 2);
+    // img.convertTo(img, CV_32F);
+    torch::Tensor img_tensor =
+        torch::from_blob(img.data, {1, img.rows, img.cols, 3}, torch::kByte);
+    img_tensor = img_tensor.permute({0, 3, 1, 2});
+    img_tensor = img_tensor.toType(torch::kFloat);
+    img_tensor = img_tensor.div(255);
+    std::cout << img_tensor.sizes() << "\n";
+
+    // end = clock();
+    // dur = (double)(end - start);
+    // printf("Use Time:%f\n", (dur / CLOCKS_PER_SEC));
+
+    // std::string net_path = "../res/models/mosaic_position.pt";
+    // torch::jit::script::Module net;
+    // try{
+    //     // if (!isfile(net_path)){
+    //     //     std::cerr<<"model does not exist\n";
+    //     // }
+
+    //     net = torch::jit::load(net_path);
+    // }
+    // catch(const std::exception& e){
+    //     std::cerr << "error loading the model\n";
+    //     return -1;
+    // }
+
+    // torch::Tensor example = torch::ones({1,3,360,360});
+    // torch::Tensor output = net.forward({example}).toTensor();
+    // std::cout<<"ok"<<std::endl;
+}

cpp/utils/CMakeLists.txt

@@ -0,0 +1,14 @@
+# Set the project name
+project (utils)
+aux_source_directory(./src DIR_LIB_SRCS)
+
+add_library(${PROJECT_NAME} SHARED ${DIR_LIB_SRCS})
+
+# 添加.h文件查找路径
+target_include_directories( ${PROJECT_NAME}
+    PUBLIC ${PROJECT_SOURCE_DIR}/include
+)
+
+# 添加掉用的第三方库
+target_link_libraries(  "${PROJECT_NAME}" 
+                        "${OpenCV_LIBS}")

cpp/utils/include/data.hpp

@@ -0,0 +1,10 @@
+#ifndef DATA_H
+#define DATA_H
+#include <opencv2/opencv.hpp>
+
+namespace data {
+void normalize(cv::Mat& matrix, double mean = 0.5, double std = 0.5);
+
+}  // namespace data
+
+#endif

cpp/utils/include/util.hpp

@@ -0,0 +1,26 @@
+#ifndef UTIL_H
+#define UTIL_H
+#include <iostream>
+#include <list>
+namespace util {
+
+class Timer {
+   private:
+    clock_t tstart, tend;
+
+   public:
+    void start();
+    void end();
+};
+
+// std::string path = util::current_path();
+std::string current_path();
+
+// std::string out = util::pathjoin({path, "b", "c"});
+std::string pathjoin(const std::list<std::string>& strs);
+
+bool isfile(const std::string& name);
+
+}  // namespace util
+
+#endif

cpp/utils/src/data.cpp

@@ -0,0 +1,10 @@
+#include "data.hpp"
+#include <opencv2/opencv.hpp>
+
+namespace data {
+void normalize(cv::Mat& matrix, double mean, double std) {
+    // matrix = (matrix / 255.0 - mean) / std;
+    matrix = matrix / (255.0 * std) - mean / std;
+}
+
+}  // namespace data

cpp/utils/src/util.cpp

@@ -0,0 +1,49 @@
+#include "util.hpp"
+#include <stdio.h>
+#include <sys/stat.h>
+#include <unistd.h>
+#include <iostream>
+#include <list>
+#include <vector>
+
+namespace util {
+
+void Timer::start() {
+    tstart = clock();
+}
+void Timer::end() {
+    tend = clock();
+    double dur;
+    dur = (double)(tend - tstart);
+    std::cout << "Cost Time:" << (dur / CLOCKS_PER_SEC) << "\n";
+}
+
+std::string current_path() {
+    char* buffer;
+    buffer = getcwd(NULL, 0);
+    return buffer;
+}
+
+std::string pathjoin(const std::list<std::string>& strs) {
+    std::string res = "";
+    int cnt = 0;
+    for (std::string s : strs) {
+        if (cnt == 0) {
+            res += s;
+        } else {
+            if (s[0] != '/') {
+                res += ("/" + s);
+            } else {
+                res += s;
+            }
+        }
+        cnt++;
+    }
+    return res;
+}
+
+bool isfile(const std::string& name) {
+    struct stat buffer;
+    return (stat(name.c_str(), &buffer) == 0);
+}
+}  // namespace util

deepmosaic.py

@@ -0,0 +1,99 @@
+import os
+import sys
+import traceback
+try:
+    from cores import Options,add,clean,style
+    from util import util
+    from models import loadmodel
+except Exception as e:
+    print(e)
+    input('Please press any key to exit.\n')
+    sys.exit(0)
+
+opt = Options().getparse(test_flag = True)
+if not os.path.isdir(opt.temp_dir):
+    util.file_init(opt)
+
+def main():
+    
+    if os.path.isdir(opt.media_path):
+        files = util.Traversal(opt.media_path)
+    else:
+        files = [opt.media_path]        
+    if opt.mode == 'add':
+        netS = loadmodel.bisenet(opt,'roi')
+        for file in files:
+            opt.media_path = file
+            if util.is_img(file):
+                add.addmosaic_img(opt,netS)
+            elif util.is_video(file):
+                add.addmosaic_video(opt,netS)
+                util.clean_tempfiles(opt, tmp_init = False)
+            else:
+                print('This type of file is not supported')
+            util.clean_tempfiles(opt, tmp_init = False)
+
+    elif opt.mode == 'clean':
+        netM = loadmodel.bisenet(opt,'mosaic')
+        if opt.traditional:
+            netG = None
+        elif opt.netG == 'video':
+            netG = loadmodel.video(opt)
+        else:
+            netG = loadmodel.pix2pix(opt)
+        
+        for file in files:
+            opt.media_path = file
+            if util.is_img(file):
+                clean.cleanmosaic_img(opt,netG,netM)
+            elif util.is_video(file):
+                if opt.netG == 'video' and not opt.traditional:            
+                    clean.cleanmosaic_video_fusion(opt,netG,netM)
+                else:
+                    clean.cleanmosaic_video_byframe(opt,netG,netM)
+                util.clean_tempfiles(opt, tmp_init = False)
+            else:
+                print('This type of file is not supported')
+
+    elif opt.mode == 'style':
+        netG = loadmodel.style(opt)
+        for file in files:
+            opt.media_path = file
+            if util.is_img(file):
+                style.styletransfer_img(opt,netG)
+            elif util.is_video(file):
+                style.styletransfer_video(opt,netG)
+                util.clean_tempfiles(opt, tmp_init = False)
+            else:
+                print('This type of file is not supported')
+
+    util.clean_tempfiles(opt, tmp_init = False)
+
+if __name__ == '__main__':
+    if opt.debug:
+        main()
+        sys.exit(0)
+    try:
+        main()
+        print('Finished!')
+    except Exception as ex:
+        print('--------------------ERROR--------------------')
+        print('--------------Environment--------------')
+        print('DeepMosaics: 0.5.1')
+        print('Python:',sys.version)
+        import torch
+        print('Pytorch:',torch.__version__)
+        import cv2
+        print('OpenCV:',cv2.__version__)
+        import platform
+        print('Platform:',platform.platform())
+
+        print('--------------BUG--------------')
+        ex_type, ex_val, ex_stack = sys.exc_info()
+        print('Error Type:',ex_type)
+        print(ex_val)
+        for stack in traceback.extract_tb(ex_stack):
+            print(stack)
+        input('Please press any key to exit.\n')
+        #util.clean_tempfiles(tmp_init = False)
+        sys.exit(0)

docs/Release_notes.txt

@@ -0,0 +1,43 @@
+DeepMosaics: 0.5.1
+Core building with:
+    Python: 3.7.3 (default, Apr 24 2019, 15:29:51) [MSC v.1915 64 bit (AMD64)]
+    Pytorch: 1.7.1
+    OpenCV: 4.1.2
+    Platform: Windows-10-10.0.19041-SP0
+    Driver Version: 461.40
+    CUDA:11.0
+GUI building with C#
+For more detail, please view on github: https://github.com/HypoX64/DeepMosaics
+
+Releases History
+  V0.5.1
+    Fix:
+      1.Fix Some BUGs when restore unfinished tasks.
+      2.Fix that audio and video are not synchronized when the video is too long.
+    New:
+      1.Speed up video processing by Asynchronous.
+  V0.5.0
+    1.New video model (Perform better)
+  V0.4.1
+    1.Allow unfinished tasks to be restored.
+    2.Clean cache during processing.
+    3.Support CUDA 11.0.
+  V0.4.0
+    1.Support GPU.
+    2.Preview images when processing video.
+    3.Choose start position of video.
+  V0.3.0
+    1. Support BiSeNet(Better recognition of mosaics).
+    2. New videoHD model.
+    3. Better feathering method.
+  V0.2.0
+    1. Add video model.
+    2. Now you can input chinese path
+    3. Support style transfer
+    4. Support fps limit
+  V0.1.2
+    1. Support pix2pixHD model
+  V0.1.1
+    1. Check path, can't input illegal path
+  V0.1.0
+    1. Initial release.

docs/exe_help.md

@@ -0,0 +1,112 @@
+## DeepMosaics.exe  Instructions
+**[[中文版]](./exe_help_CN.md)**
+This is a GUI version compiled in Windows.<br>
+Download this version and pre-trained model via [[Google Drive]](https://drive.google.com/open?id=1LTERcN33McoiztYEwBxMuRjjgxh4DEPs)  [[百度云,提取码1x0a]](https://pan.baidu.com/s/10rN3U3zd5TmfGpO_PEShqQ) <br>
+Video tutorial => [[youtube]](https://www.youtube.com/watch?v=1kEmYawJ_vk) [[bilibili]](https://www.bilibili.com/video/BV1QK4y1a7Av)<br>
+
+Attentions:<br>
+
+  - Require Windows_x86_64, Windows10 is better.<br>
+  - Different pre-trained models are suitable for different effects.<br>
+  - Run time depends on computer performance.<br>
+  - If output video cannot be played, you can try with [potplayer](https://daumpotplayer.com/download/).<br>
+  - GUI version update slower than source.<br>
+
+### How to install
+#### CPU version
+* 1.Download and install Microsoft Visual C++
+  https://aka.ms/vs/16/release/vc_redist.x64.exe
+#### GPU version
+Only suppport NVidia GPU above gtx1060(Driver:above 460 & CUDA:11.0)
+* 1.Download and install Microsoft Visual C++
+  https://aka.ms/vs/16/release/vc_redist.x64.exe
+* 2.Update your gpu drive to 460(or above)
+  https://www.nvidia.com/en-us/geforce/drivers/
+* 3.Download and install CUDA 11.0:
+  https://developer.nvidia.com/cuda-toolkit-archive
+
+You can also download them on BaiduNetdisk
+https://pan.baidu.com/s/10rN3U3zd5TmfGpO_PEShqQ
+Password: 1x0a
+
+### How to use
+* step 1: Choose image or video.
+* step 2: Choose model(Different pre-trained models are suitable for different effects)
+* step 3: Run program and wait.
+* step 4: Cheek reult in './result'.
+
+### Introduction to pre-trained models
+* Mosaic
+
+|               Name               |                         Description                         |
+| :------------------------------: | :---------------------------------------------------------: |
+|           add_face.pth           |          Add mosaic to all faces in images/videos.          |
+|        clean_face_HD.pth         | Clean mosaic to all faces in images/video.<br>(RAM > 8GB).  |
+|         add_youknow.pth          |      Add mosaic to ... in images/videos.      |
+| clean_youknow_resnet_9blocks.pth |     Clean mosaic to ... in images/videos.     |
+|     clean_youknow_video.pth      |        Clean mosaic to ... in videos. It is better for processing video mosaics         |
+
+
+*  Style Transfer
+
+|          Name           |                        Description                        |
+| :---------------------: | :-------------------------------------------------------: |
+| style_apple2orange.pth  | Convert apples to oranges. |
+| style_orange2apple.pth  | Convert oranges to apples |
+| style_summer2winter.pth |     Convert summer to winter.     |
+| style_winter2summer.pth | Convert winter to summer. |
+|    style_cezanne.pth    |            Convert photos/video to Paul Cézanne style.            |
+|     style_monet.pth     | Convert photos/video to Claude Monet style. |
+|     style_ukiyoe.pth     | Convert photos/video to Ukiyoe style. |
+|     style_vangogh.pth     | Convert photos/video to Van Gogh style. |
+### Annotation
+![image](../imgs/GUI_Instructions.jpg)<br>
+* 1. Choose image or video.
+* 2. Choose model(Different pre-trained models are suitable for different effects).
+* 3. Program running mode.   (auto | add | clean | style)
+* 4. Use GPU to run deep learning model. (The current version does not support gpu, if you need to use gpu please run source).
+* 5. Limit the fps of the output video(0->original fps).
+* 6. More options.
+* 7. More options can be input.
+* 8. Run program.
+* 9. Open help file.
+* 10. Sponsor our project.
+* 11. Version information.
+* 12. Open the URL on github.
+
+### Introduction to options
+If you need more effects,  use '--option your-parameters' to enter what you need.
+* Base
+
+|    Option    |                Description                 |                 Default                 |
+| :----------: | :----------------------------------------: | :-------------------------------------: |
+|  --gpu_id   |           if -1, do not use gpu            |                    0                    |
+| --media_path |         your videos or images path         |            ./imgs/ruoruo.jpg            |
+|    --mode    | program running mode(auto/clean/add/style) |                 'auto'                  |
+| --model_path |           pretrained model path            | ./pretrained_models/mosaic/add_face.pth |
+| --result_dir |      output media will be saved here       |                ./result                 |
+|    --fps     |     read and output fps, if 0-> origin     |                    0                    |
+
+* AddMosaic
+
+|      Option      |                         Description                          | Default  |
+| :--------------: | :----------------------------------------------------------: | :------: |
+|   --mosaic_mod   | type of mosaic -> squa_avg/ squa_random/ squa_avg_circle_edge/ rect_avg/random | squa_avg |
+|  --mosaic_size   |                mosaic size,if 0 -> auto size                 |    0     |
+|  --mask_extend   |                      extend mosaic area                      |    10    |
+| --mask_threshold |         threshold of recognize mosaic position 0~255         |    64    |
+
+* CleanMosaic
+
+|    Option     |                         Description                          | Default |
+| :-----------: | :----------------------------------------------------------: | :-----: |
+| --traditional | if specified, use traditional image processing methods to clean mosaic |         |
+|   --tr_blur   | ksize of blur when using traditional method, it will affect final quality |   10    |
+|   --tr_down   | downsample when using traditional method,it will affect final quality |   10    |
+| --medfilt_num |        medfilt window of mosaic movement in the video        |   11    |
+
+* Style Transfer
+
+|    Option     |             Description              | Default |
+| :-----------: | :----------------------------------: | :-----: |
+| --output_size | size of output media, if 0 -> origin |   512   |

docs/exe_help_CN.md

@@ -0,0 +1,114 @@
+## DeepMosaics.exe  使用说明
+下载程序以及预训练模型 [[Google Drive]](https://drive.google.com/open?id=1LTERcN33McoiztYEwBxMuRjjgxh4DEPs)  [[百度云,提取码1x0a]](https://pan.baidu.com/s/10rN3U3zd5TmfGpO_PEShqQ) <br>
+[视频教程](https://www.bilibili.com/video/BV1QK4y1a7Av)<br>
+
+注意事项:<br>
+
+
+  - 程序的运行要求在64位Windows操作系统,我们仅在Windows10运行过,其他版本暂未经过测试<br>
+  - 请根据需求选择合适的预训练模型进行测试<br>
+  - 运行时间取决于电脑性能,对于视频文件,我们建议使用GPU运行<br>
+  - 如果输出的视频无法播放,这边建议您尝试[potplayer](https://daumpotplayer.com/download/).<br>
+  - 相比于源码,该版本的更新将会延后.
+
+### 如何安装
+#### CPU version
+* 1.下载安装 Microsoft Visual C++
+  https://aka.ms/vs/16/release/vc_redist.x64.exe
+#### GPU version
+仅支持gtx1060及以上的NVidia显卡(要求460版本以上的驱动以及11.0版本的CUDA, 注意只能是11.0)
+* 1.Download and install Microsoft Visual C++
+  https://aka.ms/vs/16/release/vc_redist.x64.exe
+* 2.Update your gpu drive to 460(or above)
+  https://www.nvidia.com/en-us/geforce/drivers/
+* 3.Download and install CUDA 11.0:
+  https://developer.nvidia.com/cuda-toolkit-archive
+
+当然这些也能在百度云上下载
+https://pan.baidu.com/s/10rN3U3zd5TmfGpO_PEShqQ
+提取码: 1x0a
+
+### 如何使用
+
+* step 1: 选择需要处理的图片或视频
+* step 2: 选择预训练模型(不同的预训练模型有不同的效果)
+* step 3: 运行程序并等待
+* step 4: 查看结果(储存在result文件夹下)
+
+## 预训练模型说明
+当前的预训练模型分为两类——添加/移除马赛克以及风格转换.
+
+* 马赛克
+
+|              文件名              |                     描述                      |
+| :------------------------------: | :-------------------------------------------: |
+|           add_face.pth           |           对图片或视频中的脸部打码            |
+|        clean_face_HD.pth         | 对图片或视频中的脸部去码<br>(要求内存 > 8GB). |
+|         add_youknow.pth          |        对图片或视频中的...内容打码         |
+| clean_youknow_resnet_9blocks.pth |        对图片或视频中的...内容去码         |
+|     clean_youknow_video.pth      |           对视频中的...内容去码,推荐使用带有'video'的模型去除视频中的马赛克            |
+
+
+* 风格转换
+
+|          文件名        |                        描述                        |
+| :---------------------: | :-------------------------------------------------------: |
+| style_apple2orange.pth  | 苹果变橙子 |
+| style_orange2apple.pth  | 橙子变苹果 |
+| style_summer2winter.pth |     夏天变冬天     |
+| style_winter2summer.pth | 冬天变夏天 |
+|    style_cezanne.pth    |            转化为Paul Cézanne 的绘画风格            |
+|     style_monet.pth     | 转化为Claude Monet的绘画风格 |
+|     style_ukiyoe.pth     | 转化为Ukiyoe的绘画风格 |
+|     style_vangogh.pth     | 转化为Van Gogh的绘画风格 |
+
+### GUI界面注释
+![image](../imgs/GUI_Instructions.jpg)<br>
+* 1. 选择需要处理的图片或视频
+* 2. 选择预训练模型
+* 3. 程序运行模式  (auto | add | clean | style)
+* 4. 使用GPU (该版本目前不支持GPU,若需要使用GPU请使用源码运行).
+* 5. 限制输出的视频帧率(0->原始帧率).
+* 6. 更多的选项以及参数
+* 7. 自行输入更多参数，详见下文
+* 8. 运行
+* 9. 打开帮助文件
+* 10. 支持我们
+* 11. 版本信息
+* 12. 打开项目的github页面
+
+### 参数说明
+如果需要更多的效果,  请按照 '--option your-parameters' 输入所需要的参数
+* 基本
+
+|    选项    |        描述         |                 默认                 |
+| :----------: | :------------------------: | :-------------------------------------: |
+|  --gpu_id   |   if -1, do not use gpu    |                    0                    |
+| --media_path | 需要处理的视频或者照片的路径 |            ./imgs/ruoruo.jpg            |
+|    --mode    |    运行模式(auto/clean/add/style)    |                 'auto'                  |
+| --model_path |   预训练模型的路径    | ./pretrained_models/mosaic/add_face.pth |
+| --result_dir | 保存路径 |                 ./result          |
+|    --fps    |    限制视频输出的fps，0则为默认    |                 0                  |
+*  添加马赛克
+
+|    选项    |        描述       |                 默认                 |
+| :----------: | :------------------------: | :-------------------------------------: |
+| --mosaic_mod | 马赛克类型 -> squa_avg/ squa_random/ squa_avg_circle_edge/ rect_avg/random |                    squa_avg                    |
+| --mosaic_size | 马赛克大小，０则为自动 |            0            |
+|    --mask_extend    |    拓展马赛克区域    |         10  |
+| --mask_threshold | 马赛克区域识别阈值 0~255,越小越容易被判断为马赛克区域 | 64 |
+
+* 去除马赛克
+
+|    选项    |        描述       |                 默认                 |
+| :----------: | :------------------------: | :-------------------------------------: |
+| --traditional | 如果输入这个参数则使用传统方法清除马赛克 |                                        |
+| --tr_blur | 传统方法模糊尺寸 |            10            |
+|    --tr_down    |    传统方法下采样尺寸    |         10  |
+| --medfilt_num | medfilt window of mosaic movement in the video | 11 |
+
+* 风格转换
+
+|    选项    |        描述       |                 默认                 |
+| :----------: | :------------------------: | :-------------------------------------: |
+| --output_size | 输出媒体的尺寸，如果是０则为原始尺寸 |512|

docs/options_introduction.md

@@ -0,0 +1,41 @@
+## Introduction to options
+If you need more effects,  use '--option your-parameters' to enter what you need.
+
+### Base
+
+|    Option    |        Description         |                 Default                 |
+| :----------: | :------------------------: | :-------------------------------------: |
+|  --gpu_id   |   if -1, do not use gpu    |                    0                    |
+| --media_path | your videos or images path |            ./imgs/ruoruo.jpg            |
+| --start_time | start position of video, default is the beginning of video | '00:00:00' |
+| --last_time | limit the duration of the video, default is the entire video | '00:00:00' |
+|    --mode    |    program running mode(auto/clean/add/style)    |                 'auto'                  |
+| --model_path |   pretrained model path    | ./pretrained_models/mosaic/add_face.pth |
+| --result_dir |  output media will be saved here|                 ./result          |
+| --temp_dir | Temporary files will go here | ./tmp |
+|    --fps    |    read and output fps, if 0-> origin    |                 0                  |
+| --no_preview | if specified,do not preview images when processing video. eg.(when run it on server) | Flase |
+
+### AddMosaic
+
+|    Option    |        Description         |                 Default                 |
+| :----------: | :------------------------: | :-------------------------------------: |
+| --mosaic_mod | type of mosaic -> squa_avg/ squa_random/ squa_avg_circle_edge/ rect_avg/random |                    squa_avg                    |
+| --mosaic_size | mosaic size,if 0 -> auto size |            0            |
+|    --mask_extend    |    extend mosaic area    |         10  |
+| --mask_threshold | threshold of recognize mosaic position 0~255 | 64 |
+
+### CleanMosaic
+
+|    Option    |        Description         |                 Default                 |
+| :----------: | :------------------------: | :-------------------------------------: |
+| --traditional | if specified, use traditional image processing methods to clean mosaic |                                        |
+| --tr_blur | ksize of blur when using traditional method, it will affect final quality |            10            |
+|    --tr_down    |    downsample when using traditional method,it will affect final quality    |         10  |
+| --medfilt_num | medfilt window of mosaic movement in the video | 11 |
+
+### Style Transfer
+
+|    Option    |        Description         |                 Default                 |
+| :----------: | :------------------------: | :-------------------------------------: |
+| --output_size | size of output media, if 0 -> origin |512|

docs/options_introduction_CN.md

@@ -0,0 +1,41 @@
+## 参数说明
+如果需要更多的效果,  请按照 '--option your-parameters' 输入所需要的参数
+
+### 基本
+
+|    选项    |        描述         |                 默认                 |
+| :----------: | :------------------------: | :-------------------------------------: |
+|  --gpu_id   |   if -1, do not use gpu    |                    0                    |
+| --media_path | 需要处理的视频或者照片的路径 |            ./imgs/ruoruo.jpg            |
+| --start_time | 视频开始处理的位置，默认从头开始 | '00:00:00' |
+| --last_time | 处理的视频时长，默认是整个视频 | '00:00:00' |
+|    --mode    |    运行模式(auto/clean/add/style)    |                 'auto'                  |
+| --model_path |   预训练模型的路径    | ./pretrained_models/mosaic/add_face.pth |
+| --result_dir | 保存路径 |                 ./result          |
+| --temp_dir | 临时文件存储目录 | ./tmp |
+|    --fps    |    限制视频输出的fps，0则为默认    |                 0                  |
+| --no_preview | 如果输入，将不会在处理视频时播放实时预览．比如当你在服务器运行的时候 | Flase |
+
+### 添加马赛克
+
+|    选项    |        描述       |                 默认                 |
+| :----------: | :------------------------: | :-------------------------------------: |
+| --mosaic_mod | 马赛克类型 -> squa_avg/ squa_random/ squa_avg_circle_edge/ rect_avg/random |                    squa_avg                    |
+| --mosaic_size | 马赛克大小，０则为自动 |            0            |
+|    --mask_extend    |    拓展马赛克区域    |         10  |
+| --mask_threshold | 马赛克区域识别阈值 0~255 | 64 |
+
+### 去除马赛克
+
+|    选项    |        描述       |                 默认                 |
+| :----------: | :------------------------: | :-------------------------------------: |
+| --traditional | 如果输入这个参数则使用传统方法清除马赛克 |                                        |
+| --tr_blur | 传统方法模糊尺寸 |            10            |
+|    --tr_down    |    传统方法下采样尺寸    |         10  |
+| --medfilt_num | medfilt window of mosaic movement in the video | 11 |
+
+### 风格转换
+
+|    选项    |        描述       |                 默认                 |
+| :----------: | :------------------------: | :-------------------------------------: |
+| --output_size | 输出媒体的尺寸，如果是０则为原始尺寸 |512|

docs/pre-trained_models_introduction.md

@@ -0,0 +1,28 @@
+## Introduction to pre-trained models
+The current pre-trained models are divided into two categories(Add/Clean mosaic and StyleTransfer).
+Download  pre-trained model via [[Google Drive]](https://drive.google.com/open?id=1LTERcN33McoiztYEwBxMuRjjgxh4DEPs)  [[百度云,提取码1x0a]](https://pan.baidu.com/s/10rN3U3zd5TmfGpO_PEShqQ) <br>
+
+### Mosaic
+
+|               Name               |                       Description                       |
+| :------------------------------: | :-----------------------------------------------------: |
+|           add_face.pth           |         Add mosaic to  faces in images/videos.          |
+|        clean_face_HD.pth         | Clean mosaic to  faces in images/video.<br>(RAM > 8GB). |
+|         add_youknow.pth          |          Add mosaic to  ... in images/videos.           |
+| clean_youknow_resnet_9blocks.pth |         Clean mosaic to  ... in images/videos.          |
+|     clean_youknow_video.pth      |             Clean mosaic to  ... in videos. It is better for processing video mosaics              |
+
+
+### Style Transfer
+
+|          Name           |                        Description                        |
+| :---------------------: | :-------------------------------------------------------: |
+| style_apple2orange.pth  | Convert apples to oranges. |
+| style_orange2apple.pth  | Convert oranges to apples |
+| style_summer2winter.pth |     Convert summer to winter.     |
+| style_winter2summer.pth | Convert winter to summer. |
+|    style_cezanne.pth    |            Convert photos/video to Paul Cézanne style.            |
+|     style_monet.pth     | Convert photos/video to Claude Monet style. |
+|     style_ukiyoe.pth     | Convert photos/video to Ukiyoe style. |
+|     style_vangogh.pth     | Convert photos/video to Van Gogh style. |
+

docs/pre-trained_models_introduction_CN.md

@@ -0,0 +1,28 @@
+## 预训练模型说明
+当前的预训练模型分为两类——添加/移除马赛克以及风格转换.
+可以通过以下方式下载预训练模型 [[Google Drive]](https://drive.google.com/open?id=1LTERcN33McoiztYEwBxMuRjjgxh4DEPs)  [[百度云,提取码1x0a]](https://pan.baidu.com/s/10rN3U3zd5TmfGpO_PEShqQ) <br>
+
+### 马赛克
+
+|              文件名              |                     描述                      |
+| :------------------------------: | :-------------------------------------------: |
+|           add_face.pth           |           对图片或视频中的脸部打码            |
+|        clean_face_HD.pth         | 对图片或视频中的脸部去码<br>(要求内存 > 8GB). |
+|         add_youknow.pth          |          对图片或视频中的...内容打码          |
+| clean_youknow_resnet_9blocks.pth |          对图片或视频中的...内容去码          |
+|     clean_youknow_video.pth      |             对视频中的...内容去码,推荐使用带有'video'的模型去除视频中的马赛克               |
+
+
+### 风格转换
+
+|          文件名        |                        描述                        |
+| :---------------------: | :-------------------------------------------------------: |
+| style_apple2orange.pth  | 苹果变橙子 |
+| style_orange2apple.pth  | 橙子变苹果 |
+| style_summer2winter.pth |     夏天变冬天     |
+| style_winter2summer.pth | 冬天变夏天 |
+|    style_cezanne.pth    |            转化为Paul Cézanne 的绘画风格            |
+|     style_monet.pth     | 转化为Claude Monet的绘画风格 |
+|     style_ukiyoe.pth     | 转化为Ukiyoe的绘画风格 |
+|     style_vangogh.pth     | 转化为Van Gogh的绘画风格 |
+

docs/training_with_your_own_dataset.md

@@ -0,0 +1,77 @@
+# Training with your own dataset
+Training with your own dataset requires a GPU with 6G memory (above GTX1060).<br>
+We will make "face" as an example. If you don't have any picture, you can download [CelebA](http://mmlab.ie.cuhk.edu.hk/projects/CelebA.html) or [WIDER](http://mmlab.ie.cuhk.edu.hk/projects/WIDERFace/WiderFace_Results.html).
+
+## Getting Started
+#### Prerequisites
+  - Linux, Mac OS, Windows
+  - Python 3.6+
+  - [ffmpeg 3.4.6](http://ffmpeg.org/)
+  - [Pytorch 1.0+](https://pytorch.org/)
+  - NVIDIA GPU(with more than 6G memory) + CUDA CuDNN<br>
+#### Dependencies
+This code depends on opencv-python, torchvision, matplotlib, tensorboardX, scikit-image available via conda install.
+```bash
+# or
+pip install -r requirements.txt
+```
+#### Clone this repo
+```bash
+git clone https://github.com/HypoX64/DeepMosaics
+cd DeepMosaics
+```
+## Make training datasets
+```bash
+cd make_datasets
+```
+### Add mosaic dataset
+Please generate mask from images which you want to add mosaic(number of images should be above 1000). And then put the images in ```face/origin_image```, and masks in ```face/mask```.<br>
+* You can use ```draw_mask.py```to generate them.
+```bash
+python draw_mask.py --datadir 'dir for your pictures' --savedir ../datasets/draw/face
+#Press the left mouse button to draw the mask .  Press 'S' to save mask, 'A' to reduce  brush size, 'D' to increase brush size, 'W' to cancel drawing.
+```
+* If you want to get images from videos, you can use ```get_image_from_video.py```
+```bash
+python get_image_from_video.py --datadir 'dir for your videos' --savedir ../datasets/video2image --fps 1
+```
+### Clean mosaic dataset
+We provide several methods for generating clean mosaic datasets. However, for better effect, we recommend train a addmosaic model in a small data  first and use it to automatically generate datasets in a big data. (recommend: Method 2(for image) & Method 4(for video))
+* Method 1: Use drawn mask to make pix2pix(HD) datasets   (Require``` origin_image``` and ```mask```)
+```bash
+python make_pix2pix_dataset.py --datadir ../datasets/draw/face --hd --outsize 512 --fold 1 --name face --savedir ../datasets/pix2pix/face --mod drawn --minsize 128 --square
+```
+* Method 2: Use addmosaic model to make pix2pix(HD) datasets (Require addmosaic pre-trained model)
+```bash
+python make_pix2pix_dataset.py --datadir 'dir for your pictures' --hd --outsize 512 --fold 1 --name face --savedir ../datasets/pix2pix/face --mod network --model_path ../pretrained_models/mosaic/add_face.pth --minsize 128 --square --mask_threshold 128
+```
+* Method 3: Use Irregular Masks to make pix2pix(HD) datasets (Require [Irregular Masks](https://nv-adlr.github.io/publication/partialconv-inpainting))
+```bash
+python make_pix2pix_dataset.py --datadir 'dir for your pictures' --hd --outsize 512 --fold 1 --name face --savedir ../datasets/pix2pix/face --mod irregular --irrholedir ../datasets/Irregular_Holes_mask --square
+```
+* Method 4: Use addmosaic model to make video datasets (Require addmosaic pre-trained model. This is better for processing video mosaics)
+```bash
+python make_video_dataset.py --model_path ../pretrained_models/mosaic/add_face.pth --gpu_id 0 --datadir 'dir for your videos' --savedir ../datasets/video/face
+```
+## Training
+### Add
+```bash
+cd train/add
+python train.py --gpu_id 0 --dataset ../../datasets/draw/face --savename face --loadsize 512 --finesize 360 --batchsize 16
+```
+### Clean
+* For image datasets (generated by ```make_pix2pix_dataset.py```)
+We use [pix2pix](https://github.com/junyanz/pytorch-CycleGAN-and-pix2pix) or [pix2pixHD](https://github.com/NVIDIA/pix2pixHD) to train model. We just take pix2pixHD as an example.
+```bash
+git clone https://github.com/NVIDIA/pix2pixHD
+cd pix2pixHD
+pip install dominate
+python train.py --name face --resize_or_crop resize_and_crop --loadSize 563 --fineSize 512 --label_nc 0 --no_instance --dataroot ../datasets/pix2pix/face
+```
+* For video datasets (generated by ```make_video_dataset.py```)
+```bash
+cd train/clean
+python train.py --dataset ../../datasets/video/face --savename face --n_blocks 4 --lambda_GAN 0.01 --loadsize 286 --finesize 256 --batchsize 16 --n_layers_D 2 --num_D 3 --n_epoch 200 --gpu_id 4,5,6,7 --load_thread 16
+```
+## Testing
+Put saved network to ```./pretrained_models/mosaic/``` and rename it as ```add_face.pth``` or ```clean_face_HD.pth``` or ```clean_face_video_HD.pth```and then run ```deepmosaic.py --model_path ./pretrained_models/mosaic/your_model_name```

make_datasets/cut_video.py

@@ -0,0 +1,32 @@
+import os
+import numpy as np
+import cv2
+import random
+import csv
+
+import sys
+sys.path.append("..")
+from util import util,ffmpeg
+from util import image_processing as impro
+
+files = util.Traversal('/media/hypo/Media/download')
+videos = util.is_videos(files)
+
+
+
+useable_videos = []
+video_dict = {}
+reader = csv.reader(open('./csv/video_used_time.csv'))
+for line in reader:
+    useable_videos.append(line[0])
+    video_dict[line[0]]=line[1:]
+
+in_cnt = 0
+out_cnt = 1
+for video in videos:
+    if os.path.basename(video) in useable_videos:
+
+        for i in range(len(video_dict[os.path.basename(video)])):
+            ffmpeg.cut_video(video, video_dict[os.path.basename(video)][i], '00:00:05', './video/'+'%04d'%out_cnt+'.mp4')
+            out_cnt +=1
+        in_cnt += 1          

make_datasets/draw_mask.py

@@ -0,0 +1,96 @@
+import cv2
+import numpy as np
+import datetime
+import os
+import random
+
+import sys
+sys.path.append("..")
+from cores import Options
+from util import util
+from util import image_processing as impro
+
+
+opt = Options()
+opt.parser.add_argument('--datadir',type=str,default=' ', help='your images dir')
+opt.parser.add_argument('--savedir',type=str,default='../datasets/draw/face', help='')
+opt = opt.getparse()
+
+mask_savedir = os.path.join(opt.savedir,'mask')
+img_savedir = os.path.join(opt.savedir,'origin_image')
+util.makedirs(mask_savedir)
+util.makedirs(img_savedir)
+
+filepaths = util.Traversal(opt.datadir)
+filepaths = util.is_imgs(filepaths)
+random.shuffle(filepaths)
+print('find image:',len(filepaths))
+
+# mouse callback function
+drawing = False # true if mouse is pressed
+ix,iy = -1,-1
+brushsize = 20
+def draw_circle(event,x,y,flags,param):
+    global ix,iy,drawing,brushsize
+
+    if event == cv2.EVENT_LBUTTONDOWN:
+        drawing = True
+        ix,iy = x,y
+
+    elif event == cv2.EVENT_MOUSEMOVE:
+        if drawing == True:
+            cv2.circle(img_drawn,(x,y),brushsize,(0,255,0),-1)
+
+    elif event == cv2.EVENT_LBUTTONUP:
+        drawing = False
+        cv2.circle(img_drawn,(x,y),brushsize,(0,255,0),-1)
+
+def makemask(img_drawn):
+    # starttime = datetime.datetime.now()
+    mask = np.zeros(img_drawn.shape, np.uint8)
+    for row in range(img_drawn.shape[0]):
+        for col in range(img_drawn.shape[1]):
+            # if (img_drawn[row,col,:] == [0,255,0]).all(): #too slow
+            if img_drawn[row,col,0] == 0:
+                if img_drawn[row,col,1] == 255:
+                    if img_drawn[row,col,2] == 0:
+                        mask[row,col,:] = [255,255,255]
+    return mask
+
+cnt = 0
+for file in filepaths:
+    try:
+        cnt += 1
+        img = impro.imread(file,loadsize=512)
+        img_drawn = img.copy()
+        cv2.namedWindow('image')
+        cv2.setMouseCallback('image',draw_circle) #MouseCallback
+        while(1):
+
+            cv2.imshow('image',img_drawn)
+            k = cv2.waitKey(1) & 0xFF
+            if k == ord('s'):
+                
+                img_drawn = impro.resize(img_drawn,256)
+                mask = makemask(img_drawn)
+                cv2.imwrite(os.path.join(mask_savedir,os.path.splitext(os.path.basename(file))[0]+'.png'),mask)
+                cv2.imwrite(os.path.join(img_savedir,os.path.basename(file)),img)   
+                print('Saved:',os.path.join(mask_savedir,os.path.splitext(os.path.basename(file))[0]+'.png'),mask)
+                # cv2.destroyAllWindows()
+                print('remain:',len(filepaths)-cnt)
+                brushsize = 20
+                break
+            elif k == ord('a'):
+                brushsize -= 5
+                if brushsize<5:
+                    brushsize = 5
+                print('brushsize:',brushsize)
+            elif k == ord('d'):
+                brushsize += 5
+                print('brushsize:',brushsize)
+            elif k == ord('w'):
+                print('remain:',len(filepaths)-cnt)
+                break
+    except Exception as e:
+        print(file,e)
+