#!/usr/local/bin/python import glob import filecmp import hashlib import os.path import platform import copy import sys import shutil import operator import json import datetime import math import logging as log from os.path import normpath import os import inspect StartTime = datetime.datetime.now() EndTime = datetime.datetime.now() # --- Strings def ToUpper(string): return string.upper() def InsertStringInFilePath(file_path, string, file_extension): ext_len = len(file_extension) + 1 file_path_p = file_path[:-ext_len] + '-' + string + file_path[-ext_len:] return file_path_p # --- Strings def LoadClassFromModule(module_name, class_name): __module = __import__(module_name) if __module: __class = getattr(__module, class_name) if __class: return __class else: log.error("No class " + class_name + " found inside module " + module_name) return None else: log.error("No module " + module_name + " found") return None def LoadFunctionFromModule(module_name, function_name): __module = __import__(module_name) if __module: __function = getattr(__module, function_name) if __function: return __function else: log.error("No function " + function_name + " found inside module " + module_name) return None else: log.error("No module " + module_name + " found") return None def Log2(x): return math.log(x, 2) def FormatFloatNumber(num, i_format = '{:0>5d}', d_format = '{:.2f}' ): __str = i_format.format(int(num)) __str = __str + d_format.format(num - int(num))[1:] return __str def GetPlatform(): return sys.platform def GetOsName(): return os.name def GetPlatformSystem(): return platform.system() def IsOsLinux(): return GetPlatformSystem() == "Linux" def IsOsDos(): return GetPlatformSystem() == "Windows" def IsOsMac(): return GetPlatformSystem() == "Darwin" def GetNullFile(): if IsOsDos(): return "NUL" else: return "/dev/null" def CompareFiles(file1, file2): return filecmp.cmp(file1, file2) def AppendFile(file, text): return open(file, "a").write(text + "\n") def WriteFile(file, text): return open(file, "w").write(text) def ReadFile(file): AssertFileExists(file) return open(file, "r").read() def GenerateUniqueFileName(filename): from hashlib import md5 from time import localtime return "%s_%s" % (md5(str(localtime())).hexdigest(), filename) def FindInHash(key, hash, default): if key not in hash.keys(): hash.update( { key : copy.deepcopy(default) } ) return hash[key] def GetJsonString(object): return json.dumps(object) def WriteJsonFile(filepath, object): log.info("Writing JSON file " + filepath) json.dump(object, open(filepath, 'w'), indent = 4, ) def ReadJsonString(string): log.info("Reading JSON string: " + string) return json.loads(string) def ReadJsonFile(filepath): log.info("Reading JSON file " + filepath) return ReadJsonString(ReadFile(filepath)) def LoadPythonFile(file_path): AssertFileExists(file_path) if globals is None: globals = {} globals.update({ "__file__": file_path, "__name__": "__main__", }) with open(file_path, 'rb') as file_path: exec(compile(file_path.read(), file_path, 'exec'), globals, locals) def Assert(condition, message = None): if condition == False: if message == None: log.fatal("Assert Failed") else: log.fatal("Assert Failed: " + message) def GetAbsoluteFilePaths( dir_path, file_names ): __file_path_list = [] for __file_name in file_names: __file_path = dir_path + "/" + __file_name AssertFileExists(__file_path) __file_path_list.append(__file_path) return __file_path_list def GetFileExt(x): x = os.path.splitext(x)[1] # Get extension return x[1:] def GetFilePathWithoutExt(x): x = os.path.splitext(x)[0] # Remove extension return x def GetFileBaseName(x): x = os.path.basename( x ) # Remove complete path and only get file name x = os.path.splitext(x)[0] # Remove extension return x def GetDirBaseName(x): return os.path.basename( os.path.dirname(x) ) # Remove complete path and only get file name def GetFileName(x): x = os.path.basename( x ) # Remove complete path and only get file name return x def AssertFileExists(file, msg=None): if msg is None: msg = "File " + str(file) + " does not exist" Assert(FileExists(file), msg) def AssertFileDoesNotExist(file): Assert(FileExists(file) == False, "File " + str(file) + " does not exist" ) def AssertDirExists(dir): Assert(DirExists(dir), "Directory " + str(dir) + " does not exist" ) def GetFileSize(file): return os.path.getsize(file) def IsEmptyFile(file): if GetFileSize(file) == 0: return True else: return False def RemoveFileIfEmpty(file): if IsEmptyFile(file): RemoveFile(file) def RemoveFile(file): if FileExists(file): os.remove(file) log.info( 'RemoveFile ' + file ) def CopyFileSafe(old_file, new_file): AssertFileExists(old_file) if FileExists(new_file): log.fatal("File " + new_file + " already exists. Cannot overwrite it.") shutil.copy(old_file, new_file) log.info( 'CopyFileSafe ' + old_file + " to " + new_file ) def CopyFile(old_file, new_file): AssertFileExists(old_file) shutil.copy(old_file, new_file) log.info( 'CopyFile ' + old_file + " to " + new_file ) def MoveFile(old_file, new_file): AssertFileExists(old_file) log.info( 'MoveFile ' + old_file + " to " + new_file ) os.rename( old_file, new_file) def MoveDir(old_dir, new_dir): AssertDirExists(old_dir) log.info( 'MoveDir ' + old_dir + " to " + new_dir ) os.rename( old_dir, new_dir) def RemoveDir(dir): if os.path.isdir(dir) == True: log.info( 'RemoveDir ' + dir ) shutil.rmtree(dir) def GetExecutableFilePath(file_path): if IsOsDos(): if GetFileExt( file_path ) != "exe": return file_path + ".exe" else: return file_path else: return file_path def FileExists(file): if file is None or file == "": return False if os.path.isfile(file): return True else: return False def DirExists(dir): if os.path.isdir(dir): return True else: return False def GetRealPath(x): return os.path.realpath(x) def GetFullPath(file_or_dir): if DirExists(file_or_dir): file_or_dir = os.path.realpath(file_or_dir) if file_or_dir[-1] == '/': return file_or_dir else: return file_or_dir + "/" elif FileExists(file_or_dir): return os.path.realpath(file_or_dir) return file_or_dir def GetParentDir(dir): return GetFullPath( os.path.normpath(os.path.join(dir, "..")) ) def GetCurrentDir(): return GetFullPath( os.getcwd() ) def ChangeDir(dir, quiet = False): os.chdir( dir ) if quiet == False: log.info ('ChangeDir ' + dir ) def MakeFreshDir(dir): if os.path.isdir(dir) == True: RemoveDir(dir) MakeDir(dir) def MakeFreshDirRecursive(path): RemoveDir(path) MakeDirRecursive(path) def MakeDir(dir): if os.path.isdir(dir) == False: os.mkdir(dir) log.info ('MakeDir ' + dir ) def MakeDirRecursive(path): sub_path = os.path.dirname(path) if not DirExists(sub_path) and sub_path != "": MakeDirRecursive(sub_path) if not DirExists(path) and path != "": MakeDir(path) def GetDirName(x): x = os.path.basename( normpath(x) ) # Remove complete path and only get file name return x def GetFileListInDir(dir_path, file_ext, recursive = True): AssertDirExists(dir_path) __file_list = [] if recursive: for __sub_dir_path, __dirs, __files in os.walk(dir_path): for __file_name in __files: __file_path = FormatFilePath( __sub_dir_path + '/' + __file_name ) __file_ext = GetFileExt( __file_path ) if __file_ext.upper() == file_ext.upper(): __file_list.append( GetFullPath(__file_path) ) else: __file_list = glob.glob(os.path.join( dir_path, '*.' + file_ext )) return __file_list def BuildFileList(file_or_folder_list, file_ext): __file_list = [] for __file_or_folder in file_or_folder_list: if DirExists( __file_or_folder ): # Add all files in directory to the project __file_list_in_dir = GetFileListInDir( __file_or_folder, file_ext ) for __file in __file_list_in_dir: __file_list.append( __file ) elif GetFileExt(__file_or_folder).upper() == file_ext.upper(): __file_list.append( GetFullPath(__file_or_folder) ) return __file_list # --- Other functions def QuietExit(retcode = 0): sys.exit(retcode) def Exit(retcode = 0): if retcode != 0: log.fatal("Exit with return code: " + str(retcode)) else: QuietExit(0) def Clip(x, min, max): if x > max: return max elif x < min: return min else: return x def Sign(x): if x > 0: return 1 elif x < 0: return -1 else: return 0 def GetFileResolution(filepath): input_dir = os.path.dirname(os.path.realpath(filepath)) input_dir_basename = os.path.basename( FormatDirPath( input_dir ) ) file_basename = os.path.basename( filepath ) # Remove complete path and only get file name file_basename = os.path.splitext(file_basename)[0] # Remove extension itemslist = file_basename.split("_") # Try to get resolution from file name if len(itemslist) > 1: resolution = itemslist[-1] if resolution.find('x') >= 0 : itemslist = resolution.split('x'); if len(itemslist) > 1: return int(itemslist[0]), int(itemslist[1]) # Try to get resolution from parent folder name if input_dir_basename.find('x') >= 0 : itemslist = input_dir_basename.split('x'); if len(itemslist) > 1: return int(itemslist[0]), int(itemslist[1]) return 3000, 3000 # -- Default resolution def WriteFolderMd5Sum(input_dir, file_ext, output_file, output_full_paths = False ): __md5sum_list = GetFolderMd5Sum( input_dir, file_ext ) __md5_text = "" for __file_md5sum in __md5sum_list: __md5_text = __md5_text + __file_md5sum['md5sum'] if output_full_paths: __md5_text = __md5_text + " " + __file_md5sum['file_path'] else: __md5_text = __md5_text + " " + __file_md5sum['file_name'] __md5_text = __md5_text + "\n" WriteFile( output_file, __md5_text ) return __md5sum_list def GetFolderMd5Sum(input_dir, file_ext ): __md5sum_list = [] for __sub_dir_path, __dirs, __files in os.walk(input_dir): for __file_name in __files: __file_path = FormatFilePath( __sub_dir_path + '/' + __file_name ) if GetFileExt(__file_path) == file_ext.lower() or GetFileExt(__file_path) == file_ext.upper(): __md5sum_list.append( { 'file_path' : __file_path, 'file_name' : __file_name, 'md5sum' : GetFileMd5Sum(__file_path) } ) return __md5sum_list def GetFileMd5Sum(file_path, block_size = 256*128 ): md5 = hashlib.md5() with open(file_path,'rb') as f: for chunk in iter(lambda: f.read(block_size), b''): md5.update(chunk) return md5.hexdigest() def FormatFilePath(file_path): file_path = file_path.rstrip('\\') # Remove \\ at the end file_path = file_path.rstrip('/') # Remove / at the end file_path = file_path.replace('\\','/') # Replace backslash with forward slash # Convert relative path to absolute path file_path = os.path.abspath(file_path) file_path = file_path.replace('\\','/') # Replace backslash with forward slash # -- Make sure file_path exists before returning correct path AssertFileExists(file_path) return file_path def GetAllSubDirs(dir_path): return [GetFullPath(dir_path + o) for o in os.listdir(dir_path) if os.path.isdir(os.path.join(dir_path,o))] def FindAllFiles(dir_path, file_extension): res = [] for file_name in os.listdir(dir_path): file_path = GetFullPath(dir_path) + file_name if FileExists(file_path) and GetFileExt(file_path).upper() == file_extension.upper(): res.append(file_path) return res def FormatDirPath(dir_path, directory_must_exist = True): dir_path = dir_path.rstrip('\\') # Remove \\ at the end dir_path = dir_path.rstrip('/') # Remove / at the end dir_path = dir_path.replace('\\','/') # Replace backslash with forward slash dir_path = os.path.abspath(dir_path) dir_path = dir_path.replace('\\','/') # Replace backslash with forward slash if directory_must_exist: # -- Make sure directory exists before returning correct path AssertDirExists(dir_path) return dir_path def IsNumpyArray(x): return operator.isNumberType(x) def IsInsidePackage(): # Always assuming that root git dir is two levels over common directory git_dir = GetParentDir(__file__) + "/../../" + ".git" return DirExists(git_dir) == False def CheckForCommands( command_list, verbsoe = True ): for __command in command_list: if GetFullPathOfCommand(__command) == None: if verbsoe: log.fatal("Command " + __command + " not found. Check if these apps are installed: " + str(command_list)) return False return True def GetFullPathOfCommand(command): __command = "" if IsOsDos(): __command = "where.exe " + command else: __command = "which " + command __output = ExecuteCommand(__command, True) if __output["returncode"]: return None else: __path = __output['stdout'].strip() log.info("Found " + command + ": " + __path) return FormatFilePath(__path) def IsMyFunction(function_name): if (not function_name.startswith('_')) and inspect.isfunction(globals()[function_name]): return True else: return False def GetFunctionArguments(function_name): if not inspect.getargspec(globals()[function_name]).args: return "()" else: __arg_info = inspect.getargspec(globals()[function_name]) if __arg_info.defaults: __defaults_len = len(__arg_info.defaults) for __index in range(__defaults_len): __arg_list_index = len(__arg_info.args) - __index - 1 __arg_info.args[__arg_list_index] = str(__arg_info.args[__arg_list_index]) + " = " + str(__arg_info.defaults[__defaults_len - __index - 1]) return str(__arg_info.args) def PrintFunctions(template = None): if template: print("Printing all available functions with word " + template) else: print("Printing all available functions") __count = 0 for __function_name in globals().keys(): if IsMyFunction(__function_name): if (template == None) or (template != None and __function_name.find(template) >= 0): print(__function_name + GetFunctionArguments(__function_name)) __count = __count + 1 print(str(__count) + " Functions Found") def ConvertStringArgumentToObject(argument): if argument == "None": return None if argument == "True": return True if argument == "False": return False try: return float(argument) except: pass try: return int(argument) except: pass return argument def GetModelDirPath(model_dir_path, source_dir_path): if DirExists(model_dir_path): return FormatDirPath(model_dir_path, True) model_dir_path = FormatDirPath(source_dir_path, False) + "/" + "models" if DirExists(model_dir_path): return FormatDirPath(model_dir_path, True) model_dir_path = os.environ.get('ROLL_MODELS_DIR') if model_dir_path is not None: if DirExists(model_dir_path): return FormatDirPath(model_dir_path, True) log.fatal("Could not find model directory - please make sure to specify model_dir_path") return "" def GetFontsDirPath(fonts_dir_path, source_dir_path): if DirExists(fonts_dir_path): return FormatDirPath(fonts_dir_path, True) fonts_dir_path = FormatDirPath(source_dir_path, False) + "/" + "fonts" if DirExists(fonts_dir_path): return FormatDirPath(fonts_dir_path, True) fonts_dir_path = os.environ.get('ROLL_FONTS_DIR') if fonts_dir_path is not None: if DirExists(fonts_dir_path): return FormatDirPath(fonts_dir_path, True) log.fatal("Could not find fonts directory - please make sure to specify fonts_dir_path") return ""