Spaces:
Build error
Build error
File size: 922 Bytes
06c8a6d b6fee08 06c8a6d b6fee08 06c8a6d b6fee08 |
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 |
import os
import glob
from multiprocessing import Pool
import timeit
from tqdm import tqdm
def ensure_folder_exists(path):
if not os.path.exists(path):
os.makedirs(path)
def find_files(pattern):
files = glob.glob(pattern)
return files
def save_to_file(file_name, content):
with open(file_name, 'w', encoding='utf-8') as f:
f.write(content)
# print("Saved: ", file_name)
def multi_process_list(input_list, function, num_workers=20):
with Pool(num_workers) as p:
result = list(tqdm(p.imap_unordered(function, input_list), total=len(input_list)))
return result
def process_list(input_list, function):
# process with process bar
result = []
for item in tqdm(input_list):
result.append(function(item))
return result
def read_txt_content(file):
with open(file, 'r', encoding='utf-8') as f:
text = f.read()
return text |