File size: 5,552 Bytes
a65138c | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 193 194 195 196 197 198 199 200 201 202 203 204 205 206 207 208 209 210 211 212 213 214 | # -*- coding: utf-8 -*-
from __future__ import with_statement
import logging
from contextlib import contextmanager
logger = logging.getLogger(__name__)
def open_storage(path, mode='r', default_storage_factory=None):
import os.path
import zipfile
import fs
import _zipfile
if os.path.exists(path):
if os.path.isdir(path):
return fs.FileSystemStorage(path)
elif zipfile.is_zipfile(path):
return _zipfile.ZipFileStorage(path, mode)
raise IOError('%s: unsupported storage' % path)
elif callable(default_storage_factory):
return default_storage_factory(path, mode)
else:
raise IOError('%s: cannot open' % path)
def resolve_path(folder, path):
''' resolve path in a folder
:param folder: a folder (should have ``__getitem__``)
:param path: path to resolve
:returns: a node or None
'''
if path in ('', '/'):
return folder
import os
path_segments = path.split(os.sep)
assert len(path_segments) > 0
for seg in path_segments:
if not hasattr(folder, '__getitem__'):
return None
try:
node = folder[seg]
except KeyError:
return None
else:
folder = node
return node
def makedirs(folder, path):
from path import split as path_split
path_segments = path_split(path)
for seg in path_segments:
if seg in folder:
folder = folder[seg]
else:
folder = folder.folder(seg)
return folder
def makedirs_to_file(folder, path):
import os.path
dirname, name = os.path.split(path)
if name == '':
msg = '%s: invalid path' % path
raise ValueError(msg)
folder = makedirs(folder, dirname)
return folder.file(name)
def put_file(node, path, **kwargs):
if hasattr(node, 'put'):
node.put(path, **kwargs)
elif hasattr(node, 'open'):
with file(path, 'rb') as f:
with node.open('w') as g:
stream_copy(f, g)
else:
raise IOError('node supports neither put nor open')
def get_file(node, path):
if hasattr(node, 'get'):
node.get(path)
elif hasattr(node, 'open'):
with node.open() as f:
with file(path, 'wb') as g:
stream_copy(f, g)
else:
raise IOError('node supports neither get nor open')
@contextmanager
def openable_path_on_filesystem(node, writeback=False):
if hasattr(node, 'path_on_filesystem'):
with node.path_on_filesystem() as path:
yield path
else:
import os
from tempfile import mkstemp
fd, path = mkstemp()
try:
with node.open() as f:
with os.fdopen(fd, 'w+') as g:
fd = None
stream_copy(f, g)
# fd closed
yield path
if writeback:
put_file(node, path)
finally:
if fd is not None:
os.close(fd)
os.unlink(path)
def copy_file(src_node, dst_node):
if (hasattr(dst_node, 'path_on_filesystem') and
hasattr(src_node, 'get')):
with openable_path_on_filesystem(dst_node, writeback=True) as dst_path:
get_file(src_node, dst_path)
with openable_path_on_filesystem(src_node) as src_path:
put_file(dst_node, src_path)
def stream_copy(src, dst):
while True:
data = src.read(4096)
if len(data) == 0:
return
dst.write(data)
def iterate_nodes_recursively(folder, folder_path=''):
import os.path
for name in folder:
node = folder[name]
node_path = os.path.join(folder_path, name)
yield node_path, node
if hasattr(node, '__iter__') and hasattr(node, '__getitem__'):
for x in iterate_nodes_recursively(node, node_path):
yield x
def iterate_files_recursively(folder, folder_path=''):
for path, node in iterate_nodes_recursively(folder, folder_path):
if hasattr(node, 'open'):
yield path, node
def ls_main():
doc = '''Usage: storage-ls [options] <storage>
--help Show this screen.
'''
from docopt import docopt
args = docopt(doc)
root_path = args['<storage>'] or '.'
with open_storage(root_path, 'r') as stg:
for path, node in iterate_files_recursively(stg):
print path
def put_main():
doc = '''Usage: storage-put [options] <storage> <path> <file>
--help Show this screen.
'''
from docopt import docopt
args = docopt(doc)
import os.path
root_path = args['<storage>'] or '.'
with open_storage(root_path, 'a') as stg:
path = args['<path>']
file_path = args['<file>']
if os.path.exists(file_path):
node = makedirs_to_file(stg, path)
put_file(node, file_path)
else:
logger.error('%s: not found', file_path)
raise SystemExit(1)
def get_main():
doc = '''Usage: storage-get [options] <storage> <path> <file>
--help Show this screen.
'''
from docopt import docopt
args = docopt(doc)
root_path = args['<storage>'] or '.'
with open_storage(root_path, 'a') as stg:
path = args['<path>']
file_path = args['<file>']
node = resolve_path(stg, path)
if node:
get_file(node, file_path)
else:
logger.error('%s: not found', path)
raise SystemExit(1)
|