File size: 2,806 Bytes
a10ba7f | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 | import os
import tarfile
import io
def join_and_extract(parts_dir, output_dir, target_rec='rec_000'):
"""
Joins tar parts and extracts only the target recording to save space.
"""
parts = sorted([f for f in os.listdir(parts_dir) if 'Gaze360.tar.part' in f])
if not parts:
print("No tar parts found!")
return
print(f"Found parts: {parts}")
os.makedirs(output_dir, exist_ok=True)
# We use a custom stream class to join parts without writing a huge file to disk
class JoinedStream(io.RawIOBase):
def __init__(self, parts_dir, parts):
self.parts_dir = parts_dir
self.parts = parts
self.current_part_idx = 0
self.current_file = open(os.path.join(parts_dir, parts[0]), 'rb')
def readinto(self, b):
n = self.current_file.readinto(b)
if n == 0 and self.current_part_idx < len(self.parts) - 1:
self.current_file.close()
self.current_part_idx += 1
print(f"Switching to {self.parts[self.current_part_idx]}...")
self.current_file = open(os.path.join(self.parts_dir, self.parts[self.current_part_idx]), 'rb')
n = self.current_file.readinto(b)
return n
def readable(self):
return True
print(f"Opening joined stream and searching for {target_rec}...")
stream = JoinedStream(parts_dir, parts)
try:
with tarfile.open(fileobj=stream, mode='r|') as tar:
for member in tar:
# Gaze360 tar structure usually has imgs/rec_xxx/head/...
if target_rec in member.name and member.isfile():
# Flatten the path a bit for our local structure: data/raw/imgs/rec_xxx/...
# Original might be Gaze360/imgs/rec_000/...
parts = member.name.split('/')
if 'imgs' in parts:
idx = parts.index('imgs')
rel_path = os.path.join(*parts[idx+1:])
target_path = os.path.join(output_dir, rel_path)
os.makedirs(os.path.dirname(target_path), exist_ok=True)
with open(target_path, 'wb') as f:
f.write(tar.extractfile(member).read())
# Optional: limit extraction to first 500 images of the recording for speed/space
# if "head" in member.name and "000500.jpg" in member.name: break
except Exception as e:
print(f"Note: Stream ended or error occurred: {e}")
finally:
stream.current_file.close()
if __name__ == "__main__":
join_and_extract('data/raw', 'data/raw/imgs', target_rec='rec_000')
|