File size: 2,351 Bytes
b0c0df0 |
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 |
import json
import os
import pandas as pd
from datasets import Dataset, Features, Image, Sequence, Value
from tqdm import tqdm
# Define the features for the dataset
features = Features(
{
"video": Value(dtype="string"),
"caption": Value(dtype="string"),
"timestamp": Sequence(Value(dtype="float16")), # Use Sequence for lists
}
)
df_items = {
"video": [],
"caption": [],
"timestamp": [],
}
# Load json file
json_path = "/opt/tiger/lmms-eval/lmms_eval/tasks/charades_sta/temporal_grounding_charades.json"
with open(json_path, "r") as f:
data = json.load(f)
# Iterate over the rows of the data
for cur_meta in data:
video = cur_meta["video"]
caption = cur_meta["caption"]
timestamp = cur_meta["timestamp"]
# import pdb;pdb.set_trace()
df_items["video"].append(video)
df_items["caption"].append(caption)
df_items["timestamp"].append(timestamp)
import pdb
pdb.set_trace()
df_items = pd.DataFrame(df_items)
dataset = Dataset.from_pandas(df_items, features=features)
hub_dataset_path = "lmms-lab/charades_sta"
dataset.push_to_hub(repo_id=hub_dataset_path, split="test")
# # upload the *zip to huggingface
# from huggingface_hub import HfApi
# def upload_zip_to_huggingface(repo_id, zip_path, commit_message="Upload ZIP file"):
# """
# Uploads a ZIP file to a Hugging Face dataset repository.
# Args:
# repo_id (str): The dataset repository ID (e.g., "your-username/your-dataset").
# zip_path (str): Path to the ZIP file to upload.
# commit_message (str): Commit message for the upload.
# """
# api = HfApi()
# # Upload file to the dataset repo
# api.upload_file(
# path_or_fileobj=zip_path,
# path_in_repo=zip_path.split("/")[-1], # Store with the same filename
# repo_id=repo_id,
# repo_type="dataset",
# commit_message=commit_message
# )
# print(f"Successfully uploaded {zip_path} to {repo_id}")
# # Example Usage for upload all zip in directory
# import os
# directory_path = "/home/tiger/split_zips"
# # Iterate over all files in the directory
# for filename in os.listdir(directory_path):
# if filename.endswith(".zip"):
# file_path = os.path.join(directory_path, filename)
# upload_zip_to_huggingface("lmms-lab/charades_sta", file_path)
|