Delete gemma4_arrow_encrypted_v1
Browse files- gemma4_arrow_encrypted_v1/data-00000-of-00007.arrow +0 -3
- gemma4_arrow_encrypted_v1/data-00001-of-00007.arrow +0 -3
- gemma4_arrow_encrypted_v1/data-00002-of-00007.arrow +0 -3
- gemma4_arrow_encrypted_v1/data-00003-of-00007.arrow +0 -3
- gemma4_arrow_encrypted_v1/data-00004-of-00007.arrow +0 -3
- gemma4_arrow_encrypted_v1/data-00005-of-00007.arrow +0 -3
- gemma4_arrow_encrypted_v1/data-00006-of-00007.arrow +0 -3
- gemma4_arrow_encrypted_v1/dataset_info.json +0 -36
- gemma4_arrow_encrypted_v1/encryption_manifest.json +0 -120
- gemma4_arrow_encrypted_v1/state.json +0 -31
gemma4_arrow_encrypted_v1/data-00000-of-00007.arrow
DELETED
|
@@ -1,3 +0,0 @@
|
|
| 1 |
-
version https://git-lfs.github.com/spec/v1
|
| 2 |
-
oid sha256:4d37095765bc4268e9732772db23da5a68438c7b6889b908fb97ecd3bc13224b
|
| 3 |
-
size 495883992
|
|
|
|
|
|
|
|
|
|
|
|
gemma4_arrow_encrypted_v1/data-00001-of-00007.arrow
DELETED
|
@@ -1,3 +0,0 @@
|
|
| 1 |
-
version https://git-lfs.github.com/spec/v1
|
| 2 |
-
oid sha256:9dd6f89826a9048e071083504b77806103e93864a345c72cd42e6dedc343bbf7
|
| 3 |
-
size 488137592
|
|
|
|
|
|
|
|
|
|
|
|
gemma4_arrow_encrypted_v1/data-00002-of-00007.arrow
DELETED
|
@@ -1,3 +0,0 @@
|
|
| 1 |
-
version https://git-lfs.github.com/spec/v1
|
| 2 |
-
oid sha256:47801511d65b6031667bfc4a3ec04f7b1b290ed9380bbf3e4d95d50188763615
|
| 3 |
-
size 486410264
|
|
|
|
|
|
|
|
|
|
|
|
gemma4_arrow_encrypted_v1/data-00003-of-00007.arrow
DELETED
|
@@ -1,3 +0,0 @@
|
|
| 1 |
-
version https://git-lfs.github.com/spec/v1
|
| 2 |
-
oid sha256:4efedd3cbe294ccd4bf764abc2d8e56ef96a36efb201cce7653145de39a644d4
|
| 3 |
-
size 477873064
|
|
|
|
|
|
|
|
|
|
|
|
gemma4_arrow_encrypted_v1/data-00004-of-00007.arrow
DELETED
|
@@ -1,3 +0,0 @@
|
|
| 1 |
-
version https://git-lfs.github.com/spec/v1
|
| 2 |
-
oid sha256:49739dc0a6f8f1b864595119b91fa0f76f0759effd19b203c51415c238b6affb
|
| 3 |
-
size 489371208
|
|
|
|
|
|
|
|
|
|
|
|
gemma4_arrow_encrypted_v1/data-00005-of-00007.arrow
DELETED
|
@@ -1,3 +0,0 @@
|
|
| 1 |
-
version https://git-lfs.github.com/spec/v1
|
| 2 |
-
oid sha256:519b1f86746ccea10352c738acdccbd84bfc937af5602dde65dc19b9670d4bc5
|
| 3 |
-
size 491112272
|
|
|
|
|
|
|
|
|
|
|
|
gemma4_arrow_encrypted_v1/data-00006-of-00007.arrow
DELETED
|
@@ -1,3 +0,0 @@
|
|
| 1 |
-
version https://git-lfs.github.com/spec/v1
|
| 2 |
-
oid sha256:ba95651f30bb6c1e63a7846ba51fe165c5ee2ea6a6543577869d18501465f5ba
|
| 3 |
-
size 464851360
|
|
|
|
|
|
|
|
|
|
|
|
gemma4_arrow_encrypted_v1/dataset_info.json
DELETED
|
@@ -1,36 +0,0 @@
|
|
| 1 |
-
{
|
| 2 |
-
"citation": "",
|
| 3 |
-
"description": "",
|
| 4 |
-
"features": {
|
| 5 |
-
"idx": {
|
| 6 |
-
"dtype": "int64",
|
| 7 |
-
"_type": "Value"
|
| 8 |
-
},
|
| 9 |
-
"sample_id": {
|
| 10 |
-
"dtype": "string",
|
| 11 |
-
"_type": "Value"
|
| 12 |
-
},
|
| 13 |
-
"image_enc": {
|
| 14 |
-
"dtype": "binary",
|
| 15 |
-
"_type": "Value"
|
| 16 |
-
},
|
| 17 |
-
"conversation_enc": {
|
| 18 |
-
"dtype": "binary",
|
| 19 |
-
"_type": "Value"
|
| 20 |
-
},
|
| 21 |
-
"width": {
|
| 22 |
-
"dtype": "int32",
|
| 23 |
-
"_type": "Value"
|
| 24 |
-
},
|
| 25 |
-
"height": {
|
| 26 |
-
"dtype": "int32",
|
| 27 |
-
"_type": "Value"
|
| 28 |
-
},
|
| 29 |
-
"image_format": {
|
| 30 |
-
"dtype": "string",
|
| 31 |
-
"_type": "Value"
|
| 32 |
-
}
|
| 33 |
-
},
|
| 34 |
-
"homepage": "",
|
| 35 |
-
"license": ""
|
| 36 |
-
}
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
gemma4_arrow_encrypted_v1/encryption_manifest.json
DELETED
|
@@ -1,120 +0,0 @@
|
|
| 1 |
-
{
|
| 2 |
-
"dataset_name": "three_source_encrypted_multimodal",
|
| 3 |
-
"dataset_version": 1,
|
| 4 |
-
"serialization": "Hugging Face Datasets / Apache Arrow",
|
| 5 |
-
"sample_count": 11674,
|
| 6 |
-
"image_preprocessing": {
|
| 7 |
-
"min_dimension": 200,
|
| 8 |
-
"max_dimension": 1024,
|
| 9 |
-
"minimum_rule": "exclude if width < 200 OR height < 200",
|
| 10 |
-
"orientation": "EXIF corrected",
|
| 11 |
-
"color_space": "RGB",
|
| 12 |
-
"resampling": "LANCZOS"
|
| 13 |
-
},
|
| 14 |
-
"text_preprocessing": {
|
| 15 |
-
"applied": false,
|
| 16 |
-
"description": "Source instruction and answer text preserved."
|
| 17 |
-
},
|
| 18 |
-
"encryption": {
|
| 19 |
-
"cipher": "AES-256-GCM",
|
| 20 |
-
"nonce_bytes": 12,
|
| 21 |
-
"kdf": "scrypt",
|
| 22 |
-
"kdf_salt_base64": "Xau0NYaTez4bfCIohiriKQ==",
|
| 23 |
-
"scrypt_n": 16384,
|
| 24 |
-
"scrypt_r": 8,
|
| 25 |
-
"scrypt_p": 1,
|
| 26 |
-
"key_length_bytes": 32,
|
| 27 |
-
"aad_format": "gemma4-v1:{sample_id}:{field_name}",
|
| 28 |
-
"encrypted_fields": [
|
| 29 |
-
"image_enc",
|
| 30 |
-
"conversation_enc"
|
| 31 |
-
]
|
| 32 |
-
},
|
| 33 |
-
"arrow_schema": {
|
| 34 |
-
"idx": "int64 (global, zero-based row number after shuffle)",
|
| 35 |
-
"sample_id": "string",
|
| 36 |
-
"image_enc": "binary",
|
| 37 |
-
"conversation_enc": "binary",
|
| 38 |
-
"width": "int32",
|
| 39 |
-
"height": "int32",
|
| 40 |
-
"image_format": "string"
|
| 41 |
-
},
|
| 42 |
-
"conversion_statistics": {
|
| 43 |
-
"processed": 11674,
|
| 44 |
-
"skipped": 31
|
| 45 |
-
},
|
| 46 |
-
"source_repositories": [
|
| 47 |
-
"codeShare/civitai-natural-language-prompts-and-images",
|
| 48 |
-
"codeShare/adcom_captioned_ai_images",
|
| 49 |
-
"codeShare/sankaku_captioned_images"
|
| 50 |
-
],
|
| 51 |
-
"created_unix": 1794853788.1802542,
|
| 52 |
-
"row_order": {
|
| 53 |
-
"method": "global_random_shuffle_after_combining_all_sources",
|
| 54 |
-
"seed": 20261116,
|
| 55 |
-
"physically_materialized": true,
|
| 56 |
-
"includes_existing_destination_records": true,
|
| 57 |
-
"idx": "fresh contiguous zero-based global index assigned after shuffle"
|
| 58 |
-
},
|
| 59 |
-
"last_append": {
|
| 60 |
-
"source_repositories": [
|
| 61 |
-
"codeShare/civitai-natural-language-prompts-and-images",
|
| 62 |
-
"codeShare/adcom_captioned_ai_images",
|
| 63 |
-
"codeShare/sankaku_captioned_images"
|
| 64 |
-
],
|
| 65 |
-
"records_added_by_source": {
|
| 66 |
-
"civitai": 3783,
|
| 67 |
-
"adcom": 3590,
|
| 68 |
-
"sankaku": 4301
|
| 69 |
-
},
|
| 70 |
-
"records_skipped_by_source": {
|
| 71 |
-
"civitai": 0,
|
| 72 |
-
"adcom": 0,
|
| 73 |
-
"sankaku": 31
|
| 74 |
-
},
|
| 75 |
-
"records_filtered_out_before_processing": {
|
| 76 |
-
"adcom_prompt_remove_or_improve": 426,
|
| 77 |
-
"sankaku_written_user_or_summary_with_empty_prompt_and_thinking": 0,
|
| 78 |
-
"civitai_empty_user_input": 0,
|
| 79 |
-
"civitai_empty_expected_output": 0,
|
| 80 |
-
"adcom_empty_user_input": 0,
|
| 81 |
-
"adcom_empty_expected_output": 0,
|
| 82 |
-
"sankaku_empty_user_input": 0,
|
| 83 |
-
"sankaku_empty_expected_output": 7,
|
| 84 |
-
"existing_destination": {
|
| 85 |
-
"empty_user_text": 0,
|
| 86 |
-
"empty_expected_output": 0,
|
| 87 |
-
"image_too_small": 0,
|
| 88 |
-
"invalid_records": 0
|
| 89 |
-
}
|
| 90 |
-
},
|
| 91 |
-
"image_too_small_by_source": {
|
| 92 |
-
"civitai": 0,
|
| 93 |
-
"adcom": 0,
|
| 94 |
-
"sankaku": 31
|
| 95 |
-
},
|
| 96 |
-
"existing_destination_filter": {
|
| 97 |
-
"empty_user_text": 0,
|
| 98 |
-
"empty_expected_output": 0,
|
| 99 |
-
"image_too_small": 0,
|
| 100 |
-
"invalid_records": 0
|
| 101 |
-
},
|
| 102 |
-
"verification": {
|
| 103 |
-
"checked": 30,
|
| 104 |
-
"passed": 30,
|
| 105 |
-
"failed": 0
|
| 106 |
-
},
|
| 107 |
-
"shuffle": {
|
| 108 |
-
"method": "global_random_shuffle_after_combining_all_sources",
|
| 109 |
-
"seed": 20261116,
|
| 110 |
-
"physically_materialized": true,
|
| 111 |
-
"idx": "fresh contiguous zero-based global index assigned after shuffle",
|
| 112 |
-
"first_30_row_source_mix": {
|
| 113 |
-
"civitai": 12,
|
| 114 |
-
"adcom": 13,
|
| 115 |
-
"sankaku": 5
|
| 116 |
-
}
|
| 117 |
-
},
|
| 118 |
-
"created_unix": 1794855993.5074668
|
| 119 |
-
}
|
| 120 |
-
}
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
gemma4_arrow_encrypted_v1/state.json
DELETED
|
@@ -1,31 +0,0 @@
|
|
| 1 |
-
{
|
| 2 |
-
"_data_files": [
|
| 3 |
-
{
|
| 4 |
-
"filename": "data-00000-of-00007.arrow"
|
| 5 |
-
},
|
| 6 |
-
{
|
| 7 |
-
"filename": "data-00001-of-00007.arrow"
|
| 8 |
-
},
|
| 9 |
-
{
|
| 10 |
-
"filename": "data-00002-of-00007.arrow"
|
| 11 |
-
},
|
| 12 |
-
{
|
| 13 |
-
"filename": "data-00003-of-00007.arrow"
|
| 14 |
-
},
|
| 15 |
-
{
|
| 16 |
-
"filename": "data-00004-of-00007.arrow"
|
| 17 |
-
},
|
| 18 |
-
{
|
| 19 |
-
"filename": "data-00005-of-00007.arrow"
|
| 20 |
-
},
|
| 21 |
-
{
|
| 22 |
-
"filename": "data-00006-of-00007.arrow"
|
| 23 |
-
}
|
| 24 |
-
],
|
| 25 |
-
"_fingerprint": "e03ed2983d348704",
|
| 26 |
-
"_format_columns": null,
|
| 27 |
-
"_format_kwargs": {},
|
| 28 |
-
"_format_type": null,
|
| 29 |
-
"_output_all_columns": false,
|
| 30 |
-
"_split": null
|
| 31 |
-
}
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|