codeShare commited on
Commit
ffc8c55
·
verified ·
1 Parent(s): 2a1cc7b

Delete gemma4_arrow_encrypted_v1

Browse files
gemma4_arrow_encrypted_v1/data-00000-of-00007.arrow DELETED
@@ -1,3 +0,0 @@
1
- version https://git-lfs.github.com/spec/v1
2
- oid sha256:4d37095765bc4268e9732772db23da5a68438c7b6889b908fb97ecd3bc13224b
3
- size 495883992
 
 
 
 
gemma4_arrow_encrypted_v1/data-00001-of-00007.arrow DELETED
@@ -1,3 +0,0 @@
1
- version https://git-lfs.github.com/spec/v1
2
- oid sha256:9dd6f89826a9048e071083504b77806103e93864a345c72cd42e6dedc343bbf7
3
- size 488137592
 
 
 
 
gemma4_arrow_encrypted_v1/data-00002-of-00007.arrow DELETED
@@ -1,3 +0,0 @@
1
- version https://git-lfs.github.com/spec/v1
2
- oid sha256:47801511d65b6031667bfc4a3ec04f7b1b290ed9380bbf3e4d95d50188763615
3
- size 486410264
 
 
 
 
gemma4_arrow_encrypted_v1/data-00003-of-00007.arrow DELETED
@@ -1,3 +0,0 @@
1
- version https://git-lfs.github.com/spec/v1
2
- oid sha256:4efedd3cbe294ccd4bf764abc2d8e56ef96a36efb201cce7653145de39a644d4
3
- size 477873064
 
 
 
 
gemma4_arrow_encrypted_v1/data-00004-of-00007.arrow DELETED
@@ -1,3 +0,0 @@
1
- version https://git-lfs.github.com/spec/v1
2
- oid sha256:49739dc0a6f8f1b864595119b91fa0f76f0759effd19b203c51415c238b6affb
3
- size 489371208
 
 
 
 
gemma4_arrow_encrypted_v1/data-00005-of-00007.arrow DELETED
@@ -1,3 +0,0 @@
1
- version https://git-lfs.github.com/spec/v1
2
- oid sha256:519b1f86746ccea10352c738acdccbd84bfc937af5602dde65dc19b9670d4bc5
3
- size 491112272
 
 
 
 
gemma4_arrow_encrypted_v1/data-00006-of-00007.arrow DELETED
@@ -1,3 +0,0 @@
1
- version https://git-lfs.github.com/spec/v1
2
- oid sha256:ba95651f30bb6c1e63a7846ba51fe165c5ee2ea6a6543577869d18501465f5ba
3
- size 464851360
 
 
 
 
gemma4_arrow_encrypted_v1/dataset_info.json DELETED
@@ -1,36 +0,0 @@
1
- {
2
- "citation": "",
3
- "description": "",
4
- "features": {
5
- "idx": {
6
- "dtype": "int64",
7
- "_type": "Value"
8
- },
9
- "sample_id": {
10
- "dtype": "string",
11
- "_type": "Value"
12
- },
13
- "image_enc": {
14
- "dtype": "binary",
15
- "_type": "Value"
16
- },
17
- "conversation_enc": {
18
- "dtype": "binary",
19
- "_type": "Value"
20
- },
21
- "width": {
22
- "dtype": "int32",
23
- "_type": "Value"
24
- },
25
- "height": {
26
- "dtype": "int32",
27
- "_type": "Value"
28
- },
29
- "image_format": {
30
- "dtype": "string",
31
- "_type": "Value"
32
- }
33
- },
34
- "homepage": "",
35
- "license": ""
36
- }
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
gemma4_arrow_encrypted_v1/encryption_manifest.json DELETED
@@ -1,120 +0,0 @@
1
- {
2
- "dataset_name": "three_source_encrypted_multimodal",
3
- "dataset_version": 1,
4
- "serialization": "Hugging Face Datasets / Apache Arrow",
5
- "sample_count": 11674,
6
- "image_preprocessing": {
7
- "min_dimension": 200,
8
- "max_dimension": 1024,
9
- "minimum_rule": "exclude if width < 200 OR height < 200",
10
- "orientation": "EXIF corrected",
11
- "color_space": "RGB",
12
- "resampling": "LANCZOS"
13
- },
14
- "text_preprocessing": {
15
- "applied": false,
16
- "description": "Source instruction and answer text preserved."
17
- },
18
- "encryption": {
19
- "cipher": "AES-256-GCM",
20
- "nonce_bytes": 12,
21
- "kdf": "scrypt",
22
- "kdf_salt_base64": "Xau0NYaTez4bfCIohiriKQ==",
23
- "scrypt_n": 16384,
24
- "scrypt_r": 8,
25
- "scrypt_p": 1,
26
- "key_length_bytes": 32,
27
- "aad_format": "gemma4-v1:{sample_id}:{field_name}",
28
- "encrypted_fields": [
29
- "image_enc",
30
- "conversation_enc"
31
- ]
32
- },
33
- "arrow_schema": {
34
- "idx": "int64 (global, zero-based row number after shuffle)",
35
- "sample_id": "string",
36
- "image_enc": "binary",
37
- "conversation_enc": "binary",
38
- "width": "int32",
39
- "height": "int32",
40
- "image_format": "string"
41
- },
42
- "conversion_statistics": {
43
- "processed": 11674,
44
- "skipped": 31
45
- },
46
- "source_repositories": [
47
- "codeShare/civitai-natural-language-prompts-and-images",
48
- "codeShare/adcom_captioned_ai_images",
49
- "codeShare/sankaku_captioned_images"
50
- ],
51
- "created_unix": 1794853788.1802542,
52
- "row_order": {
53
- "method": "global_random_shuffle_after_combining_all_sources",
54
- "seed": 20261116,
55
- "physically_materialized": true,
56
- "includes_existing_destination_records": true,
57
- "idx": "fresh contiguous zero-based global index assigned after shuffle"
58
- },
59
- "last_append": {
60
- "source_repositories": [
61
- "codeShare/civitai-natural-language-prompts-and-images",
62
- "codeShare/adcom_captioned_ai_images",
63
- "codeShare/sankaku_captioned_images"
64
- ],
65
- "records_added_by_source": {
66
- "civitai": 3783,
67
- "adcom": 3590,
68
- "sankaku": 4301
69
- },
70
- "records_skipped_by_source": {
71
- "civitai": 0,
72
- "adcom": 0,
73
- "sankaku": 31
74
- },
75
- "records_filtered_out_before_processing": {
76
- "adcom_prompt_remove_or_improve": 426,
77
- "sankaku_written_user_or_summary_with_empty_prompt_and_thinking": 0,
78
- "civitai_empty_user_input": 0,
79
- "civitai_empty_expected_output": 0,
80
- "adcom_empty_user_input": 0,
81
- "adcom_empty_expected_output": 0,
82
- "sankaku_empty_user_input": 0,
83
- "sankaku_empty_expected_output": 7,
84
- "existing_destination": {
85
- "empty_user_text": 0,
86
- "empty_expected_output": 0,
87
- "image_too_small": 0,
88
- "invalid_records": 0
89
- }
90
- },
91
- "image_too_small_by_source": {
92
- "civitai": 0,
93
- "adcom": 0,
94
- "sankaku": 31
95
- },
96
- "existing_destination_filter": {
97
- "empty_user_text": 0,
98
- "empty_expected_output": 0,
99
- "image_too_small": 0,
100
- "invalid_records": 0
101
- },
102
- "verification": {
103
- "checked": 30,
104
- "passed": 30,
105
- "failed": 0
106
- },
107
- "shuffle": {
108
- "method": "global_random_shuffle_after_combining_all_sources",
109
- "seed": 20261116,
110
- "physically_materialized": true,
111
- "idx": "fresh contiguous zero-based global index assigned after shuffle",
112
- "first_30_row_source_mix": {
113
- "civitai": 12,
114
- "adcom": 13,
115
- "sankaku": 5
116
- }
117
- },
118
- "created_unix": 1794855993.5074668
119
- }
120
- }
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
gemma4_arrow_encrypted_v1/state.json DELETED
@@ -1,31 +0,0 @@
1
- {
2
- "_data_files": [
3
- {
4
- "filename": "data-00000-of-00007.arrow"
5
- },
6
- {
7
- "filename": "data-00001-of-00007.arrow"
8
- },
9
- {
10
- "filename": "data-00002-of-00007.arrow"
11
- },
12
- {
13
- "filename": "data-00003-of-00007.arrow"
14
- },
15
- {
16
- "filename": "data-00004-of-00007.arrow"
17
- },
18
- {
19
- "filename": "data-00005-of-00007.arrow"
20
- },
21
- {
22
- "filename": "data-00006-of-00007.arrow"
23
- }
24
- ],
25
- "_fingerprint": "e03ed2983d348704",
26
- "_format_columns": null,
27
- "_format_kwargs": {},
28
- "_format_type": null,
29
- "_output_all_columns": false,
30
- "_split": null
31
- }