codeShare commited on
Commit
2cfcd89
·
verified ·
1 Parent(s): 1267647

Delete gemma4_arrow_encrypted_v1

Browse files
gemma4_arrow_encrypted_v1/README.md DELETED
@@ -1,54 +0,0 @@
1
- # Gemma 4 Encrypted Sankaku Multimodal Dataset
2
-
3
- ## Source
4
-
5
- codeShare/sankaku_captioned_images
6
-
7
- ## Conversation format
8
-
9
- User:
10
- - Original instruction text
11
- - Corresponding decrypted image
12
-
13
- Assistant:
14
- - Original thinking_response
15
- - Original written_response
16
-
17
- The assistant text is constructed using:
18
-
19
- thinking_response + two newline characters + written_response
20
-
21
- Original text is preserved without cleaning or normalization.
22
-
23
- ## Storage
24
-
25
- Hugging Face Datasets / Apache Arrow.
26
-
27
- ## Schema
28
-
29
- - sample_id: string
30
- - image_enc: binary
31
- - conversation_enc: binary
32
- - width: int32
33
- - height: int32
34
- - image_format: string
35
-
36
- ## Encryption
37
-
38
- - AES-256-GCM
39
- - Independent encryption for image and conversation
40
- - 12-byte nonce
41
- - Scrypt key derivation
42
- - Authenticated sample IDs and field names
43
-
44
- ## Image preprocessing
45
-
46
- - EXIF correction
47
- - RGB conversion
48
- - Maximum dimension 1024
49
- - LANCZOS resizing
50
-
51
- ## Security
52
-
53
- The output password is not stored in the repository.
54
- Arrow files contain encrypted images and conversations.
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
gemma4_arrow_encrypted_v1/append_report.json DELETED
@@ -1,14 +0,0 @@
1
- {
2
- "source_repository": "codeShare/sankaku_captioned_images",
3
- "destination_repository": "codeShare/kromoss-lora-exports",
4
- "existing_records": 0,
5
- "new_records": 1973,
6
- "skipped_records": 5,
7
- "final_records": 1973,
8
- "verification": {
9
- "checked": 30,
10
- "passed": 30,
11
- "failed": 0
12
- },
13
- "created_unix": 1794611117.224032
14
- }
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
gemma4_arrow_encrypted_v1/data-00000-of-00001.arrow DELETED
@@ -1,3 +0,0 @@
1
- version https://git-lfs.github.com/spec/v1
2
- oid sha256:e35630410be176004b8d4d5affc5fcaf715588f5115283cc332fcd1df788d92e
3
- size 425455408
 
 
 
 
gemma4_arrow_encrypted_v1/dataset_info.json DELETED
@@ -1,32 +0,0 @@
1
- {
2
- "citation": "",
3
- "description": "",
4
- "features": {
5
- "sample_id": {
6
- "dtype": "string",
7
- "_type": "Value"
8
- },
9
- "image_enc": {
10
- "dtype": "binary",
11
- "_type": "Value"
12
- },
13
- "conversation_enc": {
14
- "dtype": "binary",
15
- "_type": "Value"
16
- },
17
- "width": {
18
- "dtype": "int32",
19
- "_type": "Value"
20
- },
21
- "height": {
22
- "dtype": "int32",
23
- "_type": "Value"
24
- },
25
- "image_format": {
26
- "dtype": "string",
27
- "_type": "Value"
28
- }
29
- },
30
- "homepage": "",
31
- "license": ""
32
- }
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
gemma4_arrow_encrypted_v1/encryption_manifest.json DELETED
@@ -1,61 +0,0 @@
1
- {
2
- "dataset_name": "gemma4_encrypted_multimodal",
3
- "dataset_version": 1,
4
- "serialization": "Hugging Face Datasets / Apache Arrow",
5
- "sample_count": 1973,
6
- "image_preprocessing": {
7
- "max_dimension": 1024,
8
- "orientation": "EXIF corrected",
9
- "color_space": "RGB",
10
- "resampling": "LANCZOS",
11
- "jpeg_quality": 95,
12
- "webp_quality": 95
13
- },
14
- "text_preprocessing": {
15
- "applied": false,
16
- "description": "Original instruction and assistant text preserved."
17
- },
18
- "encryption": {
19
- "cipher": "AES-256-GCM",
20
- "nonce_bytes": 12,
21
- "kdf": "scrypt",
22
- "kdf_salt_base64": "6O5eNt828QAeylf9xDiMyg==",
23
- "scrypt_n": 16384,
24
- "scrypt_r": 8,
25
- "scrypt_p": 1,
26
- "key_length_bytes": 32,
27
- "aad_format": "gemma4-v1:{sample_id}:{field_name}",
28
- "encrypted_fields": [
29
- "image_enc",
30
- "conversation_enc"
31
- ]
32
- },
33
- "arrow_schema": {
34
- "sample_id": "string",
35
- "image_enc": "binary",
36
- "conversation_enc": "binary",
37
- "width": "int32",
38
- "height": "int32",
39
- "image_format": "string"
40
- },
41
- "conversion_statistics": {
42
- "processed": 1973,
43
- "skipped": 5
44
- },
45
- "source_repositories": [
46
- "codeShare/sankaku_captioned_images"
47
- ],
48
- "created_unix": 1794611028.8531384,
49
- "last_append": {
50
- "source_repository": "codeShare/sankaku_captioned_images",
51
- "source_parquet": "preprocessed/captions.parquet",
52
- "records_added": 1973,
53
- "records_skipped": 5,
54
- "verification": {
55
- "checked": 30,
56
- "passed": 30,
57
- "failed": 0
58
- },
59
- "created_unix": 1794611113.429829
60
- }
61
- }
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
gemma4_arrow_encrypted_v1/processing_errors.json DELETED
@@ -1,42 +0,0 @@
1
- [
2
- {
3
- "idx": "8",
4
- "sample_id": "sankaku_captioned_v1_8_r0000000008",
5
- "json_file": "preprocessed/images-000000000000-000000000059.jsonl",
6
- "byte_offset": 2270563,
7
- "byte_length": 286554,
8
- "error": "ValueError('Both assistant response fields are empty.')"
9
- },
10
- {
11
- "idx": "10",
12
- "sample_id": "sankaku_captioned_v1_10_r0000000010",
13
- "json_file": "preprocessed/images-000000000000-000000000059.jsonl",
14
- "byte_offset": 2720887,
15
- "byte_length": 191656,
16
- "error": "ValueError('Both assistant response fields are empty.')"
17
- },
18
- {
19
- "idx": "31",
20
- "sample_id": "sankaku_captioned_v1_31_r0000000031",
21
- "json_file": "preprocessed/images-000000000000-000000000059.jsonl",
22
- "byte_offset": 8117592,
23
- "byte_length": 349660,
24
- "error": "ValueError('Both assistant response fields are empty.')"
25
- },
26
- {
27
- "idx": "33",
28
- "sample_id": "sankaku_captioned_v1_33_r0000000033",
29
- "json_file": "preprocessed/images-000000000000-000000000059.jsonl",
30
- "byte_offset": 8784983,
31
- "byte_length": 173942,
32
- "error": "ValueError('Both assistant response fields are empty.')"
33
- },
34
- {
35
- "idx": "96",
36
- "sample_id": "sankaku_captioned_v1_96_r0000000096",
37
- "json_file": "preprocessed/images-000000000060-000000000119.jsonl",
38
- "byte_offset": 10933402,
39
- "byte_length": 361263,
40
- "error": "ValueError('Both assistant response fields are empty.')"
41
- }
42
- ]
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
gemma4_arrow_encrypted_v1/state.json DELETED
@@ -1,13 +0,0 @@
1
- {
2
- "_data_files": [
3
- {
4
- "filename": "data-00000-of-00001.arrow"
5
- }
6
- ],
7
- "_fingerprint": "80ba9419c155b131",
8
- "_format_columns": null,
9
- "_format_kwargs": {},
10
- "_format_type": null,
11
- "_output_all_columns": false,
12
- "_split": null
13
- }