Delete gemma4_arrow_encrypted_v1
Browse files- gemma4_arrow_encrypted_v1/README.md +0 -54
- gemma4_arrow_encrypted_v1/append_report.json +0 -14
- gemma4_arrow_encrypted_v1/data-00000-of-00001.arrow +0 -3
- gemma4_arrow_encrypted_v1/dataset_info.json +0 -32
- gemma4_arrow_encrypted_v1/encryption_manifest.json +0 -61
- gemma4_arrow_encrypted_v1/processing_errors.json +0 -42
- gemma4_arrow_encrypted_v1/state.json +0 -13
gemma4_arrow_encrypted_v1/README.md
DELETED
|
@@ -1,54 +0,0 @@
|
|
| 1 |
-
# Gemma 4 Encrypted Sankaku Multimodal Dataset
|
| 2 |
-
|
| 3 |
-
## Source
|
| 4 |
-
|
| 5 |
-
codeShare/sankaku_captioned_images
|
| 6 |
-
|
| 7 |
-
## Conversation format
|
| 8 |
-
|
| 9 |
-
User:
|
| 10 |
-
- Original instruction text
|
| 11 |
-
- Corresponding decrypted image
|
| 12 |
-
|
| 13 |
-
Assistant:
|
| 14 |
-
- Original thinking_response
|
| 15 |
-
- Original written_response
|
| 16 |
-
|
| 17 |
-
The assistant text is constructed using:
|
| 18 |
-
|
| 19 |
-
thinking_response + two newline characters + written_response
|
| 20 |
-
|
| 21 |
-
Original text is preserved without cleaning or normalization.
|
| 22 |
-
|
| 23 |
-
## Storage
|
| 24 |
-
|
| 25 |
-
Hugging Face Datasets / Apache Arrow.
|
| 26 |
-
|
| 27 |
-
## Schema
|
| 28 |
-
|
| 29 |
-
- sample_id: string
|
| 30 |
-
- image_enc: binary
|
| 31 |
-
- conversation_enc: binary
|
| 32 |
-
- width: int32
|
| 33 |
-
- height: int32
|
| 34 |
-
- image_format: string
|
| 35 |
-
|
| 36 |
-
## Encryption
|
| 37 |
-
|
| 38 |
-
- AES-256-GCM
|
| 39 |
-
- Independent encryption for image and conversation
|
| 40 |
-
- 12-byte nonce
|
| 41 |
-
- Scrypt key derivation
|
| 42 |
-
- Authenticated sample IDs and field names
|
| 43 |
-
|
| 44 |
-
## Image preprocessing
|
| 45 |
-
|
| 46 |
-
- EXIF correction
|
| 47 |
-
- RGB conversion
|
| 48 |
-
- Maximum dimension 1024
|
| 49 |
-
- LANCZOS resizing
|
| 50 |
-
|
| 51 |
-
## Security
|
| 52 |
-
|
| 53 |
-
The output password is not stored in the repository.
|
| 54 |
-
Arrow files contain encrypted images and conversations.
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
gemma4_arrow_encrypted_v1/append_report.json
DELETED
|
@@ -1,14 +0,0 @@
|
|
| 1 |
-
{
|
| 2 |
-
"source_repository": "codeShare/sankaku_captioned_images",
|
| 3 |
-
"destination_repository": "codeShare/kromoss-lora-exports",
|
| 4 |
-
"existing_records": 0,
|
| 5 |
-
"new_records": 1973,
|
| 6 |
-
"skipped_records": 5,
|
| 7 |
-
"final_records": 1973,
|
| 8 |
-
"verification": {
|
| 9 |
-
"checked": 30,
|
| 10 |
-
"passed": 30,
|
| 11 |
-
"failed": 0
|
| 12 |
-
},
|
| 13 |
-
"created_unix": 1794611117.224032
|
| 14 |
-
}
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
gemma4_arrow_encrypted_v1/data-00000-of-00001.arrow
DELETED
|
@@ -1,3 +0,0 @@
|
|
| 1 |
-
version https://git-lfs.github.com/spec/v1
|
| 2 |
-
oid sha256:e35630410be176004b8d4d5affc5fcaf715588f5115283cc332fcd1df788d92e
|
| 3 |
-
size 425455408
|
|
|
|
|
|
|
|
|
|
|
|
gemma4_arrow_encrypted_v1/dataset_info.json
DELETED
|
@@ -1,32 +0,0 @@
|
|
| 1 |
-
{
|
| 2 |
-
"citation": "",
|
| 3 |
-
"description": "",
|
| 4 |
-
"features": {
|
| 5 |
-
"sample_id": {
|
| 6 |
-
"dtype": "string",
|
| 7 |
-
"_type": "Value"
|
| 8 |
-
},
|
| 9 |
-
"image_enc": {
|
| 10 |
-
"dtype": "binary",
|
| 11 |
-
"_type": "Value"
|
| 12 |
-
},
|
| 13 |
-
"conversation_enc": {
|
| 14 |
-
"dtype": "binary",
|
| 15 |
-
"_type": "Value"
|
| 16 |
-
},
|
| 17 |
-
"width": {
|
| 18 |
-
"dtype": "int32",
|
| 19 |
-
"_type": "Value"
|
| 20 |
-
},
|
| 21 |
-
"height": {
|
| 22 |
-
"dtype": "int32",
|
| 23 |
-
"_type": "Value"
|
| 24 |
-
},
|
| 25 |
-
"image_format": {
|
| 26 |
-
"dtype": "string",
|
| 27 |
-
"_type": "Value"
|
| 28 |
-
}
|
| 29 |
-
},
|
| 30 |
-
"homepage": "",
|
| 31 |
-
"license": ""
|
| 32 |
-
}
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
gemma4_arrow_encrypted_v1/encryption_manifest.json
DELETED
|
@@ -1,61 +0,0 @@
|
|
| 1 |
-
{
|
| 2 |
-
"dataset_name": "gemma4_encrypted_multimodal",
|
| 3 |
-
"dataset_version": 1,
|
| 4 |
-
"serialization": "Hugging Face Datasets / Apache Arrow",
|
| 5 |
-
"sample_count": 1973,
|
| 6 |
-
"image_preprocessing": {
|
| 7 |
-
"max_dimension": 1024,
|
| 8 |
-
"orientation": "EXIF corrected",
|
| 9 |
-
"color_space": "RGB",
|
| 10 |
-
"resampling": "LANCZOS",
|
| 11 |
-
"jpeg_quality": 95,
|
| 12 |
-
"webp_quality": 95
|
| 13 |
-
},
|
| 14 |
-
"text_preprocessing": {
|
| 15 |
-
"applied": false,
|
| 16 |
-
"description": "Original instruction and assistant text preserved."
|
| 17 |
-
},
|
| 18 |
-
"encryption": {
|
| 19 |
-
"cipher": "AES-256-GCM",
|
| 20 |
-
"nonce_bytes": 12,
|
| 21 |
-
"kdf": "scrypt",
|
| 22 |
-
"kdf_salt_base64": "6O5eNt828QAeylf9xDiMyg==",
|
| 23 |
-
"scrypt_n": 16384,
|
| 24 |
-
"scrypt_r": 8,
|
| 25 |
-
"scrypt_p": 1,
|
| 26 |
-
"key_length_bytes": 32,
|
| 27 |
-
"aad_format": "gemma4-v1:{sample_id}:{field_name}",
|
| 28 |
-
"encrypted_fields": [
|
| 29 |
-
"image_enc",
|
| 30 |
-
"conversation_enc"
|
| 31 |
-
]
|
| 32 |
-
},
|
| 33 |
-
"arrow_schema": {
|
| 34 |
-
"sample_id": "string",
|
| 35 |
-
"image_enc": "binary",
|
| 36 |
-
"conversation_enc": "binary",
|
| 37 |
-
"width": "int32",
|
| 38 |
-
"height": "int32",
|
| 39 |
-
"image_format": "string"
|
| 40 |
-
},
|
| 41 |
-
"conversion_statistics": {
|
| 42 |
-
"processed": 1973,
|
| 43 |
-
"skipped": 5
|
| 44 |
-
},
|
| 45 |
-
"source_repositories": [
|
| 46 |
-
"codeShare/sankaku_captioned_images"
|
| 47 |
-
],
|
| 48 |
-
"created_unix": 1794611028.8531384,
|
| 49 |
-
"last_append": {
|
| 50 |
-
"source_repository": "codeShare/sankaku_captioned_images",
|
| 51 |
-
"source_parquet": "preprocessed/captions.parquet",
|
| 52 |
-
"records_added": 1973,
|
| 53 |
-
"records_skipped": 5,
|
| 54 |
-
"verification": {
|
| 55 |
-
"checked": 30,
|
| 56 |
-
"passed": 30,
|
| 57 |
-
"failed": 0
|
| 58 |
-
},
|
| 59 |
-
"created_unix": 1794611113.429829
|
| 60 |
-
}
|
| 61 |
-
}
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
gemma4_arrow_encrypted_v1/processing_errors.json
DELETED
|
@@ -1,42 +0,0 @@
|
|
| 1 |
-
[
|
| 2 |
-
{
|
| 3 |
-
"idx": "8",
|
| 4 |
-
"sample_id": "sankaku_captioned_v1_8_r0000000008",
|
| 5 |
-
"json_file": "preprocessed/images-000000000000-000000000059.jsonl",
|
| 6 |
-
"byte_offset": 2270563,
|
| 7 |
-
"byte_length": 286554,
|
| 8 |
-
"error": "ValueError('Both assistant response fields are empty.')"
|
| 9 |
-
},
|
| 10 |
-
{
|
| 11 |
-
"idx": "10",
|
| 12 |
-
"sample_id": "sankaku_captioned_v1_10_r0000000010",
|
| 13 |
-
"json_file": "preprocessed/images-000000000000-000000000059.jsonl",
|
| 14 |
-
"byte_offset": 2720887,
|
| 15 |
-
"byte_length": 191656,
|
| 16 |
-
"error": "ValueError('Both assistant response fields are empty.')"
|
| 17 |
-
},
|
| 18 |
-
{
|
| 19 |
-
"idx": "31",
|
| 20 |
-
"sample_id": "sankaku_captioned_v1_31_r0000000031",
|
| 21 |
-
"json_file": "preprocessed/images-000000000000-000000000059.jsonl",
|
| 22 |
-
"byte_offset": 8117592,
|
| 23 |
-
"byte_length": 349660,
|
| 24 |
-
"error": "ValueError('Both assistant response fields are empty.')"
|
| 25 |
-
},
|
| 26 |
-
{
|
| 27 |
-
"idx": "33",
|
| 28 |
-
"sample_id": "sankaku_captioned_v1_33_r0000000033",
|
| 29 |
-
"json_file": "preprocessed/images-000000000000-000000000059.jsonl",
|
| 30 |
-
"byte_offset": 8784983,
|
| 31 |
-
"byte_length": 173942,
|
| 32 |
-
"error": "ValueError('Both assistant response fields are empty.')"
|
| 33 |
-
},
|
| 34 |
-
{
|
| 35 |
-
"idx": "96",
|
| 36 |
-
"sample_id": "sankaku_captioned_v1_96_r0000000096",
|
| 37 |
-
"json_file": "preprocessed/images-000000000060-000000000119.jsonl",
|
| 38 |
-
"byte_offset": 10933402,
|
| 39 |
-
"byte_length": 361263,
|
| 40 |
-
"error": "ValueError('Both assistant response fields are empty.')"
|
| 41 |
-
}
|
| 42 |
-
]
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
gemma4_arrow_encrypted_v1/state.json
DELETED
|
@@ -1,13 +0,0 @@
|
|
| 1 |
-
{
|
| 2 |
-
"_data_files": [
|
| 3 |
-
{
|
| 4 |
-
"filename": "data-00000-of-00001.arrow"
|
| 5 |
-
}
|
| 6 |
-
],
|
| 7 |
-
"_fingerprint": "80ba9419c155b131",
|
| 8 |
-
"_format_columns": null,
|
| 9 |
-
"_format_kwargs": {},
|
| 10 |
-
"_format_type": null,
|
| 11 |
-
"_output_all_columns": false,
|
| 12 |
-
"_split": null
|
| 13 |
-
}
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|