Add tokenizer files and update documentation

- Added tokenizer.json, tokenizer_config.json, special_tokens_map.json, spiece.model
- Updated README with comprehensive tokenizer usage examples
- Updated config.json to include tokenizer file information
- Now provides complete package for text-to-text generation
- Track spiece.model with Git LFS

.gitattributes

@@ -1 +1,2 @@
 *.mlpackage/** filter=lfs diff=lfs merge=lfs -text
+*.model filter=lfs diff=lfs merge=lfs -text

README.md

@@ -15,9 +15,16 @@ This repository contains CoreML versions of Google's FLAN-T5 Base model, optimiz
 
 ## Files
 
+### Model Files
 - `flan_t5_base_encoder.mlpackage` - T5 Encoder component
 - `flan_t5_base_decoder.mlpackage` - T5 Decoder component
 
+### Tokenizer Files
+- `tokenizer.json` - Fast tokenizer configuration
+- `tokenizer_config.json` - Tokenizer metadata and settings
+- `special_tokens_map.json` - Special token mappings
+- `spiece.model` - SentencePiece model for tokenization
+
 ## Model Architecture
 
 FLAN-T5 is an encoder-decoder transformer model that has been converted into two separate CoreML models:
@@ -83,38 +90,71 @@ let decoderOutput = try decoderModel.prediction(from: decoderInput)
 let logits = decoderOutput.featureValue(for: "logits")?.multiArrayValue
 ```
 
-### Python Usage (for testing)
+### Python Usage with Tokenizer
 
 ```python
 import coremltools as ct
 import numpy as np
+from transformers import T5Tokenizer
+
+# Load tokenizer
+tokenizer = T5Tokenizer.from_pretrained("./")  # Uses local tokenizer files
 
-# Load models
+# Load CoreML models
 encoder = ct.models.MLModel("flan_t5_base_encoder.mlpackage")
 decoder = ct.models.MLModel("flan_t5_base_decoder.mlpackage")
 
-# Example inference
-input_ids = np.random.randint(0, 32128, (1, 512)).astype(np.int32)
-attention_mask = np.ones((1, 512), dtype=np.int32)
+# Example text-to-text generation
+input_text = "translate English to French: Hello, how are you?"
+inputs = tokenizer(input_text, return_tensors="np", padding="max_length", 
+                  truncation=True, max_length=512)
 
 # Run encoder
 encoder_output = encoder.predict({
-    "input_ids": input_ids,
-    "attention_mask": attention_mask
+    "input_ids": inputs["input_ids"].astype(np.int32),
+    "attention_mask": inputs["attention_mask"].astype(np.int32)
 })
 hidden_states = encoder_output["hidden_states"]
 
-# Run decoder
-decoder_input_ids = np.random.randint(0, 32128, (1, 512)).astype(np.int32)
-decoder_attention_mask = np.ones((1, 512), dtype=np.int32)
+# For decoder, start with decoder start token
+decoder_input_ids = np.array([[tokenizer.pad_token_id]], dtype=np.int32)
+decoder_input_ids = np.pad(decoder_input_ids, ((0, 0), (0, 511)), 
+                          constant_values=tokenizer.pad_token_id).astype(np.int32)
+decoder_attention_mask = (decoder_input_ids != tokenizer.pad_token_id).astype(np.int32)
 
+# Run decoder
 decoder_output = decoder.predict({
     "decoder_input_ids": decoder_input_ids,
     "encoder_hidden_states": hidden_states,
     "decoder_attention_mask": decoder_attention_mask,
-    "encoder_attention_mask": attention_mask
+    "encoder_attention_mask": inputs["attention_mask"].astype(np.int32)
 })
 logits = decoder_output["logits"]
+
+# Get predicted token (greedy decoding)
+predicted_token_id = np.argmax(logits[0, 0, :])
+print(f"Predicted token: {tokenizer.decode([predicted_token_id])}")
+```
+
+### Basic Tokenizer Usage
+
+```python
+from transformers import T5Tokenizer
+
+# Load tokenizer from downloaded files
+tokenizer = T5Tokenizer.from_pretrained("./")
+
+# Tokenize text
+text = "summarize: The quick brown fox jumps over the lazy dog."
+tokens = tokenizer(text, return_tensors="np", padding="max_length", 
+                  truncation=True, max_length=512)
+
+print(f"Input IDs shape: {tokens['input_ids'].shape}")
+print(f"Attention mask shape: {tokens['attention_mask'].shape}")
+
+# Decode tokens back to text
+decoded = tokenizer.decode(tokens['input_ids'][0], skip_special_tokens=True)
+print(f"Decoded: {decoded}")
 ```
 
 ## Model Capabilities

config.json

@@ -34,5 +34,11 @@
   "model_files": {
     "encoder": "flan_t5_base_encoder.mlpackage",
     "decoder": "flan_t5_base_decoder.mlpackage"
+  },
+  "tokenizer_files": {
+    "tokenizer": "tokenizer.json",
+    "tokenizer_config": "tokenizer_config.json",
+    "special_tokens_map": "special_tokens_map.json",
+    "spiece_model": "spiece.model"
   }
 }

special_tokens_map.json

@@ -0,0 +1,107 @@
+{
+  "additional_special_tokens": [
+    "<extra_id_0>",
+    "<extra_id_1>",
+    "<extra_id_2>",
+    "<extra_id_3>",
+    "<extra_id_4>",
+    "<extra_id_5>",
+    "<extra_id_6>",
+    "<extra_id_7>",
+    "<extra_id_8>",
+    "<extra_id_9>",
+    "<extra_id_10>",
+    "<extra_id_11>",
+    "<extra_id_12>",
+    "<extra_id_13>",
+    "<extra_id_14>",
+    "<extra_id_15>",
+    "<extra_id_16>",
+    "<extra_id_17>",
+    "<extra_id_18>",
+    "<extra_id_19>",
+    "<extra_id_20>",
+    "<extra_id_21>",
+    "<extra_id_22>",
+    "<extra_id_23>",
+    "<extra_id_24>",
+    "<extra_id_25>",
+    "<extra_id_26>",
+    "<extra_id_27>",
+    "<extra_id_28>",
+    "<extra_id_29>",
+    "<extra_id_30>",
+    "<extra_id_31>",
+    "<extra_id_32>",
+    "<extra_id_33>",
+    "<extra_id_34>",
+    "<extra_id_35>",
+    "<extra_id_36>",
+    "<extra_id_37>",
+    "<extra_id_38>",
+    "<extra_id_39>",
+    "<extra_id_40>",
+    "<extra_id_41>",
+    "<extra_id_42>",
+    "<extra_id_43>",
+    "<extra_id_44>",
+    "<extra_id_45>",
+    "<extra_id_46>",
+    "<extra_id_47>",
+    "<extra_id_48>",
+    "<extra_id_49>",
+    "<extra_id_50>",
+    "<extra_id_51>",
+    "<extra_id_52>",
+    "<extra_id_53>",
+    "<extra_id_54>",
+    "<extra_id_55>",
+    "<extra_id_56>",
+    "<extra_id_57>",
+    "<extra_id_58>",
+    "<extra_id_59>",
+    "<extra_id_60>",
+    "<extra_id_61>",
+    "<extra_id_62>",
+    "<extra_id_63>",
+    "<extra_id_64>",
+    "<extra_id_65>",
+    "<extra_id_66>",
+    "<extra_id_67>",
+    "<extra_id_68>",
+    "<extra_id_69>",
+    "<extra_id_70>",
+    "<extra_id_71>",
+    "<extra_id_72>",
+    "<extra_id_73>",
+    "<extra_id_74>",
+    "<extra_id_75>",
+    "<extra_id_76>",
+    "<extra_id_77>",
+    "<extra_id_78>",
+    "<extra_id_79>",
+    "<extra_id_80>",
+    "<extra_id_81>",
+    "<extra_id_82>",
+    "<extra_id_83>",
+    "<extra_id_84>",
+    "<extra_id_85>",
+    "<extra_id_86>",
+    "<extra_id_87>",
+    "<extra_id_88>",
+    "<extra_id_89>",
+    "<extra_id_90>",
+    "<extra_id_91>",
+    "<extra_id_92>",
+    "<extra_id_93>",
+    "<extra_id_94>",
+    "<extra_id_95>",
+    "<extra_id_96>",
+    "<extra_id_97>",
+    "<extra_id_98>",
+    "<extra_id_99>"
+  ],
+  "eos_token": "</s>",
+  "pad_token": "<pad>",
+  "unk_token": "<unk>"
+}

spiece.model

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:d60acb128cf7b7f2536e8f38a5b18a05535c9e14c7a355904270e15b0945ea86
+size 791656

test_download/downloaded_models/.gitattributes

@@ -0,0 +1 @@
+*.mlpackage/** filter=lfs diff=lfs merge=lfs -text

test_download/downloaded_models/README.md

@@ -0,0 +1,162 @@
+# FLAN-T5 Base CoreML Models
+
+This repository contains CoreML versions of Google's FLAN-T5 Base model, optimized for efficient inference on Apple devices (macOS/iOS).
+
+## Model Details
+
+- **Base Model**: [google/flan-t5-base](https://huggingface.co/google/flan-t5-base)
+- **Architecture**: T5 (Text-to-Text Transfer Transformer)
+- **Model Size**: 
+  - Encoder: 215MB
+  - Decoder: 324MB
+- **Framework**: CoreML (.mlpackage format)
+- **Precision**: FP16 for optimal performance
+- **Deployment Target**: iOS 15+ / macOS 12+
+
+## Files
+
+- `flan_t5_base_encoder.mlpackage` - T5 Encoder component
+- `flan_t5_base_decoder.mlpackage` - T5 Decoder component
+
+## Model Architecture
+
+FLAN-T5 is an encoder-decoder transformer model that has been converted into two separate CoreML models:
+
+### Encoder
+- **Input**: `input_ids` (shape: [1, 512], dtype: int32), `attention_mask` (shape: [1, 512], dtype: int32)
+- **Output**: `hidden_states` (shape: [1, 512, 768], dtype: float32)
+
+### Decoder  
+- **Inputs**: 
+  - `decoder_input_ids` (shape: [1, 512], dtype: int32)
+  - `encoder_hidden_states` (shape: [1, 512, 768], dtype: float32)
+  - `decoder_attention_mask` (shape: [1, 512], dtype: int32)
+  - `encoder_attention_mask` (shape: [1, 512], dtype: int32)
+- **Output**: `logits` (shape: [1, 512, 32128], dtype: float32)
+
+## Usage
+
+### Download Models
+
+```bash
+# Download both models
+huggingface-cli download mazhewitt/flan-t5-base-coreml --local-dir ./models
+
+# Or download individual models
+huggingface-cli download mazhewitt/flan-t5-base-coreml flan_t5_base_encoder.mlpackage --local-dir ./models
+huggingface-cli download mazhewitt/flan-t5-base-coreml flan_t5_base_decoder.mlpackage --local-dir ./models
+```
+
+### Swift/iOS Usage
+
+```swift
+import CoreML
+
+// Load models
+guard let encoderURL = Bundle.main.url(forResource: "flan_t5_base_encoder", withExtension: "mlpackage"),
+      let decoderURL = Bundle.main.url(forResource: "flan_t5_base_decoder", withExtension: "mlpackage") else {
+    fatalError("Models not found")
+}
+
+let encoderModel = try MLModel(contentsOf: encoderURL)
+let decoderModel = try MLModel(contentsOf: decoderURL)
+
+// Create prediction inputs
+let encoderInput = try MLDictionaryFeatureProvider(dictionary: [
+    "input_ids": MLMultiArray(/* your input tokens */),
+    "attention_mask": MLMultiArray(/* your attention mask */)
+])
+
+// Run encoder
+let encoderOutput = try encoderModel.prediction(from: encoderInput)
+let hiddenStates = encoderOutput.featureValue(for: "hidden_states")?.multiArrayValue
+
+// Run decoder with encoder outputs
+let decoderInput = try MLDictionaryFeatureProvider(dictionary: [
+    "decoder_input_ids": MLMultiArray(/* decoder input tokens */),
+    "encoder_hidden_states": hiddenStates!,
+    "decoder_attention_mask": MLMultiArray(/* decoder attention mask */),
+    "encoder_attention_mask": MLMultiArray(/* encoder attention mask */)
+])
+
+let decoderOutput = try decoderModel.prediction(from: decoderInput)
+let logits = decoderOutput.featureValue(for: "logits")?.multiArrayValue
+```
+
+### Python Usage (for testing)
+
+```python
+import coremltools as ct
+import numpy as np
+
+# Load models
+encoder = ct.models.MLModel("flan_t5_base_encoder.mlpackage")
+decoder = ct.models.MLModel("flan_t5_base_decoder.mlpackage")
+
+# Example inference
+input_ids = np.random.randint(0, 32128, (1, 512)).astype(np.int32)
+attention_mask = np.ones((1, 512), dtype=np.int32)
+
+# Run encoder
+encoder_output = encoder.predict({
+    "input_ids": input_ids,
+    "attention_mask": attention_mask
+})
+hidden_states = encoder_output["hidden_states"]
+
+# Run decoder
+decoder_input_ids = np.random.randint(0, 32128, (1, 512)).astype(np.int32)
+decoder_attention_mask = np.ones((1, 512), dtype=np.int32)
+
+decoder_output = decoder.predict({
+    "decoder_input_ids": decoder_input_ids,
+    "encoder_hidden_states": hidden_states,
+    "decoder_attention_mask": decoder_attention_mask,
+    "encoder_attention_mask": attention_mask
+})
+logits = decoder_output["logits"]
+```
+
+## Model Capabilities
+
+FLAN-T5 has been instruction-tuned and can perform various text-to-text tasks:
+
+- **Text Summarization**: "summarize: [text]"
+- **Translation**: "translate English to French: [text]"
+- **Question Answering**: "answer the question: [question] context: [context]"
+- **General Instructions**: Direct natural language instructions
+
+## Performance Considerations
+
+- **Memory**: Encoder (~215MB) + Decoder (~324MB) = ~539MB total
+- **Precision**: FP16 for balance of accuracy and performance
+- **Sequence Length**: Maximum 512 tokens
+- **Device Compatibility**: Apple Neural Engine, GPU, or CPU depending on availability
+
+## Conversion Details
+
+- **Source Framework**: PyTorch/Transformers
+- **Conversion Tool**: CoreML Tools 8.3.0
+- **Date**: July 2025
+- **Torch Version**: 2.7.1 (with compatibility warnings handled)
+
+## License
+
+This model follows the same license as the original FLAN-T5 model. Please refer to the [original model card](https://huggingface.co/google/flan-t5-base) for licensing details.
+
+## Citation
+
+If you use these models, please cite the original FLAN-T5 paper:
+
+```bibtex
+@article{chung2022scaling,
+  title={Scaling instruction-finetuned language models},
+  author={Chung, Hyung Won and Hou, Le and Longpre, Shayne and Zoph, Barret and Tay, Yi and Fedus, William and Li, Eric and Wang, Xuezhi and Mostafazadeh, Nasrin and Shen, Jianmo and others},
+  journal={arXiv preprint arXiv:2210.11416},
+  year={2022}
+}
+```
+
+## Issues and Support
+
+For issues specific to these CoreML conversions, please open an issue in this repository. For general FLAN-T5 questions, refer to the original model repository.

test_download/downloaded_models/config.json

@@ -0,0 +1,38 @@
+{
+  "model_type": "t5",
+  "framework": "coreml",
+  "base_model": "google/flan-t5-base",
+  "conversion_info": {
+    "converted_by": "coremltools",
+    "version": "8.3.0",
+    "date": "2025-07-20",
+    "precision": "float16",
+    "deployment_target": "iOS15"
+  },
+  "architecture": {
+    "encoder": {
+      "input_shape": {
+        "input_ids": [1, 512],
+        "attention_mask": [1, 512]
+      },
+      "output_shape": {
+        "hidden_states": [1, 512, 768]
+      }
+    },
+    "decoder": {
+      "input_shape": {
+        "decoder_input_ids": [1, 512],
+        "encoder_hidden_states": [1, 512, 768],
+        "decoder_attention_mask": [1, 512],
+        "encoder_attention_mask": [1, 512]
+      },
+      "output_shape": {
+        "logits": [1, 512, 32128]
+      }
+    }
+  },
+  "model_files": {
+    "encoder": "flan_t5_base_encoder.mlpackage",
+    "decoder": "flan_t5_base_decoder.mlpackage"
+  }
+}

test_download/downloaded_models/flan_t5_base_decoder.mlpackage/Data/com.apple.CoreML/model.mlmodel

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:91e68f516139c4694bf48b4ad7d9478899ef060ba23ae5230fb7ea0ccc7925f6
+size 1011351

test_download/downloaded_models/flan_t5_base_decoder.mlpackage/Data/com.apple.CoreML/weights/weight.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:89887d8432bddf1353aba61261a58f4187bca6687d064422cf85f4e056b208e0
+size 338959296

test_download/downloaded_models/flan_t5_base_decoder.mlpackage/Manifest.json

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:171ccb76696a53e4566d20ac21d5a4f2e61ec8c1e158ff72a170523d22a79064
+size 617

test_download/downloaded_models/flan_t5_base_encoder.mlpackage/Data/com.apple.CoreML/model.mlmodel

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:4c30d7f442e7a938d05dd29e63e938ea2b72fe84748bcb82440e8ad658996a9b
+size 144373

test_download/downloaded_models/flan_t5_base_encoder.mlpackage/Data/com.apple.CoreML/weights/weight.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:7c2977bd6f2cb06e53067c0012b9944f44ef425f4a55e5a133fae9ca74c32876
+size 225560704

test_download/downloaded_models/flan_t5_base_encoder.mlpackage/Manifest.json

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:2221222f34d57f6aa55cdacc2f6cc1bfdd5e190fb0fcc0227c77ce06ccc40658
+size 617

tokenizer_config.json

@@ -0,0 +1,113 @@
+{
+  "additional_special_tokens": [
+    "<extra_id_0>",
+    "<extra_id_1>",
+    "<extra_id_2>",
+    "<extra_id_3>",
+    "<extra_id_4>",
+    "<extra_id_5>",
+    "<extra_id_6>",
+    "<extra_id_7>",
+    "<extra_id_8>",
+    "<extra_id_9>",
+    "<extra_id_10>",
+    "<extra_id_11>",
+    "<extra_id_12>",
+    "<extra_id_13>",
+    "<extra_id_14>",
+    "<extra_id_15>",
+    "<extra_id_16>",
+    "<extra_id_17>",
+    "<extra_id_18>",
+    "<extra_id_19>",
+    "<extra_id_20>",
+    "<extra_id_21>",
+    "<extra_id_22>",
+    "<extra_id_23>",
+    "<extra_id_24>",
+    "<extra_id_25>",
+    "<extra_id_26>",
+    "<extra_id_27>",
+    "<extra_id_28>",
+    "<extra_id_29>",
+    "<extra_id_30>",
+    "<extra_id_31>",
+    "<extra_id_32>",
+    "<extra_id_33>",
+    "<extra_id_34>",
+    "<extra_id_35>",
+    "<extra_id_36>",
+    "<extra_id_37>",
+    "<extra_id_38>",
+    "<extra_id_39>",
+    "<extra_id_40>",
+    "<extra_id_41>",
+    "<extra_id_42>",
+    "<extra_id_43>",
+    "<extra_id_44>",
+    "<extra_id_45>",
+    "<extra_id_46>",
+    "<extra_id_47>",
+    "<extra_id_48>",
+    "<extra_id_49>",
+    "<extra_id_50>",
+    "<extra_id_51>",
+    "<extra_id_52>",
+    "<extra_id_53>",
+    "<extra_id_54>",
+    "<extra_id_55>",
+    "<extra_id_56>",
+    "<extra_id_57>",
+    "<extra_id_58>",
+    "<extra_id_59>",
+    "<extra_id_60>",
+    "<extra_id_61>",
+    "<extra_id_62>",
+    "<extra_id_63>",
+    "<extra_id_64>",
+    "<extra_id_65>",
+    "<extra_id_66>",
+    "<extra_id_67>",
+    "<extra_id_68>",
+    "<extra_id_69>",
+    "<extra_id_70>",
+    "<extra_id_71>",
+    "<extra_id_72>",
+    "<extra_id_73>",
+    "<extra_id_74>",
+    "<extra_id_75>",
+    "<extra_id_76>",
+    "<extra_id_77>",
+    "<extra_id_78>",
+    "<extra_id_79>",
+    "<extra_id_80>",
+    "<extra_id_81>",
+    "<extra_id_82>",
+    "<extra_id_83>",
+    "<extra_id_84>",
+    "<extra_id_85>",
+    "<extra_id_86>",
+    "<extra_id_87>",
+    "<extra_id_88>",
+    "<extra_id_89>",
+    "<extra_id_90>",
+    "<extra_id_91>",
+    "<extra_id_92>",
+    "<extra_id_93>",
+    "<extra_id_94>",
+    "<extra_id_95>",
+    "<extra_id_96>",
+    "<extra_id_97>",
+    "<extra_id_98>",
+    "<extra_id_99>"
+  ],
+  "eos_token": "</s>",
+  "extra_ids": 100,
+  "model_max_length": 512,
+  "name_or_path": "google/t5-v1_1-base",
+  "pad_token": "<pad>",
+  "sp_model_kwargs": {},
+  "special_tokens_map_file": "/home/younes_huggingface_co/.cache/huggingface/hub/models--google--t5-v1_1-base/snapshots/650d7745bf1e502d6949b22cc19155cd656d3d4e/special_tokens_map.json",
+  "tokenizer_class": "T5Tokenizer",
+  "unk_token": "<unk>"
+}