| |
| |
| |
| |
|
|
| from typing import List, Union |
|
|
| import torch |
| from dinov3.thirdparty.CLIP.clip.simple_tokenizer import SimpleTokenizer |
|
|
|
|
| class Tokenizer(SimpleTokenizer): |
| def __init__(self, vocab_path: str): |
| SimpleTokenizer.__init__(self, bpe_path=vocab_path) |
|
|
| def tokenize( |
| self, texts: Union[str, List[str]], context_length: int = 77 |
| ) -> torch.LongTensor: |
| """ |
| Returns the tokenized representation of given input string(s) |
| |
| Parameters |
| ---------- |
| texts : Union[str, List[str]] |
| An input string or a list of input strings to tokenize |
| context_length : int |
| The context length to use; all CLIP models use 77 as the context length |
| |
| Returns |
| ------- |
| A two-dimensional tensor containing the resulting tokens, shape = [number of input strings, context_length] |
| """ |
| if isinstance(texts, str): |
| texts = [texts] |
| sot_token = self.encoder["<|startoftext|>"] |
| eot_token = self.encoder["<|endoftext|>"] |
| all_tokens = [[sot_token] + self.encode(text) + [eot_token] for text in texts] |
| result = torch.zeros(len(all_tokens), context_length, dtype=torch.long) |
|
|
| for i, tokens in enumerate(all_tokens): |
| if len(tokens) > context_length: |
| tokens = tokens[:context_length] |
| tokens[-1] = eot_token |
| result[i, : len(tokens)] = torch.tensor(tokens) |
|
|
| return result |
|
|
|
|
| def get_tokenizer(bpe_path_or_url: str) -> Tokenizer | None: |
| import urllib |
| from io import BytesIO |
|
|
| from .tokenizer import Tokenizer |
|
|
| if urllib.parse.urlparse(bpe_path_or_url).scheme: |
| try: |
| with urllib.request.urlopen(bpe_path_or_url) as response: |
| file_buf = BytesIO(response.read()) |
| return Tokenizer(vocab_path=file_buf) |
| except Exception as e: |
| raise FileNotFoundError( |
| f"Failed to download file from url {bpe_path_or_url} with error last: {e}" |
| ) |
| else: |
| with open(bpe_path_or_url, "rb") as f: |
| file_buf = BytesIO(f.read()) |
| return Tokenizer(vocab_path=file_buf) |
|
|
