susiegriggo commited on
Commit
4d60816
·
verified ·
1 Parent(s): ac7b2ad

Delete fold_4/best_macro_auprc_model/tokenization_ntv3.py with huggingface_hub

Browse files
fold_4/best_macro_auprc_model/tokenization_ntv3.py DELETED
@@ -1,234 +0,0 @@
1
- from __future__ import annotations
2
-
3
- import json
4
- import os
5
- from typing import Any, Dict, Iterable, List, Optional, Sequence, Tuple
6
-
7
- from transformers import PreTrainedTokenizer
8
-
9
-
10
- def _as_str(token: Any) -> Optional[str]:
11
- """Return the plain string representation for tokens/AddedTokens."""
12
- if token is None:
13
- return None
14
- if isinstance(token, str):
15
- return token
16
- return getattr(token, "content", str(token))
17
-
18
-
19
- class _BaseNTv3Tokenizer(PreTrainedTokenizer):
20
- """Shared convenience implementation for the NTv3 tokenizers."""
21
-
22
- vocab_files_names = {"vocab_file": "vocab.json"}
23
- model_input_names = ["input_ids"]
24
-
25
- def __init__(
26
- self,
27
- *,
28
- vocab_file: Optional[str],
29
- unk_token: str,
30
- pad_token: str,
31
- mask_token: str,
32
- cls_token: str,
33
- eos_token: str,
34
- bos_token: str,
35
- default_tokens: Sequence[str],
36
- standard_tokens: Iterable[str],
37
- **kwargs: Any,
38
- ) -> None:
39
- if vocab_file is None:
40
- token_to_id = {tok: idx for idx, tok in enumerate(default_tokens)}
41
- else:
42
- if not os.path.isfile(vocab_file):
43
- raise ValueError(f"Can't find a vocab file at path '{vocab_file}'.")
44
- with open(vocab_file, "r", encoding="utf-8") as handle:
45
- loaded = json.load(handle)
46
- if isinstance(loaded, dict):
47
- token_to_id = {str(tok): int(idx) for tok, idx in loaded.items()}
48
- else:
49
- token_to_id = {str(tok): idx for idx, tok in enumerate(loaded)}
50
-
51
- self._token_to_id: Dict[str, int] = dict(token_to_id)
52
- self._id_to_token: Dict[int, str] = {
53
- int(idx): tok for tok, idx in self._token_to_id.items()
54
- }
55
-
56
- super().__init__(
57
- unk_token=unk_token,
58
- pad_token=pad_token,
59
- mask_token=mask_token,
60
- cls_token=cls_token,
61
- eos_token=eos_token,
62
- bos_token=bos_token,
63
- **kwargs,
64
- )
65
-
66
- self._unk_token_str = _as_str(self.unk_token)
67
- self._pad_token_str = _as_str(self.pad_token)
68
- self._mask_token_str = _as_str(self.mask_token)
69
- self._cls_token_str = _as_str(self.cls_token)
70
- self._eos_token_str = _as_str(self.eos_token)
71
- self._bos_token_str = _as_str(self.bos_token)
72
-
73
- self._special_token_strings = {
74
- tok
75
- for tok in (
76
- self._unk_token_str,
77
- self._pad_token_str,
78
- self._mask_token_str,
79
- self._cls_token_str,
80
- self._eos_token_str,
81
- self._bos_token_str,
82
- )
83
- if tok is not None
84
- }
85
-
86
- self._standard_tokens = {str(tok) for tok in standard_tokens}
87
- self._unk_literal = (
88
- self._unk_token_str if self._unk_token_str in self._token_to_id else "<unk>"
89
- )
90
-
91
- # ------------------------------------------------------------------
92
- # Hugging Face required interface
93
- # ------------------------------------------------------------------
94
- def get_vocab(self) -> Dict[str, int]:
95
- return dict(self._token_to_id)
96
-
97
- @property
98
- def vocab_size(self) -> int:
99
- return len(self._token_to_id)
100
-
101
- # Sub-classes implement `_tokenize`.
102
-
103
- def _convert_token_to_id(self, token: str) -> int:
104
- if self._unk_literal in self._token_to_id:
105
- return self._token_to_id.get(token, self._token_to_id[self._unk_literal])
106
- return self._token_to_id.get(token, 0)
107
-
108
- def _convert_id_to_token(self, index: int) -> str:
109
- idx = int(index)
110
- return self._id_to_token.get(idx, self._unk_literal)
111
-
112
- def build_inputs_with_special_tokens(
113
- self, token_ids_0: List[int], token_ids_1: Optional[List[int]] = None
114
- ) -> List[int]:
115
- if token_ids_1 is None:
116
- return token_ids_0
117
- return token_ids_0 + token_ids_1
118
-
119
- def get_special_tokens_mask(
120
- self,
121
- token_ids_0: List[int],
122
- token_ids_1: Optional[List[int]] = None,
123
- already_has_special_tokens: bool = False,
124
- ) -> List[int]:
125
- length = len(token_ids_0) + (len(token_ids_1) if token_ids_1 else 0)
126
- if not already_has_special_tokens:
127
- return [0] * length
128
- return [
129
- 1 if self._convert_id_to_token(idx) in self._special_token_strings else 0
130
- for idx in token_ids_0
131
- ] + (
132
- [
133
- 1
134
- if self._convert_id_to_token(idx) in self._special_token_strings
135
- else 0
136
- for idx in (token_ids_1 or [])
137
- ]
138
- )
139
-
140
- def create_token_type_ids_from_sequences(
141
- self, token_ids_0: List[int], token_ids_1: Optional[List[int]] = None
142
- ) -> List[int]:
143
- return [0] * (len(token_ids_0) + (len(token_ids_1) if token_ids_1 else 0))
144
-
145
- # ------------------------------------------------------------------
146
- # Persistence helpers
147
- # ------------------------------------------------------------------
148
- def save_vocabulary(
149
- self, save_directory: str, filename_prefix: Optional[str] = None
150
- ) -> Tuple[str]:
151
- os.makedirs(save_directory, exist_ok=True)
152
- filename = self.vocab_files_names["vocab_file"]
153
- if filename_prefix:
154
- filename = f"{filename_prefix}-{filename}"
155
- path = os.path.join(save_directory, filename)
156
- with open(path, "w", encoding="utf-8") as handle:
157
- json.dump(self._token_to_id, handle, ensure_ascii=False, indent=2)
158
- return (path,)
159
-
160
- # ------------------------------------------------------------------
161
- # Optional niceties
162
- # ------------------------------------------------------------------
163
- def convert_tokens_to_string(self, tokens: List[str]) -> str:
164
- return "".join(tokens)
165
-
166
- def _decode(
167
- self,
168
- token_ids: List[int],
169
- skip_special_tokens: bool = False,
170
- clean_up_tokenization_spaces: Optional[bool] = None,
171
- spaces_between_special_tokens: bool = True,
172
- **kwargs: Any,
173
- ) -> str:
174
- output_tokens: List[str] = []
175
- for idx in token_ids:
176
- token = self._convert_id_to_token(idx)
177
- if skip_special_tokens and token in self._special_token_strings:
178
- continue
179
- output_tokens.append(token)
180
- return self.convert_tokens_to_string(output_tokens)
181
-
182
-
183
- class NTv3Tokenizer(_BaseNTv3Tokenizer):
184
- """Character-level tokenizer for NTv3 DNA sequences."""
185
-
186
- def __init__(
187
- self,
188
- vocab_file: Optional[str] = None,
189
- unk_token: str = "<unk>",
190
- pad_token: str = "<pad>",
191
- mask_token: str = "<mask>",
192
- cls_token: str = "<cls>",
193
- eos_token: str = "<eos>",
194
- bos_token: str = "<bos>",
195
- **kwargs: Any,
196
- ) -> None:
197
- dna_tokens = ("A", "T", "C", "G", "N")
198
- default_tokens = (
199
- unk_token,
200
- pad_token,
201
- mask_token,
202
- cls_token,
203
- eos_token,
204
- bos_token,
205
- *dna_tokens,
206
- )
207
- super().__init__(
208
- vocab_file=vocab_file,
209
- unk_token=unk_token,
210
- pad_token=pad_token,
211
- mask_token=mask_token,
212
- cls_token=cls_token,
213
- eos_token=eos_token,
214
- bos_token=bos_token,
215
- default_tokens=default_tokens,
216
- standard_tokens=dna_tokens,
217
- **kwargs,
218
- )
219
-
220
- def _tokenize(self, text: str) -> List[str]:
221
- tokens: List[str] = []
222
- for char in text:
223
- candidate = char.upper()
224
- if candidate in self._standard_tokens or candidate in self._token_to_id:
225
- tokens.append(candidate)
226
- else:
227
- tokens.append(self._unk_literal)
228
- return tokens
229
-
230
-
231
- __all__ = [
232
- "NTv3Tokenizer",
233
- ]
234
-