Spaces:
Sleeping
Sleeping
Update model.py
Browse files
model.py
CHANGED
|
@@ -15,6 +15,11 @@ from torchvision import transforms
|
|
| 15 |
from tqdm import tqdm
|
| 16 |
import torchvision.transforms.functional as TF
|
| 17 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 18 |
TOKEN_REGEX = re.compile(
|
| 19 |
r"(\\[a-zA-Z]+(?:\*)?)|(\\\{|\\\}|\\.)|([0-9]+(?:\.[0-9]+)?(?:[eE][+-]?[0-9]+)?)|([A-Za-z]+)|([+\-*/=<>!~^_&|%])|([{}()\[\],.;:?'])|(\s+)|(\S)"
|
| 20 |
)
|
|
|
|
| 15 |
from tqdm import tqdm
|
| 16 |
import torchvision.transforms.functional as TF
|
| 17 |
|
| 18 |
+
MAX_SEQ_LEN = 512
|
| 19 |
+
NUM_NESTED_LEVELS = 10
|
| 20 |
+
MAX_IDENTIFIER_LEN = 15
|
| 21 |
+
PAD_TOKEN, SOS_TOKEN, EOS_TOKEN, UNK_TOKEN = '<PAD>', '<SOS>', '<EOS>', '<UNK>'
|
| 22 |
+
|
| 23 |
TOKEN_REGEX = re.compile(
|
| 24 |
r"(\\[a-zA-Z]+(?:\*)?)|(\\\{|\\\}|\\.)|([0-9]+(?:\.[0-9]+)?(?:[eE][+-]?[0-9]+)?)|([A-Za-z]+)|([+\-*/=<>!~^_&|%])|([{}()\[\],.;:?'])|(\s+)|(\S)"
|
| 25 |
)
|