Update app.py
Browse files
app.py
CHANGED
|
@@ -22,7 +22,6 @@ kp = KeyPhraseTransformer()
|
|
| 22 |
|
| 23 |
dataset = load_dataset("Unknown92/Resume_dataset")
|
| 24 |
|
| 25 |
-
|
| 26 |
tokenizer = AutoTokenizer.from_pretrained("bert-base-cased")
|
| 27 |
|
| 28 |
def tokenize_function(examples):
|
|
@@ -30,8 +29,8 @@ def tokenize_function(examples):
|
|
| 30 |
|
| 31 |
tokenized_datasets = dataset.map(tokenize_function, batched=True)
|
| 32 |
|
| 33 |
-
small_train_dataset = tokenized_datasets["
|
| 34 |
-
small_eval_dataset = tokenized_datasets["
|
| 35 |
|
| 36 |
model = AutoModelForSequenceClassification.from_pretrained("bert-base-cased", num_labels=5)
|
| 37 |
training_args = TrainingArguments(output_dir="test_trainer", evaluation_strategy="epoch")
|
|
|
|
| 22 |
|
| 23 |
dataset = load_dataset("Unknown92/Resume_dataset")
|
| 24 |
|
|
|
|
| 25 |
tokenizer = AutoTokenizer.from_pretrained("bert-base-cased")
|
| 26 |
|
| 27 |
def tokenize_function(examples):
|
|
|
|
| 29 |
|
| 30 |
tokenized_datasets = dataset.map(tokenize_function, batched=True)
|
| 31 |
|
| 32 |
+
small_train_dataset = tokenized_datasets["Train"].shuffle(seed=42).select(range(200))
|
| 33 |
+
small_eval_dataset = tokenized_datasets["Test"].shuffle(seed=42).select(range(200))
|
| 34 |
|
| 35 |
model = AutoModelForSequenceClassification.from_pretrained("bert-base-cased", num_labels=5)
|
| 36 |
training_args = TrainingArguments(output_dir="test_trainer", evaluation_strategy="epoch")
|