Update app.py
Browse files
app.py
CHANGED
|
@@ -36,7 +36,7 @@ def setup_training():
|
|
| 36 |
# Load your dataset with the specified configuration
|
| 37 |
logging.info("Loading the MarbleX dataset")
|
| 38 |
dataset = load_dataset("Oranblock/marblex_dataset", "config1") # Replace "config1" with the appropriate config name
|
| 39 |
-
logging.info(f"Dataset loaded. Train size: {len(dataset['train'])},
|
| 40 |
|
| 41 |
# Load tokenizer and model
|
| 42 |
logging.info(f"Loading tokenizer and model: {config['model_name']}")
|
|
@@ -80,7 +80,7 @@ def setup_training():
|
|
| 80 |
model=model,
|
| 81 |
args=training_args,
|
| 82 |
train_dataset=tokenized_datasets["train"],
|
| 83 |
-
eval_dataset=tokenized_datasets["
|
| 84 |
tokenizer=tokenizer,
|
| 85 |
compute_metrics=compute_metrics
|
| 86 |
)
|
|
|
|
| 36 |
# Load your dataset with the specified configuration
|
| 37 |
logging.info("Loading the MarbleX dataset")
|
| 38 |
dataset = load_dataset("Oranblock/marblex_dataset", "config1") # Replace "config1" with the appropriate config name
|
| 39 |
+
logging.info(f"Dataset loaded. Train size: {len(dataset['train'])}, Test size: {len(dataset['test'])}")
|
| 40 |
|
| 41 |
# Load tokenizer and model
|
| 42 |
logging.info(f"Loading tokenizer and model: {config['model_name']}")
|
|
|
|
| 80 |
model=model,
|
| 81 |
args=training_args,
|
| 82 |
train_dataset=tokenized_datasets["train"],
|
| 83 |
+
eval_dataset=tokenized_datasets["test"], # Use 'test' split for evaluation
|
| 84 |
tokenizer=tokenizer,
|
| 85 |
compute_metrics=compute_metrics
|
| 86 |
)
|