File size: 1,505 Bytes
d7c73a3 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 | model:
base_url: "http://127.0.0.1:8000"
max_tokens: 512
temperature: 0.1
timeout: 30
datasets:
benchmark_dataset:
file_path: "ner_benchmark_dataset.jsonl"
sample_size: 100 # Use first 100 examples for quick benchmarking
instruction_field: "instruction"
input_field: "input"
expected_output_field: "response"
metrics:
# Primary metrics for HuggingFace
entity_recognition:
name: "Entity Recognition F1 Score"
description: "F1 score for named entity recognition accuracy"
type: "f1"
precision:
name: "Precision Score"
description: "Precision for entity recognition"
type: "precision"
recall:
name: "Recall Score"
description: "Recall for entity recognition"
type: "recall"
latency:
name: "Average Latency"
description: "Average response time in milliseconds"
type: "latency"
# Entity type specific performance
entity_types:
person:
name: "Person Entity Recognition"
keywords: ["PERSON", "person", "Person"]
organization:
name: "Organization Entity Recognition"
keywords: ["ORG", "organization", "Organization"]
location:
name: "Location Entity Recognition"
keywords: ["LOC", "location", "Location"]
miscellaneous:
name: "Miscellaneous Entity Recognition"
keywords: ["MISC", "miscellaneous", "Miscellaneous"]
output:
results_file: "benchmarks.txt"
detailed_results_file: "benchmark_results.json"
include_examples: true
max_examples: 10
|