config.yaml

annotation_task_name: Sentiment Analysis with Quality Control
task_dir: .
output_annotation_dir: annotation_output/quality-control-example
output_annotation_format: json
data_files:
- data/quality-control-example.json
item_properties:
  id_key: id
  text_key: text
annotation_schemes:
- name: sentiment
  annotation_type: radio
  labels:
  - name: positive
    tooltip: Positive sentiment - happy, satisfied, enthusiastic
  - name: negative
    tooltip: Negative sentiment - unhappy, dissatisfied, angry
  - name: neutral
    tooltip: Neutral sentiment - factual, balanced, no strong emotion
  description: What is the overall sentiment of this text?
attention_checks:
  enabled: true
  items_file: data/attention-checks.json
  frequency: 10
  min_response_time: 2.0
  failure_handling:
    warn_threshold: 2
    warn_message: Please read each item carefully before selecting your answer. Quality
      responses are important for this task.
    block_threshold: 5
    block_message: Unfortunately, you have been unable to complete the attention check
      items correctly. Your session has ended.
gold_standards:
  enabled: true
  items_file: data/gold-standards.json
  mode: mixed
  frequency: 20
  accuracy:
    min_threshold: 0.7
    evaluation_count: 5
  auto_promote:
    enabled: true
    min_annotators: 3
    agreement_threshold: 1.0
pre_annotation:
  enabled: true
  field: predictions
  allow_modification: true
  show_confidence: true
  highlight_low_confidence: 0.7
agreement_metrics:
  enabled: true
  min_overlap: 2
  refresh_interval: 60
ui:
  show_progress_bar: true
  show_statistics: true
site_dir: default
require_no_password: true
user_config:
  allow_all_users: true