{
  "$defs": {
    "EvaluationMetadata": {
      "additionalProperties": false,
      "description": "EvaluationMetadata contains all the info on how an EvaluationReport came to be.\nThere are two OCR-D *workflows* involved:\n  - ocr_workflow: The workflow which produced the OCR results to evaluate\n  - eval_workflow: The workflow run to evaluate OCR and GT\n\nThere are three OCR-D *workspaces* involved:\n  - gt_workspace: The workspace containing the GT\n  - ocr_workspace: The workspace containing the OCR results from ocr_workflow\n  - eval_workspace: The workspace on which the eval_workflow was run\n",
      "properties": {
        "document_metadata": {
          "properties": {
            "fonts": {
              "items": {
                "enum": [
                  "antiqua",
                  "fraktur"
                ],
                "type": "string"
              },
              "type": "array"
            },
            "layout": {
              "enum": [
                "simple",
                "complex"
              ],
              "type": "string"
            },
            "number_of_pages": {
              "description": "Number of pages in this work (i.e. the number of images in the gt_workspace)",
              "type": "number"
            },
            "publication_century": {
              "description": "Century he document was originally published",
              "pattern": "[12][0-9]{3}-[12][0-9]{3}",
              "type": "string"
            },
            "publication_decade": {
              "description": "Decade the document was originally published",
              "pattern": "[12][0-9]{3}-[12][0-9]{3}",
              "type": "string"
            },
            "publication_year": {
              "description": "Year he document was originally published",
              "type": "number"
            }
          },
          "title": "Bibliographical and typographical metadata about the work to be evaluated",
          "type": "object"
        },
        "eval_tool": {
          "description": "Human readable name and version of evaluation tool used (for UI",
          "type": "string"
        },
        "eval_workflow": {
          "allOf": [
            {
              "$ref": "#/$defs/LabeledUrl"
            }
          ],
          "description": "The OCR-D workflow that produced the eval_workspace"
        },
        "eval_workspace": {
          "allOf": [
            {
              "$ref": "#/$defs/LabeledUrl"
            }
          ],
          "description": "The workspace containing the evaluation results"
        },
        "gt_workspace": {
          "allOf": [
            {
              "$ref": "#/$defs/LabeledUrl"
            }
          ],
          "description": "The workspace containing the GT"
        },
        "ocr_workflow": {
          "allOf": [
            {
              "$ref": "#/$defs/LabeledUrl"
            }
          ],
          "description": "The OCR-D workflow that produced the ocr_workspace"
        },
        "ocr_workspace": {
          "allOf": [
            {
              "$ref": "#/$defs/LabeledUrl"
            }
          ],
          "description": "The workspace containing the OCR"
        },
        "provenance": {
          "description": "Information on which tools in which version were used in determining metrics",
          "properties": {
            "paramters": {
              "description": "Parameters passed to the evaluation processor",
              "type": "object"
            }
          },
          "type": "object"
        },
        "workflow_model": {
          "description": "Human readable name of the main model used for recognition in the OCR workflow (for UI)",
          "type": "string"
        },
        "workflow_steps": {
          "description": "Human readable description of the individual steps in the workflow (for UI)",
          "patternProperties": {
            "^[0-9]+$": {
              "description": "Description of this workflow step",
              "type": "string"
            }
          },
          "type": "object"
        }
      },
      "required": [
        "ocr_workflow",
        "ocr_workspace",
        "eval_workflow",
        "eval_workspace",
        "gt_workspace",
        "document_metadata"
      ],
      "title": "Metadata about one evaluation",
      "type": "object"
    },
    "EvaluationMetrics": {
      "cer": {
        "description": "CER calculated over the text of a whole page (in by_page) or combined text of all pages (in document_wide)"
      },
      "cer_mean": {
        "description": "Arithmetic mean of the page-wise CER (in document_wide) or regions on a page (in by_page)"
      },
      "cer_median": {
        "description": "Median of the page-wise CER (in document_wide) or regions on a page (in by_page)"
      },
      "cer_range": {
        "items": {
          "description": "Minimum and maximum of CER calculated over the text of a whole page (in by_page) or combined text of all pages (in document_wide)",
          "type": "number"
        },
        "maxItems": 2,
        "minItems": 2,
        "type": "array"
      },
      "cer_standard_deviation": {
        "description": "Standard deviation the page-wise CER (in document_wide) or regions on a page (in by_page)"
      },
      "cpu_time": {
        "description": "Cumulative CPU time used for processing workflow"
      },
      "pages_per_minute": {
        "description": "Number of pages processed per minute"
      },
      "wall_time": {
        "description": "Actual time needed for processing workflow"
      },
      "wer": {
        "description": "CER calculated over the text of a whole page (in by_page) or combined text of all pages (in document_wide)"
      }
    },
    "EvaluationReport": {
      "additionalProperties": false,
      "description": "The metrics measured for this document",
      "properties": {
        "by_page": {
          "description": "Metrics page-by-page",
          "items": {
            "allOf": [
              {
                "properties": {
                  "page_id": {
                    "description": "PAGE ID",
                    "type": "string"
                  }
                }
              }
            ],
            "type": "object"
          },
          "type": "array"
        },
        "document_wide": {
          "description": "Document-wide metrics",
          "type": "object"
        }
      },
      "type": "object"
    },
    "LabeledUrl": {
      "properties": {
        "@id": {
          "description": "URL of the thing",
          "format": "uri",
          "type": "string"
        },
        "label": {
          "description": "Description of the thing for UI purposes",
          "type": "string"
        }
      },
      "required": [
        "@id"
      ],
      "type": "object"
    }
  },
  "$id": "https://ocr-d.de/en/spec/ocrd_eval.schema.json",
  "$schema": "https://json-schema.org/draft/2019-09/schema",
  "description": "- All references to URL are JSON-LD-like objects with at least an `@id`\n  property referencing the URL and `label` for a human-readable label to be\n  used in the UI\n",
  "items": {
    "allOf": [
      {
        "$ref": "#/$defs/LabeledUrl"
      },
      {
        "properties": {
          "evaluation": {
            "$ref": "#/$defs/EvaluationReport"
          },
          "metadata": {
            "$ref": "#/$defs/EvaluationMetadata"
          }
        }
      }
    ],
    "required": [
      "@id",
      "label",
      "metadata",
      "evaluation"
    ],
    "unevaluatedProperties": false
  },
  "title": "A list of evaluations for OCR-D",
  "type": "array"
}