ocrd_eval.schema.yml

$schema: https://json-schema.org/draft/2019-09/schema
$id: https://ocr-d.de/en/spec/ocrd_eval.schema.json

title: A List of Evaluations for OCR-D
description: >
  - All references to URL are JSON-LD-like objects with at least an `@id`
    property referencing the URL and `label` for a human-readable label to be
    used in the UI.
type: array
items:
  required: ['@id', 'label', 'metadata', 'evaluation_results']
  unevaluatedProperties: false
  allOf:
    - { '$ref': '#/$defs/LabeledUrl' }
    - properties:
        metadata: { '$ref': '#/$defs/EvaluationMetadata' }
        evaluation_results: { '$ref': '#/$defs/EvaluationReport' }

# Reusable definitions
$defs:

  LabeledUrl:
    type: object
    required: ['@id']
    properties:
      '@id':
        type: string
        format: uri
        description: URL of the thing
      label:
        type: string
        description: Description of the thing for UI purposes

  EvaluationMetadata:
    type: object
    title: Metadata about one evaluation
    additionalProperties: false
    description: >
      EvaluationMetadata contains all the info on how an EvaluationReport came to be.

      There are two OCR-D *workflows* involved:
        - ocr_workflow: The workflow which produced the OCR results to evaluate
        - eval_workflow: The workflow run to evaluate OCR and GT

      There are three OCR-D *workspaces* involved:
        - gt_workspace: The workspace containing the GT
        - ocr_workspace: The workspace containing the OCR results from ocr_workflow
        - eval_workspace: The workspace on which the eval_workflow was run

    required:
      - ocr_workflow
      - ocr_workspace
      - eval_workflow
      - eval_workspace
      - gt_workspace
      - document_metadata

    properties:

      ocr_workflow:
        allOf:  [{ '$ref': '#/$defs/LabeledUrl' }]
        description: The OCR-D workflow that produced the ocr_workspace

      ocr_workspace:
        allOf:  [{ '$ref': '#/$defs/LabeledUrl' }]
        description: The workspace containing the OCR

      eval_workflow:
        allOf:  [{ '$ref': '#/$defs/LabeledUrl' }]
        description: The OCR-D workflow that produced the eval_workspace

      eval_workspace:
        allOf:  [{ '$ref': '#/$defs/LabeledUrl' }]
        description: The workspace containing the evaluation results

      gt_workspace:
        allOf:  [{ '$ref': '#/$defs/LabeledUrl' }]
        description: The workspace containing the GT

      workflow_steps:
        type: array
        description: Human readable description of the individual steps and their parameters in the workflow (for UI)
        minItems: 1
        items:
          type: object
          properties:
            id:
              type: string
              description: The name of the processor used for this workflow step
              pattern: '^ocrd-[a-z\-]+'
            params:
              type: object
              description: A map of parameters and their values applied to the processor used for this workflow step
          required: ['id', 'params']

      workflow_model:
        type: string
        description: Human readable name of the main model used for recognition in the OCR workflow (for UI)

      eval_tool:
        type: string
        description: Human readable name and version of evaluation tool used (for UI)

      document_metadata:
        type: object
        title: Bibliographical and typographical metadata about the work to be evaluated
        properties:

          publication_year:
            type: number
            description: Year the document was originally published

          publication_century:
            type: string
            description: Century the document was originally published
            pattern: '[12][0-9]{3}-[12][0-9]{3}'

          publication_decade:
            type: string
            description: Decade the document was originally published
            pattern: '[12][0-9]{2}0-[12][0-9]{2}0'

          number_of_pages:
            type: number
            description: Number of pages in this work (i.e. the number of images in the gt_workspace)

          layout:
            type: string
            enum: ['simple', 'complex']

          fonts:
            type: array
            items:
              type: string
              enum: ['antiqua', 'textura', 'gotico-antiqua', 'rotunda', 'italic', 'bastarda', 'greek', 'schwabacher', 'hebrew', 'fraktur']

      provenance:
        type: object
        description: Information on which tools in which version were used in determining metrics
        properties:
          parameters:
            type: object
            description: Parameters passed to the evaluation processor

  EvaluationReport:
    type: object
    additionalProperties: false
    description: The metrics measured for this document
    properties:
      document_wide:
        type: object
        description: Document-wide metrics
        allOf: [
          { $ref: '#$defs/DocumentEvaluationMetrics' },
          { $ref: '#$defs/CommonEvaluationMetrics' }
        ]
        unevaluatedProperties: false
      by_page:
        type: array
        description: Metrics page-by-page
        items:
          type: object
          allOf: [
            { $ref: '#$defs/CommonEvaluationMetrics' },
            { $ref: '#$defs/PageId' }
          ] 
          unevaluatedProperties: false

  PageId:
    type: object
    properties:
      page_id:
        type: string
        description: PAGE ID

  CommonEvaluationMetrics:
    type: object
    properties:
      cer_mean:
        type: number
        description: Arithmetic mean of the page-wise CER (in document_wide) or regions on a page (in by_page)

      wer:
        type: number
        description: CER calculated over the text of a whole page (in by_page) or combined text of all pages (in document_wide)


  DocumentEvaluationMetrics:
    type: object
    properties:
      cer_median:
        type: number
        description: Median of the page-wise CER (in document_wide) or regions on a page (in by_page)

      cer_range:
        type: array
        minItems: 2
        maxItems: 2
        items:
          type: number
          description: Minimum and maximum of CER calculated over the text of a whole page (in by_page) or combined text of all pages (in document_wide)

      cer_standard_deviation:
        type: number
        description: Standard deviation the page-wise CER (in document_wide) or regions on a page (in by_page)

      wall_time:
        type: number
        description: Actual time needed for processing workflow

      cpu_time:
        type: number
        description: Cumulative CPU time used for processing workflow

      pages_per_minute:
        type: number
        description: Number of pages processed per minute