1. Data Processing.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import json\n",
    "from os.path import join as pj\n",
    "\n",
    "data_folder = \"data/json\"\n",
    "# file_path = \"fully_annotated_3200.json\"\n",
    "file_path = \"hau_adjudicated.json\"\n",
    "\n",
    "json_data = json.load(open(pj(data_folder, file_path)))\n",
    "\n",
    "english = json_data[:120]\n",
    "print(len(english))\n",
    "\n",
    "language = json_data[120:]\n",
    "print(len(language))\n",
    "\n",
    "language[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "# analysis the raw input data\n",
    "# 1. the number of entities in each text, average number of entities in each text\n",
    "# 2. the number of tokens in each text, average number of tokens in each text\n",
    "# 3. the number of unique entities, the number of unique tokens\n",
    "# 4. the distribution of entities\n",
    "# 5. the percentage of disagreement between different labelers\n",
    "# 6. the percentage of disagreement between different labelers after removing the entities that are not in the majority\n",
    "\n",
    "# sampel input: [{'id': 122174876,\n",
    "#  'annotations': [{'id': 40955949,\n",
    "#    'completed_by': {'id': 10959},\n",
    "#    'result': [\n",
    "#     {'id': 'tMINc6tPvI',\n",
    "#      'type': 'labels',\n",
    "#      'value': {'end': 40, 'text': 'Niger', 'start': 35, 'labels': ['COUNTRY']},\n",
    "#      'origin': 'manual',\n",
    "#      'to_name': 'text',\n",
    "#      'from_name': 'label'},\n",
    "#     {'id': 'PAzFajPUzl',\n",
    "#      'type': 'labels',\n",
    "#      'value': {'end': 20,\n",
    "#       'text': 'kan soket',\n",
    "#       'start': 11,\n",
    "#       'labels': ['SHOPPING_ITEM']},\n",
    "#      'origin': 'manual',\n",
    "#      'to_name': 'text',\n",
    "#      'from_name': 'label'}],},\n",
    "#   {'id': 40928216,\n",
    "#    'completed_by': {'id': 10676},\n",
    "#    'result': [{'id': 'XSqLuKQWqt',\n",
    "#      'type': 'labels',\n",
    "#      'value': {'end': 40, 'text': 'Niger', 'start': 35, 'labels': ['COUNTRY']},\n",
    "#      'origin': 'manual',\n",
    "#      'to_name': 'text',\n",
    "#      'from_name': 'label'}]},\n",
    "#  'agreement': 66.66666666666666,\n",
    "#  'data': {'text': 'wanne irin kan soket zan buƙata a  Niger?'},\n",
    "#  'inner_id': 121,\n",
    "#  'total_annotations': 2,}, ...]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "('N', '?', 'Niger')"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "t = \"wanne irin kan soket zan buƙata a  Niger?\"\n",
    "t[35], t[40], t[35:40]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "avg_entities: 1.244375\n",
      "avg_tokens: 9.480625\n",
      "unique_entities: 3229\n",
      "unique_tokens: 2329\n",
      "disagreement_percentage: 0.03088900050226017\n",
      "annotator_error_rates: {10676: 0.015773660490736103,\n",
      " 10959: 0.009796533534287867,\n",
      " 23838: 0.0052763819095477385}\n",
      "disagreed_items: []\n"
     ]
    }
   ],
   "source": [
    "import json\n",
    "from collections import Counter\n",
    "from typing import List, Dict, Any\n",
    "from collections import Counter, defaultdict\n",
    "\n",
    "sptoken = \"|\"\n",
    "\n",
    "\n",
    "def compare_entity(e1, e2):\n",
    "    # 'end': 40, 'text': 'Niger', 'start': 35, 'labels': ['COUNTRY']\n",
    "    if isinstance(e1, str):\n",
    "        e1 = decode_entity(e1)\n",
    "    if isinstance(e2, str):\n",
    "        e2 = decode_entity(e2)\n",
    "    return (\n",
    "        e1[\"start\"] == e2[\"start\"]\n",
    "        and e1[\"end\"] == e2[\"end\"]\n",
    "        and e1[\"labels\"] == e2[\"labels\"]\n",
    "    )\n",
    "\n",
    "\n",
    "def encode_entity(entity):\n",
    "    return f\"{entity['start']}{sptoken}{entity['end']}{sptoken}{entity['labels'][0]}{sptoken}{entity['text']}\"\n",
    "\n",
    "\n",
    "def decode_entity(entity_str):\n",
    "    start, end, label, text = entity_str.split(sptoken, 3)\n",
    "    return {\n",
    "        \"start\": int(start),\n",
    "        \"end\": int(end),\n",
    "        \"labels\": [label],\n",
    "        \"text\": text,  # .strip()\n",
    "    }\n",
    "\n",
    "\n",
    "def check_overlap(entity1, entity2):\n",
    "    \"\"\"Check if two entities overlap based on their start and end positions.\"\"\"\n",
    "    if isinstance(entity1, str):\n",
    "        entity1 = decode_entity(entity1)\n",
    "    if isinstance(entity2, str):\n",
    "        entity2 = decode_entity(entity2)\n",
    "    return (\n",
    "        entity1[\"start\"] < entity2[\"end\"] and entity1[\"end\"] > entity2[\"start\"]\n",
    "    ) or (entity2[\"start\"] < entity1[\"end\"] and entity2[\"end\"] > entity1[\"start\"])\n",
    "\n",
    "\n",
    "def clean_entity(e):\n",
    "    return {\n",
    "        \"id\": e[\"id\"],\n",
    "        \"result\": [\n",
    "            [anno[\"id\"]] + [r[\"value\"] for r in anno[\"result\"]]\n",
    "            for anno in e[\"annotations\"]\n",
    "        ],\n",
    "        \"agreement\": e[\"agreement\"],\n",
    "        \"text\": e[\"data\"][\"text\"],\n",
    "        \"inner_id\": e[\"inner_id\"],\n",
    "        \"total_annotations\": e[\"total_annotations\"],\n",
    "    }\n",
    "\n",
    "\n",
    "def show_entity(e):\n",
    "    # fancy print with colors\n",
    "    # print(f\"Entity: {e['text']} ({e['start']}, {e['end']}) - {e['labels']}\")\n",
    "    # print(f\"Annotated by: {e['completed_by']}\")\n",
    "    # print(f\"Agreement: {e['agreement']}%\")\n",
    "    # print(f\"Total Annotations: {e['total_annotations']}\")\n",
    "    # print(\"Annotations:\")\n",
    "    # for r in e['result']:\n",
    "    #     print(f\"  {r['labels'][0]}: {r['text']}\")\n",
    "    # print()\n",
    "    pp(e)\n",
    "\n",
    "\n",
    "def analyze_data(data: List[Dict[str, Any]]):\n",
    "    total_entities = 0\n",
    "    total_tokens = 0\n",
    "    all_entities = []\n",
    "    all_tokens = set()\n",
    "    entity_distribution = Counter()\n",
    "    total_disagreements = 0\n",
    "    annotator_errors = defaultdict(int)\n",
    "    annotator_total = defaultdict(int)\n",
    "    disagreed_items = []\n",
    "    # majority_vote_results = []\n",
    "\n",
    "    for item in data:\n",
    "        text = item[\"data\"][\"text\"]\n",
    "        tokens = text.split()\n",
    "        total_tokens += len(tokens)\n",
    "        all_tokens.update(tokens)\n",
    "\n",
    "        annotations = item[\"annotations\"]\n",
    "\n",
    "        # Collect all entities from all annotators\n",
    "        item_entities = []\n",
    "        annotator_entities = defaultdict(list)\n",
    "        for annotation in annotations:\n",
    "            annotator_id = annotation[\"completed_by\"][\"id\"]\n",
    "            entities = [\n",
    "                encode_entity(result[\"value\"]) for result in annotation[\"result\"]\n",
    "            ]\n",
    "            item_entities.extend(entities)\n",
    "            all_entities.extend(entities)\n",
    "            annotator_entities[annotator_id] = entities\n",
    "            annotator_total[annotator_id] += len(entities)\n",
    "\n",
    "        # Perform majority voting\n",
    "        entity_counts = Counter(item_entities)\n",
    "        majority_threshold = len(annotations) / 2\n",
    "        majority_entities = [\n",
    "            entity\n",
    "            for entity, count in entity_counts.items()\n",
    "            if count > majority_threshold\n",
    "        ]\n",
    "\n",
    "        total_entities += len(majority_entities)\n",
    "        entity_distribution.update(majority_entities)\n",
    "\n",
    "        # Calculate disagreements and errors\n",
    "        disagreed_entities = []\n",
    "        for entity, count in entity_counts.items():\n",
    "            # if count <= majority_threshold:\n",
    "            if count <= majority_threshold:\n",
    "\n",
    "                total_disagreements += 1\n",
    "                for annotator_id, annotator_ents in annotator_entities.items():\n",
    "                    if (entity in annotator_ents and count <= majority_threshold) or (\n",
    "                        entity not in annotator_ents and count > majority_threshold\n",
    "                    ):\n",
    "                        annotator_errors[annotator_id] += 1\n",
    "                if not any(\n",
    "                    check_overlap(entity, maj_entity)\n",
    "                    for maj_entity in majority_entities\n",
    "                ):\n",
    "                    disagreed_entities.append(\n",
    "                        {\n",
    "                            \"entity\": decode_entity(entity),\n",
    "                            \"annotators\": [\n",
    "                                {\n",
    "                                    \"id\": annotator_id,\n",
    "                                    \"included\": entity in annotator_ents,\n",
    "                                }\n",
    "                                for annotator_id, annotator_ents in annotator_entities.items()\n",
    "                            ],\n",
    "                        }\n",
    "                    )\n",
    "\n",
    "        # # Create majority vote result\n",
    "        # majority_vote_results.append({\n",
    "        #     'id': item['id'],\n",
    "        #     'inner_id': item['inner_id'],\n",
    "        #     'data': item['data'],\n",
    "        #     'annotations': [{\n",
    "        #         'id': f\"majority_vote_{item['inner_id']}\",\n",
    "        #         'completed_by': {'id': 'MAJORITY_VOTE'},\n",
    "        #         'result': [\n",
    "        #             {\n",
    "        #                 'id': f\"majority_{i}\",\n",
    "        #                 'type': 'labels',\n",
    "        #                 'value': decode_entity(entity),\n",
    "        #                 'origin': 'manual',\n",
    "        #                 'to_name': 'text',\n",
    "        #                 'from_name': 'label'\n",
    "        #             }\n",
    "        #             for i, entity in enumerate(majority_entities)\n",
    "        #         ]\n",
    "        #     }],\n",
    "        #     'agreement': (len(majority_entities) / len(item_entities)) * 100 if item_entities else 100\n",
    "        # })\n",
    "\n",
    "        # # filter out the disagreed entities wich overlap the start end position in the majority vote\n",
    "        # disagreed_entities = [entity for entity in disagreed_entities if not any(\n",
    "        #     compare_entity(entity['entity'], majority_entity) for majority_entity in majority_entities\n",
    "        # )]\n",
    "\n",
    "        if disagreed_entities:\n",
    "            disagreed_items.append(\n",
    "                {\n",
    "                    \"inner_id\": item[\"inner_id\"],\n",
    "                    \"text\": text,\n",
    "                    \"entities\": disagreed_entities,\n",
    "                    \"agreement\": item[\"agreement\"],\n",
    "                }\n",
    "            )\n",
    "\n",
    "        # disagreed_items[-1]['entities'] = disagreed_entities\n",
    "\n",
    "    num_texts = len(data)\n",
    "    avg_entities = total_entities / num_texts\n",
    "    avg_tokens = total_tokens / num_texts\n",
    "    unique_entities = len(set(all_entities))\n",
    "    unique_tokens = len(all_tokens)\n",
    "    disagreement_percentage = (\n",
    "        (total_disagreements / total_entities) if total_entities > 0 else 0\n",
    "    )\n",
    "\n",
    "    # Calculate error rates for each annotator\n",
    "    annotator_error_rates = {\n",
    "        annotator_id: (errors / annotator_total[annotator_id])\n",
    "        for annotator_id, errors in annotator_errors.items()\n",
    "    }\n",
    "\n",
    "    return {\n",
    "        \"avg_entities\": avg_entities,\n",
    "        \"avg_tokens\": avg_tokens,\n",
    "        \"unique_entities\": unique_entities,\n",
    "        \"unique_tokens\": unique_tokens,\n",
    "        \"entity_distribution\": {\n",
    "            encode_entity(decode_entity(k)): v\n",
    "            for k, v in dict(entity_distribution).items()\n",
    "        },\n",
    "        \"disagreement_percentage\": disagreement_percentage,\n",
    "        \"annotator_error_rates\": annotator_error_rates,\n",
    "        \"disagreed_items\": disagreed_items,\n",
    "        # \"majority_vote_results\": majority_vote_results\n",
    "    }\n",
    "\n",
    "\n",
    "# Analyze the sample data\n",
    "results = analyze_data(language)\n",
    "# Print the results\n",
    "from pprint import pprint as pp\n",
    "\n",
    "for key, value in results.items():\n",
    "    # pp(f\"{key}: {value}\")\n",
    "    if key == \"entity_distribution\":\n",
    "        # for k, v in value.items():\n",
    "        #     print(f\"{k}: {v}\")\n",
    "        continue\n",
    "    print(key, end=\": \")\n",
    "    pp(value)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'agreement': 100.0,\n",
      " 'id': 122174877,\n",
      " 'inner_id': 122,\n",
      " 'result': [[40932827,\n",
      "             {'end': 14, 'labels': ['FOOD_ITEM'], 'start': 10, 'text': 'kifi'},\n",
      "             {'end': 28,\n",
      "              'labels': ['LANGUAGE_NAME'],\n",
      "              'start': 18,\n",
      "              'text': 'yaren Igbo'}],\n",
      "            [40955950,\n",
      "             {'end': 14, 'labels': ['FOOD_ITEM'], 'start': 10, 'text': 'kifi'},\n",
      "             {'end': 28,\n",
      "              'labels': ['LANGUAGE_NAME'],\n",
      "              'start': 18,\n",
      "              'text': 'yaren Igbo'}],\n",
      "            [40928229,\n",
      "             {'end': 14, 'labels': ['FOOD_ITEM'], 'start': 10, 'text': 'kifi'},\n",
      "             {'end': 28,\n",
      "              'labels': ['LANGUAGE_NAME'],\n",
      "              'start': 18,\n",
      "              'text': 'yaren Igbo'}]],\n",
      " 'text': 'Ya zan ce kifi da yaren Igbo?',\n",
      " 'total_annotations': 3}\n"
     ]
    }
   ],
   "source": [
    "show_entity(clean_entity(language[1]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'id': 122174876,\n",
       " 'annotations': [{'id': 40932812,\n",
       "   'completed_by': {'id': 23838,\n",
       "    'email': 'amabubakar0605@gmail.com',\n",
       "    'first_name': '',\n",
       "    'last_name': ''},\n",
       "   'result': [{'id': 'UrdSe-5Biz',\n",
       "     'type': 'labels',\n",
       "     'value': {'end': 40, 'text': 'Niger', 'start': 35, 'labels': ['COUNTRY']},\n",
       "     'origin': 'manual',\n",
       "     'to_name': 'text',\n",
       "     'from_name': 'label'},\n",
       "    {'id': '-KeRREaTOJ',\n",
       "     'type': 'labels',\n",
       "     'value': {'end': 20,\n",
       "      'text': 'kan soket',\n",
       "      'start': 11,\n",
       "      'labels': ['SHOPPING_ITEM']},\n",
       "     'origin': 'manual',\n",
       "     'to_name': 'text',\n",
       "     'from_name': 'label'}],\n",
       "   'reviews': [],\n",
       "   'was_cancelled': False,\n",
       "   'ground_truth': False,\n",
       "   'created_at': '2024-07-13T06:48:32.843766Z',\n",
       "   'updated_at': '2024-08-04T18:15:50.553212Z',\n",
       "   'draft_created_at': '2024-07-13T06:48:32.307344Z',\n",
       "   'lead_time': 66.908,\n",
       "   'prediction': {},\n",
       "   'result_count': 0,\n",
       "   'unique_id': 'c79e74e8-f5b3-4362-aa0c-d918242869f4',\n",
       "   'import_id': None,\n",
       "   'last_action': 'updated',\n",
       "   'task': 122174876,\n",
       "   'project': 71325,\n",
       "   'updated_by': 23838,\n",
       "   'parent_prediction': None,\n",
       "   'parent_annotation': None,\n",
       "   'last_created_by': 23838},\n",
       "  {'id': 40955949,\n",
       "   'completed_by': {'id': 10959,\n",
       "    'email': 'rukynas08@gmail.com',\n",
       "    'first_name': '',\n",
       "    'last_name': ''},\n",
       "   'result': [{'id': 'tMINc6tPvI',\n",
       "     'type': 'labels',\n",
       "     'value': {'end': 40, 'text': 'Niger', 'start': 35, 'labels': ['COUNTRY']},\n",
       "     'origin': 'manual',\n",
       "     'to_name': 'text',\n",
       "     'from_name': 'label'},\n",
       "    {'id': 'PAzFajPUzl',\n",
       "     'type': 'labels',\n",
       "     'value': {'end': 20,\n",
       "      'text': 'kan soket',\n",
       "      'start': 11,\n",
       "      'labels': ['SHOPPING_ITEM']},\n",
       "     'origin': 'manual',\n",
       "     'to_name': 'text',\n",
       "     'from_name': 'label'}],\n",
       "   'reviews': [],\n",
       "   'was_cancelled': False,\n",
       "   'ground_truth': False,\n",
       "   'created_at': '2024-07-14T15:26:36.893626Z',\n",
       "   'updated_at': '2024-08-01T18:33:19.320954Z',\n",
       "   'draft_created_at': None,\n",
       "   'lead_time': 30.428,\n",
       "   'prediction': {},\n",
       "   'result_count': 0,\n",
       "   'unique_id': 'e2ba71f4-ad41-49af-90f7-914f6c53a326',\n",
       "   'import_id': None,\n",
       "   'last_action': 'updated',\n",
       "   'task': 122174876,\n",
       "   'project': 71325,\n",
       "   'updated_by': 10959,\n",
       "   'parent_prediction': None,\n",
       "   'parent_annotation': None,\n",
       "   'last_created_by': 10959},\n",
       "  {'id': 40928216,\n",
       "   'completed_by': {'id': 10676,\n",
       "    'email': 'saminaliyu@gmail.com',\n",
       "    'first_name': '',\n",
       "    'last_name': ''},\n",
       "   'result': [{'id': 'XSqLuKQWqt',\n",
       "     'type': 'labels',\n",
       "     'value': {'end': 40, 'text': 'Niger', 'start': 35, 'labels': ['COUNTRY']},\n",
       "     'origin': 'manual',\n",
       "     'to_name': 'text',\n",
       "     'from_name': 'label'},\n",
       "    {'id': 'F5g8ZFgkYe',\n",
       "     'type': 'labels',\n",
       "     'value': {'end': 20,\n",
       "      'text': 'kan soket',\n",
       "      'start': 11,\n",
       "      'labels': ['SHOPPING_ITEM']},\n",
       "     'origin': 'manual',\n",
       "     'to_name': 'text',\n",
       "     'from_name': 'label'}],\n",
       "   'reviews': [],\n",
       "   'was_cancelled': False,\n",
       "   'ground_truth': False,\n",
       "   'created_at': '2024-07-12T19:51:36.670814Z',\n",
       "   'updated_at': '2024-07-22T14:02:24.410888Z',\n",
       "   'draft_created_at': None,\n",
       "   'lead_time': 34.166,\n",
       "   'prediction': {},\n",
       "   'result_count': 0,\n",
       "   'unique_id': 'f017c188-0055-48b3-9e16-a593d3851a6f',\n",
       "   'import_id': None,\n",
       "   'last_action': 'updated',\n",
       "   'task': 122174876,\n",
       "   'project': 71325,\n",
       "   'updated_by': 10676,\n",
       "   'parent_prediction': None,\n",
       "   'parent_annotation': None,\n",
       "   'last_created_by': 10676}],\n",
       " 'file_upload': '3af37b34-hau_text.txt',\n",
       " 'drafts': [],\n",
       " 'predictions': [],\n",
       " 'agreement': 100.0,\n",
       " 'data': {'text': 'wanne irin kan soket zan buƙata a  Niger?'},\n",
       " 'meta': {},\n",
       " 'created_at': '2024-07-12T16:31:35.846000Z',\n",
       " 'updated_at': '2024-08-04T18:15:50.640365Z',\n",
       " 'inner_id': 121,\n",
       " 'total_annotations': 3,\n",
       " 'cancelled_annotations': 0,\n",
       " 'total_predictions': 0,\n",
       " 'comment_count': 0,\n",
       " 'unresolved_comment_count': 0,\n",
       " 'last_comment_updated_at': None,\n",
       " 'project': 71325,\n",
       " 'updated_by': 23838,\n",
       " 'comment_authors': []}"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "language[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'agreement': 100.0,\n",
      " 'id': 122174876,\n",
      " 'inner_id': 121,\n",
      " 'result': [[40932812,\n",
      "             {'end': 40, 'labels': ['COUNTRY'], 'start': 35, 'text': 'Niger'},\n",
      "             {'end': 20,\n",
      "              'labels': ['SHOPPING_ITEM'],\n",
      "              'start': 11,\n",
      "              'text': 'kan soket'}],\n",
      "            [40955949,\n",
      "             {'end': 40, 'labels': ['COUNTRY'], 'start': 35, 'text': 'Niger'},\n",
      "             {'end': 20,\n",
      "              'labels': ['SHOPPING_ITEM'],\n",
      "              'start': 11,\n",
      "              'text': 'kan soket'}],\n",
      "            [40928216,\n",
      "             {'end': 40, 'labels': ['COUNTRY'], 'start': 35, 'text': 'Niger'},\n",
      "             {'end': 20,\n",
      "              'labels': ['SHOPPING_ITEM'],\n",
      "              'start': 11,\n",
      "              'text': 'kan soket'}]],\n",
      " 'text': 'wanne irin kan soket zan buƙata a  Niger?',\n",
      " 'total_annotations': 3}\n"
     ]
    }
   ],
   "source": [
    "show_entity(clean_entity(language[0]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "120it [00:00, 26725.24it/s]\n"
     ]
    }
   ],
   "source": [
    "def majority_vote(data: List[Dict[str, Any]]) -> List[Dict[str, Any]]:\n",
    "    all_result = []\n",
    "\n",
    "    for item in data:\n",
    "        # Initialize the new item with the original data\n",
    "        new_item = {\n",
    "            \"id\": item[\"id\"],\n",
    "            \"inner_id\": item[\"inner_id\"],\n",
    "            \"data\": item[\"data\"],\n",
    "            \"total_annotations\": item[\"total_annotations\"],\n",
    "        }\n",
    "        # Collect all annotations\n",
    "        all_annotations = []\n",
    "        for annotation in item[\"annotations\"]:\n",
    "            for result in annotation[\"result\"]:\n",
    "                all_annotations.append(\n",
    "                    {\n",
    "                        \"text\": result[\"value\"][\"text\"],\n",
    "                        \"start\": result[\"value\"][\"start\"],\n",
    "                        \"end\": result[\"value\"][\"end\"],\n",
    "                        \"labels\": tuple(\n",
    "                            result[\"value\"][\"labels\"]\n",
    "                        ),  # Convert list to tuple for hashing\n",
    "                    }\n",
    "                )\n",
    "\n",
    "        # Count occurrences of each annotation\n",
    "        annotation_counts = Counter(tuple(a.items()) for a in all_annotations)\n",
    "\n",
    "        # Select annotations with majority vote\n",
    "        majority_threshold = len(item[\"annotations\"]) / 2\n",
    "        majority_annotations = [\n",
    "            dict(a)\n",
    "            for a, count in annotation_counts.items()\n",
    "            if count > majority_threshold\n",
    "        ]\n",
    "\n",
    "        # Convert labels back to list\n",
    "        for annotation in majority_annotations:\n",
    "            annotation[\"labels\"] = list(annotation[\"labels\"])\n",
    "\n",
    "        # Create the new annotation result\n",
    "        new_annotation = {\n",
    "            \"id\": f\"majority_vote_{item['id']}\",\n",
    "            \"completed_by\": {\"id\": \"MAJORITY_VOTE\"},\n",
    "            \"result\": [\n",
    "                {\n",
    "                    \"id\": f\"majority_{i}\",\n",
    "                    \"type\": \"labels\",\n",
    "                    \"value\": annotation,\n",
    "                    \"origin\": \"manual\",\n",
    "                    \"to_name\": \"text\",\n",
    "                    \"from_name\": \"label\",\n",
    "                }\n",
    "                for i, annotation in enumerate(majority_annotations)\n",
    "            ],\n",
    "        }\n",
    "\n",
    "        # Calculate agreement\n",
    "        total_annotations = sum(annotation_counts.values())\n",
    "        agreed_annotations = sum(\n",
    "            count for count in annotation_counts.values() if count > majority_threshold\n",
    "        )\n",
    "        agreement = (\n",
    "            (agreed_annotations / total_annotations) * 100\n",
    "            if total_annotations > 0\n",
    "            else 100\n",
    "        )\n",
    "\n",
    "        # Add the new annotation and agreement to the item\n",
    "        new_item[\"annotations\"] = [new_annotation]\n",
    "        new_item[\"agreement\"] = agreement\n",
    "\n",
    "        all_result.append(new_item)\n",
    "\n",
    "    return all_result\n",
    "\n",
    "\n",
    "from mlds.utils.converter import convert_to_conll2003\n",
    "\n",
    "lang = \"eng\"\n",
    "\n",
    "convert_to_conll2003(majority_vote(english), pj(\"./data\", \"conll\", f\"{lang}.conll\"))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "9607it [00:00, 26989.66it/s]\n",
      "3200it [00:00, 21890.94it/s]\n"
     ]
    }
   ],
   "source": [
    "from mlds.utils.converter import convert_to_conll2003\n",
    "\n",
    "lang = \"hau\"\n",
    "convert_to_conll2003(language, pj(\"./data\", \"conll\", f\"{lang}.conll\"))\n",
    "convert_to_conll2003(\n",
    "    majority_vote(language), pj(\"./data\", \"conll\", f\"{lang}_majority.conll\")\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "# convert to jsonl\n",
    "# sample output: {\"example_id\": \"test-00003330\", \"language\": \"pcm\", \"text\": \"Most of de people who dey opposed to Prez Akufo-Addo en decision say within 3 weeks of lockdown, total number of cases for Ghana rise from around 100 catch 1024.\", \"spans\": [{\"start_byte\": 42, \"limit_byte\": 52, \"label\": \"PER\"}, {\"start_byte\": 76, \"limit_byte\": 83, \"label\": \"DATE\"}, {\"start_byte\": 123, \"limit_byte\": 128, \"label\": \"LOC\"}], \"target\": \"PER: Akufo-Addo $$ DATE: 3 weeks $$ LOC: Ghana\"}\n",
    "\n",
    "\n",
    "def convert_to_jsonl(data: List[Dict[str, Any]], output_file: str, lang: str):\n",
    "    with open(output_file, \"w\") as f:\n",
    "        for item in data:\n",
    "            text = item[\"data\"][\"text\"]\n",
    "            spans = []\n",
    "            for annotation in item[\"annotations\"]:\n",
    "                for result in annotation[\"result\"]:\n",
    "                    value = result[\"value\"]\n",
    "                    spans.append(\n",
    "                        {\n",
    "                            \"start_byte\": value[\"start\"],\n",
    "                            \"limit_byte\": value[\"end\"],\n",
    "                            \"label\": value[\"labels\"][0],\n",
    "                        }\n",
    "                    )\n",
    "            f.write(\n",
    "                json.dumps(\n",
    "                    {\n",
    "                        \"example_id\": str(item[\"inner_id\"]),\n",
    "                        \"language\": lang,\n",
    "                        \"text\": text,\n",
    "                        \"spans\": spans,\n",
    "                        \"target\": \" $$ \".join(\n",
    "                            [\n",
    "                                f\"{s['label']}: {text[s['start_byte']:s['limit_byte']]}\"\n",
    "                                for s in spans\n",
    "                            ]\n",
    "                        ),\n",
    "                    }\n",
    "                )\n",
    "                + \"\\n\"\n",
    "            )\n",
    "\n",
    "\n",
    "# convert_to_jsonl(english, pj(\"./data\", \"jsonl\", \"eng.jsonl\"), \"eng\")\n",
    "# convert_to_jsonl(language, pj(\"./data\", \"jsonl\", \"hau.jsonl\"), \"hau\")\n",
    "convert_to_jsonl(\n",
    "    majority_vote(language), pj(\"./data\", \"jsonl\", \"hau_majority.jsonl\"), \"hau\"\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'id': 122174876,\n",
       " 'result': [[40932812,\n",
       "   {'end': 40, 'text': 'Niger', 'start': 35, 'labels': ['COUNTRY']},\n",
       "   {'end': 20, 'text': 'kan soket', 'start': 11, 'labels': ['SHOPPING_ITEM']}],\n",
       "  [40955949,\n",
       "   {'end': 40, 'text': 'Niger', 'start': 35, 'labels': ['COUNTRY']},\n",
       "   {'end': 20, 'text': 'kan soket', 'start': 11, 'labels': ['SHOPPING_ITEM']}],\n",
       "  [40928216,\n",
       "   {'end': 40, 'text': 'Niger', 'start': 35, 'labels': ['COUNTRY']},\n",
       "   {'end': 20,\n",
       "    'text': 'kan soket',\n",
       "    'start': 11,\n",
       "    'labels': ['SHOPPING_ITEM']}]],\n",
       " 'agreement': 100.0,\n",
       " 'text': 'wanne irin kan soket zan buƙata a  Niger?',\n",
       " 'inner_id': 121,\n",
       " 'total_annotations': 3}"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "clean_entity(language[0])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "3200"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(set([clean_entity(x)[\"text\"] for x in language]))  # match them"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "ieg",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.12.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}