preprocessing.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 192,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import fnmatch\n",
    "import os\n",
    "import csv\n",
    "\n",
    "from nltk.tokenize import WordPunctTokenizer\n",
    "from nltk.tokenize.treebank import TreebankWordDetokenizer"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 83,
   "metadata": {},
   "outputs": [],
   "source": [
    "def merge(filepath1, filepath2, newpath):\n",
    "    data = data2 = \"\"\n",
    "\n",
    "# Reading data from file1\n",
    "    with open(filepath1) as fp:\n",
    "        data = fp.read()\n",
    "  \n",
    "# Reading data from file2\n",
    "    with open(filepath2) as fp:\n",
    "        data2 = fp.read()\n",
    "  \n",
    "# Merging 2 files\n",
    "# To add the data of file2\n",
    "# from next line\n",
    "    data += \"\\n\"\n",
    "    data += data2\n",
    "  \n",
    "    with open (newpath, 'w') as fp:\n",
    "        fp.write(data)\n",
    "        \n",
    "path = \"../../data/conll2003/deu.all\"\n",
    "\n",
    "with open(path) as p:\n",
    "    data = p.read()\n",
    "\n",
    "# for i in range(len(lines)):\n",
    "#     if \"DOCSTART\" in lines[i]:\n",
    "#         end = i\n",
    "#         doc = lines[start:end]\n",
    "#         start = end\n",
    "#         filenumber += 1\n",
    "#         newpath = \"../../data/conll2003/split/deu.\" + str(filenumber)\n",
    "#         with open (newpath, 'w') as fp:\n",
    "#             fp.write(doc)\n",
    "new = data.split(\"-DOCSTART- -X- -X- -X- O\")\n",
    "\n",
    "# for element in new:\n",
    "for i in range(len(new)):\n",
    "    newpath = \"../../data/conll2003/splitdeu/deu.\" + str(i)\n",
    "    with open (newpath, 'w') as fp:\n",
    "        fp.write(new[i])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "# conll2003 English: merge all data together, split by document\n",
    "path = \"../../data/conll2003\"\n",
    "conlleng1 = path + '/eng.testa'\n",
    "conlleng2 = path + '/eng.testb'\n",
    "conllengnew = path + '/eng.test'\n",
    "merge(conlleng1, conlleng2, conllengnew)\n",
    "conlleng3 = path + '/eng.train'\n",
    "conllengnew2 = path + '/eng.all'\n",
    "merge(conlleng3, conllengnew, conllengnew2)\n",
    "path = \"../../data/conll2003/eng.all\"\n",
    "\n",
    "with open(path) as p:\n",
    "    data = p.read()\n",
    "\n",
    "new = data.split(\"-DOCSTART- -X- -X- O\")\n",
    "\n",
    "for i in range(len(new)):\n",
    "    newpath = \"../../data/conll2003/spliteng/eng.\" + str(i)\n",
    "    with open (newpath, 'w') as fp:\n",
    "        fp.write(new[i])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "# conll2003 German: merge all data together, split by document\n",
    "path = \"../../data/conll2003\"\n",
    "conlldeu1 = path + '/deu.train'\n",
    "conlldeu2 = path + '/deu.dev'\n",
    "conlldeunew = path + '/deu.traindev'\n",
    "merge(conlldeu1, conlldeu2, conlldeunew)\n",
    "conlldeunew2 = path + '/deu.all'\n",
    "conlldeu3 = path + '/deu.testa'\n",
    "merge(conlldeunew, conlldeu3, conlldeunew2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "# sec-filings: merge all data together, split by document\n",
    "path = \"../../data/SEC-filings/CONLL-format/data\"\n",
    "sectrain = path + '/train/FIN5.txt'\n",
    "sectest = path + '/test/FIN3.txt'\n",
    "secnew = path + '/eng.all'\n",
    "merge(sectrain, sectest, secnew)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 64,
   "metadata": {},
   "outputs": [],
   "source": [
    "path = \"../../data/SEC-filings/CONLL-format/data/eng.all\"\n",
    "\n",
    "with open(path) as p:\n",
    "    data = p.read()\n",
    "\n",
    "new = data.split(\"-DOCSTART- -X- O O\")\n",
    "\n",
    "for i in range(len(new)):\n",
    "    newpath = \"../../data/SEC-filings/CONLL-format/data/split/eng.\" + str(i)\n",
    "    with open (newpath, 'w') as fp:\n",
    "        fp.write(new[i])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "# GermEval 2014\n",
    "\n",
    "path = \"../../data/GermEval2014\"\n",
    "germdev = path + '/NER-de-dev.tsv'\n",
    "germtest = path + '/NER-de-test.tsv'\n",
    "germtrain = path + '/NER-de-train.tsv'\n",
    "germdevtest = path + '/deu.devtest.tsv'\n",
    "merge(germdev, germtest, germdevtest)\n",
    "germnew = path + '/deu.all.tsv'\n",
    "merge(germdevtest, germtrain, germnew)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 57,
   "metadata": {},
   "outputs": [],
   "source": [
    "# GermEval 2014 csv\n",
    "# ge = pd.read_csv('../../data/GermEval2014/deu.all.tsv', sep = '\\t')\n",
    "\n",
    "# pos = ['UNK'] * (len(ge))\n",
    "# ge['pos'] = pos\n",
    "# ge\n",
    "# ge.to_csv('../../data/GermEval2014/deu.all.new.tsv', sep = '\\t', index = False)\n",
    "\n",
    "with open('../../data/GermEval2014/deu.all.pos.tsv') as de:\n",
    "    lines = de.readlines()\n",
    "dat = []\n",
    "j = 0\n",
    "for i in range(len(lines)):\n",
    "    if lines[i].startswith('#'):\n",
    "        dat.append(lines[j:i])\n",
    "        j = i\n",
    "\n",
    "for i in range(10):#len(dat)\n",
    "    newpath = \"../../data/GermEval2014/split/deu.\" + str(i)\n",
    "    with open (newpath, 'w') as fp:\n",
    "        for line in dat[i]:\n",
    "            if not line.startswith('#'):\n",
    "                fp.write(line)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 59,
   "metadata": {},
   "outputs": [],
   "source": [
    "with open('../../data/GermEval2014/deu.all.pos.tsv') as de:\n",
    "    lines = de.readlines()\n",
    "newlines = []\n",
    "for line in lines:\n",
    "    if not line.startswith('#'):\n",
    "        newlines.append(line)\n",
    "with open('../../data/GermEval2014/deu.all.pos.nohashtag.tsv', 'w') as deh:\n",
    "    deh.writelines(newlines)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 61,
   "metadata": {},
   "outputs": [],
   "source": [
    "with open('../../data/GermEval2014/NER-de-dev.new.tsv') as de:\n",
    "    lines = de.readlines()\n",
    "newlines = []\n",
    "for line in lines:\n",
    "    if not line.startswith('#'):\n",
    "        newlines.append(line)\n",
    "with open('../../data/GermEval2014/NER-de-dev.nohashtag.new.tsv', 'w') as deh:\n",
    "    deh.writelines(newlines)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 63,
   "metadata": {},
   "outputs": [],
   "source": [
    "with open('../../data/GermEval2014/NER-de-test.pos.tsv') as de:\n",
    "    lines = de.readlines()\n",
    "newlines = []\n",
    "for line in lines:\n",
    "    if not line.startswith('#'):\n",
    "        newlines.append(line)\n",
    "with open('../../data/GermEval2014/NER-de-test.pos.nohashtag.tsv', 'w') as deh:\n",
    "    deh.writelines(newlines)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 60,
   "metadata": {},
   "outputs": [],
   "source": [
    "path = '../../data/GermEval2014/deu.all.pos.nohashtag.tsv'\n",
    "\n",
    "with open(path) as p:\n",
    "    data = p.read()\n",
    "\n",
    "new = data.split(\"\\n\\n\")\n",
    "\n",
    "for i in range(len(new)):\n",
    "    newpath = \"../../data/GermEval2014/split/deu.\" + str(i)\n",
    "    with open (newpath, 'w') as fp:\n",
    "        fp.write(new[i])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 56,
   "metadata": {},
   "outputs": [],
   "source": [
    "path = \"../../data/GermEval2014/deu.all.tsv\"\n",
    "newlines = []\n",
    "\n",
    "with open(path, 'r') as p:\n",
    "    for x in p.readlines():\n",
    "        if x != '\\n':\n",
    "            newlines.append(''.join([x.strip(), '\\tUNK', '\\n']))\n",
    "        else:\n",
    "            newlines.append('\\n')\n",
    "\n",
    "newpath = \"../../data/GermEval2014/deu.all.pos.tsv\"\n",
    "with open(newpath, 'w') as np:\n",
    "    np.writelines(newlines)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 62,
   "metadata": {},
   "outputs": [],
   "source": [
    "path = \"../../data/GermEval2014/NER-de-test.tsv\"\n",
    "newlines = []\n",
    "\n",
    "with open(path, 'r') as p:\n",
    "    for x in p.readlines():\n",
    "        if x != '\\n':\n",
    "            newlines.append(''.join([x.strip(), '\\tUNK', '\\n']))\n",
    "        else:\n",
    "            newlines.append('\\n')\n",
    "\n",
    "newpath = \"../../data/GermEval2014/NER-de-test.pos.tsv\"\n",
    "with open(newpath, 'w') as np:\n",
    "    np.writelines(newlines)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# def ppgermeval(path, newpath):\n",
    "#     with open(path, 'r') as p:\n",
    "#         data = p.read()\n",
    "#     newdata = data\n",
    "#     newdata = newdata.replace('I-RR','I-PER')\n",
    "#     newdata = newdata.replace('B-RR','B-PER')\n",
    "#     newdata = newdata.replace('I-AN','I-PER')\n",
    "#     newdata = newdata.replace('B-AN','B-PER')\n",
    "#     newdata = newdata.replace('B-LD','B-LOC')\n",
    "#     newdata = newdata.replace('I-LD','I-LOC')\n",
    "#     newdata = newdata.replace('B-ST','B-LOC')\n",
    "#     newdata = newdata.replace('I-ST','I-LOC')\n",
    "#     newdata = newdata.replace('B-STR','B-LOC')\n",
    "#     newdata = newdata.replace('I-STR','I-LOC')\n",
    "#     newdata = newdata.replace('B-LDS','B-LOC')\n",
    "#     newdata = newdata.replace('I-LDS','I-LOC')\n",
    "#     newdata = newdata.replace('B-UN','B-ORG')\n",
    "#     newdata = newdata.replace('I-UN','I-ORG')\n",
    "#     newdata = newdata.replace('B-INN','B-ORG')\n",
    "#     newdata = newdata.replace('I-INN','I-ORG')\n",
    "#     newdata = newdata.replace('B-GRT','B-ORG')\n",
    "#     newdata = newdata.replace('I-GRT','I-ORG')\n",
    "#     newdata = newdata.replace('B-MRK','B-ORG')\n",
    "#     newdata = newdata.replace('I-MRK','I-ORG')\n",
    "#     newdata = newdata.replace('B-GS','B-NRM')\n",
    "#     newdata = newdata.replace('I-GS','I-NRM')\n",
    "#     newdata = newdata.replace('B-VO','B-NRM')\n",
    "#     newdata = newdata.replace('I-VO','I-NRM')\n",
    "#     newdata = newdata.replace('B-EUN','B-NRM')\n",
    "#     newdata = newdata.replace('I-EUN','I-NRM')\n",
    "#     newdata = newdata.replace('I-VS','I-REG')\n",
    "#     newdata = newdata.replace('B-VS','B-REG')\n",
    "#     newdata = newdata.replace('I-VT','I-REG')\n",
    "#     newdata = newdata.replace('B-VT','B-REG')\n",
    "# # The tags RS and LIT are identical in coarse-grained and fine-grained, so no preprocessing required.\n",
    "#     with open(newpath, 'w') as np:\n",
    "#         np.write(newdata)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [],
   "source": [
    "# LegalER\n",
    "\n",
    "path = \"../../data/LegalER/dataset_courts/bag.conll\"\n",
    "\n",
    "with open(path) as p:\n",
    "    data = p.read()\n",
    "\n",
    "new = data.split(\"\\n\\n\")\n",
    "\n",
    "# 加个\\n\n",
    "for i in range(len(new)):\n",
    "    newpath = path = \"../../data/LegalER/dataset_courts/split/deu.\" + str(i)\n",
    "    with open (newpath, 'w') as fp:\n",
    "        fp.write(new[i])\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "metadata": {},
   "outputs": [],
   "source": [
    "path = \"../../data/LegalER/dataset_courts/bag.conll\"\n",
    "newlines = []\n",
    "\n",
    "with open(path, 'r') as p:\n",
    "    for x in p.readlines():\n",
    "        if x != '\\n':\n",
    "            newlines.append(''.join([x.strip(), ' UNK', '\\n']))\n",
    "        else:\n",
    "            newlines.append('\\n')\n",
    "\n",
    "newpath = \"../../data/LegalER/dataset_courts/bag.new.conll\"\n",
    "with open(newpath, 'w') as np:\n",
    "    np.writelines(newlines)\n",
    "    \n",
    "path = \"../../data/LegalER/dataset_courts/bfh.conll\"\n",
    "newlines = []\n",
    "\n",
    "with open(path, 'r') as p:\n",
    "    for x in p.readlines():\n",
    "        if x != '\\n':\n",
    "            newlines.append(''.join([x.strip(), ' UNK', '\\n']))\n",
    "        else:\n",
    "            newlines.append('\\n')\n",
    "            \n",
    "newpath = \"../../data/LegalER/dataset_courts/bfh.new.conll\"\n",
    "with open(newpath, 'w') as np:\n",
    "    np.writelines(newlines)\n",
    "\n",
    "path = \"../../data/LegalER/dataset_courts/bgh.conll\"\n",
    "newlines = []\n",
    "\n",
    "with open(path, 'r') as p:\n",
    "    for x in p.readlines():\n",
    "        if x != '\\n':\n",
    "            newlines.append(''.join([x.strip(), ' UNK', '\\n']))\n",
    "        else:\n",
    "            newlines.append('\\n')\n",
    "            \n",
    "newpath = \"../../data/LegalER/dataset_courts/bgh.pos.conll\"\n",
    "with open(newpath, 'w') as np:\n",
    "    np.writelines(newlines)\n",
    "\n",
    "path = \"../../data/LegalER/dataset_courts/bpatg.conll\"\n",
    "newlines = []\n",
    "\n",
    "with open(path, 'r') as p:\n",
    "    for x in p.readlines():\n",
    "        if x != '\\n':\n",
    "            newlines.append(''.join([x.strip(), ' UNK', '\\n']))\n",
    "        else:\n",
    "            newlines.append('\\n')\n",
    "            \n",
    "newpath = \"../../data/LegalER/dataset_courts/bpatg.pos.conll\"\n",
    "with open(newpath, 'w') as np:\n",
    "    np.writelines(newlines)\n",
    "\n",
    "path = \"../../data/LegalER/dataset_courts/bsg.conll\"\n",
    "newlines = []\n",
    "\n",
    "with open(path, 'r') as p:\n",
    "    for x in p.readlines():\n",
    "        if x != '\\n':\n",
    "            newlines.append(''.join([x.strip(), ' UNK', '\\n']))\n",
    "        else:\n",
    "            newlines.append('\\n')\n",
    "            \n",
    "newpath = \"../../data/LegalER/dataset_courts/bsg.pos.conll\"\n",
    "with open(newpath, 'w') as np:\n",
    "    np.writelines(newlines)\n",
    "    \n",
    "path = \"../../data/LegalER/dataset_courts/bverfg.conll\"\n",
    "newlines = []\n",
    "\n",
    "with open(path, 'r') as p:\n",
    "    for x in p.readlines():\n",
    "        if x != '\\n':\n",
    "            newlines.append(''.join([x.strip(), ' UNK', '\\n']))\n",
    "        else:\n",
    "            newlines.append('\\n')\n",
    "            \n",
    "newpath = \"../../data/LegalER/dataset_courts/bverfg.pos.conll\"\n",
    "with open(newpath, 'w') as np:\n",
    "    np.writelines(newlines)\n",
    "\n",
    "path = \"../../data/LegalER/dataset_courts/bverwg.conll\"\n",
    "newlines = []\n",
    "\n",
    "with open(path, 'r') as p:\n",
    "    for x in p.readlines():\n",
    "        if x != '\\n':\n",
    "            newlines.append(''.join([x.strip(), ' UNK', '\\n']))\n",
    "        else:\n",
    "            newlines.append('\\n')\n",
    "            \n",
    "newpath = \"../../data/LegalER/dataset_courts/bverwg.pos.conll\"\n",
    "with open(newpath, 'w') as np:\n",
    "    np.writelines(newlines)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 155,
   "metadata": {},
   "outputs": [],
   "source": [
    "path = '../../data/LegalER/dataset_courts/'\n",
    "merge(path + 'bag.conll', path + 'bfh.conll', path + 'deu.zwi1')\n",
    "merge(path + 'deu.zwi1', path + 'bgh.conll', path + 'deu.zwi2')\n",
    "merge(path + 'deu.zwi2', path + 'bpatg.conll', path + 'deu.zwi3')\n",
    "merge(path + 'deu.zwi3', path + 'bsg.conll', path + 'deu.zwi4')\n",
    "merge(path + 'deu.zwi4', path + 'bverfg.conll', path + 'deu.zwi5')\n",
    "merge(path + 'deu.zwi5', path + 'bverwg.conll', path + 'deu.all')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 86,
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "# # filenames = ['../../data/LegalER/dataset_courts/coarse-grained/bag.pos.conll', '../../data/LegalER/dataset_courts/coarse-grained/bfh.pos.conll', path + 'bgh.pos.conll', path + 'bpatg.pos.conll'], '../../data/LegalER/dataset_courts/coarse-grained/bsg.pos.conll', '../../data/LegalER/dataset_courts/coarse-grained/bverfg.pos.conll', '../../data/LegalER/dataset_courts/coarse-grained/bverwg.pos.conll'  \n",
    "# with open('../../data/LegalER/dataset_courts/coarse-grained/deu.all', 'w') as outfile:\n",
    "# #     for fname in filenames:\n",
    "#     with open('../../data/LegalER/dataset_courts/coarse-grained/bag.pos.conll', 'rb') as infile:\n",
    "#         outfile.write(infile.read())\n",
    "path = '../../data/LegalER/dataset_courts/coarse-grained/'\n",
    "merge(path + 'bag.pos.conll', path + 'bfh.pos.conll', path + 'deu.zwi1')\n",
    "merge(path + 'deu.zwi1', path + 'bgh.pos.conll', path + 'deu.zwi2')\n",
    "merge(path + 'deu.zwi2', path + 'bpatg.pos.conll', path + 'deu.zwi3')\n",
    "merge(path + 'deu.zwi3', path + 'bsg.pos.conll', path + 'deu.zwi4')\n",
    "merge(path + 'deu.zwi4', path + 'bverfg.pos.conll', path + 'deu.zwi5')\n",
    "merge(path + 'deu.zwi5', path + 'bverwg.pos.conll', path + 'deu.all')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 89,
   "metadata": {},
   "outputs": [],
   "source": [
    "path = \"../../data/LegalER/dataset_courts/coarse-grained/deu.all\"\n",
    "\n",
    "with open(path) as p:\n",
    "    data = p.read()\n",
    "\n",
    "new = data.split(\"\\n\\n\")\n",
    "\n",
    "\n",
    "for i in range(len(new)):\n",
    "    newpath = path = \"../../data/LegalER/dataset_courts/coarse-grained/split/deu.\" + str(i)\n",
    "    with open (newpath, 'w') as fp:\n",
    "        fp.write(new[i])\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Prozesskostenhilfe O\n",
      " UNK\n"
     ]
    }
   ],
   "source": [
    "print(lines[0] + ' UNK')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 51,
   "metadata": {},
   "outputs": [],
   "source": [
    "def pplegaler(path, newpath):\n",
    "    with open(path, 'r') as p:\n",
    "        data = p.read()\n",
    "    newdata = data\n",
    "    newdata = newdata.replace('I-RR','I-PER')\n",
    "    newdata = newdata.replace('B-RR','B-PER')\n",
    "    newdata = newdata.replace('I-AN','I-PER')\n",
    "    newdata = newdata.replace('B-AN','B-PER')\n",
    "    newdata = newdata.replace('B-LD','B-LOC')\n",
    "    newdata = newdata.replace('I-LD','I-LOC')\n",
    "    newdata = newdata.replace('B-ST','B-LOC')\n",
    "    newdata = newdata.replace('I-ST','I-LOC')\n",
    "    newdata = newdata.replace('B-STR','B-LOC')\n",
    "    newdata = newdata.replace('I-STR','I-LOC')\n",
    "    newdata = newdata.replace('B-LDS','B-LOC')\n",
    "    newdata = newdata.replace('I-LDS','I-LOC')\n",
    "    newdata = newdata.replace('B-UN','B-ORG')\n",
    "    newdata = newdata.replace('I-UN','I-ORG')\n",
    "    newdata = newdata.replace('B-INN','B-ORG')\n",
    "    newdata = newdata.replace('I-INN','I-ORG')\n",
    "    newdata = newdata.replace('B-GRT','B-ORG')\n",
    "    newdata = newdata.replace('I-GRT','I-ORG')\n",
    "    newdata = newdata.replace('B-MRK','B-ORG')\n",
    "    newdata = newdata.replace('I-MRK','I-ORG')\n",
    "    newdata = newdata.replace('B-GS','B-NRM')\n",
    "    newdata = newdata.replace('I-GS','I-NRM')\n",
    "    newdata = newdata.replace('B-VO','B-NRM')\n",
    "    newdata = newdata.replace('I-VO','I-NRM')\n",
    "    newdata = newdata.replace('B-EUN','B-NRM')\n",
    "    newdata = newdata.replace('I-EUN','I-NRM')\n",
    "    newdata = newdata.replace('I-VS','I-REG')\n",
    "    newdata = newdata.replace('B-VS','B-REG')\n",
    "    newdata = newdata.replace('I-VT','I-REG')\n",
    "    newdata = newdata.replace('B-VT','B-REG')\n",
    "# The tags RS and LIT are identical in coarse-grained and fine-grained, so no preprocessing required.\n",
    "    with open(newpath, 'w') as np:\n",
    "        np.write(newdata)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 52,
   "metadata": {},
   "outputs": [],
   "source": [
    "path = \"../../data/LegalER/dataset_courts/bag.new.conll\"\n",
    "newpath = \"../../data/LegalER/dataset_courts/coarse-grained/bag.new.conll\"\n",
    "\n",
    "pplegaler(path, newpath)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 53,
   "metadata": {},
   "outputs": [],
   "source": [
    "path = \"../../data/LegalER/dataset_courts/bfh.new.conll\"\n",
    "newpath = \"../../data/LegalER/dataset_courts/coarse-grained/bfh.new.conll\"\n",
    "\n",
    "pplegaler(path, newpath)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 74,
   "metadata": {},
   "outputs": [],
   "source": [
    "path = \"../../data/LegalER/dataset_courts/bgh.pos.conll\"\n",
    "newpath = \"../../data/LegalER/dataset_courts/coarse-grained/bgh.pos.conll\"\n",
    "\n",
    "pplegaler(path, newpath)\n",
    "\n",
    "path = \"../../data/LegalER/dataset_courts/bpatg.pos.conll\"\n",
    "newpath = \"../../data/LegalER/dataset_courts/coarse-grained/bpatg.pos.conll\"\n",
    "\n",
    "pplegaler(path, newpath)\n",
    "\n",
    "path = \"../../data/LegalER/dataset_courts/bsg.pos.conll\"\n",
    "newpath = \"../../data/LegalER/dataset_courts/coarse-grained/bsg.pos.conll\"\n",
    "\n",
    "pplegaler(path, newpath)\n",
    "\n",
    "path = \"../../data/LegalER/dataset_courts/bverfg.pos.conll\"\n",
    "newpath = \"../../data/LegalER/dataset_courts/coarse-grained/bverfg.pos.conll\"\n",
    "\n",
    "pplegaler(path, newpath)\n",
    "\n",
    "path = \"../../data/LegalER/dataset_courts/bverwg.pos.conll\"\n",
    "newpath = \"../../data/LegalER/dataset_courts/coarse-grained/bverwg.pos.conll\"\n",
    "\n",
    "pplegaler(path, newpath)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 69,
   "metadata": {},
   "outputs": [],
   "source": [
    "#WNUT17\n",
    "path = '../../data/WNUT17/CONLL-format/data/train/wnut17train.conll'\n",
    "newlines = []\n",
    "\n",
    "with open(path, 'r') as p:\n",
    "    for x in p.readlines():\n",
    "        if x != '\\n':\n",
    "            newlines.append(''.join([x.strip(), '\\tUNK', '\\n']))\n",
    "        else:\n",
    "            newlines.append('\\n')\n",
    "            \n",
    "newpath = '../../data/WNUT17/CONLL-format/data/train/wnut17train.pos.conll'\n",
    "with open(newpath, 'w') as np:\n",
    "    np.writelines(newlines)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 70,
   "metadata": {},
   "outputs": [],
   "source": [
    "path = '../../data/WNUT17/CONLL-format/data/test/emerging.test.annotated'\n",
    "\n",
    "newlines = []\n",
    "\n",
    "with open(path, 'r') as p:\n",
    "    for x in p.readlines():\n",
    "        if x != '\\n':\n",
    "            newlines.append(''.join([x.strip(), '\\tUNK', '\\n']))\n",
    "        else:\n",
    "            newlines.append('\\n')\n",
    "\n",
    "newpath = '../../data/WNUT17/CONLL-format/data/test/emerging.test.pos.annotated'\n",
    "\n",
    "with open(newpath, 'w') as np:\n",
    "    np.writelines(newlines)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 92,
   "metadata": {},
   "outputs": [],
   "source": [
    "path = '../../data/WNUT17/CONLL-format/data/dev/emerging.dev.conll'\n",
    "newlines = []\n",
    "\n",
    "with open(path, 'r') as p:\n",
    "    for x in p.readlines():\n",
    "        if x != '\\n':\n",
    "            newlines.append(''.join([x.strip(), '\\tUNK', '\\n']))\n",
    "        else:\n",
    "            newlines.append('\\n')\n",
    "            \n",
    "newpath = '../../data/WNUT17/CONLL-format/data/dev/emerging.dev.pos.conll'\n",
    "with open(newpath, 'w') as np:\n",
    "    np.writelines(newlines)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 71,
   "metadata": {},
   "outputs": [],
   "source": [
    "path = \"../../data/EuropeanaNewspapers/enp_DE.lft.bio/enp_DE.lft.bio\"\n",
    "newlines = []\n",
    "\n",
    "with open(path, 'r') as p:\n",
    "    for x in p.readlines():\n",
    "        if x != '\\n':\n",
    "            newlines.append(''.join([x.strip(), ' UNK', '\\n']))\n",
    "        else:\n",
    "            newlines.append('\\n')\n",
    "            \n",
    "newpath = \"../../data/EuropeanaNewspapers/enp_DE.lft.bio/enp_DE.lft.pos.bio\"\n",
    "with open(newpath, 'w') as np:\n",
    "    np.writelines(newlines)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 72,
   "metadata": {},
   "outputs": [],
   "source": [
    "path = \"../../data/EuropeanaNewspapers/enp_DE.onb.bio/enp_DE.onb.bio\"\n",
    "newlines = []\n",
    "\n",
    "with open(path, 'r') as p:\n",
    "    for x in p.readlines():\n",
    "        if x != '\\n':\n",
    "            newlines.append(''.join([x.strip(), ' UNK', '\\n']))\n",
    "        else:\n",
    "            newlines.append('\\n')\n",
    "            \n",
    "newpath = \"../../data/EuropeanaNewspapers/enp_DE.onb.bio/enp_DE.onb.pos.bio\"\n",
    "with open(newpath, 'w') as np:\n",
    "    np.writelines(newlines)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 90,
   "metadata": {},
   "outputs": [],
   "source": [
    "fin = open('../../data/WNUT17/CONLL-format/data/train/wnut17train.pos.conll', \"rt\")\n",
    "#output file to write the result to\n",
    "fout = open('../../data/WNUT17/CONLL-format/data/train/wnut17train.pos.pp.conll', \"wt\")\n",
    "#for each line in the input file\n",
    "for line in fin:\n",
    "    #read replace the string and write to output file\n",
    "    fout.write(line.replace('creative-work', 'creativework'))\n",
    "#close input and output files\n",
    "fin.close()\n",
    "fout.close()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 91,
   "metadata": {},
   "outputs": [],
   "source": [
    "fin = open('../../data/WNUT17/CONLL-format/data/test/emerging.test.pos.annotated', \"rt\")\n",
    "#output file to write the result to\n",
    "fout = open('../../data/WNUT17/CONLL-format/data/test/emerging.test.pos.pp.annotated', \"wt\")\n",
    "#for each line in the input file\n",
    "for line in fin:\n",
    "    #read replace the string and write to output file\n",
    "    fout.write(line.replace('creative-work', 'creativework'))\n",
    "#close input and output files\n",
    "fin.close()\n",
    "fout.close()\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 93,
   "metadata": {},
   "outputs": [],
   "source": [
    "fin = open('../../data/WNUT17/CONLL-format/data/dev/emerging.dev.pos.conll', \"rt\")\n",
    "#output file to write the result to\n",
    "fout = open('../../data/WNUT17/CONLL-format/data/dev/emerging.dev.pos.pp.conll', \"wt\")\n",
    "#for each line in the input file\n",
    "for line in fin:\n",
    "    #read replace the string and write to output file\n",
    "    fout.write(line.replace('creative-work', 'creativework'))\n",
    "#close input and output files\n",
    "fin.close()\n",
    "fout.close()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 94,
   "metadata": {},
   "outputs": [],
   "source": [
    "merge('../../data/WNUT17/CONLL-format/data/dev/emerging.dev.pos.pp.conll', '../../data/WNUT17/CONLL-format/data/train/wnut17train.pos.pp.conll', '../../data/WNUT17/CONLL-format/data/eng.zwi')\n",
    "merge('../../data/WNUT17/CONLL-format/data/eng.zwi', '../../data/WNUT17/CONLL-format/data/test/emerging.test.pos.pp.annotated', '../../data/WNUT17/CONLL-format/data/eng.all')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 154,
   "metadata": {},
   "outputs": [],
   "source": [
    "merge('../../data/WNUT17/CONLL-format/data/dev/emerging.dev.conll', '../../data/WNUT17/CONLL-format/data/train/wnut17train.conll', '../../data/WNUT17/CONLL-format/data/engnopos.zwi')\n",
    "merge('../../data/WNUT17/CONLL-format/data/engnopos.zwi', '../../data/WNUT17/CONLL-format/data/test/emerging.test.annotated', '../../data/WNUT17/CONLL-format/data/engnopos.all')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 95,
   "metadata": {},
   "outputs": [],
   "source": [
    "path = '../../data/WNUT17/CONLL-format/data/eng.all'\n",
    "\n",
    "with open(path) as p:\n",
    "    data = p.read()\n",
    "\n",
    "new = data.split(\"\\n\\n\")\n",
    "\n",
    "for i in range(len(new)):\n",
    "    newpath = \"../../data/WNUT17/CONLL-format/data/split/eng.\" + str(i)\n",
    "    with open (newpath, 'w') as fp:\n",
    "        fp.write(new[i])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 139,
   "metadata": {},
   "outputs": [],
   "source": [
    "def filecount(dir):\n",
    "    num_files = len(fnmatch.filter(os.listdir(dir),'eng.*')) + len(fnmatch.filter(os.listdir(dir),'deu.*'))\n",
    "    return num_files"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 103,
   "metadata": {},
   "outputs": [],
   "source": [
    "conll = pd.read_csv('../../data/conll2003/deu.all', sep = ' ')\n",
    "conll = conll.drop(conll.columns[1], axis = 1)\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 109,
   "metadata": {},
   "outputs": [],
   "source": [
    "path = '../../data/conll2003/splitdeu'\n",
    "for i in range(1,10):\n",
    "    df = pd.read_csv(path + '/deu.' + str(i), sep = ' ')\n",
    "    df = df.drop(df.columns[[1,3]], axis = 1)\n",
    "    df.to_csv(path + '/final/deu.' + str(i), sep = ' ', index = False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 153,
   "metadata": {},
   "outputs": [],
   "source": [
    "path = '../../data/GermEval2014/split'\n",
    "for i in range(filecount(path)):\n",
    "    df = pd.read_csv(path + '/deu.' + str(i), quoting= csv.QUOTE_NONE, sep = '\\t')\n",
    "#     df = df.drop(df.columns[[0,3]], axis = 1)\n",
    "    cols = list(df.columns)\n",
    "    cols[0], cols[1] = cols[1], cols[0]\n",
    "    cols[2], cols[4] = cols[4], cols[2]\n",
    "    df = df[cols]\n",
    "    df.to_csv(path + '/final/deu.' + str(i), sep = ' ', index = False)\n",
    "    if \"deu.25026\" in path:\n",
    "        break"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 134,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "1980 UNK O\n",
      "kam UNK O\n",
      "der UNK O\n",
      "Crown UNK B-OTH\n",
      "als UNK O\n",
      "Versuch UNK O\n",
      "von UNK O\n",
      "Toyota UNK B-ORG\n",
      ", UNK O\n",
      "sich UNK O\n",
      "in UNK O\n",
      "der UNK O\n",
      "Oberen UNK O\n",
      "Mittelklasse UNK O\n",
      "zu UNK O\n",
      "etablieren UNK O\n",
      ", UNK O\n",
      "auch UNK O\n",
      "nach UNK O\n",
      "Deutschland UNK B-LOC\n",
      ". UNK O\n",
      "\n"
     ]
    }
   ],
   "source": [
    "f = open(\"../../data/GermEval2014/split/final/deu.1\", \"r\")\n",
    "print(f.read())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 149,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "31300\n"
     ]
    }
   ],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 150,
   "metadata": {},
   "outputs": [],
   "source": [
    "path = \"../../data/conll2003/deu.all\"\n",
    "\n",
    "with open(path) as p:\n",
    "    data = p.read()\n",
    "\n",
    "germevals = filecount('../../data/GermEval2014/split/final')\n",
    "\n",
    "new = data.split(\"-DOCSTART- -X- -X- -X- O\")\n",
    "\n",
    "for i in range(len(new)):\n",
    "    newpath = \"../../data/GermEval2014/split/final/deu.\" + str(germevals + i)\n",
    "    with open (newpath, 'w') as fp:\n",
    "        fp.write(new[i])\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 157,
   "metadata": {},
   "outputs": [],
   "source": [
    "path1 = \"../../data/conll2003/eng.all\"\n",
    "path2 = \"../../data/SEC-filings/CONLL-format/data/eng.all\"\n",
    "path3 = \"../../data/eng.conllsec\"\n",
    "merge(path1, path2, path3)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 168,
   "metadata": {},
   "outputs": [],
   "source": [
    "with open(path3) as f:\n",
    "    lines = f.readlines()\n",
    "newlines = []\n",
    "for line in lines:\n",
    "    if line == '\\n': newlines.append(line)\n",
    "    else:\n",
    "        elements = line.split(' ')\n",
    "        newelements = [elements[0], elements[3]]\n",
    "        newline = ' '.join(newelements)\n",
    "        newlines.append(newline)\n",
    "newlines = ''.join(newlines)\n",
    "with open('../../data/eng.pp.conllsec', 'w') as nf:\n",
    "    nf.write(newlines)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 170,
   "metadata": {},
   "outputs": [],
   "source": [
    "with open('../../data/WNUT17/CONLL-format/data/engnopos.all') as f:\n",
    "    lines = f.readlines()\n",
    "newlines = []\n",
    "for line in lines:\n",
    "    if line == '\\n': newlines.append(line)\n",
    "    else:\n",
    "        elements = line.split('\\t')\n",
    "        newline = ' '.join(elements)\n",
    "        newlines.append(newline)\n",
    "newlines = ''.join(newlines)\n",
    "with open('../../data/WNUT17/CONLL-format/data/engnopossepspace.all', 'w') as nf:\n",
    "    nf.write(newlines)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 171,
   "metadata": {},
   "outputs": [],
   "source": [
    "# inconsistant labels!!!!\n",
    "merge('../../data/eng.pp.conllsec', '../../data/WNUT17/CONLL-format/data/engnopossepspace.all', '../../data/engall.all')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 172,
   "metadata": {},
   "outputs": [],
   "source": [
    "with open('../../data/conll2003/deu.all') as f:\n",
    "    lines = f.readlines()\n",
    "newlines = []\n",
    "for line in lines:\n",
    "    if line == '\\n': newlines.append(line)\n",
    "    else:\n",
    "        elements = line.split(' ')\n",
    "        newelements = [elements[0], elements[4]]\n",
    "        newline = ' '.join(newelements)\n",
    "        newlines.append(newline)\n",
    "newlines = ''.join(newlines)\n",
    "with open('../../data/conll2003/deu.wordnetagonly.all', 'w') as nf:\n",
    "    nf.write(newlines)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 179,
   "metadata": {},
   "outputs": [],
   "source": [
    "with open('../../data/GermEval2014/deu.all.tsv') as f:\n",
    "    lines = f.readlines()\n",
    "newlines = []\n",
    "for line in lines:\n",
    "    if line == '\\n': newlines.append(line)\n",
    "    elif line.startswith('#'): continue\n",
    "    else:\n",
    "        elements = line.split('\\t')\n",
    "        newelements = [elements[1], elements[2], '\\n']\n",
    "        newline = ' '.join(newelements)\n",
    "        newlines.append(newline)\n",
    "newlines = ''.join(newlines)\n",
    "with open('../../data/GermEval2014/deu.wordnetagonly.all', 'w') as nf:\n",
    "    nf.write(newlines)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 180,
   "metadata": {},
   "outputs": [],
   "source": [
    "with open('../../data/GermEval2014/deu.wordnetagonly.all', 'r') as p:\n",
    "    data = p.read()\n",
    "newdata = data\n",
    "\n",
    "newdata = newdata.replace('PERderiv','MISC')\n",
    "newdata = newdata.replace('PERpart','MISC')\n",
    "newdata = newdata.replace('LOCpart','MISC')\n",
    "newdata = newdata.replace('LOCderiv','MISC')\n",
    "newdata = newdata.replace('ORGpart','MISC')\n",
    "newdata = newdata.replace('ORGderiv','MISC')\n",
    "newdata = newdata.replace('OTHpart','MISC')\n",
    "newdata = newdata.replace('OTHderiv','MISC')\n",
    "newdata = newdata.replace('OTH','MISC')\n",
    "# The tags RS and LIT are identical in coarse-grained and fine-grained, so no preprocessing required.\n",
    "with open('../../data/GermEval2014/deu.wordnetagonly.jointpp.all', 'w') as np:\n",
    "    np.write(newdata)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 181,
   "metadata": {},
   "outputs": [],
   "source": [
    "merge('../../data/conll2003/deu.wordnetagonly.all', '../../data/GermEval2014/deu.wordnetagonly.jointpp.all', '../../data/conllgermeval.all')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# with open('../../data/conllgermeval.all') as f:\n",
    "#     lines = f.readlines()\n",
    "# for line in lines:\n",
    "#     if line.startswith()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 183,
   "metadata": {},
   "outputs": [],
   "source": [
    "with open('../../data/conll2003/deu_utf.testb', 'r') as p:\n",
    "    lines = p.readlines()\n",
    "#     newlines = []\n",
    "    sentence = \"\"\n",
    "    sentences = \"\"\n",
    "    labels = []\n",
    "    for line in lines:\n",
    "        if line.startswith('-DOCSTART-') or line.startswith('#'): continue\n",
    "        elif line == '\\n':\n",
    "            if labels == [] and sentence == \"\": continue\n",
    "#             tokens = tokenizer(sentence.lstrip())\n",
    "            sentences += sentence + '\\n'\n",
    "#             TRAINING_DATA.append((tokens,labels))\n",
    "            sentence = \"\"\n",
    "            labels = []   \n",
    "        else:\n",
    "            elements = line.split(' ')\n",
    "            word = elements[0]\n",
    "            label = elements[4][:-1]\n",
    "            if label == 'I-PER' or label == 'B-PER' : word = 'PER'\n",
    "            elif label == 'I-LOC' or label == 'B-LOC': word = 'LOC'\n",
    "            sentence += word + ' '\n",
    "with open('../../data/conll2003/deu_testb.anonymized', 'w') as np:\n",
    "    np.write(sentences)        \n",
    "# newdata = data\n",
    "\n",
    "# newdata = newdata.replace('PERderiv','MISC')\n",
    "# newdata = newdata.replace('PERpart','MISC')\n",
    "# newdata = newdata.replace('LOCpart','MISC')\n",
    "# newdata = newdata.replace('LOCderiv','MISC')\n",
    "# newdata = newdata.replace('ORGpart','MISC')\n",
    "# newdata = newdata.replace('ORGderiv','MISC')\n",
    "# newdata = newdata.replace('OTHpart','MISC')\n",
    "# newdata = newdata.replace('OTHderiv','MISC')\n",
    "# newdata = newdata.replace('OTH','MISC')\n",
    "# # The tags RS and LIT are identical in coarse-grained and fine-grained, so no preprocessing required.\n",
    "# with open('../../data/GermEval2014/deu.wordnetagonly.jointpp.all', 'w') as np:\n",
    "#     np.write(newdata)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 187,
   "metadata": {},
   "outputs": [],
   "source": [
    "with open('../../data/conll2003/deu_utf.testb', 'r') as p:\n",
    "    lines = p.readlines()\n",
    "#     newlines = []\n",
    "    sentence = \"\"\n",
    "    sentences = \"\"\n",
    "    labels = []\n",
    "    for line in lines:\n",
    "        if line.startswith('-DOCSTART-') or line.startswith('#'): continue\n",
    "        elif line == '\\n':\n",
    "            if labels == [] and sentence == \"\": continue\n",
    "#             tokens = tokenizer(sentence.lstrip())\n",
    "            sentences += sentence + '\\n'\n",
    "#             TRAINING_DATA.append((tokens,labels))\n",
    "            sentence = \"\"\n",
    "            labels = []   \n",
    "        else:\n",
    "            elements = line.split(' ')\n",
    "            word = elements[0]\n",
    "            label = elements[4][:-1]\n",
    "            sentence += word + ' '\n",
    "with open('../../data/conll2003/anon_testdata.gt', 'w') as np:\n",
    "    np.write(sentences) "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 194,
   "metadata": {},
   "outputs": [],
   "source": [
    "with open('../../data/conll2003/anon_testdata.gt', 'r') as np:\n",
    "    lines = np.readlines()\n",
    "    newlines = ''\n",
    "    for line in lines:\n",
    "        tokens = WordPunctTokenizer().tokenize(line)\n",
    "        newline = TreebankWordDetokenizer().detokenize(tokens)\n",
    "        newlines += newline + '\\n'\n",
    "with open('../../data/conll2003/anon_testdata.dt.gt', 'w') as nfp:\n",
    "    nfp.writelines(newlines)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 186,
   "metadata": {},
   "outputs": [
    {
     "ename": "UnicodeDecodeError",
     "evalue": "'utf-8' codec can't decode byte 0xfc in position 484: invalid start byte",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m----------------------------------------------\u001b[0m",
      "\u001b[0;31mUnicodeDecodeError\u001b[0mTraceback (most recent call last)",
      "\u001b[0;32m<ipython-input-186-37d0f29bf11b>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m\u001b[0m\n\u001b[1;32m      1\u001b[0m \u001b[0;32mwith\u001b[0m \u001b[0mopen\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m'../../data/conll2003/deu.testb'\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m'r'\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;32mas\u001b[0m \u001b[0mp\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m----> 2\u001b[0;31m     \u001b[0mlines\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mp\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mreadlines\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m      3\u001b[0m \u001b[0;31m#     newlines = []\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      4\u001b[0m     \u001b[0msentence\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;34m\"\"\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      5\u001b[0m     \u001b[0msentences\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;34m\"\"\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/usr/lib/python3.5/codecs.py\u001b[0m in \u001b[0;36mdecode\u001b[0;34m(self, input, final)\u001b[0m\n\u001b[1;32m    319\u001b[0m         \u001b[0;31m# decode input (taking the buffer into account)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    320\u001b[0m         \u001b[0mdata\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mbuffer\u001b[0m \u001b[0;34m+\u001b[0m \u001b[0minput\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 321\u001b[0;31m         \u001b[0;34m(\u001b[0m\u001b[0mresult\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mconsumed\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_buffer_decode\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mdata\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0merrors\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mfinal\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    322\u001b[0m         \u001b[0;31m# keep undecoded input until the next call\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    323\u001b[0m         \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mbuffer\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mdata\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mconsumed\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mUnicodeDecodeError\u001b[0m: 'utf-8' codec can't decode byte 0xfc in position 484: invalid start byte"
     ]
    }
   ],
   "source": [
    "with open('../../data/conll2003/deu.testb', 'r') as p:\n",
    "    lines = p.readlines()\n",
    "#     newlines = []\n",
    "    sentence = \"\"\n",
    "    sentences = \"\"\n",
    "    labels = []\n",
    "    for line in lines:\n",
    "        if line.startswith('-DOCSTART-') or line.startswith('#'): continue\n",
    "        elif line == '\\n':\n",
    "            if labels == [] and sentence == \"\": continue\n",
    "#             tokens = tokenizer(sentence.lstrip())\n",
    "            sentences += sentence + '\\n'\n",
    "#             TRAINING_DATA.append((tokens,labels))\n",
    "            sentence = \"\"\n",
    "            labels = []   \n",
    "        else:\n",
    "            elements = line.split(' ')\n",
    "            word = elements[0]\n",
    "            label = elements[4][:-1]\n",
    "            sentence += word + ' '\n",
    "with open('../../data/conll2003/anon_testdata', 'w') as np:\n",
    "    np.write(sentences) "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.5.2"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}