diff --git a/jsonl_preprocess.ipynb b/jsonl_preprocess.ipynb new file mode 100644 index 0000000..8b9f6a3 --- /dev/null +++ b/jsonl_preprocess.ipynb @@ -0,0 +1,117 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Script for transforming csv file into jsonl for db_in and training a spaCy model. \n", + "This is the most straightforward setup, where every abstract has an 'accept' annotation for its\n", + "corresponding journal, and a 'reject' annotation for every other journal.\n", + "\n", + "https://support.prodi.gy/t/best-practices-for-text-classifier-annotations/135" + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "metadata": {}, + "outputs": [], + "source": [ + "import pandas as pd\n", + "from tqdm import tqdm\n", + "import json" + ] + }, + { + "cell_type": "code", + "execution_count": 4, + "metadata": {}, + "outputs": [], + "source": [ + "#csv with 'abstract' column and 'journal code' column\n", + "df = pd.read_csv('/path/to/abstract.csv')\n", + "journal_list = df['journal code'].unique().tolist()" + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "metadata": {}, + "outputs": [], + "source": [ + "#generate list of json accept annotations for every abstract\n", + "accept_list = []\n", + "for i in tqdm(range(0,df.shape[0])):\n", + " accept_list.append({\"answer\":\"accept\",\"text\":r'\"'+df['abstract'][i]+'\"',\"label\":df['journal code'][i]}) " + ] + }, + { + "cell_type": "code", + "execution_count": 6, + "metadata": {}, + "outputs": [], + "source": [ + "def jsonlReject(text):\n", + " return {\"answer\":\"reject\",\"text\":r'\"'+text+'\"',\"label\":code}" + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "metadata": {}, + "outputs": [], + "source": [ + "#generate list of json reject annotations for every abstract - \n", + "#with every abstract rejected by every journal to which it does not correspond\n", + "reject_list = []\n", + "for code in tqdm(journal_list):\n", + " rej_code_bool = df['journal code'] != code\n", + " rej_df = df[rej_code_bool]\n", + " reject_list.extend(rej_df['abstract'].apply(lambda text:jsonlReject(text)))" + ] + }, + { + "cell_type": "code", + "execution_count": 34, + "metadata": {}, + "outputs": [], + "source": [ + "jsonl = accept_list\n", + "jsonl.extend(reject_list)" + ] + }, + { + "cell_type": "code", + "execution_count": 29, + "metadata": {}, + "outputs": [], + "source": [ + "with open('/path/to/abstract.jsonl', 'w') as outfile:\n", + " for entry in jsonl:\n", + " json.dump(entry, outfile)\n", + " outfile.write('\\n')" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.6.6" + } + }, + "nbformat": 4, + "nbformat_minor": 2 +}