textes_encodes_analyse.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Introduction\n",
    "\n",
    "Utiliser l'environnement _emigration_, cf. les instructions de la page d'accueil du dépôt et le fichier _environment.yml_"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pprint, re, os, glob\n",
    "from lxml import etree as etree\n",
    "import pandas as pd\n",
    "from matplotlib import pyplot as plt\n",
    "from itertools import groupby\n",
    "import sqlite3"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Liste des fichiers à traiter\n",
    "\n",
    "* Définir les chemins des dossiers où se trouvent les fichiers\n",
    "* Lister les fichiers du dossier XML"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "### Chemin des documents en .docx\n",
    "folder_path = 'fichiers/'\n",
    "### Chemin du dossier des documents transformes \n",
    "folder_path_xml = 'fichiers_xml/'\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "l_docs = []\n",
    "for d in os.listdir(folder_path_xml):\n",
    "    # vérifie que le document ne commence pas avec un '.', particularité MacOS pour docs ouverts\n",
    "    if d[0] != '.':\n",
    "        if('.html' in d):\n",
    "            l_docs.append(d)\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "len(l_docs), l_docs[:5]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Exploration des documents produits\n",
    "\n",
    "* transformer en XML les fichier HTML\n",
    "* en cas d'erreur (problèmes d'encodage ou autre) une liste des fichiers qui posent problème est établie: variable _bad_files_ \n",
    "* il faut les ouvrir dans un éditeur de textes HTML/XML et les inspecter individuellement, puis corriger les erreurs directement dans le fichier word et refaire toute la procédure (afin de pouvoir continuer à travailler avec les originaux si on souhaite les enrichir)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "glob_list = [f for f in glob.glob(folder_path_xml + \"*.html\")]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "len(glob_list)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "glob_list[-3:]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "eparser = etree.ETCompatXMLParser(encoding='iso-8859-5')\n",
    "\n",
    "\n",
    "bad_files =[]\n",
    "ok_files = []\n",
    "for f in glob_list:\n",
    "    try:\n",
    "        a = etree.parse(f,parser=eparser)\n",
    "        ok_files.append(f)\n",
    "    except Exception as exc :\n",
    "        print(exc)\n",
    "        bad_files.append(f)\n",
    "        \n",
    "        \n",
    "        "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "print(str(len(glob_list)) + ' : ' + str(len(ok_files)) + ' OK + ' + str(len(bad_files)) + ' not OK') "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "bad_files"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Exploration des _tags_"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Tags "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "tag_list = []\n",
    "for doc in glob_list:\n",
    "    doc_name = doc.replace(folder_path_xml, '')\n",
    "    if doc not in bad_files:\n",
    "        file = etree.parse(doc)\n",
    "\n",
    "        ## //tei:*[contains(\"ptr name rs\", name())]'\n",
    "        \"\"\"\n",
    "        # Exemple:\n",
    "        \n",
    "        elements = file.xpath('//tei:name',\n",
    "                              namespaces={'tei':'http://www.tei-c.org/ns/1.0'})\n",
    "        \"\"\"\n",
    "        tags = file.xpath('//span[@class = \"tags\"]')\n",
    "        i = 0\n",
    "        for t in tags:\n",
    "            if t.text : \n",
    "                if_1 = t.text \n",
    "            else: \n",
    "                if_1 = ''\n",
    "            tag_list.append([t.text, doc_name])    \n",
    "            ## print(str(i) + ' ' + t.tag + ' : ' + if_1 + ' (' + doc_name + ')')\n",
    "            i += 1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "print(len(tag_list))\n",
    "tag_list[89:105]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "splitted_tag_doc_list = []\n",
    "for e in tag_list:\n",
    "    if '/' in e[0]:\n",
    "        splitted_tag_doc_list.append([e[0].split('/'), e[1]])\n",
    "    else:\n",
    "        splitted_tag_doc_list.append([[e[0]], e[1]])\n",
    "#    print(b + ' : ' + a )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "splitted_tag_doc_list[89:105]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true,
    "tags": []
   },
   "outputs": [],
   "source": [
    "### Chercher les codages avec plusieurs termes\n",
    "\n",
    "termes_multiples = []\n",
    "for t in splitted_tag_doc_list:\n",
    "    if len(t[0]) > 1:\n",
    "        termes_multiples.append([len(t[0]), t[0], t[1]])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "len(termes_multiples), termes_multiples[10:15]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "###  Créer un entrée par tag–document, \n",
    "# il y a donc moins de lignes que les tags encodées\n",
    "# un tag par document, logique SQL: DISTINCT\n",
    "\n",
    "tag_doc_list = []\n",
    "tag_doc_list_distinct = []\n",
    "for t in splitted_tag_doc_list:\n",
    "    for e in t[0]:\n",
    "        tag_doc_list.append([e, t[1]])\n",
    "        if [e, t[1]] not in tag_doc_list_distinct:\n",
    "            tag_doc_list_distinct.append([e, t[1]])\n",
    "        else:\n",
    "            pass"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "### Liste complète avec éventuelle répétition dans le même document\n",
    "tag_doc_list[89:100]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "### Si on souhaite ne retenir qu'un tag par document\n",
    "tag_doc_list_distinct[50:60]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "### Liste des tags utilisés (en ordre alphabétique)\n",
    "tl = sorted(list(set([e[0] for e in tag_doc_list])))\n",
    "len(tl), tl[:5]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "### Afficher toute la liste\n",
    "print(tl)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Effectifs des tags utilisés"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "### Liste complète des tags, répétitions dans les documents compris\n",
    "# sin on ne souhaite pas de répétition utiliser la liste 'tag_doc_list_distinct'\n",
    "all_tags = [t[0] for t in tag_doc_list]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "results = [[value, len(list(freq))] for value, freq in groupby(sorted(all_tags))]\n",
    "print(results[:3])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "results.sort(key=lambda x:x[1], reverse=True) \n",
    "for r in results[:10]:\n",
    "    print(r)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "### Transfromer la liste en data frame (Pandas)\n",
    "df_results = pd.DataFrame(results, columns = ['tag','eff'])\n",
    "df_results.head(3)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "### Afficher les mesure de centralité et dispersion des fréquences des tags utilisés\n",
    "# on constate une très grande dispérsion, un classement serait utile en vue de l'analyse\n",
    "df_results.describe().round(1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_results = df_results.set_index('tag')\n",
    "df_results.head(3)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "ax = df_results.sort_values(by='eff', ascending=True)\\\n",
    "        .plot(kind='barh',figsize=(15, 60), \n",
    "              color='#BDDAEC', width=0.90)\n",
    "# plt.yticks(df_results.index,df_results['tag'])\n",
    "#y = df_results.eff.value_counts(ascending=True)\n",
    "\n",
    "eff = df_results.sort_values(by='eff', ascending=True).eff\n",
    "\n",
    "for i, value in enumerate(eff):\n",
    "        ax.text(1, i -.2, str(value), size=10)\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Tags and texts"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true,
    "tags": []
   },
   "outputs": [],
   "source": [
    "tags_texts_list = []\n",
    "for doc in glob_list:  ### dans le premier texte   [:1]\n",
    "    doc_name = doc.replace(folder_path_xml, '')\n",
    "    if doc not in bad_files:\n",
    "        file = etree.parse(doc)\n",
    "\n",
    "        ## //tei:*[contains(\"ptr name rs\", name())]'\n",
    "        \"\"\"\n",
    "        elements = file.xpath('//tei:name',\n",
    "                              namespaces={'tei':'http://www.tei-c.org/ns/1.0'})\n",
    "        \"\"\"\n",
    "        spans = file.xpath('//span[@class = \"semAnn\"][./span/@class = \"tags\"]')\n",
    "        print(len(spans))\n",
    "        i = 0\n",
    "\n",
    "        # while i < 5:\n",
    "        for s in spans:\n",
    "            tags = [e.text.split('/') for e in s.xpath('span[@class = \"tags\"]')]\n",
    "            print(tags)\n",
    "            for tag in tags[0]:\n",
    "                print([i,tag, tags[0]])\n",
    "                tags_texts_list.append([i, doc_name,  \", \".join(tags[0]), tag, \"\".join(s.xpath('./text()')) ])\n",
    "            i += 1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true,
    "tags": []
   },
   "outputs": [],
   "source": [
    "tags_texts_list[85:90]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_tags_texts_list = pd.DataFrame(tags_texts_list, columns=['id', 'document', 'tags_originaux', 'tag', 'texte'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "chemin = 'resultats/tags_texts_list.csv'\n",
    "df_tags_texts_list.to_csv(chemin, index=False)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Exporter dans table sqlite"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "chemin = 'resultats/tags_and_texts.sqlite'\n",
    "con = sqlite3.connect(chemin)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "### !!! Attention la table sera écrasée et réécrite entièremen à chaque exécution !!!\n",
    "df_tags_texts_list.to_sql('tags_texts_list', con=con, if_exists='append')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "con.close()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "emigration",
   "language": "python",
   "name": "emigration"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.11"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}