LSTM with real data.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import os\n",
    "os.environ[\"OMP_NUM_THREADS\"] = \"20\" \n",
    "os.environ[\"MKL_NUM_THREADS\"] = \"22\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/CDBDX/remy.garnier/.virtualenvs/python_3.6_TS/lib/python3.6/site-packages/pandas/compat/__init__.py:85: UserWarning: Could not import the lzma module. Your installed Python is incomplete. Attempting to use lzma compression will result in a RuntimeError.\n",
      "  warnings.warn(msg)\n",
      "/home/CDBDX/remy.garnier/.virtualenvs/python_3.6_TS/lib/python3.6/site-packages/pandas/compat/__init__.py:85: UserWarning: Could not import the lzma module. Your installed Python is incomplete. Attempting to use lzma compression will result in a RuntimeError.\n",
      "  warnings.warn(msg)\n"
     ]
    }
   ],
   "source": [
    "import pandas as pd\n",
    "import numpy as np\n",
    "import matplotlib.pyplot as plt\n",
    "import torch\n",
    "import torch.nn as nn\n",
    "import datetime as dt\n",
    "import torch.nn.functional as F\n",
    "from src import STD_DATA_DIR\n",
    "from sklearn.preprocessing import MinMaxScaler"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Prédiction des ventes par réseau de neurones récurrents\n",
    "\n",
    "Objectif : \n",
    "- Construire un réseau de neurones récurrents (LSTM) pour prédire les prochaines ventes sur quelques catégories\n",
    "- Comparer les approches avec & sans concurrence\n",
    "- Identifier les catégories pertinentes pour ce type d'approche\n",
    "\n",
    "## I Données\n",
    "#### I.A Importation des données & premier traitements "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "data = pd.read_csv(STD_DATA_DIR /'dt_for_nn.csv', sep = ';')\n",
    "data.set_index(['product','date'], inplace = True)\n",
    "data = data[data.index.get_level_values(1) > '2017-01-01']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "backfills_cols = ['prix_moy','prix_min_marche_moy','marge_srp','min_marche','min_mp','pump','marge_pump_avg']\n",
    "for col in backfills_cols:\n",
    "    data[col] = data[col].groupby(level = 0).fillna(method = 'bfill')\n",
    "    data[col] = data[col].groupby(level = 0).fillna(method = 'ffill')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/CDBDX/remy.garnier/.virtualenvs/python_3.6_TS/lib/python3.6/site-packages/IPython/core/interactiveshell.py:3049: DtypeWarning: Columns (1,8) have mixed types. Specify dtype option on import or set low_memory=False.\n",
      "  interactivity=interactivity, compiler=compiler, result=result)\n"
     ]
    }
   ],
   "source": [
    "#del(data)\n",
    "data = pd.read_csv(STD_DATA_DIR / 'filtered_dt_for_nn', sep = ';')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#data.to_csv(STD_DATA_DIR / 'filtered_dt_for_nn', sep = ';')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 49,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'SOIN DU SIÈGE',\n",
       " 'Chauffage et brumisateur ext',\n",
       " 'Accessoires electro',\n",
       " 'Divers Composants',\n",
       " 'Jeux de Bain - Hochets',\n",
       " 'Plant de légume',\n",
       " 'ACCESSOIRES ET DETERGENTS',\n",
       " '***HS***Bougie Photophore',\n",
       " 'Ventilateurs/Refroidissement',\n",
       " 'Autre outil électroportatif',\n",
       " 'Interrupteur - prise - douille',\n",
       " 'Nourriture poisson',\n",
       " 'Talc',\n",
       " 'Classeur',\n",
       " 'Couchage chien',\n",
       " \"TV LED 4K de plus de 55''\",\n",
       " 'Voitures Miniatures - CS',\n",
       " 'Peinture - Dessin - Coloriage',\n",
       " 'Biberons',\n",
       " 'Cages et accessoires rongeur',\n",
       " 'Courant Porteur - répéteurs',\n",
       " 'Ampoule fluorescente',\n",
       " 'Cave multi-température (P)',\n",
       " 'Accessoire de porte',\n",
       " 'Tournevis - Visseuse électric',\n",
       " 'Poeles wok sauteuse',\n",
       " 'Pot crayon-Sous main-Corbeille',\n",
       " \"TV LED FHD moins de 24''\",\n",
       " 'Poussette multiple',\n",
       " 'CAble Jack',\n",
       " 'Rangement chambre Adulte',\n",
       " 'Robots radiocommandés',\n",
       " 'Matériel soudage',\n",
       " 'Accessoires disques durs',\n",
       " 'Fenêtre abattante',\n",
       " 'Thermomètre',\n",
       " 'Etabli - tréteau',\n",
       " 'Receveur douche',\n",
       " 'Phytosanitaire',\n",
       " 'Accessoire salle de bain',\n",
       " 'Mixeur -  Batteur',\n",
       " 'Billards',\n",
       " 'Broyeur',\n",
       " 'Accessoires beauté féminine',\n",
       " \"TV LED 4K entre 51 et 55''\",\n",
       " 'Habitat cheval',\n",
       " 'Petit Rangement chambre',\n",
       " 'Cleaning',\n",
       " 'ETUIS ET HOUSSES',\n",
       " 'Autres radiocommandés',\n",
       " 'Parasol',\n",
       " 'Figurines à collectionner',\n",
       " 'Groupe 0+ / 1 - CS',\n",
       " 'Matériel basse - cour',\n",
       " 'CAble RCA',\n",
       " 'Linge de plage',\n",
       " 'Salon de jardin',\n",
       " 'Réservoir wc et cuvette',\n",
       " 'Seau et Serpillère',\n",
       " 'DIVERS ACCESSOIRES TABLETTES',\n",
       " 'Brumisateur - Rafraichisseur',\n",
       " 'Coffret outils à main',\n",
       " 'Sorbetières - Machines à glace',\n",
       " 'Ingredients DLC',\n",
       " \"TV LED 4K entre 33 et 41''\",\n",
       " 'Couveuses',\n",
       " 'Griffoirs et arbres à chat',\n",
       " 'Modelage - Moulage',\n",
       " 'Decoration de Fête',\n",
       " 'Maquettes - modeles reduits',\n",
       " 'Stickers',\n",
       " 'Lit superposés\\xa0 - Mezzanines',\n",
       " 'Sur-matelas',\n",
       " 'Boîtiers PC',\n",
       " 'Table de mixage analogique',\n",
       " \"TV LED FHD de plus de 55''\",\n",
       " 'Tronçonneuse',\n",
       " 'Coffret repas',\n",
       " 'Accessoires',\n",
       " 'Casque et Micro',\n",
       " 'Correspondance',\n",
       " 'Lecteur MP3',\n",
       " 'Playmobil',\n",
       " 'Accessoire wc',\n",
       " \"Portable 11-12''\",\n",
       " 'Tampon - Etiqueteuse',\n",
       " 'Jouets Educatifs - CS',\n",
       " 'Lave Linge (E)',\n",
       " 'Console',\n",
       " 'Planches de poussettes',\n",
       " 'Détection - mesure',\n",
       " 'Trotteur',\n",
       " 'Tapis comtemporain',\n",
       " 'Peintures - Couleurs',\n",
       " 'wc suspendu',\n",
       " 'Veilleuse',\n",
       " 'Décoration - Supports',\n",
       " 'Presse agrume - Centrifugeuse',\n",
       " 'Huile additif moteur - Danger',\n",
       " 'Perforateur- Burineur',\n",
       " 'Télécommande',\n",
       " 'Amplificateurs',\n",
       " 'Plinthe',\n",
       " \"Lits d'appoint\",\n",
       " 'Ensemble memoire',\n",
       " \"Porte d'intérieur\",\n",
       " 'Imprimante 3D',\n",
       " 'Guitares',\n",
       " 'Pompe de forage',\n",
       " 'CODODOS',\n",
       " 'Ventilateur mobile',\n",
       " 'Lave vaisselle (E)',\n",
       " 'Commodes - plan à langer',\n",
       " 'Accessoires beauté masculine',\n",
       " 'Rangement chambre Enfant',\n",
       " 'Hivernage des végétaux',\n",
       " 'Chauffage élec à inertie sèche',\n",
       " 'Batterie et percussions',\n",
       " 'Accessoires cuisson',\n",
       " 'Multimédia / Loisirs',\n",
       " 'Textiles cheval',\n",
       " 'Mémo-Loto-Domino-7 Famil-Magne',\n",
       " 'Table basse',\n",
       " 'Boîtiers externes',\n",
       " 'Gamelle et distributeur chien',\n",
       " 'Lego',\n",
       " 'Tondeuse autoportée',\n",
       " 'Pack électroportatif',\n",
       " 'Cuiseur à riz',\n",
       " 'Ballon eau chaude - chauffeau',\n",
       " 'iMac',\n",
       " 'Scie stationnaire',\n",
       " 'Console extensible',\n",
       " 'Emballage cadeau',\n",
       " 'Mijoteur - Wok',\n",
       " \"Gestion de l'énergie\",\n",
       " 'Dinette - Cuisine - CS',\n",
       " 'Gazon  artificiel',\n",
       " 'Dressage chat',\n",
       " 'Chemise - Sous chemise',\n",
       " 'Fondue',\n",
       " 'Home Studio',\n",
       " 'Tapis Fantaisie',\n",
       " 'Matelas',\n",
       " 'Jeu scientifique - Expérience',\n",
       " 'Surligneur - Marqueur',\n",
       " 'Friteuse',\n",
       " 'Pinceaux et couteaux',\n",
       " 'Table à repasser',\n",
       " 'CALCULATRICE',\n",
       " 'Théière - chocolatière - bouil',\n",
       " 'Plant de fraisier',\n",
       " 'Porteurs - Trotteurs - CS',\n",
       " 'Soin du corps',\n",
       " 'Eclairage extérieur',\n",
       " 'Perceuse',\n",
       " 'Plante verte',\n",
       " 'Volet roulant',\n",
       " 'TAPIS VYNILE',\n",
       " 'Intercalaire-Pochette perforee',\n",
       " 'Agenda et calendrier',\n",
       " \"Récupérateur d'eau de pluie\",\n",
       " \"Stations d'accueil\",\n",
       " 'Cuisine complète',\n",
       " 'Piscine autostable tubulaire',\n",
       " 'Marchande - Tâche Ménagère',\n",
       " 'Accessoire mobilier jardin',\n",
       " 'Ensemble repas de jardin',\n",
       " 'Ensembles Home Cinema',\n",
       " 'Lits simples',\n",
       " 'Ponceuse',\n",
       " 'Compote - petit pot gouter',\n",
       " 'Mémoires RAM',\n",
       " 'Groupe 0+',\n",
       " 'Drap Housse',\n",
       " 'Plaid',\n",
       " 'Céréales',\n",
       " 'Nourriture oiseau',\n",
       " 'Tour de rangement',\n",
       " 'CLIC CLAC',\n",
       " 'Garages-Parkings-Circuits - CS',\n",
       " 'Aquariums',\n",
       " 'Lampe à poser',\n",
       " \"Desserte - Table d'appoint\",\n",
       " 'Chancelière auto',\n",
       " 'Transmetteur - Amplificateur',\n",
       " 'Carrelage intérieur',\n",
       " 'Mobiles',\n",
       " 'Soins et hygiène cheval',\n",
       " '***HS***Récepteur-Décodeur TNT',\n",
       " 'Tetines et sucettes',\n",
       " 'Tête de lit Adulte',\n",
       " 'Découpage - Collage',\n",
       " 'Jouets à tirer-Radiocommandés',\n",
       " 'Groupe 0+ - CS',\n",
       " 'Sommier tapissier',\n",
       " 'Perceuse à colonne',\n",
       " 'Ombrelles',\n",
       " 'Logiciels Pro',\n",
       " 'Matelas mousse',\n",
       " \"TV LED FHD entre 42 et 47''\",\n",
       " 'Colles - Danger',\n",
       " 'Chauffage élec inertie fluide',\n",
       " 'Correcteur - Effaceur',\n",
       " 'Objet suspendu',\n",
       " 'Hotte',\n",
       " 'Cave à vin de service (E)',\n",
       " 'BANQUETTE CONVERTIBLE',\n",
       " 'Intra-auriculaires',\n",
       " 'Fauteuil Relaxation',\n",
       " 'Autre décoration murale',\n",
       " 'Parabole - Antenne RAteau',\n",
       " 'Coussins',\n",
       " 'Affiche poster',\n",
       " 'Outillage pneumatique',\n",
       " 'Beauté - Bijoux',\n",
       " '27\" et plus',\n",
       " 'Platines CD',\n",
       " 'PROTECTION LITERIE',\n",
       " 'Babyphone',\n",
       " 'Autres',\n",
       " 'Climatisation mobile',\n",
       " \"TV LED FHD entre 24 et 29''\",\n",
       " 'Coffret et trousse de soin',\n",
       " 'Casseroles',\n",
       " 'Autres accessoires',\n",
       " 'Crédence et fond de hotte',\n",
       " 'Jeu apprentissage - Expérience',\n",
       " 'Barrière De Lit',\n",
       " 'Toners',\n",
       " 'Grill - Pierrade',\n",
       " 'Accessoires Repassage',\n",
       " 'Instruments de musique',\n",
       " 'Transport et accessoires chat',\n",
       " 'MINI PC',\n",
       " 'Sterilisateurs',\n",
       " 'Horloges',\n",
       " 'CAble optique',\n",
       " 'Motoculteur - motobineuse',\n",
       " 'Comme les Héros',\n",
       " 'Clés 128 Go ou plus',\n",
       " 'Défonceuse',\n",
       " 'Dictaphones',\n",
       " 'Anti-mousse',\n",
       " 'Autres mobiliers',\n",
       " 'Bougies Senteurs',\n",
       " 'Lutte fourmis - insectes',\n",
       " 'Lecteurs Blu-Ray',\n",
       " 'Crochet - access de suspension',\n",
       " 'Wifi / Bluetooth / Sans Fil',\n",
       " 'All in One',\n",
       " 'Lit parapluie couffin',\n",
       " 'Platines Vinyles',\n",
       " 'Loisir du chien',\n",
       " 'Micro ondes (E)',\n",
       " 'Accessoires Beauté Bien etre',\n",
       " 'Cave à vin vieillissement (P)',\n",
       " 'Ensemble claviers et souris',\n",
       " 'Lit parapluie combiné',\n",
       " 'Mosaïque',\n",
       " 'Portail coulissant',\n",
       " 'Décoration Chambre',\n",
       " 'Protection incendie',\n",
       " 'Papier peint',\n",
       " 'Access installation électrique',\n",
       " 'Alimentations',\n",
       " 'Gestion de la lumière',\n",
       " 'Sommier',\n",
       " 'Raccord - robinet - arrosoir',\n",
       " \"Jeux d'éveil\",\n",
       " 'Lit parapluie simple',\n",
       " 'Construction plastique',\n",
       " 'Acces sol intérieur - colle',\n",
       " 'Micro ondes (P)',\n",
       " 'Terreau',\n",
       " 'Accessoires hygiène et soins',\n",
       " 'Refrigerateur (P)',\n",
       " 'Divers Réseau',\n",
       " 'Anneaux de dentition',\n",
       " \"TV LED FHD entre 30 et 32''\",\n",
       " 'Manettes, Joysticks, Volants',\n",
       " 'Webcams, surveillance',\n",
       " 'Accessoire volet',\n",
       " 'Antenne Intérieure',\n",
       " 'Inférieur ou égal à 22\"',\n",
       " 'Ecouteurs sports',\n",
       " 'Batteries',\n",
       " 'Body',\n",
       " 'Autres jeux de bar',\n",
       " 'Sieges de table',\n",
       " 'Tondeuse électrique',\n",
       " 'Film et tunnel de forçage',\n",
       " 'Jeu-construction-Habilete - CS',\n",
       " 'Isolation',\n",
       " 'Sous-couche - enduit - colle',\n",
       " \"Jeux d'Apprentissage\",\n",
       " 'Cartes mères',\n",
       " 'Combustible',\n",
       " 'Décoration de noël',\n",
       " 'Pate à modeler - CS',\n",
       " 'Protege document-Couvre livre',\n",
       " 'Aspirateur à main',\n",
       " 'Protection des mains',\n",
       " 'Plante fleur - graminé - bulbe',\n",
       " 'Evier',\n",
       " 'Sécurité - Danger chien',\n",
       " 'Armoires Adulte',\n",
       " 'Papier dessin',\n",
       " 'Jus de fruits',\n",
       " 'Alimentation et évacuation eau',\n",
       " 'Cadre',\n",
       " 'Deshumificateur',\n",
       " 'Poubelle à pédale',\n",
       " 'Accessoires chaises hautes',\n",
       " 'Acces fenêtre et baie vitrée',\n",
       " \"TV LED 4K entre 48 et 50''\",\n",
       " 'Photophore Bougeoir',\n",
       " 'Luminothérapie',\n",
       " \"TV LED FHD entre 48 et 50''\",\n",
       " 'Mini four (P)',\n",
       " 'Carillon - sonnette',\n",
       " 'Scie à bûches',\n",
       " 'Loisir - Jouets rongeur',\n",
       " 'Lanterne',\n",
       " 'Vestiaire meubles à chaussures',\n",
       " 'Coupe bordures',\n",
       " 'Robinetterie lavabo',\n",
       " 'Sécurité - Danger chat',\n",
       " 'BALANCELLE',\n",
       " 'Marquise',\n",
       " 'Groupe 2 / 3 - CS',\n",
       " 'Enclos - Clapiers',\n",
       " 'Accessoires Froid',\n",
       " 'Meuleuse',\n",
       " 'Garages - Parkings - Circuits',\n",
       " 'Elagueuse',\n",
       " 'Abri en bois',\n",
       " 'Poire',\n",
       " 'Bases de fixation - CS',\n",
       " 'CAble Péritel',\n",
       " 'Boîte - coffre',\n",
       " 'Casque VR',\n",
       " 'Puzzles',\n",
       " 'Groupe électrogène',\n",
       " \"TV LED FHD entre 51 et 55''\",\n",
       " 'Compresseur',\n",
       " 'Brosse - pinceau - rouleau',\n",
       " '***HS***Accessoires de cuisine',\n",
       " 'Etagère murale',\n",
       " 'Ensemble relaxation',\n",
       " 'Nourriture Chien',\n",
       " 'Plante artificielle décorative',\n",
       " 'Trieur - Porte vues',\n",
       " 'Fille',\n",
       " 'Congelateur (P)',\n",
       " 'CRÈME DE SOIN',\n",
       " 'Chaine Hi-Fi',\n",
       " 'Graveurs DVD + Lecteur',\n",
       " 'Autres cAbles-connectique',\n",
       " 'Platines vinyle',\n",
       " 'Cheminée',\n",
       " 'CEINTURE DE GROSSESSE',\n",
       " \"Pompe d'arrosage\",\n",
       " 'Accessoires de cuisines',\n",
       " \"Traitement de l'eau\",\n",
       " 'Petit rangement salon séjour',\n",
       " \"Sport et jeux d'extérieur\",\n",
       " 'Jouets Educatifs Et Musicaux',\n",
       " 'Accessoires literie',\n",
       " 'Barbecue',\n",
       " 'Tour de lit',\n",
       " 'Mémo-Loto-Domino',\n",
       " 'Cabine de douche',\n",
       " 'Accessoire cuisine',\n",
       " 'Cactée et plante grasse',\n",
       " 'Lampe torche - lampe frontale',\n",
       " 'wc à poser',\n",
       " 'Accessoires Petit Déjeuner',\n",
       " 'Chauffage élec à panneau rayon',\n",
       " 'Autre Plein Air',\n",
       " 'Antiparasitaires chien',\n",
       " 'Rangement Bureau Adulte',\n",
       " 'Nourriture Chats',\n",
       " 'DISQUE SSD',\n",
       " 'CAble HDMI - Splitter',\n",
       " 'Ardoise-Tapis magique-Tableau',\n",
       " 'Amplis et effet',\n",
       " 'Lits combinés',\n",
       " 'Basses & Guitares',\n",
       " 'Couteaux',\n",
       " 'Tapis classique',\n",
       " 'Robots mixeurs',\n",
       " 'Box domotique',\n",
       " 'Nurserie - Poussettes',\n",
       " 'Disques SSD externes',\n",
       " 'Transats et anneaux de bain',\n",
       " 'Chaussure - botte de sécurité',\n",
       " '***HS***Star Wars',\n",
       " 'Chromebook',\n",
       " 'Accessoire pour semi',\n",
       " 'Accessoires de lit',\n",
       " 'Accessoire Sécurité Domestique',\n",
       " 'Bureau Enfant',\n",
       " 'Barre de son',\n",
       " 'Lampes LED',\n",
       " 'Chariot - diable',\n",
       " 'Fontaine intérieur Jardin zen',\n",
       " 'Chambre adulte',\n",
       " \"TV LED HD moins de 24''\",\n",
       " 'Kit solaire -photovoltaïque',\n",
       " 'Pots et reducteurs',\n",
       " 'Rasage',\n",
       " 'Friandise et complément chat',\n",
       " 'Amplis & Effets',\n",
       " 'Matériel DJ',\n",
       " 'Coton',\n",
       " 'Miroir',\n",
       " 'Feutrecoloriage Crayoncouleur',\n",
       " 'Quincaillerie du meuble',\n",
       " 'Produits spéciaux',\n",
       " 'Cartes son',\n",
       " 'Mobilier enfant',\n",
       " 'Arbuste',\n",
       " 'Circuits électrique',\n",
       " 'Marchande - Tâche ménagère',\n",
       " 'Raccordement wc',\n",
       " 'Décapeur - décolleuse',\n",
       " 'Smart Box TV - Clé miracast',\n",
       " 'Switch/Hub USB/KVM',\n",
       " 'Serre tunnel et maraîchère',\n",
       " 'BOITE AUX LETTRES',\n",
       " 'Poubelle encastrable',\n",
       " 'Routeurs - modems',\n",
       " 'Lave Linge (P)',\n",
       " 'Logiciels',\n",
       " 'Parquet',\n",
       " 'Cartes à jouer - collectionner',\n",
       " 'Table de cuisson (E)',\n",
       " 'Instruments à cordes',\n",
       " 'Centrale vapeur',\n",
       " 'Tapis de souris Gaming',\n",
       " 'Coffre et armoire de rangement',\n",
       " 'CONSERVATION',\n",
       " 'Vitrine-Vaisselier-argentier',\n",
       " 'Composteur',\n",
       " 'Accessoires table à repasser',\n",
       " 'Divers Périphériques PC',\n",
       " 'Expresso',\n",
       " 'Purificateur-Lampe anti-insect',\n",
       " 'Huile de bain - shampoing',\n",
       " 'Lit enfant',\n",
       " 'Enceintes',\n",
       " 'Rangement bureau',\n",
       " 'Chaise Salle À Manger',\n",
       " 'Bibliothèque - étagère',\n",
       " 'Hochet',\n",
       " 'Linge de table',\n",
       " 'Lingettes',\n",
       " 'Soins et hygiène chien',\n",
       " 'Sommier lattes',\n",
       " 'Consoles-Tablettes éducatives',\n",
       " 'Maisons de Poupees - CS',\n",
       " 'Tableau électrique - composant',\n",
       " 'Clés 8 Go ou moins',\n",
       " 'Scarificateur',\n",
       " 'Motorisation portail',\n",
       " 'Housse de rangement',\n",
       " 'Soin et hygiène',\n",
       " 'Mixte',\n",
       " 'Cahier',\n",
       " 'Décodeurs TNT',\n",
       " 'Gamelle et distributeur chat',\n",
       " 'De 23\" à 25\"',\n",
       " 'Poupées mannequin',\n",
       " 'Support TV mural Fixe',\n",
       " 'Divers classement',\n",
       " 'Piscine acier',\n",
       " 'Epilation',\n",
       " 'Baby Foot',\n",
       " 'Plastification',\n",
       " 'Colle - Adhesif',\n",
       " 'Doudous',\n",
       " 'CAble coaxial',\n",
       " 'Observation, météo, cadre num',\n",
       " 'Ardoise et craie',\n",
       " 'Accessoires Lavage',\n",
       " 'Outil à main de jardin',\n",
       " 'Soins et hygiène reptile',\n",
       " 'Ensemble lit complet',\n",
       " 'TEST ADN',\n",
       " 'Tapis stylisés Orient',\n",
       " 'Tonnelle',\n",
       " 'Instruments à cordes (Quatuor)',\n",
       " 'Accessoires Cuisson GEM',\n",
       " 'Caméra IP',\n",
       " 'Purificateur',\n",
       " 'Tondeuse robot',\n",
       " 'Fouet à lait',\n",
       " 'Housse de couette',\n",
       " 'Autres instruments à cordes',\n",
       " 'Garçon',\n",
       " 'Robot patissier',\n",
       " 'Chauffage à bois',\n",
       " 'Tablettes sup à 10 pouces',\n",
       " 'Valet de chambre / portant',\n",
       " 'Accessibilité et sécurité SdB',\n",
       " 'Têtes à Coiffer',\n",
       " 'Ampli',\n",
       " 'Fer à repasser',\n",
       " 'Clés 16 Go',\n",
       " 'Coffre-fort - armoire à fusils',\n",
       " 'ONDULEURS ET PARAFOUDRES',\n",
       " 'Tapis de cuisine',\n",
       " 'ACCESSOIRES MATERNITÉ',\n",
       " 'Ensemble mousse',\n",
       " 'Poubelle de sdb',\n",
       " 'BZ',\n",
       " 'Enceintes - Haut-parleurs',\n",
       " 'Transats',\n",
       " \"Jeux d'Imitation\",\n",
       " 'Armoires Enfant',\n",
       " 'Cafetière filtre',\n",
       " 'Machine à bière - tireuse',\n",
       " 'Véhicule à Thème',\n",
       " 'Appliques',\n",
       " 'Voitures radiocommandées',\n",
       " 'Jeux construction - Habileté',\n",
       " '***HS***Meuble a chaussures',\n",
       " 'Groupe 2 / 3',\n",
       " 'Protection du corps',\n",
       " 'Nettoyeurs Vapeur',\n",
       " 'Garcon - CS',\n",
       " 'Bijoux et Perles',\n",
       " 'Accessoires biberon et sucette',\n",
       " 'Ferme - Nature',\n",
       " 'Support enceinte - barre son',\n",
       " 'Accessoires buanderie',\n",
       " 'Mediums, vernis et colles',\n",
       " 'Paroi de douche',\n",
       " 'Multicuiseur',\n",
       " 'Arbre',\n",
       " 'Jeux éducatifs électroniques',\n",
       " 'Construction magnétique',\n",
       " 'MOUCHE BÉBÉ MANUEL',\n",
       " 'CDR, DVDR, SAUVEGARDES',\n",
       " 'Poussette citadine',\n",
       " 'Claviers & Pianos',\n",
       " 'Poussette canne',\n",
       " 'Balai balayettes',\n",
       " 'Autocuiseurs',\n",
       " 'Traitement piscine - Danger',\n",
       " 'Aspirateur de chantier',\n",
       " \"Portables 13''\",\n",
       " 'Linge de lit',\n",
       " 'Ventilateur',\n",
       " 'Balnéothérapie',\n",
       " 'Lecteur CD-MP3-Radio réveil',\n",
       " 'Rideaux et voilages',\n",
       " 'Aspirateur traineau',\n",
       " 'Lecteurs MP3 & MP4 (Vidéo)',\n",
       " 'Processeurs',\n",
       " 'Vêtement chien',\n",
       " 'Traitement - Entretien',\n",
       " '***HS***Armoire-Dressing-Placa',\n",
       " 'Stérilisateur - Déshydrateur',\n",
       " 'Chassis et Chevalets',\n",
       " 'Sèche linge (P)',\n",
       " 'Paillassons',\n",
       " 'Mondes Imaginaires',\n",
       " 'Poterie-Plâtre-Pâte à sel',\n",
       " 'Autres textiles de chambre',\n",
       " 'Accessoire électroportatif',\n",
       " 'Table de jardin',\n",
       " 'Housses',\n",
       " 'Pulvérisateur et accessoire',\n",
       " 'Construction bois',\n",
       " 'Accessoires repassage couture',\n",
       " 'Accessoire mur intérieur',\n",
       " 'Cages et accessoires oiseau',\n",
       " 'Armoire et Dressing Adulte',\n",
       " \"Coussins d'allaitement\",\n",
       " 'Autres câbles',\n",
       " 'Parcs',\n",
       " 'CHAUFFEUSES',\n",
       " 'Décoration-Accessoire poisson',\n",
       " 'Braséro',\n",
       " 'Niche',\n",
       " 'Machine à glaçons',\n",
       " 'Bavoirs et langes',\n",
       " 'Lecteur DVD portable',\n",
       " 'Tablettes 7 pouces ou moins',\n",
       " 'Couchage chat',\n",
       " 'Ordinateurs Educatifs',\n",
       " 'Insecticide',\n",
       " 'Spa',\n",
       " 'Meuble TV',\n",
       " 'Linge de décoration',\n",
       " 'Cheville - vis - clou - boulon',\n",
       " 'Buffet',\n",
       " 'CAble Satellite',\n",
       " 'Ensemble latex',\n",
       " 'Entretien de la piscine',\n",
       " 'Mini PC / Barebones',\n",
       " 'Jouets électroniques',\n",
       " 'Claviers et pianos',\n",
       " 'Controleurs numériques',\n",
       " 'Couvres lits et plaids',\n",
       " 'Radio Internet, K7, CD',\n",
       " 'Raccordement gaz',\n",
       " \"TV LED HD entre 30 et 32''\",\n",
       " 'Purée de légume - plat préparé',\n",
       " 'Poussette 2 en 1',\n",
       " 'Poubelle automatique',\n",
       " 'Tapis enfant',\n",
       " 'LINGERIE HAUT',\n",
       " 'Scrapbooking',\n",
       " 'Vinyles',\n",
       " 'Eclairage chantier',\n",
       " 'Nettoyeur haute pression',\n",
       " 'Penderie souple',\n",
       " 'Casques TV - HiFi',\n",
       " 'Peluches',\n",
       " 'Housses - CS',\n",
       " 'Chambre enfant',\n",
       " \"Multifonction jet d'encre\",\n",
       " 'Accessoire sol extérieur',\n",
       " 'Cuiseur vapeur',\n",
       " 'Cave à vin de service (P)',\n",
       " 'Sofas, fauteuils et poufs',\n",
       " 'Couverts',\n",
       " 'Pack Gamer',\n",
       " 'Peluches avec fonction',\n",
       " 'Coffrets cadeau',\n",
       " 'Habillages',\n",
       " 'Carré potager',\n",
       " 'Autres objets déco',\n",
       " 'Câbles Audio / Vidéo',\n",
       " \"Portables 15''\",\n",
       " 'Jeux scientifiques',\n",
       " 'VÊTEMENT MATERNITÉ',\n",
       " 'Machine A Coudre',\n",
       " 'Meuble TV amplifié',\n",
       " 'Dinette - Cuisine',\n",
       " 'Robot nettoyeur',\n",
       " 'Piège',\n",
       " 'Autres accessoires TV-Antennes',\n",
       " 'Poupees mannequin - CS',\n",
       " 'Support vidéoprojecteur',\n",
       " 'Blender',\n",
       " 'Lampes enfants',\n",
       " 'Ensemble ressort',\n",
       " 'Vaisselle',\n",
       " 'Panoplies',\n",
       " 'Câbles USB',\n",
       " 'Piece detachee outils jardin',\n",
       " 'Jouets Educatifs',\n",
       " 'Découpage éléctrique',\n",
       " 'Fleur - Plante artificielle',\n",
       " 'Accessoires Instrument musique',\n",
       " 'Plan de travail',\n",
       " 'Jouet de bain',\n",
       " \"Imp. jet d'encre\",\n",
       " 'Crayon-Porte mine-Taillecrayon',\n",
       " \"Pompe d'evacuation\",\n",
       " 'Décorations de chambre',\n",
       " 'Canapé relaxation',\n",
       " 'Bahut - Buffet - Enfilade',\n",
       " 'Access Préparation Culinaire',\n",
       " 'Aspirateur souffleur',\n",
       " 'Câble - gaine',\n",
       " 'Haut-parleurs',\n",
       " 'Elément séparé SDB',\n",
       " 'Composition de plante',\n",
       " 'Verrerie Cache-pot Vide poche',\n",
       " 'Motorisation volet',\n",
       " 'Accessoire garage - rangement',\n",
       " 'Yaourtière - Fromagère',\n",
       " 'Balance de cuisine',\n",
       " 'TraÃ§age - Compas',\n",
       " 'Hamac de jardin',\n",
       " 'Outillage à main',\n",
       " 'Outil multifonctions',\n",
       " 'Matelas + sommier',\n",
       " 'Accessoires table',\n",
       " 'Crepière',\n",
       " 'Housse de protection',\n",
       " 'Soins et hygiène chat',\n",
       " 'Baignoire',\n",
       " 'Séjour Complet',\n",
       " 'Robinetterie de bain',\n",
       " 'Construction métal',\n",
       " 'Chevets - commodes Enfant',\n",
       " 'Cocotte',\n",
       " 'Piscines - Jeux de piscines',\n",
       " '***HS***DISQUE DUR INTERNE',\n",
       " 'Chauffeuse',\n",
       " 'Radio réveil',\n",
       " 'Robot Cuiseur',\n",
       " '***HS***Barbie',\n",
       " 'Hygiène',\n",
       " 'BERCEAUX',\n",
       " 'Chaises hautes',\n",
       " \"TV LED HD entre 24 et 29''\",\n",
       " 'Access outillage stationnaire',\n",
       " 'Bâche',\n",
       " 'Piscine autostable',\n",
       " 'Protection de la tête',\n",
       " \"Portables 13'' tactiles\",\n",
       " \"PRODUITS D'ENTRETIEN\",\n",
       " 'Machine à pain, pâtes',\n",
       " 'Balancelles',\n",
       " 'Machine sous vide',\n",
       " 'Bureau Adulte',\n",
       " 'Ampoule halogène',\n",
       " 'Lasers - Lumières - Effets',\n",
       " 'Poupons et poupees - CS',\n",
       " 'Fiche bristol',\n",
       " 'Bonnet',\n",
       " \"Portables 17'' et sup\",\n",
       " 'Grille Pain,Toasters',\n",
       " 'Desherbage',\n",
       " 'Spots',\n",
       " \"Vestiaire - meuble d'entrée\",\n",
       " 'Lutte escargots - limaces',\n",
       " 'Article de déco et bouquet',\n",
       " 'Maisons de Poupées',\n",
       " 'Cuisinière (P)',\n",
       " 'Accessoires consommables',\n",
       " 'Mondes Imaginaires - CS',\n",
       " '***HS***Imprimantes / Multifon',\n",
       " 'Pare soleil',\n",
       " 'Terrariums',\n",
       " 'Ampoule LED',\n",
       " 'Recepteurs Satellite',\n",
       " 'Vidéoprojecteurs',\n",
       " 'Fendeur',\n",
       " 'Groupe 0+ / 1 / 2 - CS',\n",
       " 'Multijeux',\n",
       " 'Abri résine et polycarbonate',\n",
       " 'Chaise Cuisine',\n",
       " 'Couture - Mercerie',\n",
       " 'Enrouleur-rallonge-multiprise',\n",
       " 'Humidificateur',\n",
       " 'NETTOYEUR SOL',\n",
       " 'Autres Petits rangement',\n",
       " 'Chauffe biberon',\n",
       " 'Barbecue charbon',\n",
       " 'Lit adulte',\n",
       " 'Soin dentaire',\n",
       " 'Tondeuse manuelle',\n",
       " 'Salle de bain complète',\n",
       " 'Déco - Accessoires reptiles',\n",
       " 'Lavante Séchante (P)',\n",
       " 'Vernis - lasure',\n",
       " 'Ecran de projection',\n",
       " 'Peluches avec fonction - CS',\n",
       " 'Poterie et platre',\n",
       " 'Pergola',\n",
       " 'Groupe 0 - nacelle',\n",
       " 'Enceintes Portables',\n",
       " 'Tricot - Crochet',\n",
       " 'Lutte rat - souris - taupe',\n",
       " 'Nourriture reptile',\n",
       " 'Piles',\n",
       " 'Antiparasitaires chat',\n",
       " 'Scie électrique',\n",
       " 'Lecteurs DVD - DivX',\n",
       " 'Sommier relaxation',\n",
       " 'Autres eveil',\n",
       " 'Friandise et complément cheval',\n",
       " 'Porte manteau - Patère',\n",
       " 'Filet - protection - paillage',\n",
       " 'Poussette canne - CS',\n",
       " 'Bout de canapé',\n",
       " 'Pèse bébé',\n",
       " 'Desserte',\n",
       " 'Claviers',\n",
       " 'Dalle et lame de terrasse',\n",
       " 'Accessoires VR',\n",
       " 'Accessoire chauffage',\n",
       " 'Batteries & percussions',\n",
       " 'Accessoires Cuisson PEM',\n",
       " \"TV LED FHD entre 33 et 41''\",\n",
       " 'Petit rangement Cuisine',\n",
       " 'Débroussailleuse',\n",
       " 'Imp. Laser',\n",
       " 'Couette',\n",
       " 'Disques durs externes',\n",
       " 'Robinetterie de douche',\n",
       " 'Manutention levage',\n",
       " 'Brise soleil fenêtre et baie',\n",
       " 'Tapis de bain',\n",
       " 'Suspension plafonnier',\n",
       " 'Boite de classement-Archivage',\n",
       " 'Sécurité et Antivirus',\n",
       " 'Serrure - verrou - cadenas',\n",
       " 'Clés 32 Go',\n",
       " 'Enregistreur',\n",
       " 'Clés 64 Go',\n",
       " 'Autres Eléments séparés',\n",
       " 'FOURNITURES DE BUREAU',\n",
       " 'Accessoire portail',\n",
       " 'Support TV mural orientable',\n",
       " 'Caméra de surveillance',\n",
       " 'Unités centrales',\n",
       " 'Four (E)',\n",
       " 'Beauté-Mode-Perles-Bijoux',\n",
       " 'Tapis Eveil - Aire De Jeu - CS',\n",
       " \"Jeux d'eveil - CS\",\n",
       " 'Bases de fixation',\n",
       " 'Broderie - Canevas',\n",
       " \"Portables < 11''\",\n",
       " 'Pistolet-lance',\n",
       " 'Poussette 2 en 1 - CS',\n",
       " 'Cuisine - Parfum - Bougie',\n",
       " 'Lustre',\n",
       " 'Autre outil stationnaire',\n",
       " 'Consommables imprimante 3D',\n",
       " 'Canapé droit fixe',\n",
       " 'Rangement Bureau Enfant',\n",
       " 'Tablettes 8 pouces',\n",
       " 'Refrigerateur Americain (P)',\n",
       " 'Chauffage Mobile',\n",
       " \"Canapé d'angle fixe\",\n",
       " 'Multifonction Laser',\n",
       " 'Tablettes 9 à 10 pouces',\n",
       " 'Gadgets - Objets insolites',\n",
       " 'Plaque de cuisson posable',\n",
       " 'Gazon',\n",
       " 'Electrostimulation',\n",
       " 'Micros',\n",
       " '***HS***DISQUE SSD',\n",
       " 'Grillage',\n",
       " 'Chauffage eau chaude',\n",
       " 'Statuette Figurine Sculpture',\n",
       " 'Jeux de societe - CS',\n",
       " 'Kit de plantation',\n",
       " 'Engrais',\n",
       " 'Etagère - Plateau',\n",
       " 'Armoire - étagère',\n",
       " 'Lave vaisselle (P)',\n",
       " '***HS***Claviers, souris, webc',\n",
       " 'Cartes graphiques',\n",
       " 'Poupons et poupées',\n",
       " 'Thermomètres',\n",
       " 'Biscuits bébé',\n",
       " 'Sac à langer',\n",
       " 'Maison toilette - Litière',\n",
       " 'Pots lumineux',\n",
       " 'Transport et accessoires chien',\n",
       " 'Sièges gamer',\n",
       " 'Petit déjeuner',\n",
       " 'Autre aménagement atelier',\n",
       " 'Tondeuses corps',\n",
       " 'Souris Gaming',\n",
       " 'Motorisation porte de garage',\n",
       " 'Canapé droit convertible',\n",
       " 'Alimentation basse - cour',\n",
       " 'ABAT-JOUR',\n",
       " 'Gros outillage de chantier',\n",
       " 'AUTRE DHP',\n",
       " 'Haie artificielle - brise vue',\n",
       " 'Etiquette - lien -  tuteur',\n",
       " 'LINGERIE BAS',\n",
       " 'Moule plat à four',\n",
       " 'DISQUES DURS INTERNES',\n",
       " 'Tuyau - enrouleur - accessoire',\n",
       " 'Macbook',\n",
       " 'Support TV Plafond',\n",
       " 'Couches',\n",
       " '***HS***Rangement déco',\n",
       " 'Transats - CS',\n",
       " 'Accessoires santé connectée',\n",
       " 'Escabeau - échelle - échafauda',\n",
       " 'Groupe 1 / 2 / 3 - CS',\n",
       " 'Pompe - vide-cave - surpresseu',\n",
       " 'Tapis de souris',\n",
       " 'Dessin Et Graphisme',\n",
       " 'Barbecue gaz',\n",
       " 'Fontaine - bassin',\n",
       " 'Centre de repassage',\n",
       " 'Peluches-Poupées-Doudous',\n",
       " 'Garage et carport',\n",
       " 'Pyjama',\n",
       " 'Instruments à vent',\n",
       " 'Groupe 0 - nacelle - CS',\n",
       " 'Accessoires déguisement',\n",
       " 'Répulsif',\n",
       " 'Lampadaires',\n",
       " 'Poussette 3 en 1',\n",
       " 'Linge de bain',\n",
       " 'Accessoire porte de garage',\n",
       " 'Transat - bain de soleil',\n",
       " 'Crêpières gril',\n",
       " 'LAVE VITRE ELECTRIQUE',\n",
       " 'Convertibles',\n",
       " 'Buste - Mannequin',\n",
       " 'Boite aux lettres - accessoire',\n",
       " 'DOCKS ET SUPPORTS',\n",
       " 'MOUCHE BÉBÉ ÉLECTRIQUE',\n",
       " 'Coiffure',\n",
       " 'Autre produit gros oeuvre',\n",
       " 'Cale bébé et plan incliné',\n",
       " 'BANQUETTE FIXE ET MERIDIENNE',\n",
       " 'Chancelieres',\n",
       " 'Autre accessoire de peinture',\n",
       " 'Accessoires aspirateur',\n",
       " 'Autre linge de lit',\n",
       " '***HS***Barbecue nomade',\n",
       " 'Souris',\n",
       " 'Balances',\n",
       " 'Chauffage élec convecteur',\n",
       " 'Cave multi-température (E)',\n",
       " 'Nourriture rongeur',\n",
       " 'Batteries, chargeurs, alimenta',\n",
       " 'Etau',\n",
       " 'Pyrogravure',\n",
       " 'Casques sans fils - Bluetooth',\n",
       " 'Lait bébé',\n",
       " \"Portables 14''\",\n",
       " 'Accessoires repas',\n",
       " 'Accessoires de sdb',\n",
       " 'Cloture - grillage',\n",
       " 'Casques Réducteurs de bruit',\n",
       " 'Chauffage pétrole gaz',\n",
       " 'Jeux de société',\n",
       " \"Tapis D'Eveil Et Aires De Jeu\",\n",
       " 'Lavabo - vasque - bidet',\n",
       " 'Chevaliers - Princesses',\n",
       " 'Beaute-Mode-Perles-Bijoux - CS',\n",
       " 'Sol vinyle',\n",
       " 'Traitement du bois',\n",
       " 'POULAILLER',\n",
       " 'Claviers Gaming',\n",
       " 'Tapis Shaggy mèches',\n",
       " 'Panier  / coffre',\n",
       " 'Matelas ressort',\n",
       " 'Textile jardinerie',\n",
       " 'Eclairage SDB',\n",
       " 'Sèche serviette',\n",
       " 'Disques durs réseau NAS',\n",
       " 'Climatisation',\n",
       " 'Bricolage - Outils',\n",
       " 'Alarme et détection',\n",
       " 'Lait - Liniment',\n",
       " \"Voile d'ombrage\",\n",
       " 'Peinture extérieure',\n",
       " 'Ciment - chaux - mortier',\n",
       " 'Robot multifonctions',\n",
       " 'Accessoires allaitement',\n",
       " 'Poussette 3 roues - CS',\n",
       " 'Fax et Télécopieurs, autres',\n",
       " 'Voitures Miniatures',\n",
       " 'Baignoires et supports',\n",
       " 'Coton Tige',\n",
       " 'Tablette Graphique',\n",
       " 'Raclette',\n",
       " 'Plante fleurie',\n",
       " 'Machine à soda',\n",
       " 'Reliage - Destruction',\n",
       " 'SOIN VISAGE BÉBÉ',\n",
       " 'Petite domotique',\n",
       " 'Table à langer',\n",
       " 'Contenant - pot - cache pot',\n",
       " 'Véhicules et trottinettes',\n",
       " 'Chevets - commodes Adulte',\n",
       " 'Gaufrier - Croque Monsieur',\n",
       " 'Peinture - aérosol - Danger',\n",
       " 'Jeux de plage',\n",
       " 'Séchoirs',\n",
       " 'Colle et adhésif',\n",
       " 'Transport cheval',\n",
       " 'Poussette 3 roues',\n",
       " 'Hélicoptères-Avions RC',\n",
       " 'Papier photo',\n",
       " 'Tableaux',\n",
       " 'Papier standard',\n",
       " 'Barrière De Sécurité',\n",
       " 'Papier-peint',\n",
       " 'Soins et hygiène basse - cour',\n",
       " 'Robots',\n",
       " 'Stylo-FeutreEcriture-Cartouche',\n",
       " 'Acquisition vidéo',\n",
       " 'Fauteuil Classique',\n",
       " 'Circuits',\n",
       " 'Note repositionnable - Post it',\n",
       " 'Rehausseurs',\n",
       " 'Escalier',\n",
       " 'Tables à manger',\n",
       " 'Assiettes',\n",
       " 'Peinture intérieure',\n",
       " 'Pompe',\n",
       " 'Livre scolaire - Dictionnaire',\n",
       " 'Santé connectée',\n",
       " 'Chauffage mobile',\n",
       " '***HS***Pouf chauffeuse banc',\n",
       " 'Concentrés',\n",
       " 'Taille-haies',\n",
       " 'Tabouret',\n",
       " 'Laisserie chien',\n",
       " ...}"
      ]
     },
     "execution_count": 49,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "set(data ['Niv. 3'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/CDBDX/remy.garnier/.virtualenvs/python_3.6_TS/lib/python3.6/site-packages/IPython/core/interactiveshell.py:3049: DtypeWarning: Columns (1,8) have mixed types. Specify dtype option on import or set low_memory=False.\n",
      "  interactivity=interactivity, compiler=compiler, result=result)\n"
     ]
    }
   ],
   "source": [
    "data = pd.read_csv(STD_DATA_DIR / 'filtered_dt_for_nn', sep = ';')\n",
    "data.set_index(['product','date'], inplace = True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### I.B Filtrage des données"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "index ='Vente lisse' #index usuel utilisé pour le filtrage\n",
    "\n",
    "def filter_by_level(data,fam, niv='Niv. 3'):\n",
    "    return data[data[niv] == fam]\n",
    "\n",
    "def add_sum(data, fam, niv= 'Niv. 3'):\n",
    "    df = filter_by_level(data, fam, niv)\n",
    "    sum_df = df.groupby(level = 1)[index].sum()\n",
    "    return df.join(sum_df, rsuffix ='_somme')\n",
    "\n",
    "def extract_data(data, fam, niv= 'Niv. 3'):\n",
    "    df = add_sum(data, fam, niv)\n",
    "    df['part_de_marché']  = df[index]/df[index+'_somme']\n",
    "    return df\n",
    "\n",
    "fam = 'Aspirateur souffleur'\n",
    "level = 'Niv. 3'\n",
    "df = add_sum(data, fam, niv=level)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "#Scaling\n",
    "cols_to_scale =['prix_moy','prix_min_marche_moy','marge_srp', 'pump']\n",
    "scaler = MinMaxScaler(feature_range=(0,10))\n",
    "df[cols_to_scale] = scaler.fit_transform(df[cols_to_scale])\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "del(data)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### I.C Adaptation des données à Pytorch"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "# filtre produit:\n",
    "nb_ventes_mini = 100 #Nombre minimal de ventes                                                        \n",
    "len_min =  10 # Nombre de semaines minimales de ventes \n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def gener_torch_dataset(data,split_date, col_drop =['Unnamed: 0','Vente lisse','min_marche', 'Vente réelle','Niv. 1','Niv. 2','Niv. 3'], horizon = 3 ):\n",
    "    target =100*(data[index]/data[index+'_somme']).shift(horizon).iloc[horizon:]\n",
    "    y_train = torch.tensor(target[target.index < split_date].values, dtype = torch.float32)\n",
    "    y_test = torch.tensor(target.values, dtype = torch.float32)\n",
    "    \n",
    "    \n",
    "    X = data[horizon:]\n",
    "    #X['truth'] = 100 * data[index]/data[index+'_somme']\n",
    "    X.drop(columns = col_drop, inplace= True)\n",
    "    X['pdM'] = 100 * data[index]/data[index+'_somme']\n",
    "    \n",
    "    n_item = X.shape[1]\n",
    "    X_train =  torch.tensor(X[X.index < split_date].values, dtype= torch.float32).reshape(-1,1,n_item)\n",
    "    X_test = torch.tensor(X.values, dtype= torch.float32).reshape(-1,1,n_item)\n",
    "    return X_train, y_train, X_test,y_test, torch.tensor(X['pdM'], dtype = torch.float32)\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_list = []\n",
    "test_list = []\n",
    "keeped_products = []\n",
    "date_train = '2019-01-01'\n",
    "products = set(df.index.get_level_values(0))\n",
    "for prod in products:\n",
    "    vt_prod = df.loc[prod].fillna(0)\n",
    "    if sum(vt_prod[index]) > nb_ventes_mini:\n",
    "        min_date = min(vt_prod[vt_prod[index] > 0].index)\n",
    "        max_date = min(date_train, max(vt_prod[vt_prod[index] > 0].index))\n",
    "        \n",
    "        min_date = dt.datetime.strptime(min_date, '%Y-%m-%d')\n",
    "        max_date = dt.datetime.strptime(max_date, '%Y-%m-%d')\n",
    "        \n",
    "        if (max_date - min_date).days // 7 > len_min:\n",
    "            keeped_products.append(prod)\n",
    "            X_train, y_train, X_test,y_test,ref = gener_torch_dataset(vt_prod[vt_prod.index > dt.datetime.strftime(min_date,'%Y-%m-%d' )], date_train)\n",
    "            train_list.append((X_train,y_train))\n",
    "            test_list.append((X_test,y_test,ref))\n",
    "            \n",
    "print('Nb de produits initial  %s ' % len(products))\n",
    "print('Nb de ventes initiales %s ' % sum(df[index]))\n",
    "print('Nb de produits conservés %s' %len(keeped_products))        \n",
    "print('Nb de ventes conservées %s' % sum(df[index].loc[keeped_products]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "X_train.shape"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## II Modèle sans concurrence\n",
    "#### II.A Réseau de neurones"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "\n",
    "class LSTM_nn(nn.Module):\n",
    "    def __init__(self, n_param,n_hidden, n_layer):\n",
    "        super().__init__()\n",
    "        self.rnn = nn.LSTM(n_param,n_hidden,n_layer)\n",
    "        self.hidden_layer = (torch.zeros(n_layer, 1,n_hidden),\n",
    "                            torch.zeros(n_layer, 1 ,n_hidden))\n",
    "        self.linear = nn.Linear(n_hidden,1)\n",
    "        self.n_hidden = n_hidden\n",
    "        \n",
    "    def forward(self,x):\n",
    "        output, self.hidden_layer = self.rnn(x, self.hidden_layer)\n",
    "        return F.softplus(self.linear(output))\n",
    "    \n",
    "    def reinitialize(self):\n",
    "        self.hidden_layer = (torch.normal(0,1,size=(n_layer, 1,self.n_hidden)),\n",
    "                            torch.normal(0,1,size=(n_layer, 1,self.n_hidden)))\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#Constantes \n",
    "\n",
    "epochs =1000\n",
    "learning_rate =  1e-3\n",
    "n_hidden = 5\n",
    "n_param = X_train.shape[2]\n",
    "n_layer =2\n",
    "\n",
    "model = LSTM_nn(n_param,n_hidden,n_layer)\n",
    "loss_function = nn.L1Loss(reduction ='sum')\n",
    "optimizer = torch.optim.Adam(model.parameters(), lr=learning_rate)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### II.B Entrainement"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from random import shuffle\n",
    "\n",
    "err_train  = []\n",
    "err_test =  []\n",
    "\n",
    "for i in range(epochs):\n",
    "    l=0\n",
    "    s=0\n",
    "    copy_train_list = train_list.copy()\n",
    "    shuffle(copy_train_list)\n",
    "    for X_train,y_train in copy_train_list:\n",
    "        optimizer.zero_grad()\n",
    "        model.reinitialize()\n",
    "        y_pred = model(X_train)\n",
    "        single_loss = loss_function(y_pred[-30:,0],y_train[-30:] )\n",
    "        single_loss.backward()\n",
    "        optimizer.step()\n",
    "        l += single_loss.item()\n",
    "        s += sum(y_train[-30:])\n",
    "        \n",
    "    l_test = 0\n",
    "    s_t = 0\n",
    "    l_ref = 0\n",
    "    for X_test, y_test,ref  in test_list:\n",
    "        model.reinitialize()\n",
    "        y_pred = model(X_test)[-50:,0]\n",
    "        single_loss = loss_function(y_pred,y_test[-50:] )\n",
    "        l_test += single_loss.item()\n",
    "        s_t+= sum(y_test[-50:])\n",
    "        \n",
    "        l_ref += loss_function(ref[-50:],y_test[-50:]).item()\n",
    "    print('Training MAPE: %.4f  Testing MAPE: %.4f Ref : %.4F' %(l/s  , l_test/s_t,l_ref/s_t ))\n",
    "    err_train.append(l)\n",
    "    err_test.append(l_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "X_train,y_train = train_list[5]\n",
    "model.reinitialize()\n",
    "model(X_train)[:,0,0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "X_train[:,0,:]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "len(copy_train_list)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "plt.figure(figsize=(12,7))\n",
    "#plt.plot(lt_test_ref,'--', label =  'Test Error Reference', color = 'red')\n",
    "#plt.plot(lt_train_ref,'--', label =  'Train Error Reference', color = 'blue')\n",
    "plt.plot(err_test, label =  'Test Error Reference', color =  'red')\n",
    "plt.plot(err_train, label =  'Train Error Reference', color =  'blue')\n",
    "plt.legend()\n",
    "plt.xlabel(\"Epoch\")\n",
    "#plt.yscale('log')\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "X_test,y_test,_ = test_list[3]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "model.reinitialize()\n",
    "y_pred = model(X_test)[-35:,0]\n",
    "y_pred"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "sum(torch.abs(y_test[-35:] - y_pred[-35,:]))/sum(y_test[-35:])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "sum(y_test[-35:])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "loss_function(y_test[-35:],y_pred[-35,:]).item()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "y_test"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## III A taille d'historique fixée\n",
    "#### III.A Construction des données"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "def gener_lagged_dataset(serie_prod, period, horizon, min_cons=10):\n",
    "    N = len(serie_prod)\n",
    "    l = []\n",
    "    for i in range(N-period-horizon+1):\n",
    "        X = torch.tensor(serie_prod.drop(columns = 'pdM').iloc[i:i+period, :].values, dtype = torch.float32).reshape(period,1,-1) \n",
    "        y = torch.tensor(serie_prod['pdM'].iloc[i+horizon:i+horizon+period], dtype = torch.float32).reshape(period,1,-1)\n",
    "        if sum(y) >=min_cons:\n",
    "            l.append([X,y])\n",
    "    return l\n",
    "\n",
    "def gener_dat(data, index, col_drop =['Unnamed: 0','Vente lisse','min_marche', 'Vente réelle','Niv. 1','Niv. 2','Niv. 3'], horizon=6):\n",
    "    X = data.copy()\n",
    "    X.drop(columns = col_drop, inplace= True)\n",
    "    X['pdM'] = 100 * (data[index]/data[index+'_somme'])\n",
    "    X['pdM_shift'] = X['pdM'].shift(horizon)\n",
    "    return X.iloc[horizon:]\n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "#paramètres \n",
    "len_min = 25\n",
    "window =20\n",
    "horizon = 4\n",
    "date_train = '2019-01-01'\n",
    "nb_ventes_mini = 100\n",
    "n_calc = 10"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Nb de produits initial  104 \n",
      "Nb de ventes initiales 21892.603540221062 \n",
      "Nb de produits conservés 32\n",
      "Nb de ventes conservées 17383.077999857356\n"
     ]
    }
   ],
   "source": [
    "\n",
    "l_train = []\n",
    "l_valid =[]\n",
    "keeped_products =[]\n",
    "\n",
    "products = set(df.index.get_level_values(0))\n",
    "for prod in products:\n",
    "    vt_prod = df.loc[prod].fillna(0)\n",
    "    if sum(vt_prod[index]) > nb_ventes_mini:\n",
    "        min_date = min(vt_prod[vt_prod[index] > 0].index)\n",
    "        max_date = min(date_train, max(vt_prod[vt_prod[index] > 0].index))\n",
    "        \n",
    "        min_date = dt.datetime.strptime(min_date, '%Y-%m-%d')\n",
    "        max_date = dt.datetime.strptime(max_date, '%Y-%m-%d')\n",
    "\n",
    "        if (max_date - min_date).days // 7 > len_min:\n",
    "            keeped_products.append(prod)\n",
    "            l_train += gener_lagged_dataset(gener_dat(vt_prod[vt_prod.index < dt.datetime.strftime(max_date,'%Y-%m-%d' )], index, horizon=horizon),window,horizon)\n",
    "            l_valid += gener_lagged_dataset(gener_dat(vt_prod[vt_prod.index > (dt.datetime.strftime(max_date - dt.timedelta(days = 7*horizon),'%Y-%m-%d' ) )], index,horizon=horizon),window,horizon)\n",
    "            \n",
    "print('Nb de produits initial  %s ' % len(products))\n",
    "print('Nb de ventes initiales %s ' % sum(df[index]))\n",
    "print('Nb de produits conservés %s' %len(keeped_products))        \n",
    "print('Nb de ventes conservées %s' % sum(df[index].loc[keeped_products]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "#Constantes\n",
    "\n",
    "epochs =200\n",
    "learning_rate =  5e-3\n",
    "n_hidden = 20\n",
    "n_param = l_train[0][0].shape[2]\n",
    "n_layer =1\n",
    "\n",
    "model = LSTM_nn(n_param,n_hidden,n_layer)\n",
    "loss_function = nn.L1Loss(reduction ='sum')\n",
    "optimizer = torch.optim.Adam(model.parameters(), lr=learning_rate)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from random import shuffle\n",
    "\n",
    "\n",
    "err_train  = []\n",
    "err_test =  []\n",
    "\n",
    "for i in range(epochs):\n",
    "    l=0\n",
    "    s=0\n",
    "    copy_train_list = l_train.copy()\n",
    "    shuffle(copy_train_list)\n",
    "    \n",
    "    optimizer.zero_grad()\n",
    "    \n",
    "    loss = 0\n",
    "    for X_train,y_train in copy_train_list:\n",
    "        model.reinitialize()\n",
    "        y_pred = model(X_train)\n",
    "        single_loss = loss_function(y_pred[-n_calc:,0],y_train[-n_calc:,0] )\n",
    "        \n",
    "        loss +=single_loss\n",
    "        l += single_loss.item()\n",
    "        s += sum(y_train[-n_calc:,0])\n",
    "    loss.backward()\n",
    "    optimizer.step()\n",
    "        \n",
    "    l_test = 0\n",
    "    s_t = 0\n",
    "    l_ref = 0\n",
    "    for X_test, y_test  in l_valid:\n",
    "        model.reinitialize()\n",
    "        y_pred = model(X_test)\n",
    "        single_loss = loss_function(y_pred[-n_calc:,0],y_test[-n_calc:,0] )\n",
    "        l_test += single_loss.item()\n",
    "        s_t+= sum(y_test[-n_calc:,0])\n",
    "    if i % 5 ==0:\n",
    "        print('Training MAPE: %.4f  Testing MAPE: %.4f' %(100 * l/s  , 100 * l_test/s_t))\n",
    "    err_train.append(l)\n",
    "    err_test.append(l_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "ename": "NameError",
     "evalue": "name 'err_test' is not defined",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mNameError\u001b[0m                                 Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-13-05c23f2e5244>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m\u001b[0m\n\u001b[1;32m      2\u001b[0m \u001b[0;31m#plt.plot(lt_test_ref,'--', label =  'Test Error Reference', color = 'red')\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      3\u001b[0m \u001b[0;31m#plt.plot(lt_train_ref,'--', label =  'Train Error Reference', color = 'blue')\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m----> 4\u001b[0;31m \u001b[0mplt\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mplot\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mlist\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mmap\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;32mlambda\u001b[0m \u001b[0mx\u001b[0m\u001b[0;34m:\u001b[0m \u001b[0;36m100\u001b[0m \u001b[0;34m*\u001b[0m \u001b[0mx\u001b[0m\u001b[0;34m/\u001b[0m\u001b[0mfloat\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0ms_t\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0merr_test\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mlabel\u001b[0m \u001b[0;34m=\u001b[0m  \u001b[0;34m'Test Error Reference'\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mcolor\u001b[0m \u001b[0;34m=\u001b[0m  \u001b[0;34m'red'\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m      5\u001b[0m \u001b[0mplt\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mplot\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mlist\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mmap\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;32mlambda\u001b[0m \u001b[0mx\u001b[0m\u001b[0;34m:\u001b[0m \u001b[0;36m100\u001b[0m \u001b[0;34m*\u001b[0m \u001b[0mx\u001b[0m\u001b[0;34m/\u001b[0m\u001b[0mfloat\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0ms\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0merr_train\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mlabel\u001b[0m \u001b[0;34m=\u001b[0m  \u001b[0;34m'Train Error Reference'\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mcolor\u001b[0m \u001b[0;34m=\u001b[0m  \u001b[0;34m'blue'\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      6\u001b[0m \u001b[0mplt\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mlegend\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mNameError\u001b[0m: name 'err_test' is not defined"
     ]
    },
    {
     "data": {
      "text/plain": [
       "<Figure size 864x504 with 0 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "plt.figure(figsize=(12,7))\n",
    "#plt.plot(lt_test_ref,'--', label =  'Test Error Reference', color = 'red')\n",
    "#plt.plot(lt_train_ref,'--', label =  'Train Error Reference', color = 'blue')\n",
    "plt.plot(list(map(lambda x: 100 * x/float(s_t), err_test)), label =  'Test Error Reference', color =  'red')\n",
    "plt.plot(list(map(lambda x: 100 * x/float(s), err_train)), label =  'Train Error Reference', color =  'blue')\n",
    "plt.legend()\n",
    "plt.xlabel(\"Epoch\")\n",
    "plt.ylabel('MAPE')\n",
    "#plt.yscale('log')\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### La Même, avec competition"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {},
   "outputs": [],
   "source": [
    "# filtre produit:\n",
    "nb_ventes_mini = 0 #Nombre minimal de ventes                                                        \n",
    "len_min =  0 # Nombre de semaines minimales de ventes \n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "def gener_lagged_dataset_mat(serie_prod, period, horizon,matrix, index, min_cons=0):\n",
    "    N = len(serie_prod)\n",
    "    for i in range(N-period-horizon+1):\n",
    "        X = torch.tensor(serie_prod.iloc[i:i+period, :].values, dtype = torch.float32).reshape(period,1,-1) \n",
    "        y = torch.tensor(serie_prod['pdM'].iloc[i+horizon:i+horizon+period], dtype = torch.float32).reshape(period,1,-1)\n",
    "        if sum(y) >=min_cons:\n",
    "            matrix[index+i].append([X,y])\n",
    "\n",
    "def gener_dat(data, index, col_drop =['Unnamed: 0','Vente lisse','min_marche', 'Vente réelle','Niv. 1','Niv. 2','Niv. 3'], horizon=6):\n",
    "    X = data.copy()\n",
    "    X.drop(columns = col_drop, inplace= True)\n",
    "    X['pdM'] = 100 * (data['Vente lisse']/data['Vente lisse'+'_somme'])\n",
    "    X['pdM_shift'] = X['pdM'].shift(horizon)\n",
    "    return X.iloc[horizon:]\n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {},
   "outputs": [],
   "source": [
    "date_start = dt.datetime.strptime( min(df.index.get_level_values(1)), '%Y-%m-%d')\n",
    "date_end_training = dt.datetime.strptime(date_train, '%Y-%m-%d')\n",
    "nb_weeks = (date_end_training - date_start).days //7\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Nb de produits initial  104 \n",
      "Nb de ventes initiales 21892.603540221062 \n",
      "Nb de produits conservés 83\n",
      "Nb de ventes conservées 21005.874692294365\n"
     ]
    }
   ],
   "source": [
    "mat_train = [[] for i in range(nb_weeks)]\n",
    "mat_test = [[] for i in range(54 + horizon)]\n",
    "\n",
    "l_train = []\n",
    "l_valid =[]\n",
    "keeped_products =[]\n",
    "\n",
    "products = set(df.index.get_level_values(0))\n",
    "for prod in products:\n",
    "    vt_prod = df.loc[prod].fillna(0)\n",
    "    if sum(vt_prod[index]) > nb_ventes_mini:\n",
    "        min_date = min(vt_prod[vt_prod[index] > 0].index)\n",
    "        max_date = min(date_train, max(vt_prod[vt_prod[index] > 0].index))\n",
    "    \n",
    "        min_date = dt.datetime.strptime(min_date, '%Y-%m-%d')\n",
    "        max_date = dt.datetime.strptime(max_date, '%Y-%m-%d')\n",
    "        index_start = (min_date-date_start).days//7\n",
    "        \n",
    "        if (max_date - min_date).days // 7 > len_min:\n",
    "            keeped_products.append(prod)\n",
    "            red_prod = vt_prod[vt_prod.index <= dt.datetime.strftime(max_date,'%Y-%m-%d' )]\n",
    "            red_prod = red_prod[red_prod.index >= dt.datetime.strftime(min_date,'%Y-%m-%d' )]\n",
    "            \n",
    "            valid_red_prod = vt_prod[vt_prod.index > (dt.datetime.strftime(date_end_training - dt.timedelta(days = 7*horizon),'%Y-%m-%d' ) )]\n",
    "\n",
    "            gener_lagged_dataset_mat(gener_dat(red_prod, index, horizon=horizon),window,horizon, mat_train,index_start)\n",
    "            gener_lagged_dataset_mat(gener_dat(valid_red_prod, index, horizon=horizon),window,horizon, mat_test,0)            \n",
    "print('Nb de produits initial  %s ' % len(products))\n",
    "print('Nb de ventes initiales %s ' % sum(df[index]))\n",
    "print('Nb de produits conservés %s' %len(keeped_products))        \n",
    "print('Nb de ventes conservées %s' % sum(df[index].loc[keeped_products]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "metadata": {},
   "outputs": [],
   "source": [
    "#Constantes\n",
    "\n",
    "epochs =2000\n",
    "learning_rate =  5e-3\n",
    "n_hidden = 20\n",
    "n_param = 10\n",
    "n_layer =1\n",
    "\n",
    "model = LSTM_nn(n_param,n_hidden,n_layer)\n",
    "loss_function = nn.L1Loss(reduction ='sum')\n",
    "optimizer = torch.optim.Adam(model.parameters(), lr=learning_rate)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/CDBDX/remy.garnier/.virtualenvs/python_3.6_TS/lib/python3.6/site-packages/ipykernel_launcher.py:18: UserWarning: To copy construct from a tensor, it is recommended to use sourceTensor.clone().detach() or sourceTensor.clone().detach().requires_grad_(True), rather than torch.tensor(sourceTensor).\n",
      "/home/CDBDX/remy.garnier/.virtualenvs/python_3.6_TS/lib/python3.6/site-packages/ipykernel_launcher.py:46: UserWarning: To copy construct from a tensor, it is recommended to use sourceTensor.clone().detach() or sourceTensor.clone().detach().requires_grad_(True), rather than torch.tensor(sourceTensor).\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Training MAPE: 85.4388 Testing MAPE: 115.2496\n",
      "Training MAPE: 85.1531 Testing MAPE: 114.9988\n",
      "Training MAPE: 85.0788 Testing MAPE: 115.4934\n",
      "Training MAPE: 84.5405 Testing MAPE: 115.2309\n",
      "Training MAPE: 84.9282 Testing MAPE: 115.2025\n",
      "Training MAPE: 84.4866 Testing MAPE: 114.7804\n",
      "Training MAPE: 84.2930 Testing MAPE: 115.1383\n",
      "Training MAPE: 84.5122 Testing MAPE: 114.8590\n",
      "Training MAPE: 84.2360 Testing MAPE: 114.3299\n",
      "Training MAPE: 83.9423 Testing MAPE: 114.5561\n",
      "Training MAPE: 84.2792 Testing MAPE: 114.5532\n",
      "Training MAPE: 83.7342 Testing MAPE: 114.3624\n",
      "Training MAPE: 83.6859 Testing MAPE: 114.2583\n",
      "Training MAPE: 83.5847 Testing MAPE: 113.8489\n",
      "Training MAPE: 83.6070 Testing MAPE: 114.4449\n",
      "Training MAPE: 83.2321 Testing MAPE: 114.0974\n",
      "Training MAPE: 83.0975 Testing MAPE: 113.5963\n",
      "Training MAPE: 82.8151 Testing MAPE: 113.3061\n",
      "Training MAPE: 82.4069 Testing MAPE: 113.1362\n",
      "Training MAPE: 82.5682 Testing MAPE: 112.8396\n",
      "Training MAPE: 82.3789 Testing MAPE: 112.4771\n",
      "Training MAPE: 82.0738 Testing MAPE: 112.2274\n",
      "Training MAPE: 81.7713 Testing MAPE: 112.0701\n",
      "Training MAPE: 81.5600 Testing MAPE: 111.8205\n",
      "Training MAPE: 81.5372 Testing MAPE: 111.5317\n",
      "Training MAPE: 81.3209 Testing MAPE: 111.4629\n",
      "Training MAPE: 81.0264 Testing MAPE: 111.1324\n",
      "Training MAPE: 80.7878 Testing MAPE: 110.9769\n",
      "Training MAPE: 80.4310 Testing MAPE: 111.1115\n",
      "Training MAPE: 80.3701 Testing MAPE: 111.1192\n",
      "Training MAPE: 79.9618 Testing MAPE: 110.7856\n",
      "Training MAPE: 79.4473 Testing MAPE: 110.4785\n",
      "Training MAPE: 79.2195 Testing MAPE: 110.8227\n",
      "Training MAPE: 78.3794 Testing MAPE: 110.0484\n",
      "Training MAPE: 78.0289 Testing MAPE: 109.1644\n",
      "Training MAPE: 77.1153 Testing MAPE: 107.6550\n",
      "Training MAPE: 76.4872 Testing MAPE: 106.9666\n",
      "Training MAPE: 76.6447 Testing MAPE: 106.9857\n",
      "Training MAPE: 76.3143 Testing MAPE: 105.9870\n",
      "Training MAPE: 75.6790 Testing MAPE: 106.2031\n",
      "Training MAPE: 75.3392 Testing MAPE: 106.5829\n",
      "Training MAPE: 74.9291 Testing MAPE: 106.2975\n",
      "Training MAPE: 74.6029 Testing MAPE: 106.1348\n",
      "Training MAPE: 74.1916 Testing MAPE: 105.3262\n",
      "Training MAPE: 73.5377 Testing MAPE: 105.4160\n",
      "Training MAPE: 72.8418 Testing MAPE: 104.4133\n",
      "Training MAPE: 72.2323 Testing MAPE: 103.5914\n",
      "Training MAPE: 72.1324 Testing MAPE: 103.2479\n",
      "Training MAPE: 71.5417 Testing MAPE: 102.9089\n",
      "Training MAPE: 71.1517 Testing MAPE: 102.0886\n",
      "Training MAPE: 70.7052 Testing MAPE: 100.8545\n",
      "Training MAPE: 70.1096 Testing MAPE: 99.7335\n",
      "Training MAPE: 69.5111 Testing MAPE: 100.0698\n",
      "Training MAPE: 68.9602 Testing MAPE: 99.9572\n",
      "Training MAPE: 68.7027 Testing MAPE: 99.4305\n",
      "Training MAPE: 68.4537 Testing MAPE: 98.0033\n",
      "Training MAPE: 67.9639 Testing MAPE: 97.0002\n",
      "Training MAPE: 67.1104 Testing MAPE: 95.3185\n",
      "Training MAPE: 66.5307 Testing MAPE: 94.4548\n",
      "Training MAPE: 66.1115 Testing MAPE: 93.7051\n",
      "Training MAPE: 65.9444 Testing MAPE: 92.3176\n",
      "Training MAPE: 65.3373 Testing MAPE: 92.5647\n",
      "Training MAPE: 65.0239 Testing MAPE: 91.8039\n",
      "Training MAPE: 64.4548 Testing MAPE: 91.1594\n",
      "Training MAPE: 64.0110 Testing MAPE: 91.0242\n",
      "Training MAPE: 63.3169 Testing MAPE: 90.8322\n",
      "Training MAPE: 63.2600 Testing MAPE: 90.5376\n",
      "Training MAPE: 62.6845 Testing MAPE: 90.0358\n",
      "Training MAPE: 62.8092 Testing MAPE: 89.4407\n",
      "Training MAPE: 62.2294 Testing MAPE: 90.2451\n",
      "Training MAPE: 62.0152 Testing MAPE: 89.0508\n",
      "Training MAPE: 61.5492 Testing MAPE: 89.2795\n",
      "Training MAPE: 61.5111 Testing MAPE: 88.9822\n",
      "Training MAPE: 61.3598 Testing MAPE: 89.1814\n",
      "Training MAPE: 61.1307 Testing MAPE: 87.9437\n",
      "Training MAPE: 60.9155 Testing MAPE: 87.7664\n",
      "Training MAPE: 60.6231 Testing MAPE: 87.1238\n",
      "Training MAPE: 60.4217 Testing MAPE: 86.5117\n",
      "Training MAPE: 60.2047 Testing MAPE: 86.6859\n",
      "Training MAPE: 60.0178 Testing MAPE: 85.4865\n",
      "Training MAPE: 59.6978 Testing MAPE: 86.1012\n",
      "Training MAPE: 59.7533 Testing MAPE: 86.0091\n",
      "Training MAPE: 59.5287 Testing MAPE: 86.3666\n",
      "Training MAPE: 59.2026 Testing MAPE: 85.5105\n",
      "Training MAPE: 59.2236 Testing MAPE: 84.8174\n",
      "Training MAPE: 58.6802 Testing MAPE: 84.1109\n",
      "Training MAPE: 58.4998 Testing MAPE: 84.2977\n",
      "Training MAPE: 58.7603 Testing MAPE: 83.2602\n",
      "Training MAPE: 58.3295 Testing MAPE: 82.9217\n",
      "Training MAPE: 58.4000 Testing MAPE: 83.0904\n",
      "Training MAPE: 58.4084 Testing MAPE: 83.2133\n",
      "Training MAPE: 58.1066 Testing MAPE: 83.1169\n",
      "Training MAPE: 57.9738 Testing MAPE: 83.3349\n",
      "Training MAPE: 57.9551 Testing MAPE: 83.0159\n",
      "Training MAPE: 57.4493 Testing MAPE: 82.1767\n",
      "Training MAPE: 57.5347 Testing MAPE: 81.8460\n",
      "Training MAPE: 57.4707 Testing MAPE: 81.7972\n",
      "Training MAPE: 57.5279 Testing MAPE: 82.0798\n",
      "Training MAPE: 57.5629 Testing MAPE: 81.1983\n",
      "Training MAPE: 57.1055 Testing MAPE: 81.0702\n",
      "Training MAPE: 57.2437 Testing MAPE: 80.6380\n",
      "Training MAPE: 57.0354 Testing MAPE: 80.6717\n",
      "Training MAPE: 56.9331 Testing MAPE: 80.1178\n",
      "Training MAPE: 56.7092 Testing MAPE: 79.9229\n",
      "Training MAPE: 56.6565 Testing MAPE: 79.5548\n",
      "Training MAPE: 56.7285 Testing MAPE: 79.6576\n",
      "Training MAPE: 56.5085 Testing MAPE: 78.9648\n",
      "Training MAPE: 56.4956 Testing MAPE: 78.7274\n",
      "Training MAPE: 56.4173 Testing MAPE: 78.8678\n",
      "Training MAPE: 56.3056 Testing MAPE: 78.6287\n",
      "Training MAPE: 56.3845 Testing MAPE: 78.7948\n",
      "Training MAPE: 56.2316 Testing MAPE: 78.7374\n",
      "Training MAPE: 56.3690 Testing MAPE: 79.0637\n",
      "Training MAPE: 56.1083 Testing MAPE: 78.4265\n",
      "Training MAPE: 56.0651 Testing MAPE: 78.3992\n",
      "Training MAPE: 56.0476 Testing MAPE: 78.8177\n",
      "Training MAPE: 55.8631 Testing MAPE: 78.3588\n",
      "Training MAPE: 55.8173 Testing MAPE: 78.5173\n",
      "Training MAPE: 55.8878 Testing MAPE: 78.0138\n",
      "Training MAPE: 55.7941 Testing MAPE: 77.8650\n",
      "Training MAPE: 55.5750 Testing MAPE: 77.3663\n",
      "Training MAPE: 55.5451 Testing MAPE: 77.6489\n",
      "Training MAPE: 55.5197 Testing MAPE: 77.6913\n",
      "Training MAPE: 55.4855 Testing MAPE: 77.9821\n",
      "Training MAPE: 55.5886 Testing MAPE: 77.8223\n",
      "Training MAPE: 55.6115 Testing MAPE: 77.8905\n",
      "Training MAPE: 55.3953 Testing MAPE: 77.1142\n",
      "Training MAPE: 55.4507 Testing MAPE: 76.3981\n",
      "Training MAPE: 55.1719 Testing MAPE: 76.0170\n",
      "Training MAPE: 54.8910 Testing MAPE: 75.6216\n",
      "Training MAPE: 55.2245 Testing MAPE: 76.0130\n",
      "Training MAPE: 55.1678 Testing MAPE: 75.8100\n",
      "Training MAPE: 55.0891 Testing MAPE: 75.9914\n",
      "Training MAPE: 55.0977 Testing MAPE: 76.3142\n",
      "Training MAPE: 54.8577 Testing MAPE: 76.5560\n",
      "Training MAPE: 54.9469 Testing MAPE: 75.9237\n",
      "Training MAPE: 54.7936 Testing MAPE: 75.7010\n",
      "Training MAPE: 54.8338 Testing MAPE: 75.5630\n",
      "Training MAPE: 54.7089 Testing MAPE: 74.6985\n",
      "Training MAPE: 54.6770 Testing MAPE: 74.8410\n",
      "Training MAPE: 54.7547 Testing MAPE: 74.7932\n",
      "Training MAPE: 54.4370 Testing MAPE: 74.5113\n",
      "Training MAPE: 54.5220 Testing MAPE: 73.9905\n",
      "Training MAPE: 54.5253 Testing MAPE: 73.3527\n",
      "Training MAPE: 54.4057 Testing MAPE: 73.6369\n",
      "Training MAPE: 54.3689 Testing MAPE: 72.9730\n",
      "Training MAPE: 54.1892 Testing MAPE: 72.7336\n",
      "Training MAPE: 54.2275 Testing MAPE: 72.9175\n",
      "Training MAPE: 54.0920 Testing MAPE: 72.9666\n",
      "Training MAPE: 54.2084 Testing MAPE: 73.1226\n",
      "Training MAPE: 53.9408 Testing MAPE: 72.8408\n",
      "Training MAPE: 53.8335 Testing MAPE: 73.1572\n",
      "Training MAPE: 53.9450 Testing MAPE: 72.4867\n",
      "Training MAPE: 53.7987 Testing MAPE: 71.7464\n",
      "Training MAPE: 53.8541 Testing MAPE: 71.4072\n",
      "Training MAPE: 53.9167 Testing MAPE: 71.5470\n",
      "Training MAPE: 53.9728 Testing MAPE: 72.1078\n",
      "Training MAPE: 53.7114 Testing MAPE: 72.5158\n",
      "Training MAPE: 53.5759 Testing MAPE: 72.5166\n",
      "Training MAPE: 53.8612 Testing MAPE: 72.3790\n",
      "Training MAPE: 53.9190 Testing MAPE: 72.2747\n",
      "Training MAPE: 53.7594 Testing MAPE: 71.4529\n",
      "Training MAPE: 53.5888 Testing MAPE: 71.4482\n",
      "Training MAPE: 53.5546 Testing MAPE: 71.1903\n",
      "Training MAPE: 53.2881 Testing MAPE: 72.1246\n",
      "Training MAPE: 53.3427 Testing MAPE: 72.4291\n",
      "Training MAPE: 53.5226 Testing MAPE: 73.0764\n",
      "Training MAPE: 53.4288 Testing MAPE: 72.5046\n",
      "Training MAPE: 53.0199 Testing MAPE: 72.1886\n",
      "Training MAPE: 53.3847 Testing MAPE: 71.5956\n",
      "Training MAPE: 53.4719 Testing MAPE: 71.4641\n",
      "Training MAPE: 53.2103 Testing MAPE: 71.5163\n",
      "Training MAPE: 53.3284 Testing MAPE: 71.4762\n",
      "Training MAPE: 53.0964 Testing MAPE: 71.9311\n",
      "Training MAPE: 53.1929 Testing MAPE: 72.0271\n",
      "Training MAPE: 53.0043 Testing MAPE: 71.3884\n",
      "Training MAPE: 52.8927 Testing MAPE: 70.8407\n",
      "Training MAPE: 53.0513 Testing MAPE: 70.4052\n",
      "Training MAPE: 52.9274 Testing MAPE: 70.3258\n",
      "Training MAPE: 52.9190 Testing MAPE: 70.3476\n",
      "Training MAPE: 52.8147 Testing MAPE: 70.4801\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Training MAPE: 52.9047 Testing MAPE: 70.7445\n",
      "Training MAPE: 52.7747 Testing MAPE: 71.2753\n",
      "Training MAPE: 52.7209 Testing MAPE: 70.4955\n",
      "Training MAPE: 52.6421 Testing MAPE: 70.0164\n",
      "Training MAPE: 52.7887 Testing MAPE: 70.1192\n",
      "Training MAPE: 52.6168 Testing MAPE: 70.4968\n",
      "Training MAPE: 52.7979 Testing MAPE: 70.8190\n",
      "Training MAPE: 52.5610 Testing MAPE: 70.4857\n",
      "Training MAPE: 52.4838 Testing MAPE: 70.5943\n",
      "Training MAPE: 52.6600 Testing MAPE: 69.6564\n",
      "Training MAPE: 52.5761 Testing MAPE: 69.2988\n",
      "Training MAPE: 52.6151 Testing MAPE: 68.8509\n",
      "Training MAPE: 52.5874 Testing MAPE: 69.4321\n",
      "Training MAPE: 52.4168 Testing MAPE: 69.7640\n",
      "Training MAPE: 52.4534 Testing MAPE: 70.5767\n",
      "Training MAPE: 52.4424 Testing MAPE: 70.3448\n",
      "Training MAPE: 52.1932 Testing MAPE: 70.5479\n",
      "Training MAPE: 52.2329 Testing MAPE: 70.4333\n"
     ]
    },
    {
     "ename": "KeyboardInterrupt",
     "evalue": "",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mKeyboardInterrupt\u001b[0m                         Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-26-20ec24918f43>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m\u001b[0m\n\u001b[1;32m     36\u001b[0m                 \u001b[0ml\u001b[0m \u001b[0;34m+=\u001b[0m \u001b[0msingle_loss\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mitem\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     37\u001b[0m                 \u001b[0ms\u001b[0m \u001b[0;34m+=\u001b[0m \u001b[0msum\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0my_train\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m-\u001b[0m\u001b[0mn_calc\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;36m0\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 38\u001b[0;31m     \u001b[0mloss\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mbackward\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     39\u001b[0m     \u001b[0moptimizer\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mstep\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     40\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/.virtualenvs/python_3.6_TS/lib/python3.6/site-packages/torch/tensor.py\u001b[0m in \u001b[0;36mbackward\u001b[0;34m(self, gradient, retain_graph, create_graph)\u001b[0m\n\u001b[1;32m    148\u001b[0m                 \u001b[0mproducts\u001b[0m\u001b[0;34m.\u001b[0m \u001b[0mDefaults\u001b[0m \u001b[0mto\u001b[0m\u001b[0;31m \u001b[0m\u001b[0;31m`\u001b[0m\u001b[0;31m`\u001b[0m\u001b[0;32mFalse\u001b[0m\u001b[0;31m`\u001b[0m\u001b[0;31m`\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    149\u001b[0m         \"\"\"\n\u001b[0;32m--> 150\u001b[0;31m         \u001b[0mtorch\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mautograd\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mbackward\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mgradient\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mretain_graph\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mcreate_graph\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    151\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    152\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0mregister_hook\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mhook\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/.virtualenvs/python_3.6_TS/lib/python3.6/site-packages/torch/autograd/__init__.py\u001b[0m in \u001b[0;36mbackward\u001b[0;34m(tensors, grad_tensors, retain_graph, create_graph, grad_variables)\u001b[0m\n\u001b[1;32m     97\u001b[0m     Variable._execution_engine.run_backward(\n\u001b[1;32m     98\u001b[0m         \u001b[0mtensors\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mgrad_tensors\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mretain_graph\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mcreate_graph\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 99\u001b[0;31m         allow_unreachable=True)  # allow_unreachable flag\n\u001b[0m\u001b[1;32m    100\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    101\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mKeyboardInterrupt\u001b[0m: "
     ]
    }
   ],
   "source": [
    "from random import shuffle\n",
    "\n",
    "\n",
    "err_train_2  = []\n",
    "err_test_2 =  []\n",
    "\n",
    "for i in range(epochs):\n",
    "    l=0\n",
    "    s=0\n",
    "    copy_train_list = mat_train.copy()\n",
    "    shuffle(copy_train_list)\n",
    "    \n",
    "    optimizer.zero_grad()\n",
    "    loss = 0\n",
    "    for mat_step in copy_train_list:\n",
    "        if len(mat_step) !=0:\n",
    "            \n",
    "            partial_sum= torch.tensor(mat_step[0][1])\n",
    "            for i in range(1,len(mat_step)):\n",
    "                partial_sum += mat_step[i][1]\n",
    "        \n",
    "            \n",
    "            model.reinitialize()\n",
    "            cp_mat_step = mat_step.copy()\n",
    "            sum_prediction = torch.ones(partial_sum.shape)\n",
    "            for X_train,_ in cp_mat_step:\n",
    "                model.reinitialize()\n",
    "                sum_prediction += model(X_train)\n",
    "\n",
    "            shuffle(cp_mat_step)\n",
    "            for (X_train,y_train) in cp_mat_step:\n",
    "                model.reinitialize()\n",
    "                y_pred = partial_sum * model(X_train)/sum_prediction\n",
    "                single_loss = loss_function(y_pred[-n_calc:,0],y_train[-n_calc:,0] )\n",
    "                loss += single_loss\n",
    "                l += single_loss.item()\n",
    "                s += sum(y_train[-n_calc:,0])\n",
    "    loss.backward()\n",
    "    optimizer.step()\n",
    "            \n",
    "    \n",
    "    l_test = 0\n",
    "    s_test = 0 \n",
    "    for mat_step in mat_test: \n",
    "        if len(mat_step) !=0:\n",
    "            partial_sum= torch.tensor(mat_step[0][1])\n",
    "            for i in range(1,len(mat_step)):\n",
    "                partial_sum += mat_step[i][1]\n",
    "            optimizer.zero_grad()\n",
    "            model.reinitialize()\n",
    "            cp_mat_step = mat_step.copy()\n",
    "\n",
    "            sum_prediction = torch.zeros(y_train.shape)\n",
    "            for X_test,_ in cp_mat_step:\n",
    "                model.reinitialize()\n",
    "                sum_prediction += model(X_test)\n",
    "\n",
    "            for (X_train,y_train) in cp_mat_step:\n",
    "                model.reinitialize()\n",
    "                y_pred = partial_sum * model(X_train)/sum_prediction\n",
    "                single_loss = loss_function(y_pred[-n_calc:,0],y_train[-n_calc:,0] )\n",
    "                l_test += single_loss.item()\n",
    "                s_test += sum(y_train[-n_calc:,0])\n",
    "    \n",
    "    print('Training MAPE: %.4f Testing MAPE: %.4f' %(100 * l/s , 100 * l_test/s_test))\n",
    "    err_train_2.append(l)\n",
    "    err_test_2.append(l_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/CDBDX/remy.garnier/.virtualenvs/python_3.6_TS/lib/python3.6/site-packages/ipykernel_launcher.py:2: UserWarning: To copy construct from a tensor, it is recommended to use sourceTensor.clone().detach() or sourceTensor.clone().detach().requires_grad_(True), rather than torch.tensor(sourceTensor).\n",
      "  \n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "tensor([[[96.6887]],\n",
       "\n",
       "        [[97.0137]],\n",
       "\n",
       "        [[92.6562]],\n",
       "\n",
       "        [[86.1099]],\n",
       "\n",
       "        [[86.0978]],\n",
       "\n",
       "        [[67.2631]],\n",
       "\n",
       "        [[62.7099]],\n",
       "\n",
       "        [[72.8341]],\n",
       "\n",
       "        [[65.6269]],\n",
       "\n",
       "        [[64.1968]],\n",
       "\n",
       "        [[62.6434]],\n",
       "\n",
       "        [[55.7985]],\n",
       "\n",
       "        [[58.1935]],\n",
       "\n",
       "        [[64.6675]],\n",
       "\n",
       "        [[66.2367]],\n",
       "\n",
       "        [[85.6285]],\n",
       "\n",
       "        [[84.7043]],\n",
       "\n",
       "        [[83.5637]],\n",
       "\n",
       "        [[75.3938]],\n",
       "\n",
       "        [[86.7936]]])"
      ]
     },
     "execution_count": 46,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "mat_step =mat_train[13]\n",
    "partial_sum= torch.tensor(mat_step[0][1])\n",
    "for i in range(1,len(mat_step)):\n",
    "    partial_sum += mat_step[i][1]\n",
    "partial_sum"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}