debugger.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [],
   "source": [
    "#import packages\n",
    "import datetime as dt \n",
    "import lxml\n",
    "from lxml import html, etree\n",
    "import requests\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "import pandas_datareader\n",
    "from pandas_datareader import data as pdr\n",
    "import time\n",
    "from time import sleep\n",
    "import string\n",
    "import yfinance as yf\n",
    "from bs4 import BeautifulSoup\n",
    "import urllib\n",
    "import io\n",
    "import os\n",
    "import pandasql\n",
    "from pandasql import sqldf\n",
    "import re\n",
    "import gc\n",
    "import zipfile\n",
    "from zipfile import ZipFile\n",
    "from io import StringIO\n",
    "import sys\n",
    "\n",
    "from os import listdir\n",
    "from os.path import isfile, join\n",
    "import psutil\n",
    "\n",
    "from collections import Counter\n",
    "import linecache\n",
    "import tracemalloc\n",
    "\n",
    "import reshape\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [],
   "source": [
    "df = pd.read_csv(\"total_finance_table.csv\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Unnamed: 0      int64\n",
       "name           object\n",
       "sic           float64\n",
       "fye           float64\n",
       "form           object\n",
       "period        float64\n",
       "fy            float64\n",
       "fp             object\n",
       "filed           int64\n",
       "ticker         object\n",
       "cik             int64\n",
       "ddate           int64\n",
       "qtrs          float64\n",
       "uom            object\n",
       "value         float64\n",
       "adsh           object\n",
       "stmt           object\n",
       "tag            object\n",
       "version        object\n",
       "plabel         object\n",
       "dtype: object"
      ]
     },
     "execution_count": 31,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.dtypes"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [],
   "source": [
    "# want to be able to get the plabels and tags and values of a company for a year where data is missing \n",
    "\n",
    "#ex1: stz, cik = 16918 has no shares outstanding data\n",
    "\n",
    "_year = 2018.0\n",
    "_ticker = 'cof' \n",
    "#_stmt = 'BS'\n",
    "\n",
    "\n",
    "ex1 = df[(df['fy'] == _year)\n",
    "        & (df['ticker'] == _ticker)\n",
    "        ]\n",
    "\n",
    "#print(ex1)\n",
    "ex1.to_csv(_ticker + '_'+str(_year)+\".csv\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "#debug df\n",
    "label_counts = df.groupby(['plabel', \n",
    "'tag', \n",
    "'stmt', \n",
    "'qtrs']).size().reset_index(name=\"frequency\")\n",
    "\n",
    "ordered_label = sqldf(\"\"\"\n",
    "SELECT \n",
    "* \n",
    "FROM \n",
    "label_counts \n",
    "WHERE frequency >= 10\n",
    "ORDER BY \n",
    "stmt, \n",
    "frequency DESC\n",
    "\"\"\")\n",
    "\n",
    "ordered_label.to_csv(\"label_counts.csv\")\n",
    "print(ordered_label)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}