STRO20_5_sparseDensity_STRO20.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "6806cab9-1e8b-4ab9-a1fc-25d9b32ab33a",
   "metadata": {},
   "outputs": [],
   "source": [
    "import os\n",
    "import pandas as pd\n",
    "from pathlib import Path\n",
    "import requests\n",
    "import json"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "a759f72c-41bd-49dc-9bad-9085e92e1e89",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Add the name of a directory to download the re-engineered data STRO 2.0\n",
    "# The re-engineered STRO data are found at: https://doi.org/10.6084/m9.figshare.27176202\n",
    "# We will download it below using the figshare API\n",
    "directory = r\"C:\\STRO200\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "d09c8b5e",
   "metadata": {},
   "outputs": [],
   "source": [
    "# This cell is based on info found at: https://help.figshare.com/article/how-to-use-the-figshare-api#api-interface\n",
    "# Because of their size (1.52 GB), it takes a few minutes to download the files.\n",
    "item_id = \"27176202\" # This is the item ID of the figshare project STRO 2.0 (re-engineered Sound Toll Registers Online data)\n",
    "BASE_URL = 'https://api.figshare.com/v2'\n",
    "file_info = [] # A blank list to hold all the file metadata\n",
    "r = requests.get(BASE_URL + '/articles/' + item_id + '/files')\n",
    "file_metadata = json.loads(r.text)\n",
    "for j in file_metadata: # Add the item id to each file record- this is used later to name a folder to save the file to\n",
    "    j['item_id'] = item_id\n",
    "    file_info.append(j) # Add the file metadata to the list\n",
    "\n",
    "# Download each file to the directory indicated above\n",
    "for k in file_info:\n",
    "    response = requests.get(BASE_URL + '/file/download/' + str(k['id']))\n",
    "    Path(directory).mkdir(exist_ok=True)\n",
    "    open(directory + '/' + k['name'], 'wb').write(response.content)\n",
    "    \n",
    "print('Done. All files are downloaded.')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "id": "bf0077cb-8d1a-4b6c-996a-737d0a080eb1",
   "metadata": {},
   "outputs": [],
   "source": [
    "def compute_shape_density(directory):\n",
    "    \"\"\"\n",
    "    Compute the shape and density of DataFrames from CSV files in a given directory.\n",
    "    Parameters: directory (str): The directory containing the CSV files.\n",
    "    Returns: A Pandas DataFrame specifying the name of each CSV file, its shape, and density.\n",
    "    \"\"\"\n",
    "    shapeDensity = []\n",
    "    for subdir, dirs, files in os.walk(directory):\n",
    "        if subdir.endswith('_MACOSX'):\n",
    "            pass\n",
    "        else:\n",
    "            try:\n",
    "                for file in files:\n",
    "                    filepath = os.path.join(subdir, file)\n",
    "                    # load data into a pandas dataframe\n",
    "                    df = pd.read_csv(filepath, sep=';', quotechar='\"', encoding=\"utf-8\", low_memory=False)\n",
    "                    # compute shape of the dataframe\n",
    "                    shape = df.shape\n",
    "                    # print(f\"The shape of {file} is: \", df.shape)\n",
    "                    # create a sparse array based on the dataframe\n",
    "                    sparseArray = df.apply(pd.arrays.SparseArray)\n",
    "                    # compute the ratio of non-sparse points to total (dense) data points\n",
    "                    density = sparseArray.sparse.density\n",
    "                    # print(f\"The density of {file} is: \", sparr.sparse.density)\n",
    "                    shapeDensity.append([file, shape, f\"{density:.2f}\"])\n",
    "            except Exception as e:\n",
    "                print(f\"Error processing {file}: {e}\")\n",
    "    return pd.DataFrame(shapeDensity, columns=['File', 'Shape', 'Density'])               "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "a72e0c35-11ab-488b-acf8-581fc8c0a771",
   "metadata": {},
   "outputs": [],
   "source": [
    "# takes a while to execute for all dataframes in STRO 2.0\n",
    "result = compute_shape_density(directory)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "3f0a7302-5196-4e02-826c-36fd3959a428",
   "metadata": {},
   "outputs": [],
   "source": [
    "result"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "base",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.18"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}