nb2-RDD-basics.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "\n",
       "        <div>\n",
       "            <p><b>SparkContext</b></p>\n",
       "\n",
       "            <p><a href=\"http://127.0.0.1:4040\">Spark UI</a></p>\n",
       "\n",
       "            <dl>\n",
       "              <dt>Version</dt>\n",
       "                <dd><code>v2.2.1</code></dd>\n",
       "              <dt>Master</dt>\n",
       "                <dd><code>local[*]</code></dd>\n",
       "              <dt>AppName</dt>\n",
       "                <dd><code>PySparkShell</code></dd>\n",
       "            </dl>\n",
       "        </div>\n",
       "        "
      ],
      "text/plain": [
       "<SparkContext master=local[*] appName=PySparkShell>"
      ]
     },
     "execution_count": 1,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "sc"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Getting the data and creating the RDD"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "data_file = \"file:///home/lygbug666/workdir/spark-py-notebooks/kddcup.data_10_percent.gz\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "raw_data = sc.textFile(data_file)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# The filter transformation"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "494021"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "raw_data.count()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "normal_raw_data = raw_data.filter(lambda x: 'normal.' in x)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "There are 97278 'normal' interactions\n",
      "Count completed in 1.605 seconds\n"
     ]
    }
   ],
   "source": [
    "from time import time\n",
    "t0 = time()\n",
    "normal_count = normal_raw_data.count()\n",
    "tt = time() - t0\n",
    "print (\"There are {} 'normal' interactions\".format(normal_count))\n",
    "print (\"Count completed in {} seconds\".format(round(tt,3)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# lazy computation"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# The map transformation"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Parse completed in 0.09266 seconds\n",
      "['0',\n",
      " 'tcp',\n",
      " 'http',\n",
      " 'SF',\n",
      " '181',\n",
      " '5450',\n",
      " '0',\n",
      " '0',\n",
      " '0',\n",
      " '0',\n",
      " '0',\n",
      " '1',\n",
      " '0',\n",
      " '0',\n",
      " '0',\n",
      " '0',\n",
      " '0',\n",
      " '0',\n",
      " '0',\n",
      " '0',\n",
      " '0',\n",
      " '0',\n",
      " '8',\n",
      " '8',\n",
      " '0.00',\n",
      " '0.00',\n",
      " '0.00',\n",
      " '0.00',\n",
      " '1.00',\n",
      " '0.00',\n",
      " '0.00',\n",
      " '9',\n",
      " '9',\n",
      " '1.00',\n",
      " '0.00',\n",
      " '0.11',\n",
      " '0.00',\n",
      " '0.00',\n",
      " '0.00',\n",
      " '0.00',\n",
      " '0.00',\n",
      " 'normal.']\n"
     ]
    }
   ],
   "source": [
    "from pprint import pprint\n",
    "csv_data = raw_data.map(lambda x: x.split(\",\"))\n",
    "t0 = time()\n",
    "head_rows = csv_data.take(5)\n",
    "tt = time() - t0\n",
    "print (\"Parse completed in {} seconds\".format(round(tt,5)))\n",
    "pprint(head_rows[0])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# round(tt,5) ５位数字"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Parse completed in 3.155 seconds\n"
     ]
    }
   ],
   "source": [
    "t0 = time()\n",
    "head_rows = csv_data.take(100000)\n",
    "tt = time() - t0\n",
    "print (\"Parse completed in {} seconds\".format(round(tt,3)))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Using map and predefined functions"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "('normal.', ['0', 'tcp', 'http', 'SF', '181', '5450', '0', '0', '0', '0', '0', '1', '0', '0', '0', '0', '0', '0', '0', '0', '0', '0', '8', '8', '0.00', '0.00', '0.00', '0.00', '1.00', '0.00', '0.00', '9', '9', '1.00', '0.00', '0.11', '0.00', '0.00', '0.00', '0.00', '0.00', 'normal.']) ('normal.', ['0', 'tcp', 'http', 'SF', '239', '486', '0', '0', '0', '0', '0', '1', '0', '0', '0', '0', '0', '0', '0', '0', '0', '0', '8', '8', '0.00', '0.00', '0.00', '0.00', '1.00', '0.00', '0.00', '19', '19', '1.00', '0.00', '0.05', '0.00', '0.00', '0.00', '0.00', '0.00', 'normal.'])\n"
     ]
    }
   ],
   "source": [
    "def parse_interaction(line):\n",
    "    elems = line.split(\",\")\n",
    "    tag = elems[41]\n",
    "    # normal, list[42]\n",
    "    return (tag, elems)\n",
    "\n",
    "key_csv_data = raw_data.map(parse_interaction)\n",
    "head_rows = key_csv_data.take(5)\n",
    "print(head_rows[0], head_rows[1])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# The collect action\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Data collected in 5.639 seconds\n"
     ]
    }
   ],
   "source": [
    "t0 = time()\n",
    "all_raw_data = raw_data.collect()\n",
    "tt = time() - t0\n",
    "print (\"Data collected in {} seconds\".format(round(tt,3)))"
   ]
  },
  {
   "cell_type": "raw",
   "metadata": {},
   "source": [
    "That took longer as any other action we used before, of course. Every Spark worker node that has a fragment of the RDD has to be coordinated in order to retrieve its part, and then reduce everything together.\n",
    "集群中所有节点需要将RDD汇总，然后一起进行reduce处理，因此比单独要慢。"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Data collected in 5.369 seconds\n",
      "There are 97278 'normal' interactions\n"
     ]
    }
   ],
   "source": [
    "# parse into key-value pairs\n",
    "key_csv_data = raw_data.map(parse_interaction)\n",
    "\n",
    "# filter normal key interactions\n",
    "normal_key_interactions = key_csv_data.filter(lambda x: x[1][41] == \"normal.\")\n",
    "# Upper function equals to \n",
    "# normal_key_interactions = key_csv_data.filter(lambda x: x[0] == \"normal.\")\n",
    "\n",
    "# collect all\n",
    "t0 = time()\n",
    "all_normal = normal_key_interactions.collect()\n",
    "tt = time() - t0\n",
    "normal_count = len(all_normal)\n",
    "print (\"Data collected in {} seconds\".format(round(tt,3)))\n",
    "print (\"There are {} 'normal' interactions\".format(normal_count))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}