Notebooks/Spark-Example-06-Join Operation on RDD.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Join Operation on RDD in Spark\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "['a', 'b', 'c', 'd', 'e']\n",
      "['AA', 'BB', 'CC', 'DD']\n",
      "[1, 1, 5, 2, 3]\n",
      "[1, 5, 5, 6]\n"
     ]
    }
   ],
   "source": [
    "# Let us create some RDDs\n",
    "\n",
    "valueRDDA = sc.parallelize([\"a\", \"b\", \"c\", \"d\", \"e\"])\n",
    "valueRDDB = sc.parallelize([\"AA\", \"BB\", \"CC\", \"DD\"])\n",
    "\n",
    "rddB = sc.parallelize([1, 1 , 5,  2, 3])\n",
    "rddC = sc.parallelize([1, 5, 5, 6])\n",
    "\n",
    "print(valueRDDA.collect())\n",
    "print(valueRDDB.collect())\n",
    "print(rddB.collect())\n",
    "print(rddC.collect())"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Zip operation on RDD\n",
    "\n",
    "We can zip the above RDDs to create new RDDs with different keys"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "RDD 1 :  [(1, 'a'), (1, 'b'), (5, 'c'), (2, 'd'), (3, 'e')]\n",
      "RDD 2 :  [(1, 'AA'), (5, 'BB'), (5, 'CC'), (6, 'DD')]\n"
     ]
    }
   ],
   "source": [
    "# Now, we can zip these RDDs to create new RDDs with different keys\n",
    "\n",
    "rdd1 = rddB.zip(valueRDDA)\n",
    "rdd2 = rddC.zip(valueRDDB)\n",
    "\n",
    "print(\"RDD 1 : \", rdd1.collect())\n",
    "print(\"RDD 2 : \", rdd2.collect())\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[(1, ('a', 'AA')), (1, ('b', 'AA')), (5, ('c', 'BB')), (5, ('c', 'CC'))]"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# We would like to see what we get from different kind of join operations \n",
    "rdd1.join(rdd2).collect()\n",
    "\n",
    "# with join we get the intersection of two sets "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[(1, ('a', 'AA')),\n",
       " (1, ('b', 'AA')),\n",
       " (2, ('d', None)),\n",
       " (3, ('e', None)),\n",
       " (5, ('c', 'BB')),\n",
       " (5, ('c', 'CC'))]"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "rdd1.leftOuterJoin(rdd2).collect()\n",
    "\n",
    "# with leftOuterJoin we get the intersection of two sets plus what is on the first left side set"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[(1, ('a', 'AA')),\n",
       " (1, ('b', 'AA')),\n",
       " (5, ('c', 'BB')),\n",
       " (5, ('c', 'CC')),\n",
       " (6, (None, 'DD'))]"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "rdd1.rightOuterJoin(rdd2).collect()\n",
    "# similar to the above but from the other side. "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[(1, ('a', 'AA')),\n",
       " (1, ('b', 'AA')),\n",
       " (2, ('d', None)),\n",
       " (3, ('e', None)),\n",
       " (5, ('c', 'BB')),\n",
       " (5, ('c', 'CC')),\n",
       " (6, (None, 'DD'))]"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "\n",
    "rdd1.fullOuterJoin(rdd2).collect()\n",
    "\n",
    "# full outer join will give all of the keys"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[('a', 'AA'), ('b', 'AA'), ('c', 'BB'), ('c', 'CC')]"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# In case that you do not need the keys after the join operations \n",
    "# you can remove the keys using a map transformation. \n",
    "\n",
    "\n",
    "result = rdd1.join(rdd2).map(lambda x: (x[1][0], x[1][1]))\n",
    "result.collect()\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}