extract_metro_network_from_osm.ipynb

{
 "cells": [
  {
   "metadata": {
    "ExecuteTime": {
     "end_time": "2024-07-05T08:01:50.369099Z",
     "start_time": "2024-07-05T08:01:50.364081Z"
    }
   },
   "cell_type": "code",
   "source": [
    "import os\n",
    "import osmnx as ox\n",
    "import networkx as nx\n",
    "\n",
    "import pandas as pd\n",
    "import geopandas as gpd\n",
    "\n",
    "from lxml import etree\n",
    "from bs4 import BeautifulSoup"
   ],
   "id": "711fbc8c5a926f00",
   "outputs": [],
   "execution_count": 140
  },
  {
   "metadata": {
    "ExecuteTime": {
     "end_time": "2024-07-05T06:48:04.682753Z",
     "start_time": "2024-07-05T06:48:04.663769Z"
    }
   },
   "cell_type": "code",
   "source": "osm_path = r'C:\\Users\\Wei Zhou\\OneDrive - National University of Singapore\\Pesearch Project\\Traffic dataset\\Singapore Geodatabase\\OSM\\singapore_osm\\singapore-2024-07-02.osm'",
   "id": "98203f99b945d246",
   "outputs": [],
   "execution_count": 4
  },
  {
   "metadata": {
    "ExecuteTime": {
     "end_time": "2024-07-05T06:48:19.936641Z",
     "start_time": "2024-07-05T06:48:05.042847Z"
    }
   },
   "cell_type": "code",
   "source": "tree = etree.parse(osm_path)",
   "id": "a5c551adda7314cb",
   "outputs": [],
   "execution_count": 5
  },
  {
   "metadata": {
    "ExecuteTime": {
     "end_time": "2024-07-05T08:00:18.900682Z",
     "start_time": "2024-07-05T08:00:18.887747Z"
    }
   },
   "cell_type": "code",
   "source": [
    "def get_relation_member(tree, osm_id):\n",
    "    \n",
    "    element = tree.getroot()\n",
    "\n",
    "    # find the root \"relation\" based on osm id\n",
    "    # xpath parser <tag_name attribute=value/>\n",
    "    relation = element.findall(f\"relation[@id='{osm_id}']\")\n",
    "    # assert len(relation) == 1, 'The relation id for PT system should be unique, but found ' + str(len(relation))\n",
    "    # the \"relation\" should be unique\n",
    "    relation = relation[0]\n",
    "    \n",
    "    relation_string = etree.tostring(relation, pretty_print=False)\n",
    "    \n",
    "    soup = BeautifulSoup(relation_string, 'xml')\n",
    "    # members of \"relation\"\n",
    "    member = []\n",
    "    for m in soup.find_all('member'):\n",
    "        if not(m.get('ref', None) is None):\n",
    "            member.append(\n",
    "                {'osm_id' : m.get('ref', None),\n",
    "                 'type'   : m.get('type', None) })\n",
    "    \n",
    "    # tags of \"relation\", contain information\n",
    "    info = {\n",
    "        'osm_id'        : osm_id,\n",
    "        'member_osm_id' : [v['osm_id'] for v in member],\n",
    "        'member_type'   : [v['type'] for v in member], \n",
    "        'member_number' : len(member)}\n",
    "    \n",
    "    for t in soup.find_all('tag'):\n",
    "        if not(t.get('k', None) is None):\n",
    "            info[t.get('k', 'Null')] = t.get('v', 'Null')\n",
    "    \n",
    "    return info\n",
    "# ======================================================================================\n"
   ],
   "id": "b5baceacc9da8ca2",
   "outputs": [],
   "execution_count": 137
  },
  {
   "metadata": {
    "ExecuteTime": {
     "end_time": "2024-07-05T07:44:14.852257Z",
     "start_time": "2024-07-05T07:44:14.807459Z"
    }
   },
   "cell_type": "code",
   "source": [
    "pt_system_osm_id = \"3149677\"\n",
    "\n",
    "info = get_relation_member(tree, osm_id=pt_system_osm_id, member_osm_type='relation')\n",
    "\n",
    "member_info = {}\n",
    "for member_osm_id, member_osm_type in zip(info['member_osm_id'].split(','), info['member_osm_type'].split(',')):\n",
    "    member_info[member_osm_id] = get_relation_member(tree, member_osm_id, member_osm_type)"
   ],
   "id": "6eb196b356f62fff",
   "outputs": [
    {
     "ename": "TypeError",
     "evalue": "get_relation_member() got an unexpected keyword argument 'member_osm_type'",
     "output_type": "error",
     "traceback": [
      "\u001B[1;31m---------------------------------------------------------------------------\u001B[0m",
      "\u001B[1;31mTypeError\u001B[0m                                 Traceback (most recent call last)",
      "Cell \u001B[1;32mIn[131], line 3\u001B[0m\n\u001B[0;32m      1\u001B[0m pt_system_osm_id \u001B[38;5;241m=\u001B[39m \u001B[38;5;124m\"\u001B[39m\u001B[38;5;124m3149677\u001B[39m\u001B[38;5;124m\"\u001B[39m\n\u001B[1;32m----> 3\u001B[0m info \u001B[38;5;241m=\u001B[39m \u001B[43mget_relation_member\u001B[49m\u001B[43m(\u001B[49m\u001B[43mtree\u001B[49m\u001B[43m,\u001B[49m\u001B[43m \u001B[49m\u001B[43mosm_id\u001B[49m\u001B[38;5;241;43m=\u001B[39;49m\u001B[43mpt_system_osm_id\u001B[49m\u001B[43m,\u001B[49m\u001B[43m \u001B[49m\u001B[43mmember_osm_type\u001B[49m\u001B[38;5;241;43m=\u001B[39;49m\u001B[38;5;124;43m'\u001B[39;49m\u001B[38;5;124;43mrelation\u001B[39;49m\u001B[38;5;124;43m'\u001B[39;49m\u001B[43m)\u001B[49m\n\u001B[0;32m      5\u001B[0m member_info \u001B[38;5;241m=\u001B[39m {}\n\u001B[0;32m      6\u001B[0m \u001B[38;5;28;01mfor\u001B[39;00m member_osm_id, member_osm_type \u001B[38;5;129;01min\u001B[39;00m \u001B[38;5;28mzip\u001B[39m(info[\u001B[38;5;124m'\u001B[39m\u001B[38;5;124mmember_osm_id\u001B[39m\u001B[38;5;124m'\u001B[39m]\u001B[38;5;241m.\u001B[39msplit(\u001B[38;5;124m'\u001B[39m\u001B[38;5;124m,\u001B[39m\u001B[38;5;124m'\u001B[39m), info[\u001B[38;5;124m'\u001B[39m\u001B[38;5;124mmember_osm_type\u001B[39m\u001B[38;5;124m'\u001B[39m]\u001B[38;5;241m.\u001B[39msplit(\u001B[38;5;124m'\u001B[39m\u001B[38;5;124m,\u001B[39m\u001B[38;5;124m'\u001B[39m)):\n",
      "\u001B[1;31mTypeError\u001B[0m: get_relation_member() got an unexpected keyword argument 'member_osm_type'"
     ]
    }
   ],
   "execution_count": 131
  },
  {
   "metadata": {
    "ExecuteTime": {
     "end_time": "2024-07-05T07:36:45.889017Z",
     "start_time": "2024-07-05T07:36:44.752225Z"
    }
   },
   "cell_type": "code",
   "source": [
    "def retrieve_metro_relation(xml_tree, osm_id):\n",
    "    '''\n",
    "    \n",
    "    '''\n",
    "    "
   ],
   "id": "7f1d7c357fd68bac",
   "outputs": [],
   "execution_count": 124
  },
  {
   "metadata": {
    "ExecuteTime": {
     "end_time": "2024-07-05T10:56:43.112568Z",
     "start_time": "2024-07-05T10:56:43.070744Z"
    }
   },
   "cell_type": "code",
   "source": [
    "def _add_tag_info(bs4soup, detail):\n",
    "\n",
    "    detail['detail'] = {}\n",
    "    \n",
    "    for elem in bs4soup.find_all('tag'):\n",
    "        if not( elem.get('k', None) is None ):\n",
    "            detail['detail'][elem.get('k', None)] = elem.get('v', None)\n",
    "    \n",
    "    return detail\n",
    "# ======================================================================\n",
    "def _parser_osm_relation_detail(bs4soup):\n",
    "    detail = {'member' : []} \n",
    "    \n",
    "    order_ix = 0\n",
    "    for elem in bs4soup.find_all('member'):\n",
    "        if not( elem.get('ref', None) is None ) :\n",
    "            detail['member'].append(\n",
    "                (elem.get('ref'), {\n",
    "                    'ix'   : order_ix,\n",
    "                    'type' : elem.get('type', None) } )\n",
    "            )\n",
    "            order_ix = order_ix + 1\n",
    "\n",
    "    # detail['member_number'] = len(detail['member'])\n",
    "    \n",
    "    # tags\n",
    "    detail = _add_tag_info(bs4soup, detail)\n",
    "    \n",
    "    return detail\n",
    "# =============================================================================\n",
    "def _parser_osm_relation_route_detail(bs4soup):\n",
    "    \n",
    "    detail = { 'member' : {\n",
    "        'node' : [],\n",
    "        'way'  : [] }}\n",
    "\n",
    "    # node\n",
    "    order_node_ix = 0\n",
    "    order_way_ix = 0\n",
    "    for elem in bs4soup.find_all('member'):\n",
    "        if not( elem.get('ref', None) is None ) and not (elem.get('type', None) is None) :\n",
    "            type = elem.get('type', None) \n",
    "            \n",
    "            if type == 'node':\n",
    "                detail['member']['node'].append(\n",
    "                    (elem.get('ref'), {\n",
    "                    'ix'   : order_node_ix,\n",
    "                    'role' : elem.get('role', None) } )\n",
    "                )\n",
    "                order_node_ix = order_node_ix + 1\n",
    "                \n",
    "            elif type == 'way':\n",
    "                detail['member']['way'].append(\n",
    "                    (elem.get('ref'), {\n",
    "                        'ix'   : order_way_ix,\n",
    "                        'role' : elem.get('role', None) } ) \n",
    "                )\n",
    "                order_way_ix = order_way_ix + 1\n",
    "            \n",
    "            else:\n",
    "                continue\n",
    "                \n",
    "    # detail['node_number'] = len(detail['member']['node'])\n",
    "    # detail['way_number']  = len(detail['member']['way'])\n",
    "    \n",
    "    # tags\n",
    "    detail = _add_tag_info(bs4soup, detail)\n",
    "\n",
    "    return detail\n",
    "# =============================================================================\n",
    "def _parser_route_stop_node(bs4soup):\n",
    "\n",
    "    bs4soup = bs4soup.find('node')\n",
    "    \n",
    "    detail = {\n",
    "        'lat' : bs4soup.get('lat', None),\n",
    "        'lon' : bs4soup.get('lon', None)  }\n",
    "    \n",
    "    # tags\n",
    "    detail = _add_tag_info(bs4soup, detail)\n",
    "    \n",
    "    return detail\n",
    "# =============================================================================\n",
    "def _parser_way(bs4soup):\n",
    "    \n",
    "    detail = {'node' : [] }\n",
    "    \n",
    "    ix = 0\n",
    "    for elem in bs4soup.find_all('nd'):\n",
    "        detail['node'].append( (\n",
    "            elem.get('ref'), {\n",
    "            'ix'  : ix })\n",
    "        )\n",
    "        ix = ix + 1\n",
    "    \n",
    "    # tags\n",
    "    detail = _add_tag_info(bs4soup, detail)\n",
    "    \n",
    "    return detail\n",
    "# =============================================================================\n",
    "def _parser_way(bs4soup):\n",
    "\n",
    "    bs4soup = bs4soup.find('node')\n",
    "    \n",
    "    detail = {\n",
    "        'lat' : bs4soup.get('lat', None),\n",
    "        'lon' : bs4soup.get('lon', None)  }\n",
    "    \n",
    "    return detail\n",
    "# =============================================================================\n",
    "def _get_xml_bs4soup(xml_tree, osm_id, dtype):\n",
    "    \n",
    "    element = tree.getroot()\n",
    "    # find the root \"relation\" based on osm id\n",
    "    # xpath parser <tag_name attribute=value/>\n",
    "    relation = element.findall(f\"{dtype}[@id='{osm_id}']\")\n",
    "    # assert len(relation) == 1, 'The relation id for PT system should be unique, but found ' + str(len(relation))\n",
    "    # the \"relation\" should be unique\n",
    "    relation = relation[0]\n",
    "    \n",
    "    relation_string = etree.tostring(relation, pretty_print=False)\n",
    "\n",
    "    soup = BeautifulSoup(relation_string, 'xml')\n",
    "    \n",
    "    return soup\n",
    "# =============================================================================\n",
    "def parser_osm_detail(bs4soup, parser_type):\n",
    "    if parser_type == 'relation':\n",
    "        detail = _parser_osm_relation_detail(soup)\n",
    "    elif parser_type == 'route':\n",
    "        detail = _parser_osm_relation_route_detail(soup)\n",
    "    elif parser_type == 'stop_node':\n",
    "        detail = _parser_route_stop_node(soup)\n",
    "    elif parser_type == 'way':\n",
    "        detail = _parser_way(soup)\n",
    "    else:\n",
    "        return None\n",
    "    return detail\n",
    "# =============================================================================\n",
    "def _decide_parser_type(bs4soup):\n",
    "    \n",
    "    for child in bs4soup.children:\n",
    "        dtype = child.name\n",
    "        break\n",
    "    \n",
    "    if dtype == 'relation':\n",
    "        if child.get('type', None) == 'route':\n",
    "            parser_type = 'route'\n",
    "        else:\n",
    "            parser_type = 'relation'\n",
    "            \n",
    "    elif dtype == 'node':\n",
    "        parser_type = 'stop_node'\n",
    "    elif dtype == 'way':\n",
    "        parser_type = 'way'\n",
    "        \n",
    "    return parser_type"
   ],
   "id": "356524bcb6c9ad6a",
   "outputs": [],
   "execution_count": 245
  },
  {
   "metadata": {},
   "cell_type": "markdown",
   "source": [
    "```\n",
    "--relation\n",
    "----relation\n",
    "------relation\n",
    "--------route\n",
    "----------node\n",
    "----------way\n",
    "------------node\n",
    "```"
   ],
   "id": "1f1e59c834431a1a"
  },
  {
   "metadata": {
    "ExecuteTime": {
     "end_time": "2024-07-05T10:56:51.813287Z",
     "start_time": "2024-07-05T10:56:51.205974Z"
    }
   },
   "cell_type": "code",
   "source": [
    "\n",
    "\n",
    "osm_id = '3149677'\n",
    "dtype = 'relation'\n",
    "\n",
    "structure = nx.DiGraph()\n",
    "\n",
    "soup = _get_xml_bs4soup(xml_tree=tree, osm_id=osm_id, dtype=dtype)\n",
    "parser_type = _decide_parser_type(soup)\n",
    "print(parser_type)\n",
    "\n",
    "detail = parser_osm_detail(soup, parser_type)\n",
    "\n",
    "\n",
    "graph = add_children_node(structure, osm_id, detail, parser_type)"
   ],
   "id": "f06591456b675579",
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "relation\n"
     ]
    }
   ],
   "execution_count": 247
  },
  {
   "metadata": {
    "ExecuteTime": {
     "end_time": "2024-07-05T10:56:48.885239Z",
     "start_time": "2024-07-05T10:56:48.870101Z"
    }
   },
   "cell_type": "code",
   "source": [
    "def add_children_node(graph, parent_node, detail, parser_type):\n",
    "\n",
    "    graph.add_node(parent_node, **detail['detail'])\n",
    "    \n",
    "    if parser_type == 'relation':\n",
    "        graph.add_nodes_from(detail['member'])\n",
    "        for node in detail['member']:\n",
    "            graph.add_edge(parent_node, node[0])\n",
    "    \n",
    "    elif parser_type == 'route':\n",
    "        graph.add_nodes_from(detail['member']['node'])\n",
    "        for node in detail['member']:\n",
    "            graph.add_edge(parent_node, node[0])\n",
    "\n",
    "        graph.add_nodes_from(detail['member']['way'])\n",
    "        for node in detail['member']:\n",
    "            graph.add_edge(parent_node, node[0])\n",
    "    \n",
    "    return graph"
   ],
   "id": "51a8deb4eb5d6744",
   "outputs": [],
   "execution_count": 246
  },
  {
   "metadata": {
    "ExecuteTime": {
     "end_time": "2024-07-05T10:56:56.867905Z",
     "start_time": "2024-07-05T10:56:56.854967Z"
    }
   },
   "cell_type": "code",
   "source": "graph.nodes(data=True)",
   "id": "5c9c19f31c463796",
   "outputs": [
    {
     "data": {
      "text/plain": [
       "NodeDataView({'3149677': {'name': 'Singapore Public Transportation', 'network': 'Singapore', 'type': 'network'}, '1152539': {'ix': 0, 'type': 'relation'}, '3149676': {'ix': 1, 'type': 'relation'}})"
      ]
     },
     "execution_count": 248,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "execution_count": 248
  },
  {
   "metadata": {},
   "cell_type": "code",
   "outputs": [],
   "execution_count": null,
   "source": "",
   "id": "94954dc6bfc13fc0"
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}