From 1e956a164958128f8d39c10fac2ab55ef7d46e24 Mon Sep 17 00:00:00 2001 From: Neil Cotie Date: Wed, 18 Oct 2023 14:22:35 +0200 Subject: [PATCH 1/8] First version for set, dict attributes --- osmnx/io.py | 29 ++++++++++++++++++++++++++++- 1 file changed, 28 insertions(+), 1 deletion(-) diff --git a/osmnx/io.py b/osmnx/io.py index 151865a09..e7e0eafa9 100644 --- a/osmnx/io.py +++ b/osmnx/io.py @@ -419,8 +419,25 @@ def _convert_node_attr_types(G, dtypes=None): Returns ------- G : networkx.MultiDiGraph + + Modified to include detection of list, dict, and set type + node attribute strings and handle their translation into + python objects of the correct type. """ for _, data in G.nodes(data=True): + # first, eval stringified lists, dicts or sets to convert them to objects + # edge attributes might have a single value, or a list if simplified + for attr, value in data.items(): + # check for stringified lists + if value.startswith("[") and value.endswith("]"): + with contextlib.suppress(SyntaxError, ValueError): + data[attr] = ast.literal_eval(value) + # check for stringified dicts or sets: both can be converted + # to python objects using ast.literal_eval() + if value.startswith("{") and value.endswith("}"): + with contextlib.suppress(SyntaxError, ValueError): + data[attr] = ast.literal_eval(value) + for attr in data.keys() & dtypes.keys(): data[attr] = dtypes[attr](data[attr]) return G @@ -440,18 +457,28 @@ def _convert_edge_attr_types(G, dtypes=None): Returns ------- G : networkx.MultiDiGraph + + Modified to include detection of list, dict, and set type + node attribute strings and handle their translation into + python objects of the correct type. """ # for each edge in the graph, eval attribute value lists and convert types for _, _, data in G.edges(data=True, keys=False): # remove extraneous "id" attribute added by graphml saving data.pop("id", None) - # first, eval stringified lists to convert them to list objects + # first, eval stringified lists, dicts or sets to convert them to objects # edge attributes might have a single value, or a list if simplified for attr, value in data.items(): + # check for stringified lists if value.startswith("[") and value.endswith("]"): with contextlib.suppress(SyntaxError, ValueError): data[attr] = ast.literal_eval(value) + # check for stringified dicts or sets: both can be converted + # to python objects using ast.literal_eval() + if value.startswith("{") and value.endswith("}"): + with contextlib.suppress(SyntaxError, ValueError): + data[attr] = ast.literal_eval(value) # next, convert attribute value types if attribute appears in dtypes for attr in data.keys() & dtypes.keys(): From 22b719340067ec0cab83ca3e9e7faad3d80602df Mon Sep 17 00:00:00 2001 From: Neil Cotie Date: Thu, 19 Oct 2023 09:43:26 +0200 Subject: [PATCH 2/8] Removed comments, compressed condition clause to single check --- osmnx/io.py | 25 ++++--------------------- 1 file changed, 4 insertions(+), 21 deletions(-) diff --git a/osmnx/io.py b/osmnx/io.py index e7e0eafa9..985c7f0c2 100644 --- a/osmnx/io.py +++ b/osmnx/io.py @@ -419,22 +419,14 @@ def _convert_node_attr_types(G, dtypes=None): Returns ------- G : networkx.MultiDiGraph - - Modified to include detection of list, dict, and set type - node attribute strings and handle their translation into - python objects of the correct type. """ for _, data in G.nodes(data=True): # first, eval stringified lists, dicts or sets to convert them to objects # edge attributes might have a single value, or a list if simplified for attr, value in data.items(): # check for stringified lists - if value.startswith("[") and value.endswith("]"): - with contextlib.suppress(SyntaxError, ValueError): - data[attr] = ast.literal_eval(value) - # check for stringified dicts or sets: both can be converted - # to python objects using ast.literal_eval() - if value.startswith("{") and value.endswith("}"): + if ((value.startswith("[") and value.endswith("]")) + or (value.startswith("{") and value.endswith("}"))): with contextlib.suppress(SyntaxError, ValueError): data[attr] = ast.literal_eval(value) @@ -457,10 +449,6 @@ def _convert_edge_attr_types(G, dtypes=None): Returns ------- G : networkx.MultiDiGraph - - Modified to include detection of list, dict, and set type - node attribute strings and handle their translation into - python objects of the correct type. """ # for each edge in the graph, eval attribute value lists and convert types for _, _, data in G.edges(data=True, keys=False): @@ -470,13 +458,8 @@ def _convert_edge_attr_types(G, dtypes=None): # first, eval stringified lists, dicts or sets to convert them to objects # edge attributes might have a single value, or a list if simplified for attr, value in data.items(): - # check for stringified lists - if value.startswith("[") and value.endswith("]"): - with contextlib.suppress(SyntaxError, ValueError): - data[attr] = ast.literal_eval(value) - # check for stringified dicts or sets: both can be converted - # to python objects using ast.literal_eval() - if value.startswith("{") and value.endswith("}"): + if ((value.startswith("[") and value.endswith("]")) + or (value.startswith("{") and value.endswith("}"))): with contextlib.suppress(SyntaxError, ValueError): data[attr] = ast.literal_eval(value) From f28d8b94e9e768d4c4c9c87e802958189a601d57 Mon Sep 17 00:00:00 2001 From: Neil Cotie Date: Thu, 19 Oct 2023 09:51:02 +0200 Subject: [PATCH 3/8] Missed two trailing spaces in previous --- osmnx/io.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/osmnx/io.py b/osmnx/io.py index 985c7f0c2..445036100 100644 --- a/osmnx/io.py +++ b/osmnx/io.py @@ -425,7 +425,7 @@ def _convert_node_attr_types(G, dtypes=None): # edge attributes might have a single value, or a list if simplified for attr, value in data.items(): # check for stringified lists - if ((value.startswith("[") and value.endswith("]")) + if ((value.startswith("[") and value.endswith("]")) or (value.startswith("{") and value.endswith("}"))): with contextlib.suppress(SyntaxError, ValueError): data[attr] = ast.literal_eval(value) @@ -458,7 +458,7 @@ def _convert_edge_attr_types(G, dtypes=None): # first, eval stringified lists, dicts or sets to convert them to objects # edge attributes might have a single value, or a list if simplified for attr, value in data.items(): - if ((value.startswith("[") and value.endswith("]")) + if ((value.startswith("[") and value.endswith("]")) or (value.startswith("{") and value.endswith("}"))): with contextlib.suppress(SyntaxError, ValueError): data[attr] = ast.literal_eval(value) From 3dd168b902090c058e14a0425d381afa663f3a0a Mon Sep 17 00:00:00 2001 From: Neil Cotie Date: Thu, 19 Oct 2023 16:18:31 +0200 Subject: [PATCH 4/8] After lint testing with automatic change --- osmnx/io.py | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/osmnx/io.py b/osmnx/io.py index 445036100..5d9ba43ab 100644 --- a/osmnx/io.py +++ b/osmnx/io.py @@ -425,11 +425,12 @@ def _convert_node_attr_types(G, dtypes=None): # edge attributes might have a single value, or a list if simplified for attr, value in data.items(): # check for stringified lists - if ((value.startswith("[") and value.endswith("]")) - or (value.startswith("{") and value.endswith("}"))): + if (value.startswith("[") and value.endswith("]")) or ( + value.startswith("{") and value.endswith("}") + ): with contextlib.suppress(SyntaxError, ValueError): data[attr] = ast.literal_eval(value) - + for attr in data.keys() & dtypes.keys(): data[attr] = dtypes[attr](data[attr]) return G @@ -458,8 +459,9 @@ def _convert_edge_attr_types(G, dtypes=None): # first, eval stringified lists, dicts or sets to convert them to objects # edge attributes might have a single value, or a list if simplified for attr, value in data.items(): - if ((value.startswith("[") and value.endswith("]")) - or (value.startswith("{") and value.endswith("}"))): + if (value.startswith("[") and value.endswith("]")) or ( + value.startswith("{") and value.endswith("}") + ): with contextlib.suppress(SyntaxError, ValueError): data[attr] = ast.literal_eval(value) From 7a024ac690ad3a111721417f781a63092a7848b9 Mon Sep 17 00:00:00 2001 From: Neil Cotie Date: Thu, 19 Oct 2023 18:03:01 +0200 Subject: [PATCH 5/8] Comment corrected --- osmnx/io.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/osmnx/io.py b/osmnx/io.py index 5d9ba43ab..6925341b5 100644 --- a/osmnx/io.py +++ b/osmnx/io.py @@ -422,7 +422,7 @@ def _convert_node_attr_types(G, dtypes=None): """ for _, data in G.nodes(data=True): # first, eval stringified lists, dicts or sets to convert them to objects - # edge attributes might have a single value, or a list if simplified + # node attributes might have a single value, or a list if simplified for attr, value in data.items(): # check for stringified lists if (value.startswith("[") and value.endswith("]")) or ( From bdf15212c79cfda38e5acf0e3c2d0dbfbd5a0e30 Mon Sep 17 00:00:00 2001 From: Neil Cotie Date: Fri, 20 Oct 2023 11:26:43 +0200 Subject: [PATCH 6/8] Added code testing for list, set and dict type attributes for nodes and edges --- tests/test_osmnx.py | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/tests/test_osmnx.py b/tests/test_osmnx.py index ebd63e8e3..32150e118 100755 --- a/tests/test_osmnx.py +++ b/tests/test_osmnx.py @@ -474,6 +474,25 @@ def test_graph_save_load(): edge_attrs = {n: bool(b) for n, b in zip(G.edges, bools)} nx.set_edge_attributes(G, edge_attrs, attr_name) + # create list, set, and dict attributes for nodes and edges + rand_ints_nodes = np.random.randint(0, 10, len(G.nodes)) + rand_ints_edges = np.random.randint(0, 10, len(G.edges)) + list_name = "test_list" + list_node_attrs = {n: [n, r] for n, r in zip(G.nodes, rand_ints_nodes)} + nx.set_node_attributes(G, list_node_attrs, list_name) + list_edge_attrs = {e: [e, r] for e, r in zip(G.edges, rand_ints_edges)} + nx.set_edge_attributes(G, list_edge_attrs, list_name) + set_name = "test_set" + set_node_attrs = {n: {n, r} for n, r in zip(G.nodes, rand_ints_nodes)} + nx.set_node_attributes(G, set_node_attrs, set_name) + set_edge_attrs = {e: {e, r} for e, r in zip(G.edges, rand_ints_edges)} + nx.set_edge_attributes(G, set_edge_attrs, set_name) + dict_name = "test_dict" + dict_node_attrs = {n: {n: r} for n, r in zip(G.nodes, rand_ints_nodes)} + nx.set_node_attributes(G, dict_node_attrs, dict_name) + dict_edge_attrs = {e: {e: r} for e, r in zip(G.edges, rand_ints_edges)} + nx.set_edge_attributes(G, dict_edge_attrs, dict_name) + # save/load graph as graphml file ox.save_graphml(G, gephi=True) ox.save_graphml(G, gephi=False) From 2fc7823b1344125f5d934f3168426731c5749a20 Mon Sep 17 00:00:00 2001 From: Geoff Boeing Date: Fri, 27 Oct 2023 13:43:15 -0700 Subject: [PATCH 7/8] update changelog --- CHANGELOG.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index d257074ee..8ee3bcbfe 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,7 +2,8 @@ ## Unreleased -- refer to latitude and longitude parameters as lat and lon consistently across package (#1068 #1069) +- fix references to latitude and longitude parameters as lat and lon consistently across package (#1068 #1069) +- fix handling dict and set attribute types when reloading GraphML files (#1075 #1077) ## 1.7.0 (2023-10-11) From 3a3ee04239ad09dea20d3577983178f70105f4b2 Mon Sep 17 00:00:00 2001 From: Geoff Boeing Date: Fri, 27 Oct 2023 13:43:22 -0700 Subject: [PATCH 8/8] code clean up --- osmnx/io.py | 8 ++++---- tests/test_osmnx.py | 24 +++++++++++------------- 2 files changed, 15 insertions(+), 17 deletions(-) diff --git a/osmnx/io.py b/osmnx/io.py index 6925341b5..5d4325228 100644 --- a/osmnx/io.py +++ b/osmnx/io.py @@ -421,10 +421,9 @@ def _convert_node_attr_types(G, dtypes=None): G : networkx.MultiDiGraph """ for _, data in G.nodes(data=True): - # first, eval stringified lists, dicts or sets to convert them to objects - # node attributes might have a single value, or a list if simplified + # first, eval stringified lists, dicts, or sets to convert them to objects + # lists, dicts, or sets would be custom attribute types added by a user for attr, value in data.items(): - # check for stringified lists if (value.startswith("[") and value.endswith("]")) or ( value.startswith("{") and value.endswith("}") ): @@ -456,8 +455,9 @@ def _convert_edge_attr_types(G, dtypes=None): # remove extraneous "id" attribute added by graphml saving data.pop("id", None) - # first, eval stringified lists, dicts or sets to convert them to objects + # first, eval stringified lists, dicts, or sets to convert them to objects # edge attributes might have a single value, or a list if simplified + # dicts or sets would be custom attribute types added by a user for attr, value in data.items(): if (value.startswith("[") and value.endswith("]")) or ( value.startswith("{") and value.endswith("}") diff --git a/tests/test_osmnx.py b/tests/test_osmnx.py index 32150e118..e4c1d280c 100755 --- a/tests/test_osmnx.py +++ b/tests/test_osmnx.py @@ -441,6 +441,8 @@ def test_endpoints(): def test_graph_save_load(): """Test saving/loading graphs to/from disk.""" + fp = Path(ox.settings.data_folder) / "graph.graphml" + # save graph as shapefile and geopackage G = ox.graph_from_point(location_point, dist=500, network_type="drive") ox.save_graph_shapefile(G, directed=True) @@ -477,29 +479,25 @@ def test_graph_save_load(): # create list, set, and dict attributes for nodes and edges rand_ints_nodes = np.random.randint(0, 10, len(G.nodes)) rand_ints_edges = np.random.randint(0, 10, len(G.edges)) - list_name = "test_list" list_node_attrs = {n: [n, r] for n, r in zip(G.nodes, rand_ints_nodes)} - nx.set_node_attributes(G, list_node_attrs, list_name) + nx.set_node_attributes(G, list_node_attrs, "test_list") list_edge_attrs = {e: [e, r] for e, r in zip(G.edges, rand_ints_edges)} - nx.set_edge_attributes(G, list_edge_attrs, list_name) - set_name = "test_set" + nx.set_edge_attributes(G, list_edge_attrs, "test_list") set_node_attrs = {n: {n, r} for n, r in zip(G.nodes, rand_ints_nodes)} - nx.set_node_attributes(G, set_node_attrs, set_name) + nx.set_node_attributes(G, set_node_attrs, "test_set") set_edge_attrs = {e: {e, r} for e, r in zip(G.edges, rand_ints_edges)} - nx.set_edge_attributes(G, set_edge_attrs, set_name) - dict_name = "test_dict" + nx.set_edge_attributes(G, set_edge_attrs, "test_set") dict_node_attrs = {n: {n: r} for n, r in zip(G.nodes, rand_ints_nodes)} - nx.set_node_attributes(G, dict_node_attrs, dict_name) + nx.set_node_attributes(G, dict_node_attrs, "test_dict") dict_edge_attrs = {e: {e: r} for e, r in zip(G.edges, rand_ints_edges)} - nx.set_edge_attributes(G, dict_edge_attrs, dict_name) + nx.set_edge_attributes(G, dict_edge_attrs, "test_dict") # save/load graph as graphml file ox.save_graphml(G, gephi=True) ox.save_graphml(G, gephi=False) - ox.save_graphml(G, gephi=False, filepath=Path(ox.settings.data_folder) / "graph.graphml") - filepath = Path(ox.settings.data_folder) / "graph.graphml" + ox.save_graphml(G, gephi=False, filepath=fp) G2 = ox.load_graphml( - filepath, + fp, graph_dtypes={attr_name: ox.io._convert_bool_string}, node_dtypes={attr_name: ox.io._convert_bool_string}, edge_dtypes={attr_name: ox.io._convert_bool_string}, @@ -524,7 +522,7 @@ def test_graph_save_load(): # test custom data types nd = {"osmid": str} ed = {"length": str, "osmid": float} - G2 = ox.load_graphml(filepath, node_dtypes=nd, edge_dtypes=ed) + G2 = ox.load_graphml(fp, node_dtypes=nd, edge_dtypes=ed) # test loading graphml from a file stream file_bytes = Path.open(Path("tests/input_data/short.graphml"), "rb").read()