From 602efb84b570e12b7b115c505b3db6d7e92a0895 Mon Sep 17 00:00:00 2001 From: Martijn Visser Date: Tue, 22 Oct 2024 15:41:25 +0200 Subject: [PATCH 01/13] Update lock file for new dependencies --- pixi.lock | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/pixi.lock b/pixi.lock index e8534b9c..2d7e4189 100644 --- a/pixi.lock +++ b/pixi.lock @@ -14706,10 +14706,12 @@ packages: name: peilbeheerst-model version: 0.1.0 path: src/peilbeheerst_model - sha256: e8c270f68d683c802990a8bb905cd665fd327b33e1d17bf3e01dfe9c49ec335e + sha256: 9ec22544800123361c4449542c548c1e3739a3f8d9d5bbe2f91a8d50d304fac5 requires_dist: + - fiona - geopandas - matplotlib + - networkx - numpy - pandas - pydantic From d64dde4c84ee6c70a1782b4afbaee3b35edff1df Mon Sep 17 00:00:00 2001 From: Martijn Visser Date: Tue, 22 Oct 2024 17:23:26 +0200 Subject: [PATCH 02/13] Run AGV shortest_path and crossings_to_ribasim --- .../Shortest_path/10_shortest_path_AGV.py | 50 ++- .../crossings_to_ribasim/AmstelGooienVecht.py | 116 +++++++ .../peilbeheerst_model/__init__.py | 15 +- .../crossings_to_ribasim.py | 294 +++--------------- .../peilbeheerst_model/shortest_path.py | 2 +- src/ribasim_nl/ribasim_nl/__init__.py | 18 +- 6 files changed, 226 insertions(+), 269 deletions(-) create mode 100644 src/peilbeheerst_model/crossings_to_ribasim/AmstelGooienVecht.py diff --git a/src/peilbeheerst_model/Shortest_path/10_shortest_path_AGV.py b/src/peilbeheerst_model/Shortest_path/10_shortest_path_AGV.py index 1d161661..c0d89760 100644 --- a/src/peilbeheerst_model/Shortest_path/10_shortest_path_AGV.py +++ b/src/peilbeheerst_model/Shortest_path/10_shortest_path_AGV.py @@ -1,7 +1,47 @@ -from peilbeheerst_model import shortest_path_waterschap +# The AGV shortest path code is identical to the other water boards. +# Below the content of shortest_path_waterschappen is spelled out to be able to +# point to the right data from the cloud storage. + +import fiona +import geopandas as gpd +import pandas as pd +from shapely.wkt import dumps + +from peilbeheerst_model.shortest_path import shortest_path +from ribasim_nl import CloudStorage waterschap = "AmstelGooienVecht" -gdf_out = shortest_path_waterschap(waterschap) -gdf_out.to_file( - f"/DATAFOLDER/projects/4750_30/Data_shortest_path/{waterschap}/{waterschap}_shortest_path.gpkg", driver="GPKG" -) + +cloud = CloudStorage() +# cloud.download_verwerkt(waterschap) +# cloud.download_verwerkt("Rijkswaterstaat") +# cloud.download_basisgegevens() + +# %% +verwerkt_dir = cloud.joinpath(waterschap, "verwerkt") + +# Load Data +# Define crossings file path +data_path = verwerkt_dir / "crossings.gpkg" + +# Load crossings file +DATA = {L: gpd.read_file(data_path, layer=L) for L in fiona.listlayers(data_path)} + +# Select rhws + +# Select RHWS peilgebeied & calculate representative point +gdf_rhws = DATA["peilgebied"].loc[DATA["peilgebied"]["peilgebied_cat"] == 1].copy() +gdf_rhws["representative_point"] = gdf_rhws.representative_point() + +# Apply aggregation level based filter +gdf_cross = ( + DATA["crossings_hydroobject_filtered"].loc[DATA["crossings_hydroobject_filtered"]["agg_links_in_use"]].copy() +) # filter aggregation level + +gdf_crossings_out = shortest_path(waterschap, DATA, gdf_cross, gdf_rhws) +# Write final output +gdf_out = gpd.GeoDataFrame(pd.concat(gdf_crossings_out)) +gdf_out["shortest_path"] = gdf_out["shortest_path"].apply(lambda geom: dumps(geom) if geom is not None else None) +gdf_out.to_file(verwerkt_dir / "shortest_path.gpkg", driver="GPKG") + +# cloud.upload_verwerkt(waterschap) diff --git a/src/peilbeheerst_model/crossings_to_ribasim/AmstelGooienVecht.py b/src/peilbeheerst_model/crossings_to_ribasim/AmstelGooienVecht.py new file mode 100644 index 00000000..a7e7c520 --- /dev/null +++ b/src/peilbeheerst_model/crossings_to_ribasim/AmstelGooienVecht.py @@ -0,0 +1,116 @@ +# Modified version of the AGV part of 02_crossings_to_ribasim.py. + +# %% + +import pandas as pd +from ribasim import Model + +from peilbeheerst_model import CrossingsToRibasim, RibasimNetwork, waterschap_data +from ribasim_nl import CloudStorage + +# %% +waterschap = "AmstelGooienVecht" +waterschap_struct = waterschap_data[waterschap] + +cloud = CloudStorage() +verwerkt_dir = cloud.joinpath(waterschap, "verwerkt") +# cloud.download_verwerkt(waterschap) +# cloud.download_basisgegevens() + +# %% + +pd.set_option("display.max_columns", None) +# warnings.filterwarnings("ignore") + + +model_characteristics = { + # model description + "waterschap": "AmstelGooienVecht", + "modelname": "repro", + "modeltype": "boezemmodel", + # define paths + "path_postprocessed_data": verwerkt_dir / "postprocessed.gpkg", + "path_crossings": verwerkt_dir / "crossings.gpkg", + "path_boezem": verwerkt_dir / "shortest_path.gpkg", + "path_Pdrive": None, + # apply filters + "crossings_layer": "crossings_hydroobject_filtered", + "in_use": True, + "agg_links_in_use": True, + "agg_areas_in_use": True, + "aggregation": True, + # data storage settings + "write_goodcloud": False, # TODO + "write_checks": True, + # numerical settings + "solver": None, + "logging": None, + "starttime": "2024-01-01 00:00:00", + "endtime": "2024-01-02 00:00:00", +} + +waterboard = CrossingsToRibasim(model_characteristics=model_characteristics) + +post_processed_data, crossings = waterboard.read_files() +post_processed_data, crossings = waterboard.routing_processor(post_processed_data, crossings) +crossings = waterboard.assign_node_ids(crossings) +edges = waterboard.create_edges(crossings) +nodes, edges = waterboard.create_nodes(crossings, edges) +edges = waterboard.embed_boezems(edges, post_processed_data, crossings) + +# create individual model parts of the network +network = RibasimNetwork(nodes=nodes, edges=edges, model_characteristics=model_characteristics) + +edge = network.edge() +basin_node, basin_profile, basin_static, basin_state, basin_area = network.basin() +pump_node, pump_static = network.pump() +tabulated_rating_curve_node, tabulated_rating_curve_static = network.tabulated_rating_curve() +level_boundary_node, level_boundary_static = network.level_boundary() +flow_boundary_node, flow_boundary_static = network.flow_boundary() +manning_resistance_node, manning_resistance_static = network.manning_resistance() +terminal_node = network.terminal() + +# linear_resistance = network.linear_resistance() +# fractional_flow = network.fractional_flow() +# outlet = network.outlet() +# discrete_control = network.discrete_control() +# pid_control = network.pid_control() + +# insert the individual model modules in an actual model +model = Model(starttime=model_characteristics["starttime"], endtime=model_characteristics["endtime"], crs="EPSG:28992") + +model.edge.df = edge + +model.basin.node.df = basin_node +model.basin.profile = basin_profile +model.basin.static = basin_static +model.basin.state = basin_state +model.basin.area = basin_area + +model.pump.node.df = pump_node +model.pump.static = pump_static + +model.tabulated_rating_curve.node.df = tabulated_rating_curve_node +model.tabulated_rating_curve.static = tabulated_rating_curve_static + +model.manning_resistance.node.df = manning_resistance_node +model.manning_resistance.static = manning_resistance_static + +model.level_boundary.node.df = level_boundary_node +model.level_boundary.static = level_boundary_static + +model.flow_boundary.node.df = flow_boundary_node +model.flow_boundary.static = flow_boundary_static + +model.terminal.node.df = terminal_node + +# add checks and metadata +checks = network.check(model, post_processed_data=post_processed_data, crossings=crossings) +model = network.add_meta_data(model, checks, post_processed_data, crossings) + +# write the result +network.WriteResults(model=model, checks=checks) + +# %% + +# cloud.upload_verwerkt(waterschap) diff --git a/src/peilbeheerst_model/peilbeheerst_model/__init__.py b/src/peilbeheerst_model/peilbeheerst_model/__init__.py index 3fd1b2ed..58ba213f 100644 --- a/src/peilbeheerst_model/peilbeheerst_model/__init__.py +++ b/src/peilbeheerst_model/peilbeheerst_model/__init__.py @@ -1,7 +1,14 @@ __version__ = "0.1.0" -from peilbeheerst_model.parse_crossings import ParseCrossings -from peilbeheerst_model.shortest_path import shortest_path_waterschap -from peilbeheerst_model.waterschappen import waterschap_data +from .crossings_to_ribasim import CrossingsToRibasim, RibasimNetwork +from .parse_crossings import ParseCrossings +from .shortest_path import shortest_path_waterschap +from .waterschappen import waterschap_data -__all__ = ["ParseCrossings", "shortest_path_waterschap", "waterschap_data"] +__all__ = [ + "CrossingsToRibasim", + "ParseCrossings", + "RibasimNetwork", + "shortest_path_waterschap", + "waterschap_data", +] diff --git a/src/peilbeheerst_model/peilbeheerst_model/crossings_to_ribasim.py b/src/peilbeheerst_model/peilbeheerst_model/crossings_to_ribasim.py index fe4d5983..715c185e 100644 --- a/src/peilbeheerst_model/peilbeheerst_model/crossings_to_ribasim.py +++ b/src/peilbeheerst_model/peilbeheerst_model/crossings_to_ribasim.py @@ -1,6 +1,4 @@ import itertools -import os -import shutil import geopandas as gpd import numpy as np @@ -136,13 +134,14 @@ def routing_processor( # define the most representative point where the basins will be located. This is always within the peilgebied polygon. # save the original peilgebied_from and _to for boezem purposes later, before changing anything due to the aggregation - crossings["peilgebied_from_original"] = crossings["peilgebied_from"] - crossings["peilgebied_to_original"] = crossings["peilgebied_to"] + crossings["peilgebied_from_original"] = crossings["peilgebied_from"].copy() + crossings["peilgebied_to_original"] = crossings["peilgebied_to"].copy() if self.model_characteristics["aggregation"]: # not all peilgebieden are part of a aggregation area. Add the missing peilgebieden to the agg_area column, so no areas will be left out - crossings["agg_area_from"].fillna(crossings["peilgebied_from"], inplace=True) - crossings["agg_area_to"].fillna(crossings["peilgebied_to"], inplace=True) + crossings.fillna( + {"agg_area_from": crossings["peilgebied_from"], "agg_area_to": crossings["peilgebied_to"]}, inplace=True + ) # change administrative detailed peilgebieden to aggregation areas crossings["peilgebied_from"] = crossings["agg_area_from"] @@ -893,8 +892,6 @@ def edge(self): edge["name"] = None edge["subnetwork_id"] = None edge["geometry"] = self.edges["line_geom"] - - # comply to Ribasim 2024.11 edge = edge.reset_index(drop=True) edge["edge_id"] = edge.index.astype(int) edge = edge.set_index("edge_id") @@ -920,6 +917,8 @@ def basin(self): basin_node["name"] = np.nan basin_node["subnetwork_id"] = np.nan basin_node["geometry"] = basin_nodes["geometry"] + basin_node = basin_node.set_index("node_id") + basin_node = gpd.GeoDataFrame(basin_node, crs="EPSG:28992") basins_area = self.nodes.loc[self.nodes["type"] == "Basin"] basins_area = basins_area[["node_id", "streefpeil", "basins_area_geom"]] @@ -955,11 +954,6 @@ def basin(self): basin_area = basin_area[["node_id", "meta_streefpeil", "geometry"]] basin_area = gpd.GeoDataFrame(basin_area, geometry="geometry").to_crs(crs="EPSG:28992") - # comply to Ribasim 2024.11 - basin_node["meta_node_id"] = basin_node["node_id"].copy().astype(int) - basin_area["meta_node_id"] = basin_area["node_id"].copy().astype(int) - basin_node = basin_node.set_index("node_id") - return basin_node, basin_profile, basin_static, basin_state, basin_area def tabulated_rating_curve(self): @@ -994,7 +988,8 @@ def tabulated_rating_curve(self): rating_curve_node["name"] = np.nan rating_curve_node["subnetwork_id"] = np.nan rating_curve_node["geometry"] = TRC_nodes["geometry"] - # rating_curve_node = rating_curve_node.reset_index(drop=True) + rating_curve_node = rating_curve_node.set_index("node_id") + rating_curve_node = gpd.GeoDataFrame(rating_curve_node, crs="EPSG:28992") rating_curve_static = pd.DataFrame() rating_curve_static["node_id"] = Qh["node_id"] @@ -1004,10 +999,6 @@ def tabulated_rating_curve(self): rating_curve_static["control_state"] = np.nan # rating_curve_static = rating_curve_static.reset_index(drop=True) - # comply to Ribasim 2024.11 - rating_curve_node["meta_node_id"] = rating_curve_node["node_id"].copy().astype(int) - rating_curve_node = rating_curve_node.set_index("node_id") - return rating_curve_node, rating_curve_static def pump(self): @@ -1027,6 +1018,8 @@ def pump(self): pump_node["name"] = np.nan pump_node["subnetwork_id"] = np.nan pump_node["geometry"] = pump_nodes["geometry"] + pump_node = pump_node.set_index("node_id") + pump_node = gpd.GeoDataFrame(pump_node, crs="EPSG:28992") pump_static = pd.DataFrame() pump_static["node_id"] = pump_nodes["node_id"] @@ -1036,10 +1029,6 @@ def pump(self): pump_static["max_flow_rate"] = np.nan pump_static["control_state"] = np.nan - # comply to Ribasim 2024.11 - pump_node["meta_node_id"] = pump_node["node_id"].copy().astype(int) - pump_node = pump_node.set_index("node_id") - return pump_node, pump_static def level_boundary(self): @@ -1061,16 +1050,14 @@ def level_boundary(self): level_boundary_node["name"] = np.nan level_boundary_node["subnetwork_id"] = np.nan level_boundary_node["geometry"] = level_boundary_nodes["geometry"] + level_boundary_node = level_boundary_node.set_index("node_id") + level_boundary_node = gpd.GeoDataFrame(level_boundary_node, crs="EPSG:28992") level_boundary_static = pd.DataFrame() level_boundary_static["node_id"] = level_boundary_nodes["node_id"] level_boundary_static["active"] = np.nan level_boundary_static["level"] = 0 - # comply to Ribasim 2024.11 - level_boundary_node["meta_node_id"] = level_boundary_node["node_id"].copy().astype(int) - level_boundary_node = level_boundary_node.set_index("node_id") - return level_boundary_node, level_boundary_static def flow_boundary(self): @@ -1092,16 +1079,14 @@ def flow_boundary(self): flow_boundary_node["name"] = np.nan flow_boundary_node["subnetwork_id"] = np.nan flow_boundary_node["geometry"] = flow_boundary_nodes["geometry"] + flow_boundary_node = flow_boundary_node.set_index("node_id") + flow_boundary_node = gpd.GeoDataFrame(flow_boundary_node, crs="EPSG:28992") flow_boundary_static = pd.DataFrame() flow_boundary_static["node_id"] = flow_boundary_nodes["node_id"] flow_boundary_static["active"] = np.nan flow_boundary_static["flow_rate"] = 0 - # comply to Ribasim 2024.11 - flow_boundary_node["meta_node_id"] = flow_boundary_node["node_id"].copy().astype(int) - flow_boundary_node = flow_boundary_node.set_index("node_id") - return flow_boundary_node, flow_boundary_static def linear_resistance(self): @@ -1133,6 +1118,8 @@ def manning_resistance(self): manning_resistance_node["name"] = np.nan manning_resistance_node["subnetwork_id"] = np.nan manning_resistance_node["geometry"] = manning_resistance_nodes["geometry"] + manning_resistance_node = manning_resistance_node.set_index("node_id") + manning_resistance_node = gpd.GeoDataFrame(manning_resistance_node, crs="EPSG:28992") manning_resistance_static = pd.DataFrame() manning_resistance_static["node_id"] = manning_resistance_nodes["node_id"] @@ -1142,10 +1129,6 @@ def manning_resistance(self): manning_resistance_static["profile_width"] = 2 manning_resistance_static["profile_slope"] = 3 - # comply to Ribasim 2024.11 - manning_resistance_node["meta_node_id"] = manning_resistance_node["node_id"].copy().astype(int) - manning_resistance_node = manning_resistance_node.set_index("node_id") - return manning_resistance_node, manning_resistance_static def fractional_flow(self): @@ -1177,10 +1160,8 @@ def terminal(self): terminal_node["name"] = np.nan terminal_node["subnetwork_id"] = np.nan terminal_node["geometry"] = terminal_nodes["geometry"] - - # comply to Ribasim 2024.11 - terminal_node["meta_node_id"] = terminal_node["node_id"].copy().astype(int) terminal_node = terminal_node.set_index("node_id") + terminal_node = gpd.GeoDataFrame(terminal_node, crs="EPSG:28992") return terminal_node @@ -1453,7 +1434,7 @@ def check(self, model, post_processed_data, crossings): ) # .loc[model.basin.state.df['node_type'] == 'Basin'] #select all basins # model.basin.node.df.index += 1 #RB: outcommented plus one basin_nodes["geometry"] = model.basin.node.df.geometry # add geometry column - basin_nodes = gpd.GeoDataFrame(basin_nodes, geometry="geometry") # convert from pd go gpd + basin_nodes = gpd.GeoDataFrame(basin_nodes, geometry="geometry", crs=model.crs) # convert from pd go gpd points_within = gpd.sjoin( basin_nodes, checks["boezem"], how="inner", predicate="within" @@ -1475,7 +1456,7 @@ def check(self, model, post_processed_data, crossings): inlaten_TRC = nodes_from_boezem.loc[ (nodes_from_boezem.meta_to_node_type == "TabulatedRatingCurve") | (nodes_from_boezem.meta_to_node_type == "Outlet") - ] + ].copy() inlaten_TRC = inlaten_TRC["to_node_id"] inlaten_TRC = model.tabulated_rating_curve.node.df.loc[ model.tabulated_rating_curve.node.df.index.isin(inlaten_TRC) # df.node_id --> df.index @@ -1489,7 +1470,7 @@ def check(self, model, post_processed_data, crossings): inlaten_TRC["meta_type_verbinding"] = "Inlaat" # inlaten_gemalen - inlaten_gemalen = nodes_from_boezem.loc[nodes_from_boezem.meta_to_node_type == "Pump"] + inlaten_gemalen = nodes_from_boezem.loc[nodes_from_boezem.meta_to_node_type == "Pump"].copy() inlaten_gemalen = inlaten_gemalen["to_node_id"] inlaten_gemalen = model.pump.node.df.loc[ model.pump.node.df.index.isin(inlaten_gemalen) @@ -1497,7 +1478,7 @@ def check(self, model, post_processed_data, crossings): inlaten_gemalen["meta_type_verbinding"] = "Inlaat" # inlaten_flowboundary - inlaten_flowboundary = nodes_to_boezem.loc[nodes_to_boezem.meta_from_node_type == "FlowBoundary"] + inlaten_flowboundary = nodes_to_boezem.loc[nodes_to_boezem.meta_from_node_type == "FlowBoundary"].copy() inlaten_flowboundary = inlaten_flowboundary["from_node_id"] inlaten_flowboundary = model.flow_boundary.node.df.loc[ model.flow_boundary.node.df.index.isin(inlaten_flowboundary) # df.node_id --> df.index @@ -1508,14 +1489,14 @@ def check(self, model, post_processed_data, crossings): uitlaten_TRC = nodes_to_boezem.loc[ (nodes_to_boezem.meta_from_node_type == "TabulatedRatingCurve") | (nodes_to_boezem.meta_from_node_type == "Outlet") - ] + ].copy() uitlaten_TRC = uitlaten_TRC["from_node_id"] uitlaten_TRC = model.tabulated_rating_curve.node.df.loc[ model.tabulated_rating_curve.node.df.index.isin(uitlaten_TRC) # df.node_id --> df.index ] # uitlaten_gemalen - uitlaten_gemalen = nodes_to_boezem.loc[nodes_to_boezem.meta_from_node_type == "Pump"] + uitlaten_gemalen = nodes_to_boezem.loc[nodes_to_boezem.meta_from_node_type == "Pump"].copy() uitlaten_gemalen = uitlaten_gemalen["from_node_id"] uitlaten_gemalen = model.pump.node.df.loc[ model.pump.node.df.index.isin(uitlaten_gemalen) @@ -1530,7 +1511,7 @@ def check(self, model, post_processed_data, crossings): uitlaten_TRC["meta_type_verbinding"] = "Uitlaat" # uitlaten_flowboundary - uitlaten_flowboundary = nodes_to_boezem.loc[nodes_to_boezem.meta_from_node_type == "FlowBoundary"] + uitlaten_flowboundary = nodes_to_boezem.loc[nodes_to_boezem.meta_from_node_type == "FlowBoundary"].copy() uitlaten_flowboundary = uitlaten_flowboundary["from_node_id"] uitlaten_flowboundary = model.flow_boundary.node.df.loc[ model.flow_boundary.node.df.index.isin(uitlaten_flowboundary) # df.node_id --> df.index @@ -1613,7 +1594,7 @@ def check(self, model, post_processed_data, crossings): (BCN_to.meta_from_node_type == "FlowBoundary") | (BCN_to.meta_from_node_type == "LevelBoundary") | (BCN_to.meta_from_node_type == "Terminal") - ] + ].copy() BCN_to["meta_type_verbinding"] = "Inlaat boundary" # look the node ids up in each table. @@ -1633,7 +1614,7 @@ def check(self, model, post_processed_data, crossings): inlaten_uitlaten = pd.concat([inlaten_uitlaten, BCN_from, BCN_to]) inlaten_uitlaten = inlaten_uitlaten.reset_index(drop=True) - inlaten_uitlaten = gpd.GeoDataFrame(inlaten_uitlaten, geometry="geometry", crs="EPSG:28992") + inlaten_uitlaten = gpd.GeoDataFrame(inlaten_uitlaten, geometry="geometry", crs=model.crs) checks["inlaten_uitlaten_boezems"] = inlaten_uitlaten return checks @@ -1751,7 +1732,7 @@ def store_data(data, output_path): _description_ """ for key in data.keys(): - data[str(key)].to_file(output_path + ".gpkg", layer=str(key), driver="GPKG") + data[str(key)].to_file(output_path.with_suffix(".gpkg"), layer=str(key), driver="GPKG") return @@ -1765,216 +1746,23 @@ def WriteResults(self, model, checks): checks : _type_ _description_ """ - path = f"../../../../Ribasim_networks/Waterschappen/{self.model_characteristics['waterschap']}" - # path = os.path.join(path, '', 'modellen', '', self.model_characteristics['waterschap'] + '_' + self.model_characteristics['modeltype']) - - ##### write the model to the Z drive ##### - if self.model_characteristics["write_Zdrive"]: - dir_path = f"../../../../Ribasim_networks/Waterschappen/{self.model_characteristics['waterschap']}/modellen/{self.model_characteristics['waterschap']}_{self.model_characteristics['modeltype']}" - - if not os.path.exists(dir_path): - os.makedirs(dir_path) - else: - for filename in os.listdir(dir_path): # delete outdated models in de original folder - file_path = os.path.join(dir_path, filename) - if os.path.isfile(file_path): - os.remove(file_path) - - path_ribasim = os.path.join( - path, - "", - "modellen", - "", - self.model_characteristics["waterschap"] + "_" + self.model_characteristics["modeltype"], - "ribasim.toml", - ) - model.write(path_ribasim) - - # print('Edges after writing to Z drive:') - # display(model.network.edge.df) - # gpd.GeoDataFrame(model.network.edge.df.geometry).plot(color='red') - # model.network.edge.df.to_file('zzl_test.gpkg') - # model.network.edge.plot() - - ##### write the checks ##### - if self.model_characteristics["write_checks"]: - RibasimNetwork.store_data( - data=checks, - # output_path = str(path + self.model_characteristics['waterschap'] + '_' + self.model_characteristics['modelname'] + '_' + self.model_characteristics['modeltype'] + '_checks')) - output_path=os.path.join( - path, - "modellen", - self.model_characteristics["waterschap"] + "_" + self.model_characteristics["modeltype"], - "database_checks", - ), - ) - - ##### write to the P drive ##### - if self.model_characteristics["write_Pdrive"]: - P_path = self.model_characteristics["path_Pdrive"] - P_path = os.path.join( - P_path, - self.model_characteristics["waterschap"], - "modellen", - self.model_characteristics["waterschap"] + "_" + self.model_characteristics["modeltype"], - self.model_characteristics["waterschap"] - + "_" - + self.model_characteristics["modelname"] - + "_" - + self.model_characteristics["modeltype"], - ) - - if not os.path.exists(P_path): - os.makedirs(P_path) + cloud = CloudStorage() + waterschap = self.model_characteristics["waterschap"] + modeltype = self.model_characteristics["modeltype"] + modellen_dir = cloud.joinpath(waterschap, "modellen") + model_dir = modellen_dir / "_repro" + model_dir.mkdir(parents=True, exist_ok=True) + model.write(model_dir / "repro.toml") - P_path = os.path.join( - P_path, - f"{self.model_characteristics['waterschap']}_{self.model_characteristics['modelname']}_{self.model_characteristics['modeltype']}_ribasim.toml", + if self.model_characteristics["write_goodcloud"]: + cloud.upload_model( + authority=waterschap, + model=waterschap + "_" + modeltype, ) - - model.write(P_path) - - # write checks to the P drive + if self.model_characteristics["write_checks"]: RibasimNetwork.store_data( data=checks, - output_path=str(P_path + "visualisation_checks"), - ) - - ##### copy symbology for the RIBASIM model ##### - if self.model_characteristics["write_symbology"]: - # dont change the paths below! - checks_symbology_path = ( - # r"../../../../Ribasim_networks/Waterschappen/Symbo_feb/modellen/Symbo_feb_poldermodel/Symbo_feb_20240219_Ribasimmodel.qlr" - r"../../../../Data_overig/QGIS_qlr/visualisation_Ribasim.qlr" - ) - checks_symbology_path_new = os.path.join( - path, - "modellen", - self.model_characteristics["waterschap"] + "_" + self.model_characteristics["modeltype"], - "visualisation_Ribasim.qlr", - ) - - # dummy string, required to replace string in the file - # checks_path_old = r"../../symbology/symbology__poldermodel_Ribasim/symbology__poldermodel.gpkg" - # # checks_path_new = os.path.join(self.model_characteristics['waterschap'] + '_' + self.model_characteristics['modelname'] + '_' + self.model_characteristics['modeltype'] + '.gpkg') - # checks_path_new = os.path.join("database.gpkg") - - # copy checks_symbology file from old dir to new dir - shutil.copy(src=checks_symbology_path, dst=checks_symbology_path_new) - - # read file - # with open(checks_symbology_path_new, encoding="utf-8") as file: - # qlr_contents = file.read() - - # # change paths in the .qlr file - # qlr_contents = qlr_contents.replace(checks_path_old, checks_path_new) - - # # write updated file - # with open(checks_symbology_path_new, "w", encoding="utf-8") as file: - # file.write(qlr_contents) - - if self.model_characteristics["write_Pdrive"]: - # write Ribasim model to the P drive - P_path = self.model_characteristics["path_Pdrive"] - P_path = os.path.join( - P_path, - self.model_characteristics["waterschap"], - "modellen", - self.model_characteristics["waterschap"] + "_" + self.model_characteristics["modeltype"], - self.model_characteristics["waterschap"] - + "_" - + self.model_characteristics["modelname"] - + "_" - + self.model_characteristics["modeltype"], - ) - - if not os.path.exists(P_path): - os.makedirs(P_path) - - P_path_ribasim = os.path.join(P_path, "ribasim.toml") - model.write(P_path_ribasim) - - shutil.copy( - src=checks_symbology_path_new, - dst=os.path.join( - P_path, - "visualisation_Ribasim.qlr", - ), - ) - - ##### copy symbology for the CHECKS data ##### - if self.model_characteristics["write_symbology"]: - # dont change the paths below! - # checks_symbology_path = r"../../../../Ribasim_networks/Waterschappen/Symbo_feb/modellen/Symbo_feb_poldermodel/Symbo_feb_20240219_checks.qlr" - checks_symbology_path = r"../../../../Data_overig/QGIS_qlr/visualisation_checks.qlr" - - checks_symbology_path_new = os.path.join( - path, - "modellen", - self.model_characteristics["waterschap"] + "_" + self.model_characteristics["modeltype"], - "visualisation_checks.qlr", - ) - - # # dummy string, required to replace string in the file - # checks_path_old = r"../../symbology/symbology__poldermodel_Ribasim/symbology__poldermodel_checks.gpkg" - # # checks_path_new = os.path.join(self.model_characteristics['waterschap'] + '_' + self.model_characteristics['modelname'] + '_' + self.model_characteristics['modeltype'] + '.gpkg') - # checks_path_new = os.path.join("HollandseDelta_classtest_poldermodel_checks.gpkg") - - # copy checks_symbology file from old dir to new dir - shutil.copy(src=checks_symbology_path, dst=checks_symbology_path_new) - - # read file - # with open(checks_symbology_path_new, encoding="utf-8") as file: - # qlr_contents = file.read() - - # change paths in the .qlr file - # qlr_contents = qlr_contents.replace(checks_path_old, checks_path_new) - - # # write updated file - # with open(checks_symbology_path_new, "w", encoding="utf-8") as file: - # file.write(qlr_contents) - - if self.model_characteristics["write_Pdrive"]: - # write Ribasim model to the P drive - P_path = self.model_characteristics["path_Pdrive"] - P_path = os.path.join( - P_path, - self.model_characteristics["waterschap"], - "modellen", - self.model_characteristics["waterschap"] + "_" + self.model_characteristics["modeltype"], - self.model_characteristics["waterschap"] - + "_" - + self.model_characteristics["modelname"] - + "_" - + self.model_characteristics["modeltype"], - ) - - if not os.path.exists(P_path): - os.makedirs(P_path) - - P_path_ribasim = os.path.join(P_path, "ribasim.toml") - model.write(P_path_ribasim) - - shutil.copy( - src=checks_symbology_path_new, - dst=os.path.join( - P_path, - "visualisation_Ribasim.qlr", - ), - ) - - if self.model_characteristics["write_goodcloud"]: - with open(self.model_characteristics["path_goodcloud_password"]) as file: - password = file.read() - - cloud_storage = CloudStorage( - password=password, - data_dir=r"../../../../Ribasim_networks/Waterschappen/", # + waterschap + '_'+ modelname + '_' + modeltype, - ) - - cloud_storage.upload_model( - authority=self.model_characteristics["waterschap"], - model=self.model_characteristics["waterschap"] + "_" + self.model_characteristics["modeltype"], + output_path=model_dir / "database_checks", ) print("Done") diff --git a/src/peilbeheerst_model/peilbeheerst_model/shortest_path.py b/src/peilbeheerst_model/peilbeheerst_model/shortest_path.py index 75a395d3..4cdfd10a 100644 --- a/src/peilbeheerst_model/peilbeheerst_model/shortest_path.py +++ b/src/peilbeheerst_model/peilbeheerst_model/shortest_path.py @@ -14,7 +14,7 @@ from shapely.ops import split from shapely.wkt import dumps -from peilbeheerst_model import waterschap_data +from .waterschappen import waterschap_data # ### Define functions # 1. splitting functions diff --git a/src/ribasim_nl/ribasim_nl/__init__.py b/src/ribasim_nl/ribasim_nl/__init__.py index ea8fb34d..27b33782 100644 --- a/src/ribasim_nl/ribasim_nl/__init__.py +++ b/src/ribasim_nl/ribasim_nl/__init__.py @@ -1,9 +1,15 @@ __version__ = "0.1.0" -from ribasim_nl.cloud import CloudStorage -from ribasim_nl.model import Model -from ribasim_nl.network import Network -from ribasim_nl.network_validator import NetworkValidator -from ribasim_nl.reset_index import reset_index +from .cloud import CloudStorage +from .model import Model +from .network import Network +from .network_validator import NetworkValidator +from .reset_index import reset_index -__all__ = ["CloudStorage", "Network", "reset_index", "Model", "NetworkValidator"] +__all__ = [ + "CloudStorage", + "Model", + "Network", + "NetworkValidator", + "reset_index", +] From 24a2e2374a3b7095c97f431c7bc0af57f929edbb Mon Sep 17 00:00:00 2001 From: rbruijnshkv Date: Tue, 19 Nov 2024 11:48:36 +0100 Subject: [PATCH 03/13] Alligning code AGV Mainly commenting out different lines --- .../preprocess_data/AmstelGooienVecht.py | 41 +++++++++++-------- 1 file changed, 23 insertions(+), 18 deletions(-) diff --git a/src/peilbeheerst_model/peilbeheerst_model/preprocess_data/AmstelGooienVecht.py b/src/peilbeheerst_model/peilbeheerst_model/preprocess_data/AmstelGooienVecht.py index db8ad3d0..89e9b8c2 100644 --- a/src/peilbeheerst_model/peilbeheerst_model/preprocess_data/AmstelGooienVecht.py +++ b/src/peilbeheerst_model/peilbeheerst_model/preprocess_data/AmstelGooienVecht.py @@ -57,18 +57,18 @@ AVG[variable] = df_var # there is one last gpkg which contains the streefpeilen (and peilgebieden) -AVG["peilgebied"] = gpd.read_file(aangeleverd_dir / "Na_levering" / "vigerende_peilgebieden.gpkg") +# AVG["peilgebied"] = gpd.read_file(aangeleverd_dir / "Na_levering" / "vigerende_peilgebieden.gpkg") -AVG["peilgebied"]["streefpeil"] = np.nan -AVG["peilgebied"]["streefpeil"] = AVG["peilgebied"]["streefpeil"].fillna(value=AVG["peilgebied"]["GPGZMRPL"]) -AVG["peilgebied"]["streefpeil"] = AVG["peilgebied"]["streefpeil"].fillna(value=AVG["peilgebied"]["IWS_GPGVASTP"]) -AVG["peilgebied"]["streefpeil"] = AVG["peilgebied"]["streefpeil"].fillna(value=AVG["peilgebied"]["IWS_GPGONDP"]) +# AVG["peilgebied"]["streefpeil"] = np.nan +# AVG["peilgebied"]["streefpeil"] = AVG["peilgebied"]["streefpeil"].fillna(value=AVG["peilgebied"]["GPGZMRPL"]) +# AVG["peilgebied"]["streefpeil"] = AVG["peilgebied"]["streefpeil"].fillna(value=AVG["peilgebied"]["IWS_GPGVASTP"]) +# AVG["peilgebied"]["streefpeil"] = AVG["peilgebied"]["streefpeil"].fillna(value=AVG["peilgebied"]["IWS_GPGONDP"]) -print( - "Number of missing streefpeilen = ", - len(AVG["peilgebied"]["streefpeil"].loc[AVG["peilgebied"]["streefpeil"].isna()]), -) +# print( +# "Number of missing streefpeilen = ", +# len(AVG["peilgebied"]["streefpeil"].loc[AVG["peilgebied"]["streefpeil"].isna()]), +# ) # fig, ax = plt.subplots() # AVG['peilgebied'].geometry.plot(ax=ax, color='cornflowerblue') @@ -81,10 +81,10 @@ # overwrite previous data AVG["stuw"] = gpd.read_file(dump_path + "/Stuw.shp") -AVG["stuw"] = AVG["stuw"].loc[AVG["stuw"].LHM == "LHM"] +# AVG["stuw"] = AVG["stuw"].loc[AVG["stuw"].LHM == "LHM"] AVG["gemaal"] = gpd.read_file(dump_path + "/Gemaal.shp") -AVG["gemaal"] = AVG["gemaal"].loc[AVG["gemaal"].LHM == "LHM"] +# AVG["gemaal"] = AVG["gemaal"].loc[AVG["gemaal"].LHM == "LHM"] AVG["duikersifonhevel"] = gpd.read_file(dump_path + "/DuikerSifonHevel.shp") AVG["hydroobject"] = gpd.read_file(dump_path + "/LHM_hydrovakken.shp") @@ -145,8 +145,8 @@ AVG["duikersifonhevel"].loc[:, "nen3610id"] = "dummy_nen3610id_duikersifonhevel_" + AVG[ "duikersifonhevel" ].index.astype(str) -AVG["duikersifonhevel"]["globalid"] = "dummy_globalid_duikersifonhevel_" + AVG["duikersifonhevel"].index.astype(str) -AVG["duikersifonhevel"] = gpd.GeoDataFrame(AVG["duikersifonhevel"]).to_crs("epsg:28992") +AVG['duikersifonhevel']['globalid'] = 'dummy_globalid_duikersifonhevel_' + AVG['duikersifonhevel'].index.astype(str) +AVG['duikersifonhevel'] = gpd.GeoDataFrame(AVG['duikersifonhevel']).to_crs('epsg:28992') # hydroobject AVG["hydroobject"] = AVG["hydroobject"][["geometry"]] @@ -156,11 +156,11 @@ AVG["hydroobject"] = gpd.GeoDataFrame(AVG["hydroobject"]).set_crs("epsg:28992") # streefpeil -AVG["streefpeil"] = pd.DataFrame() -AVG["streefpeil"]["waterhoogte"] = AVG["peilgebied"]["streefpeil"] -AVG["streefpeil"]["globalid"] = "dummy_globalid_streefpeil_" + AVG["streefpeil"].index.astype(str) -AVG["streefpeil"]["geometry"] = np.nan -AVG["streefpeil"] = gpd.GeoDataFrame(AVG["streefpeil"]).set_crs("epsg:28992") +# AVG["streefpeil"] = pd.DataFrame() +# AVG["streefpeil"]["waterhoogte"] = AVG["peilgebied"]["streefpeil"] +# AVG["streefpeil"]["globalid"] = "dummy_globalid_streefpeil_" + AVG["streefpeil"].index.astype(str) +# AVG["streefpeil"]["geometry"] = np.nan +# AVG["streefpeil"] = gpd.GeoDataFrame(AVG["streefpeil"]).set_crs("epsg:28992") # peilgebied AVG["peilgebied"]["code"] = AVG["peilgebied"]["GAFNAAM"] @@ -168,6 +168,11 @@ AVG["peilgebied"]["nen3610id"] = "dummy_nen3610id_peilgebied_" + AVG["peilgebied"].index.astype(str) AVG["peilgebied"]["globalid"] = "dummy_globalid_peilgebied_" + AVG["peilgebied"].index.astype(str) +AVG['streefpeil'] = AVG['peilgebied'][['waterhoogte', 'globalid']] +AVG['streefpeil']['code'] = 'dummy_code_streefpeil_' + AVG['streefpeil'].index.astype(str) +AVG['streefpeil']['geometry'] = None +AVG['streefpeil'] = gpd.GeoDataFrame(AVG['streefpeil'], geometry = 'geometry') + AVG["peilgebied"] = AVG["peilgebied"][["code", "nen3610id", "globalid", "geometry"]] AVG["peilgebied"] = gpd.GeoDataFrame(AVG["peilgebied"]).to_crs("epsg:28992") From 809b3b9c234c9cbfb6ac50a73e02ac36a8c9cd95 Mon Sep 17 00:00:00 2001 From: rbruijnshkv Date: Wed, 20 Nov 2024 11:00:11 +0100 Subject: [PATCH 04/13] Update preprocessing of AGV --- .../preprocess_data/AmstelGooienVecht.py | 17 ++++++++++------- 1 file changed, 10 insertions(+), 7 deletions(-) diff --git a/src/peilbeheerst_model/peilbeheerst_model/preprocess_data/AmstelGooienVecht.py b/src/peilbeheerst_model/peilbeheerst_model/preprocess_data/AmstelGooienVecht.py index 89e9b8c2..952aa5c7 100644 --- a/src/peilbeheerst_model/peilbeheerst_model/preprocess_data/AmstelGooienVecht.py +++ b/src/peilbeheerst_model/peilbeheerst_model/preprocess_data/AmstelGooienVecht.py @@ -33,6 +33,8 @@ aangeleverd_dir / "aanlevering_6maart24/data dump 6 maart LHM AGV.zip!/data dump 6 maart LHM AGV/" ).as_posix() +dump_path_peilgebied = (aangeleverd_dir / "verbeterde_LHM_gebieden_geactualiseerd/").as_posix() + verwerkt_dir.mkdir(parents=True, exist_ok=True) @@ -88,7 +90,7 @@ AVG["duikersifonhevel"] = gpd.read_file(dump_path + "/DuikerSifonHevel.shp") AVG["hydroobject"] = gpd.read_file(dump_path + "/LHM_hydrovakken.shp") -AVG["peilgebied"] = gpd.read_file(dump_path + "/LHM_gebieden.shp") +AVG["peilgebied"] = gpd.read_file(dump_path_peilgebied + "/LHM_gebieden.shp").to_crs(crs="EPSG:28992") AVG["peilgebied"].loc[AVG["peilgebied"].zomer == 0, "zomer"] = np.nan @@ -145,8 +147,8 @@ AVG["duikersifonhevel"].loc[:, "nen3610id"] = "dummy_nen3610id_duikersifonhevel_" + AVG[ "duikersifonhevel" ].index.astype(str) -AVG['duikersifonhevel']['globalid'] = 'dummy_globalid_duikersifonhevel_' + AVG['duikersifonhevel'].index.astype(str) -AVG['duikersifonhevel'] = gpd.GeoDataFrame(AVG['duikersifonhevel']).to_crs('epsg:28992') +AVG["duikersifonhevel"]["globalid"] = "dummy_globalid_duikersifonhevel_" + AVG["duikersifonhevel"].index.astype(str) +AVG["duikersifonhevel"] = gpd.GeoDataFrame(AVG["duikersifonhevel"]).to_crs("epsg:28992") # hydroobject AVG["hydroobject"] = AVG["hydroobject"][["geometry"]] @@ -167,11 +169,12 @@ AVG["peilgebied"]["geometry"] = AVG["peilgebied"]["geometry"] AVG["peilgebied"]["nen3610id"] = "dummy_nen3610id_peilgebied_" + AVG["peilgebied"].index.astype(str) AVG["peilgebied"]["globalid"] = "dummy_globalid_peilgebied_" + AVG["peilgebied"].index.astype(str) +AVG["peilgebied"]["waterhoogte"] = AVG["peilgebied"].streefpeil -AVG['streefpeil'] = AVG['peilgebied'][['waterhoogte', 'globalid']] -AVG['streefpeil']['code'] = 'dummy_code_streefpeil_' + AVG['streefpeil'].index.astype(str) -AVG['streefpeil']['geometry'] = None -AVG['streefpeil'] = gpd.GeoDataFrame(AVG['streefpeil'], geometry = 'geometry') +AVG["streefpeil"] = AVG["peilgebied"][["waterhoogte", "globalid"]] +AVG["streefpeil"]["code"] = "dummy_code_streefpeil_" + AVG["streefpeil"].index.astype(str) +AVG["streefpeil"]["geometry"] = None +AVG["streefpeil"] = gpd.GeoDataFrame(AVG["streefpeil"], geometry="geometry") AVG["peilgebied"] = AVG["peilgebied"][["code", "nen3610id", "globalid", "geometry"]] AVG["peilgebied"] = gpd.GeoDataFrame(AVG["peilgebied"]).to_crs("epsg:28992") From 3d709a365d1869d804334ff32d3f534ea4780d03 Mon Sep 17 00:00:00 2001 From: rbruijnshkv Date: Wed, 20 Nov 2024 11:00:34 +0100 Subject: [PATCH 05/13] Download only a subsection of the basisgegevens --- .../peilbeheerst_model/postprocess_data/post-process_agv.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/peilbeheerst_model/peilbeheerst_model/postprocess_data/post-process_agv.py b/src/peilbeheerst_model/peilbeheerst_model/postprocess_data/post-process_agv.py index d2fea1a3..d61f6fe4 100644 --- a/src/peilbeheerst_model/peilbeheerst_model/postprocess_data/post-process_agv.py +++ b/src/peilbeheerst_model/peilbeheerst_model/postprocess_data/post-process_agv.py @@ -24,7 +24,7 @@ cloud = CloudStorage() cloud.download_verwerkt(waterschap) cloud.download_verwerkt("Rijkswaterstaat") -cloud.download_basisgegevens() +cloud.download_basisgegevens(bronnen=["RWS_waterschaps_grenzen"]) # only download subdirectory of the basisgegevens # %% verwerkt_dir = cloud.joinpath(waterschap, "verwerkt") From c1da95e0c1c862d39942434895b32997fee92997 Mon Sep 17 00:00:00 2001 From: rbruijnshkv Date: Wed, 20 Nov 2024 12:05:55 +0100 Subject: [PATCH 06/13] Only download subsection of the the basisgegevens --- src/peilbeheerst_model/parse_crossings/AmstelGooienVecht.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/peilbeheerst_model/parse_crossings/AmstelGooienVecht.py b/src/peilbeheerst_model/parse_crossings/AmstelGooienVecht.py index 66559079..afde740d 100644 --- a/src/peilbeheerst_model/parse_crossings/AmstelGooienVecht.py +++ b/src/peilbeheerst_model/parse_crossings/AmstelGooienVecht.py @@ -10,7 +10,7 @@ cloud = CloudStorage() verwerkt_dir = cloud.joinpath(waterschap, "verwerkt") cloud.download_verwerkt(waterschap) -cloud.download_basisgegevens() +cloud.download_basisgegevens(bronnen=["KRW"]) # %% @@ -36,3 +36,5 @@ # %% cloud.upload_verwerkt(waterschap) + +# %% From a9bf339b30d4322b9919865a134800781e273817 Mon Sep 17 00:00:00 2001 From: rbruijnshkv Date: Wed, 20 Nov 2024 12:06:08 +0100 Subject: [PATCH 07/13] Test for AGV --- src/peilbeheerst_model/01_parse_crossings.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/peilbeheerst_model/01_parse_crossings.py b/src/peilbeheerst_model/01_parse_crossings.py index 54b23a7f..af240612 100644 --- a/src/peilbeheerst_model/01_parse_crossings.py +++ b/src/peilbeheerst_model/01_parse_crossings.py @@ -29,7 +29,7 @@ init_settings = waterschap_struct["init"] init_settings["logfile"] = pathlib.Path(init_settings["output_path"]).with_suffix(".log") - if waterschap not in ["HHNK"]: + if waterschap not in ["AmstelGooienVecht"]: continue # if pathlib.Path(init_settings["output_path"]).exists() and "crossings_hydroobject" in fiona.listlayers(init_settings["output_path"]): @@ -127,3 +127,5 @@ print(pd.DataFrame(reduction_results, index=waterschappen)) print(pd.DataFrame(network_results, index=waterschappen)) + +# %% From 43f528bc9782b5e0c22783b0b9dacdc749189e78 Mon Sep 17 00:00:00 2001 From: rbruijnshkv Date: Wed, 20 Nov 2024 13:14:29 +0100 Subject: [PATCH 08/13] Update AmstelGooienVecht.py --- .../crossings_to_ribasim/AmstelGooienVecht.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/peilbeheerst_model/crossings_to_ribasim/AmstelGooienVecht.py b/src/peilbeheerst_model/crossings_to_ribasim/AmstelGooienVecht.py index a7e7c520..deede12a 100644 --- a/src/peilbeheerst_model/crossings_to_ribasim/AmstelGooienVecht.py +++ b/src/peilbeheerst_model/crossings_to_ribasim/AmstelGooienVecht.py @@ -113,4 +113,6 @@ # %% -# cloud.upload_verwerkt(waterschap) +cloud.upload_verwerkt(waterschap) + +# %% From 13540ecfd447f770e5effd8f1fd1cbbf1c7d8306 Mon Sep 17 00:00:00 2001 From: rbruijnshkv Date: Wed, 20 Nov 2024 13:14:37 +0100 Subject: [PATCH 09/13] Update 10_shortest_path_AGV.py --- src/peilbeheerst_model/Shortest_path/10_shortest_path_AGV.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/peilbeheerst_model/Shortest_path/10_shortest_path_AGV.py b/src/peilbeheerst_model/Shortest_path/10_shortest_path_AGV.py index c0d89760..a8168ee6 100644 --- a/src/peilbeheerst_model/Shortest_path/10_shortest_path_AGV.py +++ b/src/peilbeheerst_model/Shortest_path/10_shortest_path_AGV.py @@ -45,3 +45,5 @@ gdf_out.to_file(verwerkt_dir / "shortest_path.gpkg", driver="GPKG") # cloud.upload_verwerkt(waterschap) + +# %% From 898d5272f0938d5bc31c93ff41d4ff36bf1731d5 Mon Sep 17 00:00:00 2001 From: rbruijnshkv Date: Thu, 30 Jan 2025 10:53:52 +0100 Subject: [PATCH 10/13] Update store data function --- .../peilbeheerst_model/general_functions.py | 46 ++++++++++++------- 1 file changed, 29 insertions(+), 17 deletions(-) diff --git a/src/peilbeheerst_model/peilbeheerst_model/general_functions.py b/src/peilbeheerst_model/peilbeheerst_model/general_functions.py index ca7fff61..d997e3ac 100644 --- a/src/peilbeheerst_model/peilbeheerst_model/general_functions.py +++ b/src/peilbeheerst_model/peilbeheerst_model/general_functions.py @@ -2,6 +2,7 @@ import geopandas as gpd import matplotlib.pyplot as plt import pandas as pd +import pyogrio def read_gpkg_layers(gpkg_path, engine="fiona", print_var=False): @@ -55,28 +56,39 @@ def show_layers_and_columns(waterschap): print() -def store_data(waterschap, output_gpkg_path): - """ - Store Geospatial Data to a GeoPackage (GPKG) File. +# def store_data(waterschap, output_gpkg_path): +# """ +# Store Geospatial Data to a GeoPackage (GPKG) File. - Parameters - ---------- - waterschap (dict): A dictionary containing GeoDataFrames to be stored in the GPKG file. - output_gpkg_path (str): The file path (including the file name without extension) to save the GPKG file. +# Parameters +# ---------- +# waterschap (dict): A dictionary containing GeoDataFrames to be stored in the GPKG file. +# output_gpkg_path (str): The file path (including the file name without extension) to save the GPKG file. - Returns - ------- - None +# Returns +# ------- +# None - This function stores geospatial data from a dictionary of GeoDataFrames into a GeoPackage (GPKG) file. +# This function stores geospatial data from a dictionary of GeoDataFrames into a GeoPackage (GPKG) file. - Parameters - ---------- - - waterschap: A dictionary where the keys represent layer names, and the values are GeoDataFrames. - - output_gpkg_path: The file path for the output GPKG file. The '.gpkg' extension is added automatically. - """ +# Parameters +# ---------- +# - waterschap: A dictionary where the keys represent layer names, and the values are GeoDataFrames. +# - output_gpkg_path: The file path for the output GPKG file. The '.gpkg' extension is added automatically. +# """ +# for key in waterschap.keys(): +# waterschap[str(key)].to_file(output_gpkg_path + ".gpkg", layer=str(key), driver="GPKG") + + +def store_data(waterschap, output_gpkg_path): for key in waterschap.keys(): - waterschap[str(key)].to_file(output_gpkg_path + ".gpkg", layer=str(key), driver="GPKG") + pyogrio.write_dataframe( + waterschap[key], + output_gpkg_path + ".gpkg", + layer=str(key), + driver="GPKG", + promote_to_multi=False, + ) def overlapping_peilgebieden(waterschap_peilgebieden): From e3d9db3a9769913f27404507e5b474adb4256681 Mon Sep 17 00:00:00 2001 From: rbruijnshkv Date: Thu, 30 Jan 2025 10:54:27 +0100 Subject: [PATCH 11/13] Utilize new export function --- .../peilbeheerst_model/parse_crossings.py | 26 ++++++++++++++++--- 1 file changed, 23 insertions(+), 3 deletions(-) diff --git a/src/peilbeheerst_model/peilbeheerst_model/parse_crossings.py b/src/peilbeheerst_model/peilbeheerst_model/parse_crossings.py index 33a36b26..4673d68d 100644 --- a/src/peilbeheerst_model/peilbeheerst_model/parse_crossings.py +++ b/src/peilbeheerst_model/peilbeheerst_model/parse_crossings.py @@ -9,6 +9,7 @@ import numpy.typing as npt import pandas as pd import pydantic +import pyogrio import shapely.ops import tqdm.auto as tqdm from shapely.geometry import LineString, MultiLineString, MultiPoint, Point, Polygon @@ -600,14 +601,33 @@ def write_crossings( # Write the input files (some with minor modifications) for layer, df in self.df_gpkg.items(): - df.to_file(output_path, layer=layer) + # df.to_file(output_path, layer=layer) + + pyogrio.write_dataframe( + df, + output_path, + layer=str(layer), + driver="GPKG", + promote_to_multi=False, + ) # Write supplied output files df_hydro.to_file(output_path, layer="crossings_hydroobject") if df_filter is not None: - df_filter.to_file(output_path, layer=f"crossings_{filterlayer}") + # df_filter.to_file(output_path, layer=f"crossings_{filterlayer}") + pyogrio.write_dataframe( + df_filter, output_path, layer=f"crossings_{filterlayer}", driver="GPKG", promote_to_multi=False + ) + if df_hydro_filter is not None: - df_hydro_filter.to_file(output_path, layer="crossings_hydroobject_filtered") + # df_hydro_filter.to_file(output_path, layer="crossings_hydroobject_filtered") + pyogrio.write_dataframe( + df_hydro_filter, + output_path, + layer="crossings_hydroobject_filtered", + driver="GPKG", + promote_to_multi=False, + ) @pydantic.validate_call(config={"strict": True}) def _classify_from_to_peilgebieden(self, pfrom: str | None, pto: str | None) -> tuple[str, str]: From 4afef4d16f971dc7dd6040686e98357a13370bda Mon Sep 17 00:00:00 2001 From: rbruijnshkv Date: Thu, 30 Jan 2025 10:54:46 +0100 Subject: [PATCH 12/13] Utilize new export function --- .../postprocess_data/post-process_agv.py | 15 +++++++++++++-- 1 file changed, 13 insertions(+), 2 deletions(-) diff --git a/src/peilbeheerst_model/peilbeheerst_model/postprocess_data/post-process_agv.py b/src/peilbeheerst_model/peilbeheerst_model/postprocess_data/post-process_agv.py index d61f6fe4..4e144642 100644 --- a/src/peilbeheerst_model/peilbeheerst_model/postprocess_data/post-process_agv.py +++ b/src/peilbeheerst_model/peilbeheerst_model/postprocess_data/post-process_agv.py @@ -9,6 +9,7 @@ import geopandas as gpd import numpy as np import pandas as pd +import pyogrio from peilbeheerst_model.general_functions import read_gpkg_layers from ribasim_nl import CloudStorage @@ -126,9 +127,19 @@ output_gpkg_path = verwerkt_dir / "postprocessed.gpkg" +# for key in AVG.keys(): +# print(key) +# AVG[str(key)].to_file(output_gpkg_path, layer=str(key), driver="GPKG") + for key in AVG.keys(): - print(key) - AVG[str(key)].to_file(output_gpkg_path, layer=str(key), driver="GPKG") + pyogrio.write_dataframe( + AVG[key], + output_gpkg_path, + layer=str(key), + driver="GPKG", + promote_to_multi=False, + ) + cloud.upload_verwerkt(waterschap) # %% From 6cb82efc74384984e27ece2bfc78c63b98887f04 Mon Sep 17 00:00:00 2001 From: rbruijnshkv Date: Thu, 30 Jan 2025 10:55:07 +0100 Subject: [PATCH 13/13] Upload to the goodcloud --- src/peilbeheerst_model/Shortest_path/10_shortest_path_AGV.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/peilbeheerst_model/Shortest_path/10_shortest_path_AGV.py b/src/peilbeheerst_model/Shortest_path/10_shortest_path_AGV.py index a8168ee6..a8314428 100644 --- a/src/peilbeheerst_model/Shortest_path/10_shortest_path_AGV.py +++ b/src/peilbeheerst_model/Shortest_path/10_shortest_path_AGV.py @@ -44,6 +44,6 @@ gdf_out["shortest_path"] = gdf_out["shortest_path"].apply(lambda geom: dumps(geom) if geom is not None else None) gdf_out.to_file(verwerkt_dir / "shortest_path.gpkg", driver="GPKG") -# cloud.upload_verwerkt(waterschap) +cloud.upload_verwerkt(waterschap) # %%