From 602efb84b570e12b7b115c505b3db6d7e92a0895 Mon Sep 17 00:00:00 2001 From: Martijn Visser Date: Tue, 22 Oct 2024 15:41:25 +0200 Subject: [PATCH 1/3] Update lock file for new dependencies --- pixi.lock | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/pixi.lock b/pixi.lock index e8534b9..2d7e418 100644 --- a/pixi.lock +++ b/pixi.lock @@ -14706,10 +14706,12 @@ packages: name: peilbeheerst-model version: 0.1.0 path: src/peilbeheerst_model - sha256: e8c270f68d683c802990a8bb905cd665fd327b33e1d17bf3e01dfe9c49ec335e + sha256: 9ec22544800123361c4449542c548c1e3739a3f8d9d5bbe2f91a8d50d304fac5 requires_dist: + - fiona - geopandas - matplotlib + - networkx - numpy - pandas - pydantic From d64dde4c84ee6c70a1782b4afbaee3b35edff1df Mon Sep 17 00:00:00 2001 From: Martijn Visser Date: Tue, 22 Oct 2024 17:23:26 +0200 Subject: [PATCH 2/3] Run AGV shortest_path and crossings_to_ribasim --- .../Shortest_path/10_shortest_path_AGV.py | 50 ++- .../crossings_to_ribasim/AmstelGooienVecht.py | 116 +++++++ .../peilbeheerst_model/__init__.py | 15 +- .../crossings_to_ribasim.py | 294 +++--------------- .../peilbeheerst_model/shortest_path.py | 2 +- src/ribasim_nl/ribasim_nl/__init__.py | 18 +- 6 files changed, 226 insertions(+), 269 deletions(-) create mode 100644 src/peilbeheerst_model/crossings_to_ribasim/AmstelGooienVecht.py diff --git a/src/peilbeheerst_model/Shortest_path/10_shortest_path_AGV.py b/src/peilbeheerst_model/Shortest_path/10_shortest_path_AGV.py index 1d16166..c0d8976 100644 --- a/src/peilbeheerst_model/Shortest_path/10_shortest_path_AGV.py +++ b/src/peilbeheerst_model/Shortest_path/10_shortest_path_AGV.py @@ -1,7 +1,47 @@ -from peilbeheerst_model import shortest_path_waterschap +# The AGV shortest path code is identical to the other water boards. +# Below the content of shortest_path_waterschappen is spelled out to be able to +# point to the right data from the cloud storage. + +import fiona +import geopandas as gpd +import pandas as pd +from shapely.wkt import dumps + +from peilbeheerst_model.shortest_path import shortest_path +from ribasim_nl import CloudStorage waterschap = "AmstelGooienVecht" -gdf_out = shortest_path_waterschap(waterschap) -gdf_out.to_file( - f"/DATAFOLDER/projects/4750_30/Data_shortest_path/{waterschap}/{waterschap}_shortest_path.gpkg", driver="GPKG" -) + +cloud = CloudStorage() +# cloud.download_verwerkt(waterschap) +# cloud.download_verwerkt("Rijkswaterstaat") +# cloud.download_basisgegevens() + +# %% +verwerkt_dir = cloud.joinpath(waterschap, "verwerkt") + +# Load Data +# Define crossings file path +data_path = verwerkt_dir / "crossings.gpkg" + +# Load crossings file +DATA = {L: gpd.read_file(data_path, layer=L) for L in fiona.listlayers(data_path)} + +# Select rhws + +# Select RHWS peilgebeied & calculate representative point +gdf_rhws = DATA["peilgebied"].loc[DATA["peilgebied"]["peilgebied_cat"] == 1].copy() +gdf_rhws["representative_point"] = gdf_rhws.representative_point() + +# Apply aggregation level based filter +gdf_cross = ( + DATA["crossings_hydroobject_filtered"].loc[DATA["crossings_hydroobject_filtered"]["agg_links_in_use"]].copy() +) # filter aggregation level + +gdf_crossings_out = shortest_path(waterschap, DATA, gdf_cross, gdf_rhws) +# Write final output +gdf_out = gpd.GeoDataFrame(pd.concat(gdf_crossings_out)) +gdf_out["shortest_path"] = gdf_out["shortest_path"].apply(lambda geom: dumps(geom) if geom is not None else None) +gdf_out.to_file(verwerkt_dir / "shortest_path.gpkg", driver="GPKG") + +# cloud.upload_verwerkt(waterschap) diff --git a/src/peilbeheerst_model/crossings_to_ribasim/AmstelGooienVecht.py b/src/peilbeheerst_model/crossings_to_ribasim/AmstelGooienVecht.py new file mode 100644 index 0000000..a7e7c52 --- /dev/null +++ b/src/peilbeheerst_model/crossings_to_ribasim/AmstelGooienVecht.py @@ -0,0 +1,116 @@ +# Modified version of the AGV part of 02_crossings_to_ribasim.py. + +# %% + +import pandas as pd +from ribasim import Model + +from peilbeheerst_model import CrossingsToRibasim, RibasimNetwork, waterschap_data +from ribasim_nl import CloudStorage + +# %% +waterschap = "AmstelGooienVecht" +waterschap_struct = waterschap_data[waterschap] + +cloud = CloudStorage() +verwerkt_dir = cloud.joinpath(waterschap, "verwerkt") +# cloud.download_verwerkt(waterschap) +# cloud.download_basisgegevens() + +# %% + +pd.set_option("display.max_columns", None) +# warnings.filterwarnings("ignore") + + +model_characteristics = { + # model description + "waterschap": "AmstelGooienVecht", + "modelname": "repro", + "modeltype": "boezemmodel", + # define paths + "path_postprocessed_data": verwerkt_dir / "postprocessed.gpkg", + "path_crossings": verwerkt_dir / "crossings.gpkg", + "path_boezem": verwerkt_dir / "shortest_path.gpkg", + "path_Pdrive": None, + # apply filters + "crossings_layer": "crossings_hydroobject_filtered", + "in_use": True, + "agg_links_in_use": True, + "agg_areas_in_use": True, + "aggregation": True, + # data storage settings + "write_goodcloud": False, # TODO + "write_checks": True, + # numerical settings + "solver": None, + "logging": None, + "starttime": "2024-01-01 00:00:00", + "endtime": "2024-01-02 00:00:00", +} + +waterboard = CrossingsToRibasim(model_characteristics=model_characteristics) + +post_processed_data, crossings = waterboard.read_files() +post_processed_data, crossings = waterboard.routing_processor(post_processed_data, crossings) +crossings = waterboard.assign_node_ids(crossings) +edges = waterboard.create_edges(crossings) +nodes, edges = waterboard.create_nodes(crossings, edges) +edges = waterboard.embed_boezems(edges, post_processed_data, crossings) + +# create individual model parts of the network +network = RibasimNetwork(nodes=nodes, edges=edges, model_characteristics=model_characteristics) + +edge = network.edge() +basin_node, basin_profile, basin_static, basin_state, basin_area = network.basin() +pump_node, pump_static = network.pump() +tabulated_rating_curve_node, tabulated_rating_curve_static = network.tabulated_rating_curve() +level_boundary_node, level_boundary_static = network.level_boundary() +flow_boundary_node, flow_boundary_static = network.flow_boundary() +manning_resistance_node, manning_resistance_static = network.manning_resistance() +terminal_node = network.terminal() + +# linear_resistance = network.linear_resistance() +# fractional_flow = network.fractional_flow() +# outlet = network.outlet() +# discrete_control = network.discrete_control() +# pid_control = network.pid_control() + +# insert the individual model modules in an actual model +model = Model(starttime=model_characteristics["starttime"], endtime=model_characteristics["endtime"], crs="EPSG:28992") + +model.edge.df = edge + +model.basin.node.df = basin_node +model.basin.profile = basin_profile +model.basin.static = basin_static +model.basin.state = basin_state +model.basin.area = basin_area + +model.pump.node.df = pump_node +model.pump.static = pump_static + +model.tabulated_rating_curve.node.df = tabulated_rating_curve_node +model.tabulated_rating_curve.static = tabulated_rating_curve_static + +model.manning_resistance.node.df = manning_resistance_node +model.manning_resistance.static = manning_resistance_static + +model.level_boundary.node.df = level_boundary_node +model.level_boundary.static = level_boundary_static + +model.flow_boundary.node.df = flow_boundary_node +model.flow_boundary.static = flow_boundary_static + +model.terminal.node.df = terminal_node + +# add checks and metadata +checks = network.check(model, post_processed_data=post_processed_data, crossings=crossings) +model = network.add_meta_data(model, checks, post_processed_data, crossings) + +# write the result +network.WriteResults(model=model, checks=checks) + +# %% + +# cloud.upload_verwerkt(waterschap) diff --git a/src/peilbeheerst_model/peilbeheerst_model/__init__.py b/src/peilbeheerst_model/peilbeheerst_model/__init__.py index 3fd1b2e..58ba213 100644 --- a/src/peilbeheerst_model/peilbeheerst_model/__init__.py +++ b/src/peilbeheerst_model/peilbeheerst_model/__init__.py @@ -1,7 +1,14 @@ __version__ = "0.1.0" -from peilbeheerst_model.parse_crossings import ParseCrossings -from peilbeheerst_model.shortest_path import shortest_path_waterschap -from peilbeheerst_model.waterschappen import waterschap_data +from .crossings_to_ribasim import CrossingsToRibasim, RibasimNetwork +from .parse_crossings import ParseCrossings +from .shortest_path import shortest_path_waterschap +from .waterschappen import waterschap_data -__all__ = ["ParseCrossings", "shortest_path_waterschap", "waterschap_data"] +__all__ = [ + "CrossingsToRibasim", + "ParseCrossings", + "RibasimNetwork", + "shortest_path_waterschap", + "waterschap_data", +] diff --git a/src/peilbeheerst_model/peilbeheerst_model/crossings_to_ribasim.py b/src/peilbeheerst_model/peilbeheerst_model/crossings_to_ribasim.py index fe4d598..715c185 100644 --- a/src/peilbeheerst_model/peilbeheerst_model/crossings_to_ribasim.py +++ b/src/peilbeheerst_model/peilbeheerst_model/crossings_to_ribasim.py @@ -1,6 +1,4 @@ import itertools -import os -import shutil import geopandas as gpd import numpy as np @@ -136,13 +134,14 @@ def routing_processor( # define the most representative point where the basins will be located. This is always within the peilgebied polygon. # save the original peilgebied_from and _to for boezem purposes later, before changing anything due to the aggregation - crossings["peilgebied_from_original"] = crossings["peilgebied_from"] - crossings["peilgebied_to_original"] = crossings["peilgebied_to"] + crossings["peilgebied_from_original"] = crossings["peilgebied_from"].copy() + crossings["peilgebied_to_original"] = crossings["peilgebied_to"].copy() if self.model_characteristics["aggregation"]: # not all peilgebieden are part of a aggregation area. Add the missing peilgebieden to the agg_area column, so no areas will be left out - crossings["agg_area_from"].fillna(crossings["peilgebied_from"], inplace=True) - crossings["agg_area_to"].fillna(crossings["peilgebied_to"], inplace=True) + crossings.fillna( + {"agg_area_from": crossings["peilgebied_from"], "agg_area_to": crossings["peilgebied_to"]}, inplace=True + ) # change administrative detailed peilgebieden to aggregation areas crossings["peilgebied_from"] = crossings["agg_area_from"] @@ -893,8 +892,6 @@ def edge(self): edge["name"] = None edge["subnetwork_id"] = None edge["geometry"] = self.edges["line_geom"] - - # comply to Ribasim 2024.11 edge = edge.reset_index(drop=True) edge["edge_id"] = edge.index.astype(int) edge = edge.set_index("edge_id") @@ -920,6 +917,8 @@ def basin(self): basin_node["name"] = np.nan basin_node["subnetwork_id"] = np.nan basin_node["geometry"] = basin_nodes["geometry"] + basin_node = basin_node.set_index("node_id") + basin_node = gpd.GeoDataFrame(basin_node, crs="EPSG:28992") basins_area = self.nodes.loc[self.nodes["type"] == "Basin"] basins_area = basins_area[["node_id", "streefpeil", "basins_area_geom"]] @@ -955,11 +954,6 @@ def basin(self): basin_area = basin_area[["node_id", "meta_streefpeil", "geometry"]] basin_area = gpd.GeoDataFrame(basin_area, geometry="geometry").to_crs(crs="EPSG:28992") - # comply to Ribasim 2024.11 - basin_node["meta_node_id"] = basin_node["node_id"].copy().astype(int) - basin_area["meta_node_id"] = basin_area["node_id"].copy().astype(int) - basin_node = basin_node.set_index("node_id") - return basin_node, basin_profile, basin_static, basin_state, basin_area def tabulated_rating_curve(self): @@ -994,7 +988,8 @@ def tabulated_rating_curve(self): rating_curve_node["name"] = np.nan rating_curve_node["subnetwork_id"] = np.nan rating_curve_node["geometry"] = TRC_nodes["geometry"] - # rating_curve_node = rating_curve_node.reset_index(drop=True) + rating_curve_node = rating_curve_node.set_index("node_id") + rating_curve_node = gpd.GeoDataFrame(rating_curve_node, crs="EPSG:28992") rating_curve_static = pd.DataFrame() rating_curve_static["node_id"] = Qh["node_id"] @@ -1004,10 +999,6 @@ def tabulated_rating_curve(self): rating_curve_static["control_state"] = np.nan # rating_curve_static = rating_curve_static.reset_index(drop=True) - # comply to Ribasim 2024.11 - rating_curve_node["meta_node_id"] = rating_curve_node["node_id"].copy().astype(int) - rating_curve_node = rating_curve_node.set_index("node_id") - return rating_curve_node, rating_curve_static def pump(self): @@ -1027,6 +1018,8 @@ def pump(self): pump_node["name"] = np.nan pump_node["subnetwork_id"] = np.nan pump_node["geometry"] = pump_nodes["geometry"] + pump_node = pump_node.set_index("node_id") + pump_node = gpd.GeoDataFrame(pump_node, crs="EPSG:28992") pump_static = pd.DataFrame() pump_static["node_id"] = pump_nodes["node_id"] @@ -1036,10 +1029,6 @@ def pump(self): pump_static["max_flow_rate"] = np.nan pump_static["control_state"] = np.nan - # comply to Ribasim 2024.11 - pump_node["meta_node_id"] = pump_node["node_id"].copy().astype(int) - pump_node = pump_node.set_index("node_id") - return pump_node, pump_static def level_boundary(self): @@ -1061,16 +1050,14 @@ def level_boundary(self): level_boundary_node["name"] = np.nan level_boundary_node["subnetwork_id"] = np.nan level_boundary_node["geometry"] = level_boundary_nodes["geometry"] + level_boundary_node = level_boundary_node.set_index("node_id") + level_boundary_node = gpd.GeoDataFrame(level_boundary_node, crs="EPSG:28992") level_boundary_static = pd.DataFrame() level_boundary_static["node_id"] = level_boundary_nodes["node_id"] level_boundary_static["active"] = np.nan level_boundary_static["level"] = 0 - # comply to Ribasim 2024.11 - level_boundary_node["meta_node_id"] = level_boundary_node["node_id"].copy().astype(int) - level_boundary_node = level_boundary_node.set_index("node_id") - return level_boundary_node, level_boundary_static def flow_boundary(self): @@ -1092,16 +1079,14 @@ def flow_boundary(self): flow_boundary_node["name"] = np.nan flow_boundary_node["subnetwork_id"] = np.nan flow_boundary_node["geometry"] = flow_boundary_nodes["geometry"] + flow_boundary_node = flow_boundary_node.set_index("node_id") + flow_boundary_node = gpd.GeoDataFrame(flow_boundary_node, crs="EPSG:28992") flow_boundary_static = pd.DataFrame() flow_boundary_static["node_id"] = flow_boundary_nodes["node_id"] flow_boundary_static["active"] = np.nan flow_boundary_static["flow_rate"] = 0 - # comply to Ribasim 2024.11 - flow_boundary_node["meta_node_id"] = flow_boundary_node["node_id"].copy().astype(int) - flow_boundary_node = flow_boundary_node.set_index("node_id") - return flow_boundary_node, flow_boundary_static def linear_resistance(self): @@ -1133,6 +1118,8 @@ def manning_resistance(self): manning_resistance_node["name"] = np.nan manning_resistance_node["subnetwork_id"] = np.nan manning_resistance_node["geometry"] = manning_resistance_nodes["geometry"] + manning_resistance_node = manning_resistance_node.set_index("node_id") + manning_resistance_node = gpd.GeoDataFrame(manning_resistance_node, crs="EPSG:28992") manning_resistance_static = pd.DataFrame() manning_resistance_static["node_id"] = manning_resistance_nodes["node_id"] @@ -1142,10 +1129,6 @@ def manning_resistance(self): manning_resistance_static["profile_width"] = 2 manning_resistance_static["profile_slope"] = 3 - # comply to Ribasim 2024.11 - manning_resistance_node["meta_node_id"] = manning_resistance_node["node_id"].copy().astype(int) - manning_resistance_node = manning_resistance_node.set_index("node_id") - return manning_resistance_node, manning_resistance_static def fractional_flow(self): @@ -1177,10 +1160,8 @@ def terminal(self): terminal_node["name"] = np.nan terminal_node["subnetwork_id"] = np.nan terminal_node["geometry"] = terminal_nodes["geometry"] - - # comply to Ribasim 2024.11 - terminal_node["meta_node_id"] = terminal_node["node_id"].copy().astype(int) terminal_node = terminal_node.set_index("node_id") + terminal_node = gpd.GeoDataFrame(terminal_node, crs="EPSG:28992") return terminal_node @@ -1453,7 +1434,7 @@ def check(self, model, post_processed_data, crossings): ) # .loc[model.basin.state.df['node_type'] == 'Basin'] #select all basins # model.basin.node.df.index += 1 #RB: outcommented plus one basin_nodes["geometry"] = model.basin.node.df.geometry # add geometry column - basin_nodes = gpd.GeoDataFrame(basin_nodes, geometry="geometry") # convert from pd go gpd + basin_nodes = gpd.GeoDataFrame(basin_nodes, geometry="geometry", crs=model.crs) # convert from pd go gpd points_within = gpd.sjoin( basin_nodes, checks["boezem"], how="inner", predicate="within" @@ -1475,7 +1456,7 @@ def check(self, model, post_processed_data, crossings): inlaten_TRC = nodes_from_boezem.loc[ (nodes_from_boezem.meta_to_node_type == "TabulatedRatingCurve") | (nodes_from_boezem.meta_to_node_type == "Outlet") - ] + ].copy() inlaten_TRC = inlaten_TRC["to_node_id"] inlaten_TRC = model.tabulated_rating_curve.node.df.loc[ model.tabulated_rating_curve.node.df.index.isin(inlaten_TRC) # df.node_id --> df.index @@ -1489,7 +1470,7 @@ def check(self, model, post_processed_data, crossings): inlaten_TRC["meta_type_verbinding"] = "Inlaat" # inlaten_gemalen - inlaten_gemalen = nodes_from_boezem.loc[nodes_from_boezem.meta_to_node_type == "Pump"] + inlaten_gemalen = nodes_from_boezem.loc[nodes_from_boezem.meta_to_node_type == "Pump"].copy() inlaten_gemalen = inlaten_gemalen["to_node_id"] inlaten_gemalen = model.pump.node.df.loc[ model.pump.node.df.index.isin(inlaten_gemalen) @@ -1497,7 +1478,7 @@ def check(self, model, post_processed_data, crossings): inlaten_gemalen["meta_type_verbinding"] = "Inlaat" # inlaten_flowboundary - inlaten_flowboundary = nodes_to_boezem.loc[nodes_to_boezem.meta_from_node_type == "FlowBoundary"] + inlaten_flowboundary = nodes_to_boezem.loc[nodes_to_boezem.meta_from_node_type == "FlowBoundary"].copy() inlaten_flowboundary = inlaten_flowboundary["from_node_id"] inlaten_flowboundary = model.flow_boundary.node.df.loc[ model.flow_boundary.node.df.index.isin(inlaten_flowboundary) # df.node_id --> df.index @@ -1508,14 +1489,14 @@ def check(self, model, post_processed_data, crossings): uitlaten_TRC = nodes_to_boezem.loc[ (nodes_to_boezem.meta_from_node_type == "TabulatedRatingCurve") | (nodes_to_boezem.meta_from_node_type == "Outlet") - ] + ].copy() uitlaten_TRC = uitlaten_TRC["from_node_id"] uitlaten_TRC = model.tabulated_rating_curve.node.df.loc[ model.tabulated_rating_curve.node.df.index.isin(uitlaten_TRC) # df.node_id --> df.index ] # uitlaten_gemalen - uitlaten_gemalen = nodes_to_boezem.loc[nodes_to_boezem.meta_from_node_type == "Pump"] + uitlaten_gemalen = nodes_to_boezem.loc[nodes_to_boezem.meta_from_node_type == "Pump"].copy() uitlaten_gemalen = uitlaten_gemalen["from_node_id"] uitlaten_gemalen = model.pump.node.df.loc[ model.pump.node.df.index.isin(uitlaten_gemalen) @@ -1530,7 +1511,7 @@ def check(self, model, post_processed_data, crossings): uitlaten_TRC["meta_type_verbinding"] = "Uitlaat" # uitlaten_flowboundary - uitlaten_flowboundary = nodes_to_boezem.loc[nodes_to_boezem.meta_from_node_type == "FlowBoundary"] + uitlaten_flowboundary = nodes_to_boezem.loc[nodes_to_boezem.meta_from_node_type == "FlowBoundary"].copy() uitlaten_flowboundary = uitlaten_flowboundary["from_node_id"] uitlaten_flowboundary = model.flow_boundary.node.df.loc[ model.flow_boundary.node.df.index.isin(uitlaten_flowboundary) # df.node_id --> df.index @@ -1613,7 +1594,7 @@ def check(self, model, post_processed_data, crossings): (BCN_to.meta_from_node_type == "FlowBoundary") | (BCN_to.meta_from_node_type == "LevelBoundary") | (BCN_to.meta_from_node_type == "Terminal") - ] + ].copy() BCN_to["meta_type_verbinding"] = "Inlaat boundary" # look the node ids up in each table. @@ -1633,7 +1614,7 @@ def check(self, model, post_processed_data, crossings): inlaten_uitlaten = pd.concat([inlaten_uitlaten, BCN_from, BCN_to]) inlaten_uitlaten = inlaten_uitlaten.reset_index(drop=True) - inlaten_uitlaten = gpd.GeoDataFrame(inlaten_uitlaten, geometry="geometry", crs="EPSG:28992") + inlaten_uitlaten = gpd.GeoDataFrame(inlaten_uitlaten, geometry="geometry", crs=model.crs) checks["inlaten_uitlaten_boezems"] = inlaten_uitlaten return checks @@ -1751,7 +1732,7 @@ def store_data(data, output_path): _description_ """ for key in data.keys(): - data[str(key)].to_file(output_path + ".gpkg", layer=str(key), driver="GPKG") + data[str(key)].to_file(output_path.with_suffix(".gpkg"), layer=str(key), driver="GPKG") return @@ -1765,216 +1746,23 @@ def WriteResults(self, model, checks): checks : _type_ _description_ """ - path = f"../../../../Ribasim_networks/Waterschappen/{self.model_characteristics['waterschap']}" - # path = os.path.join(path, '', 'modellen', '', self.model_characteristics['waterschap'] + '_' + self.model_characteristics['modeltype']) - - ##### write the model to the Z drive ##### - if self.model_characteristics["write_Zdrive"]: - dir_path = f"../../../../Ribasim_networks/Waterschappen/{self.model_characteristics['waterschap']}/modellen/{self.model_characteristics['waterschap']}_{self.model_characteristics['modeltype']}" - - if not os.path.exists(dir_path): - os.makedirs(dir_path) - else: - for filename in os.listdir(dir_path): # delete outdated models in de original folder - file_path = os.path.join(dir_path, filename) - if os.path.isfile(file_path): - os.remove(file_path) - - path_ribasim = os.path.join( - path, - "", - "modellen", - "", - self.model_characteristics["waterschap"] + "_" + self.model_characteristics["modeltype"], - "ribasim.toml", - ) - model.write(path_ribasim) - - # print('Edges after writing to Z drive:') - # display(model.network.edge.df) - # gpd.GeoDataFrame(model.network.edge.df.geometry).plot(color='red') - # model.network.edge.df.to_file('zzl_test.gpkg') - # model.network.edge.plot() - - ##### write the checks ##### - if self.model_characteristics["write_checks"]: - RibasimNetwork.store_data( - data=checks, - # output_path = str(path + self.model_characteristics['waterschap'] + '_' + self.model_characteristics['modelname'] + '_' + self.model_characteristics['modeltype'] + '_checks')) - output_path=os.path.join( - path, - "modellen", - self.model_characteristics["waterschap"] + "_" + self.model_characteristics["modeltype"], - "database_checks", - ), - ) - - ##### write to the P drive ##### - if self.model_characteristics["write_Pdrive"]: - P_path = self.model_characteristics["path_Pdrive"] - P_path = os.path.join( - P_path, - self.model_characteristics["waterschap"], - "modellen", - self.model_characteristics["waterschap"] + "_" + self.model_characteristics["modeltype"], - self.model_characteristics["waterschap"] - + "_" - + self.model_characteristics["modelname"] - + "_" - + self.model_characteristics["modeltype"], - ) - - if not os.path.exists(P_path): - os.makedirs(P_path) + cloud = CloudStorage() + waterschap = self.model_characteristics["waterschap"] + modeltype = self.model_characteristics["modeltype"] + modellen_dir = cloud.joinpath(waterschap, "modellen") + model_dir = modellen_dir / "_repro" + model_dir.mkdir(parents=True, exist_ok=True) + model.write(model_dir / "repro.toml") - P_path = os.path.join( - P_path, - f"{self.model_characteristics['waterschap']}_{self.model_characteristics['modelname']}_{self.model_characteristics['modeltype']}_ribasim.toml", + if self.model_characteristics["write_goodcloud"]: + cloud.upload_model( + authority=waterschap, + model=waterschap + "_" + modeltype, ) - - model.write(P_path) - - # write checks to the P drive + if self.model_characteristics["write_checks"]: RibasimNetwork.store_data( data=checks, - output_path=str(P_path + "visualisation_checks"), - ) - - ##### copy symbology for the RIBASIM model ##### - if self.model_characteristics["write_symbology"]: - # dont change the paths below! - checks_symbology_path = ( - # r"../../../../Ribasim_networks/Waterschappen/Symbo_feb/modellen/Symbo_feb_poldermodel/Symbo_feb_20240219_Ribasimmodel.qlr" - r"../../../../Data_overig/QGIS_qlr/visualisation_Ribasim.qlr" - ) - checks_symbology_path_new = os.path.join( - path, - "modellen", - self.model_characteristics["waterschap"] + "_" + self.model_characteristics["modeltype"], - "visualisation_Ribasim.qlr", - ) - - # dummy string, required to replace string in the file - # checks_path_old = r"../../symbology/symbology__poldermodel_Ribasim/symbology__poldermodel.gpkg" - # # checks_path_new = os.path.join(self.model_characteristics['waterschap'] + '_' + self.model_characteristics['modelname'] + '_' + self.model_characteristics['modeltype'] + '.gpkg') - # checks_path_new = os.path.join("database.gpkg") - - # copy checks_symbology file from old dir to new dir - shutil.copy(src=checks_symbology_path, dst=checks_symbology_path_new) - - # read file - # with open(checks_symbology_path_new, encoding="utf-8") as file: - # qlr_contents = file.read() - - # # change paths in the .qlr file - # qlr_contents = qlr_contents.replace(checks_path_old, checks_path_new) - - # # write updated file - # with open(checks_symbology_path_new, "w", encoding="utf-8") as file: - # file.write(qlr_contents) - - if self.model_characteristics["write_Pdrive"]: - # write Ribasim model to the P drive - P_path = self.model_characteristics["path_Pdrive"] - P_path = os.path.join( - P_path, - self.model_characteristics["waterschap"], - "modellen", - self.model_characteristics["waterschap"] + "_" + self.model_characteristics["modeltype"], - self.model_characteristics["waterschap"] - + "_" - + self.model_characteristics["modelname"] - + "_" - + self.model_characteristics["modeltype"], - ) - - if not os.path.exists(P_path): - os.makedirs(P_path) - - P_path_ribasim = os.path.join(P_path, "ribasim.toml") - model.write(P_path_ribasim) - - shutil.copy( - src=checks_symbology_path_new, - dst=os.path.join( - P_path, - "visualisation_Ribasim.qlr", - ), - ) - - ##### copy symbology for the CHECKS data ##### - if self.model_characteristics["write_symbology"]: - # dont change the paths below! - # checks_symbology_path = r"../../../../Ribasim_networks/Waterschappen/Symbo_feb/modellen/Symbo_feb_poldermodel/Symbo_feb_20240219_checks.qlr" - checks_symbology_path = r"../../../../Data_overig/QGIS_qlr/visualisation_checks.qlr" - - checks_symbology_path_new = os.path.join( - path, - "modellen", - self.model_characteristics["waterschap"] + "_" + self.model_characteristics["modeltype"], - "visualisation_checks.qlr", - ) - - # # dummy string, required to replace string in the file - # checks_path_old = r"../../symbology/symbology__poldermodel_Ribasim/symbology__poldermodel_checks.gpkg" - # # checks_path_new = os.path.join(self.model_characteristics['waterschap'] + '_' + self.model_characteristics['modelname'] + '_' + self.model_characteristics['modeltype'] + '.gpkg') - # checks_path_new = os.path.join("HollandseDelta_classtest_poldermodel_checks.gpkg") - - # copy checks_symbology file from old dir to new dir - shutil.copy(src=checks_symbology_path, dst=checks_symbology_path_new) - - # read file - # with open(checks_symbology_path_new, encoding="utf-8") as file: - # qlr_contents = file.read() - - # change paths in the .qlr file - # qlr_contents = qlr_contents.replace(checks_path_old, checks_path_new) - - # # write updated file - # with open(checks_symbology_path_new, "w", encoding="utf-8") as file: - # file.write(qlr_contents) - - if self.model_characteristics["write_Pdrive"]: - # write Ribasim model to the P drive - P_path = self.model_characteristics["path_Pdrive"] - P_path = os.path.join( - P_path, - self.model_characteristics["waterschap"], - "modellen", - self.model_characteristics["waterschap"] + "_" + self.model_characteristics["modeltype"], - self.model_characteristics["waterschap"] - + "_" - + self.model_characteristics["modelname"] - + "_" - + self.model_characteristics["modeltype"], - ) - - if not os.path.exists(P_path): - os.makedirs(P_path) - - P_path_ribasim = os.path.join(P_path, "ribasim.toml") - model.write(P_path_ribasim) - - shutil.copy( - src=checks_symbology_path_new, - dst=os.path.join( - P_path, - "visualisation_Ribasim.qlr", - ), - ) - - if self.model_characteristics["write_goodcloud"]: - with open(self.model_characteristics["path_goodcloud_password"]) as file: - password = file.read() - - cloud_storage = CloudStorage( - password=password, - data_dir=r"../../../../Ribasim_networks/Waterschappen/", # + waterschap + '_'+ modelname + '_' + modeltype, - ) - - cloud_storage.upload_model( - authority=self.model_characteristics["waterschap"], - model=self.model_characteristics["waterschap"] + "_" + self.model_characteristics["modeltype"], + output_path=model_dir / "database_checks", ) print("Done") diff --git a/src/peilbeheerst_model/peilbeheerst_model/shortest_path.py b/src/peilbeheerst_model/peilbeheerst_model/shortest_path.py index 75a395d..4cdfd10 100644 --- a/src/peilbeheerst_model/peilbeheerst_model/shortest_path.py +++ b/src/peilbeheerst_model/peilbeheerst_model/shortest_path.py @@ -14,7 +14,7 @@ from shapely.ops import split from shapely.wkt import dumps -from peilbeheerst_model import waterschap_data +from .waterschappen import waterschap_data # ### Define functions # 1. splitting functions diff --git a/src/ribasim_nl/ribasim_nl/__init__.py b/src/ribasim_nl/ribasim_nl/__init__.py index ea8fb34..27b3378 100644 --- a/src/ribasim_nl/ribasim_nl/__init__.py +++ b/src/ribasim_nl/ribasim_nl/__init__.py @@ -1,9 +1,15 @@ __version__ = "0.1.0" -from ribasim_nl.cloud import CloudStorage -from ribasim_nl.model import Model -from ribasim_nl.network import Network -from ribasim_nl.network_validator import NetworkValidator -from ribasim_nl.reset_index import reset_index +from .cloud import CloudStorage +from .model import Model +from .network import Network +from .network_validator import NetworkValidator +from .reset_index import reset_index -__all__ = ["CloudStorage", "Network", "reset_index", "Model", "NetworkValidator"] +__all__ = [ + "CloudStorage", + "Model", + "Network", + "NetworkValidator", + "reset_index", +] From 24a2e2374a3b7095c97f431c7bc0af57f929edbb Mon Sep 17 00:00:00 2001 From: rbruijnshkv Date: Tue, 19 Nov 2024 11:48:36 +0100 Subject: [PATCH 3/3] Alligning code AGV Mainly commenting out different lines --- .../preprocess_data/AmstelGooienVecht.py | 41 +++++++++++-------- 1 file changed, 23 insertions(+), 18 deletions(-) diff --git a/src/peilbeheerst_model/peilbeheerst_model/preprocess_data/AmstelGooienVecht.py b/src/peilbeheerst_model/peilbeheerst_model/preprocess_data/AmstelGooienVecht.py index db8ad3d..89e9b8c 100644 --- a/src/peilbeheerst_model/peilbeheerst_model/preprocess_data/AmstelGooienVecht.py +++ b/src/peilbeheerst_model/peilbeheerst_model/preprocess_data/AmstelGooienVecht.py @@ -57,18 +57,18 @@ AVG[variable] = df_var # there is one last gpkg which contains the streefpeilen (and peilgebieden) -AVG["peilgebied"] = gpd.read_file(aangeleverd_dir / "Na_levering" / "vigerende_peilgebieden.gpkg") +# AVG["peilgebied"] = gpd.read_file(aangeleverd_dir / "Na_levering" / "vigerende_peilgebieden.gpkg") -AVG["peilgebied"]["streefpeil"] = np.nan -AVG["peilgebied"]["streefpeil"] = AVG["peilgebied"]["streefpeil"].fillna(value=AVG["peilgebied"]["GPGZMRPL"]) -AVG["peilgebied"]["streefpeil"] = AVG["peilgebied"]["streefpeil"].fillna(value=AVG["peilgebied"]["IWS_GPGVASTP"]) -AVG["peilgebied"]["streefpeil"] = AVG["peilgebied"]["streefpeil"].fillna(value=AVG["peilgebied"]["IWS_GPGONDP"]) +# AVG["peilgebied"]["streefpeil"] = np.nan +# AVG["peilgebied"]["streefpeil"] = AVG["peilgebied"]["streefpeil"].fillna(value=AVG["peilgebied"]["GPGZMRPL"]) +# AVG["peilgebied"]["streefpeil"] = AVG["peilgebied"]["streefpeil"].fillna(value=AVG["peilgebied"]["IWS_GPGVASTP"]) +# AVG["peilgebied"]["streefpeil"] = AVG["peilgebied"]["streefpeil"].fillna(value=AVG["peilgebied"]["IWS_GPGONDP"]) -print( - "Number of missing streefpeilen = ", - len(AVG["peilgebied"]["streefpeil"].loc[AVG["peilgebied"]["streefpeil"].isna()]), -) +# print( +# "Number of missing streefpeilen = ", +# len(AVG["peilgebied"]["streefpeil"].loc[AVG["peilgebied"]["streefpeil"].isna()]), +# ) # fig, ax = plt.subplots() # AVG['peilgebied'].geometry.plot(ax=ax, color='cornflowerblue') @@ -81,10 +81,10 @@ # overwrite previous data AVG["stuw"] = gpd.read_file(dump_path + "/Stuw.shp") -AVG["stuw"] = AVG["stuw"].loc[AVG["stuw"].LHM == "LHM"] +# AVG["stuw"] = AVG["stuw"].loc[AVG["stuw"].LHM == "LHM"] AVG["gemaal"] = gpd.read_file(dump_path + "/Gemaal.shp") -AVG["gemaal"] = AVG["gemaal"].loc[AVG["gemaal"].LHM == "LHM"] +# AVG["gemaal"] = AVG["gemaal"].loc[AVG["gemaal"].LHM == "LHM"] AVG["duikersifonhevel"] = gpd.read_file(dump_path + "/DuikerSifonHevel.shp") AVG["hydroobject"] = gpd.read_file(dump_path + "/LHM_hydrovakken.shp") @@ -145,8 +145,8 @@ AVG["duikersifonhevel"].loc[:, "nen3610id"] = "dummy_nen3610id_duikersifonhevel_" + AVG[ "duikersifonhevel" ].index.astype(str) -AVG["duikersifonhevel"]["globalid"] = "dummy_globalid_duikersifonhevel_" + AVG["duikersifonhevel"].index.astype(str) -AVG["duikersifonhevel"] = gpd.GeoDataFrame(AVG["duikersifonhevel"]).to_crs("epsg:28992") +AVG['duikersifonhevel']['globalid'] = 'dummy_globalid_duikersifonhevel_' + AVG['duikersifonhevel'].index.astype(str) +AVG['duikersifonhevel'] = gpd.GeoDataFrame(AVG['duikersifonhevel']).to_crs('epsg:28992') # hydroobject AVG["hydroobject"] = AVG["hydroobject"][["geometry"]] @@ -156,11 +156,11 @@ AVG["hydroobject"] = gpd.GeoDataFrame(AVG["hydroobject"]).set_crs("epsg:28992") # streefpeil -AVG["streefpeil"] = pd.DataFrame() -AVG["streefpeil"]["waterhoogte"] = AVG["peilgebied"]["streefpeil"] -AVG["streefpeil"]["globalid"] = "dummy_globalid_streefpeil_" + AVG["streefpeil"].index.astype(str) -AVG["streefpeil"]["geometry"] = np.nan -AVG["streefpeil"] = gpd.GeoDataFrame(AVG["streefpeil"]).set_crs("epsg:28992") +# AVG["streefpeil"] = pd.DataFrame() +# AVG["streefpeil"]["waterhoogte"] = AVG["peilgebied"]["streefpeil"] +# AVG["streefpeil"]["globalid"] = "dummy_globalid_streefpeil_" + AVG["streefpeil"].index.astype(str) +# AVG["streefpeil"]["geometry"] = np.nan +# AVG["streefpeil"] = gpd.GeoDataFrame(AVG["streefpeil"]).set_crs("epsg:28992") # peilgebied AVG["peilgebied"]["code"] = AVG["peilgebied"]["GAFNAAM"] @@ -168,6 +168,11 @@ AVG["peilgebied"]["nen3610id"] = "dummy_nen3610id_peilgebied_" + AVG["peilgebied"].index.astype(str) AVG["peilgebied"]["globalid"] = "dummy_globalid_peilgebied_" + AVG["peilgebied"].index.astype(str) +AVG['streefpeil'] = AVG['peilgebied'][['waterhoogte', 'globalid']] +AVG['streefpeil']['code'] = 'dummy_code_streefpeil_' + AVG['streefpeil'].index.astype(str) +AVG['streefpeil']['geometry'] = None +AVG['streefpeil'] = gpd.GeoDataFrame(AVG['streefpeil'], geometry = 'geometry') + AVG["peilgebied"] = AVG["peilgebied"][["code", "nen3610id", "globalid", "geometry"]] AVG["peilgebied"] = gpd.GeoDataFrame(AVG["peilgebied"]).to_crs("epsg:28992")