Skip to content

Commit

Permalink
fixed hydra issue with running in notebooks
Browse files Browse the repository at this point in the history
  • Loading branch information
@taddyb
taddyb committed Mar 5, 2024
1 parent 44c908d commit 4d21d78
Show file tree
Hide file tree
Showing 6 changed files with 74 additions and 83 deletions.
17 changes: 12 additions & 5 deletions dMC/__main__.py
View file
Original file line number Diff line number Diff line change
@@ -1,4 +1,5 @@
import logging
from pathlib import Path
import time
import random
import warnings
Expand All @@ -8,7 +9,7 @@
import hydra
from injector import inject, Injector
import numpy as np
from omegaconf import DictConfig
from omegaconf import DictConfig, OmegaConf
import torch.utils.data
import torch.nn as nn

Expand Down Expand Up @@ -61,7 +62,7 @@ def run(self):


@hydra.main(
version_base=None,
version_base="1.3",
config_path="conf/",
config_name="global_settings",
)
Expand All @@ -73,7 +74,7 @@ def main(cfg: DictConfig) -> None:
:type cfg: DictConfig
:return: None
"""
_set_seed(cfg)
_set_defaults(cfg)
start = time.perf_counter()
injector = Injector([configure(cfg), Factory])
handler = injector.get(ExperimentHandler)
Expand All @@ -82,7 +83,7 @@ def main(cfg: DictConfig) -> None:
log.info(f"Experiment: {cfg.name} took {(end - start):.6f} seconds")


def _set_seed(cfg):
def _set_defaults(cfg):
torch.manual_seed(cfg.config.model.seed)
if torch.cuda.is_available():
torch.cuda.empty_cache()
Expand All @@ -93,7 +94,13 @@ def _set_seed(cfg):
torch.backends.cudnn.benchmark = False
np.random.seed(1)
random.seed(0)

OmegaConf.set_struct(cfg, False)
cfg.config.model.device = cfg.device
cfg.config.experiment.device = cfg.device
cfg.config.data.device = cfg.device
OmegaConf.set_struct(cfg, True)
return cfg


if __name__ == "__main__":
main()
9 changes: 4 additions & 5 deletions dMC/conf/global_settings.yaml
View file
Original file line number Diff line number Diff line change
@@ -1,9 +1,8 @@
defaults:
- config: 03_train_usgs_period_1a
- hydra_settings: settings
- config: 01_generate_single_synth_parameter_data
- _self_
# Global ------------------------------------------------------------------------
cwd: /path/to/your/codefolder/dMC-Juniata-hydroDL2
data_dir: /path/to/your/data/dx-2000dis1_non_merge
name: 03_train_usgs_period_1a
cwd: /data/tkb5476/projects/dMC-Juniata-hydroDL2
data_dir: /data/tkb5476/projects/dMC-Juniata-hydroDL2/flat_files/dMC-Juniata-hydroDL2/dx-2000dis1_non_merge
name: 01_generate_single_synth_parameter_data
device: cpu
20 changes: 20 additions & 0 deletions dMC/conf/hydra/settings.yaml
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,20 @@
help:
app_name: dMC-Juniata-hydroDL2
header: == ${hydra.help.app_name} ==
template: |-
${hydra.help.header}
A differentiable Muskingum Cunge routing module developed by Tadd Bindas to learn "parameterization schemes"
${hydra.help.footer}
footer: |-
Powered by Hydra (https://hydra.cc)
Use --hydra-help to view Hydra specific help
job:
name: ${name}
chdir: True
run:
dir: ${hydra.runtime.cwd}/output/${hydra.job.name}/${now:%Y-%m-%d_%H-%M-%S}
sweep:
dir: multirun
subdir: ${hydra.job.override_dirname}
19 changes: 0 additions & 19 deletions dMC/conf/hydra_settings/settings.yaml
View file

This file was deleted.

12 changes: 0 additions & 12 deletions dMC/configuration.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -27,7 +27,6 @@ def __init__(self, cfg: DictConfig, package_paths: Dict):
:param cfg: Configuration object.
:type cfg: DictConfig
"""
_set_device(cfg)
self.cfg = cfg.config
self.package_paths = package_paths
self.service_locator = self._import_and_instantiate()
Expand Down Expand Up @@ -78,14 +77,3 @@ def import_(self, class_ref, **kwargs):
log.error(f"No class found in the service locator. Check cfg file or dictionary pathing")
raise TypeError


def _set_device(cfg):
"""
Sets our device
:return:
"""
OmegaConf.set_struct(cfg, False)
cfg.config.model.device = cfg.device
cfg.config.experiment.device = cfg.device
cfg.config.data.device = cfg.device
OmegaConf.set_struct(cfg, True)
80 changes: 38 additions & 42 deletions notebooks/ExampleConfiguration.ipynb
View file
Original file line number Diff line number Diff line change
Expand Up @@ -18,7 +18,7 @@
},
{
"cell_type": "code",
"execution_count": 1,
"execution_count": 10,
"id": "549cbd99-100c-4ad9-8ff6-cef7309c0210",
"metadata": {},
"outputs": [],
Expand All @@ -29,6 +29,7 @@
"import sys\n",
"\n",
"# Pypi imported Modules\n",
"import hydra\n",
"import matplotlib.pyplot as plt\n",
"import numpy as np\n",
"from omegaconf import DictConfig, OmegaConf\n",
Expand Down Expand Up @@ -57,8 +58,7 @@
"from dMC.experiments.writer import Writer\n",
"\n",
"# Utils functions\n",
"from dMC.configuration import _set_device\n",
"from dMC.__main__ import _set_seed\n",
"from dMC.__main__ import _set_defaults\n",
"\n",
"# Required to generate data\n",
"from dMC.data.datasets.nhd_srb import NHDSRB\n",
Expand All @@ -78,45 +78,37 @@
"source": [
"## Setting up the Config\n",
"\n",
"Let's import the config files from our `dMC` directory:"
"Let's import the config files from our `dMC` directory:\n",
"\n",
"**Note** this uses the config settings from `../dMC/conf/gloal_settings.yaml`"
]
},
{
"cell_type": "code",
"execution_count": 2,
"execution_count": 11,
"id": "ac501919-2de5-4b1a-86a4-3063fd060b9f",
"metadata": {},
"outputs": [
{
"data": {
"text/plain": [
"{'defaults': [{'config': '03_train_usgs_period_1a'}, {'hydra_settings': 'settings'}, '_self_'], 'cwd': '/data/tkb5476/projects/dMC-Juniata-hydroDL2', 'data_dir': '/data/tkb5476/projects/dMC-Juniata-hydroDL2/flat_files/dMC-Juniata-hydroDL2/dx-2000dis1_non_merge', 'name': '03_train_usgs_period_1a', 'device': 'cpu', 'config': {'service_locator': {'experiment': 'generate_synthetic.GenerateSynthetic', 'data': 'nhd_srb.NHDSRB', 'observations': 'usgs.USGS', 'physics': 'explicit_mc.ExplicitMC', 'neural_network': 'single_parameters.SingleParameters'}, 'data': {'processed_dir': '${cwd}/flat_files', 'end_node': 4809, 'time': {'start': '02/01/2001 00:00:00', 'end': '09/18/2010 23:00:00', 'steps': 1344, 'tau': 9, 'batch_size': '${config.data.time.steps}'}, 'observations': {'loss_nodes': [1053, 1280, 2662, 2689, 2799, 4780, 4801, 4809], 'dir': '${data_dir}/inflow_interpolated/', 'file_name': '???'}, 'save_paths': {'edges': '${config.data.processed_dir}/${config.data.end_node}_edges.csv', 'nodes': '${config.data.processed_dir}/${config.data.end_node}_nodes.csv', 'areas': '${config.data.processed_dir}/${config.data.end_node}_areas.npy', 'q_prime': '${config.data.processed_dir}/${config.data.end_node}_tau${config.data.time.tau}_{}_{}_q_prime.csv', 'network': '${config.data.processed_dir}/${config.data.end_node}_network_matrix.csv', 'gage_locations': '${config.data.processed_dir}/gages_II_locations.csv', 'q_prime_sum': '${config.data.processed_dir}/${config.data.end_node}_tau${config.data.time.tau}_q_prime_sum.npy'}, 'csv': {'edges': '${data_dir}/graphs/edges_NaNFix.csv', 'nodes': '${data_dir}/graphs/node.csv', 'q_prime': '${data_dir}/graphs/srb_post_process.csv', 'mass_transfer_matrix': '${data_dir}/graphs/TM.csv'}}, 'experiment': {'learning_rate': 0.01, 'epochs': 100, 'warmup': 72, 'lb': [0.01, 0.0], 'ub': [0.3, 3.0], 'factor': 100, 'name': '${name}', 'save_path': '${cwd}/runs/01_synthetic_data/', 'output_cols': '${config.data.observations.loss_nodes}', 'tensorboard_dir': '${cwd}/logs/srb/${name}/${now:%Y-%m-%d}/'}, 'model': {'noise': 0.005, 'train_q': True, 'seed': 0, 'mlp': {'initialization': 'xavier_normal', 'fan': 'fan_in', 'gain': 0.7, 'hidden_size': 6, 'input_size': 8, 'output_size': 2}, 'length': {'idx': 8}, 'slope': {'idx': 2, 'min': 0.0001, 'max': 0.3}, 'velocity': {'min': 0.3, 'max': 15}, 'q_prime': {'min': 0}, 'variables': {'n': 0.03, 'p': 21.0, 'q': 0.5, 't': 3600.0, 'x': 0.3}, 'transformations': {'n': [0.01, 0.3], 'q_spatial': [0, 3]}, 'save_paths': {'areas': '${config.data.save_paths.areas}'}, 'is_base': True}}}"
"{'hydra': {'run': {'dir': 'outputs/${now:%Y-%m-%d}/${now:%H-%M-%S}'}, 'sweep': {'dir': 'multirun/${now:%Y-%m-%d}/${now:%H-%M-%S}', 'subdir': '${hydra.job.num}'}, 'launcher': {'_target_': 'hydra._internal.core_plugins.basic_launcher.BasicLauncher'}, 'sweeper': {'_target_': 'hydra._internal.core_plugins.basic_sweeper.BasicSweeper', 'max_batch_size': None, 'params': None}, 'help': {'app_name': '${hydra.job.name}', 'header': '${hydra.help.app_name} is powered by Hydra.\\n', 'footer': 'Powered by Hydra (https://hydra.cc)\\nUse --hydra-help to view Hydra specific help\\n', 'template': '${hydra.help.header}\\n== Configuration groups ==\\nCompose your configuration from those groups (group=option)\\n\\n$APP_CONFIG_GROUPS\\n\\n== Config ==\\nOverride anything in the config (foo.bar=value)\\n\\n$CONFIG\\n\\n${hydra.help.footer}\\n'}, 'hydra_help': {'template': \"Hydra (${hydra.runtime.version})\\nSee https://hydra.cc for more info.\\n\\n== Flags ==\\n$FLAGS_HELP\\n\\n== Configuration groups ==\\nCompose your configuration from those groups (For example, append hydra/job_logging=disabled to command line)\\n\\n$HYDRA_CONFIG_GROUPS\\n\\nUse '--cfg hydra' to Show the Hydra config.\\n\", 'hydra_help': '???'}, 'hydra_logging': {'version': 1, 'formatters': {'simple': {'format': '[%(asctime)s][HYDRA] %(message)s'}}, 'handlers': {'console': {'class': 'logging.StreamHandler', 'formatter': 'simple', 'stream': 'ext://sys.stdout'}}, 'root': {'level': 'INFO', 'handlers': ['console']}, 'loggers': {'logging_example': {'level': 'DEBUG'}}, 'disable_existing_loggers': False}, 'job_logging': {'version': 1, 'formatters': {'simple': {'format': '[%(asctime)s][%(name)s][%(levelname)s] - %(message)s'}}, 'handlers': {'console': {'class': 'logging.StreamHandler', 'formatter': 'simple', 'stream': 'ext://sys.stdout'}, 'file': {'class': 'logging.FileHandler', 'formatter': 'simple', 'filename': '${hydra.runtime.output_dir}/${hydra.job.name}.log'}}, 'root': {'level': 'INFO', 'handlers': ['console', 'file']}, 'disable_existing_loggers': False}, 'env': {}, 'mode': None, 'searchpath': [], 'callbacks': {}, 'output_subdir': '.hydra', 'overrides': {'hydra': [], 'task': []}, 'job': {'name': 'notebook', 'chdir': None, 'override_dirname': '', 'id': '???', 'num': '???', 'config_name': 'global_settings.yaml', 'env_set': {}, 'env_copy': [], 'config': {'override_dirname': {'kv_sep': '=', 'item_sep': ',', 'exclude_keys': []}}}, 'runtime': {'version': '1.3.2', 'version_base': '1.3', 'cwd': '/data/tkb5476/projects/dMC-Juniata-hydroDL2/notebooks', 'config_sources': [{'path': 'hydra.conf', 'schema': 'pkg', 'provider': 'hydra'}, {'path': '/data/tkb5476/projects/dMC-Juniata-hydroDL2/dMC/conf', 'schema': 'file', 'provider': 'main'}, {'path': '', 'schema': 'structured', 'provider': 'schema'}], 'output_dir': '???', 'choices': {'config': '01_generate_single_synth_parameter_data', 'hydra/env': 'default', 'hydra/callbacks': None, 'hydra/job_logging': 'default', 'hydra/hydra_logging': 'default', 'hydra/hydra_help': 'default', 'hydra/help': 'default', 'hydra/sweeper': 'basic', 'hydra/launcher': 'basic', 'hydra/output': 'default'}}, 'verbose': False}, 'config': {'service_locator': {'experiment': 'generate_synthetic.GenerateSynthetic', 'data': 'nhd_srb.NHDSRB', 'observations': 'usgs.USGS', 'physics': 'explicit_mc.ExplicitMC', 'neural_network': 'single_parameters.SingleParameters'}, 'data': {'processed_dir': '${cwd}/flat_files', 'end_node': 4809, 'time': {'start': '02/01/2001 00:00:00', 'end': '09/18/2010 23:00:00', 'steps': 1344, 'tau': 9, 'batch_size': '${config.data.time.steps}'}, 'observations': {'loss_nodes': [1053, 1280, 2662, 2689, 2799, 4780, 4801, 4809], 'dir': '${data_dir}/inflow_interpolated/', 'file_name': '???'}, 'save_paths': {'edges': '${config.data.processed_dir}/${config.data.end_node}_edges.csv', 'nodes': '${config.data.processed_dir}/${config.data.end_node}_nodes.csv', 'areas': '${config.data.processed_dir}/${config.data.end_node}_areas.npy', 'q_prime': '${config.data.processed_dir}/${config.data.end_node}_tau${config.data.time.tau}_{}_{}_q_prime.csv', 'network': '${config.data.processed_dir}/${config.data.end_node}_network_matrix.csv', 'gage_locations': '${config.data.processed_dir}/gages_II_locations.csv', 'q_prime_sum': '${config.data.processed_dir}/${config.data.end_node}_tau${config.data.time.tau}_q_prime_sum.npy'}, 'csv': {'edges': '${data_dir}/graphs/edges_NaNFix.csv', 'nodes': '${data_dir}/graphs/node.csv', 'q_prime': '${data_dir}/graphs/srb_post_process.csv', 'mass_transfer_matrix': '${data_dir}/graphs/TM.csv'}, 'device': 'cpu'}, 'experiment': {'learning_rate': 0.01, 'epochs': 100, 'warmup': 72, 'lb': [0.01, 0.0], 'ub': [0.3, 3.0], 'factor': 100, 'name': '${name}', 'save_path': '${cwd}/runs/01_synthetic_data/', 'output_cols': '${config.data.observations.loss_nodes}', 'tensorboard_dir': '${cwd}/logs/srb/${name}/${now:%Y-%m-%d}/', 'device': 'cpu'}, 'model': {'noise': 0.005, 'train_q': True, 'seed': 0, 'mlp': {'initialization': 'xavier_normal', 'fan': 'fan_in', 'gain': 0.7, 'hidden_size': 6, 'input_size': 8, 'output_size': 2}, 'length': {'idx': 8}, 'slope': {'idx': 2, 'min': 0.0001, 'max': 0.3}, 'velocity': {'min': 0.3, 'max': 15}, 'q_prime': {'min': 0}, 'variables': {'n': 0.03, 'p': 21.0, 'q': 0.5, 't': 3600.0, 'x': 0.3}, 'transformations': {'n': [0.01, 0.3], 'q_spatial': [0, 3]}, 'save_paths': {'areas': '${config.data.save_paths.areas}'}, 'is_base': True, 'device': 'cpu'}}, 'cwd': '/data/tkb5476/projects/dMC-Juniata-hydroDL2', 'data_dir': '/data/tkb5476/projects/dMC-Juniata-hydroDL2/flat_files/dMC-Juniata-hydroDL2/dx-2000dis1_non_merge', 'name': '01_generate_single_synth_parameter_data', 'device': 'cpu'}"
]
},
"execution_count": 2,
"execution_count": 11,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"cfg = OmegaConf.load(dmc_dev_path / \"dMC/conf/global_settings.yaml\")\n",
"experiment_settings = OmegaConf.load(dmc_dev_path / \"dMC/conf/config/01_generate_single_synth_parameter_data.yaml\")\n",
"cfg.config = experiment_settings\n",
"config_path = \"../dMC/conf/\"\n",
"with hydra.initialize(config_path=config_path, version_base=\"1.3\"):\n",
" cfg = hydra.compose(config_name=\"global_settings.yaml\", return_hydra_config=True)\n",
"\n",
"cfg = _set_defaults(cfg)\n",
"cfg"
]
},
{
"cell_type": "code",
"execution_count": 3,
"id": "ccfeacc2-dbd6-48e4-95e0-5af753a48c1b",
"metadata": {},
"outputs": [],
"source": [
"# Applying our global settings, and specifying an output dir as \n",
"_set_device(cfg)\n",
"_set_seed(cfg)"
]
},
{
"cell_type": "markdown",
"id": "9d3c0be6-1cab-4b46-b781-5af2e9332561",
Expand All @@ -137,17 +129,17 @@
},
{
"cell_type": "code",
"execution_count": 4,
"execution_count": 12,
"id": "6fef710d-566e-449b-abf8-5cd94ce89111",
"metadata": {},
"outputs": [
{
"data": {
"text/plain": [
"<torch.utils.data.dataloader.DataLoader at 0x7fe9a42baf50>"
"<torch.utils.data.dataloader.DataLoader at 0x7f3248d00bd0>"
]
},
"execution_count": 4,
"execution_count": 12,
"metadata": {},
"output_type": "execute_result"
}
Expand Down Expand Up @@ -178,7 +170,7 @@
},
{
"cell_type": "code",
"execution_count": 5,
"execution_count": 13,
"id": "154f92d6-4f1b-4f17-9ef4-f85ccbbc7c82",
"metadata": {},
"outputs": [
Expand All @@ -190,7 +182,7 @@
")"
]
},
"execution_count": 5,
"execution_count": 13,
"metadata": {},
"output_type": "execute_result"
}
Expand All @@ -199,13 +191,17 @@
"cfg_model = cfg.config.model\n",
"\n",
"neural_network = SingleParameters(cfg=cfg_model).to(cfg_model.device)\n",
"# neural_network = Power(cfg=cfg_model).to(cfg_model.device)\n",
"# neural_network = MLP(cfg=cfg_model).to(cfg_model.device)\n",
"# ... etc (see imports for all options)\n",
"\n",
"physics_model = ExplicitMC(cfg=cfg_model, neural_network=neural_network)\n",
"physics_model"
]
},
{
"cell_type": "code",
"execution_count": 6,
"execution_count": 14,
"id": "0ffde6ac-ae29-41af-b77b-f6fa96a385ac",
"metadata": {},
"outputs": [
Expand All @@ -216,7 +212,7 @@
"tensor(0.0300, requires_grad=True)"
]
},
"execution_count": 6,
"execution_count": 14,
"metadata": {},
"output_type": "execute_result"
}
Expand All @@ -235,25 +231,25 @@
},
{
"cell_type": "code",
"execution_count": 7,
"execution_count": 15,
"id": "eb4efb99-df66-4f98-ac01-a86838698a4e",
"metadata": {},
"outputs": [
{
"data": {
"text/plain": [
"<dMC.experiments.generate_synthetic.GenerateSynthetic at 0x7fe9a42bb710>"
"<dMC.experiments.generate_synthetic.GenerateSynthetic at 0x7f324767a110>"
]
},
"execution_count": 7,
"execution_count": 15,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"cfg_experiment = cfg.config.experiment\n",
"# writer = Writer(cfg_experiment)\n",
"experiment = GenerateSynthetic(cfg=cfg_experiment, writer=None)\n",
"writer = Writer(cfg_experiment)\n",
"experiment = GenerateSynthetic(cfg=cfg_experiment, writer=writer)\n",
"experiment"
]
},
Expand All @@ -269,14 +265,14 @@
},
{
"cell_type": "code",
"execution_count": 8,
"execution_count": 16,
"id": "6faa7b78-5f66-40e4-a6b6-6cf9801f0315",
"metadata": {},
"outputs": [
{
"data": {
"application/vnd.jupyter.widget-view+json": {
"model_id": "242174c4316e4c56aaacb3f2e198b94e",
"model_id": "b4665248457343b78dcc571259d3be99",
"version_major": 2,
"version_minor": 0
},
Expand All @@ -294,7 +290,7 @@
},
{
"cell_type": "code",
"execution_count": 9,
"execution_count": 19,
"id": "cfd0506a-c8dc-4563-8340-5e15e325d74b",
"metadata": {},
"outputs": [
Expand All @@ -304,7 +300,7 @@
"PosixPath('/data/tkb5476/projects/dMC-Juniata-hydroDL2/runs/01_synthetic_data')"
]
},
"execution_count": 9,
"execution_count": 19,
"metadata": {},
"output_type": "execute_result"
}
Expand All @@ -315,7 +311,7 @@
},
{
"cell_type": "code",
"execution_count": 10,
"execution_count": 20,
"id": "e573eb1b-e848-4806-9865-61014c16f40f",
"metadata": {},
"outputs": [
Expand Down Expand Up @@ -407,7 +403,7 @@
" <td>1.499759</td>\n",
" <td>0.235011</td>\n",
" <td>0.221742</td>\n",
" <td>1.618124</td>\n",
" <td>1.618123</td>\n",
" <td>0.993112</td>\n",
" <td>0.632635</td>\n",
" <td>1.268810</td>\n",
Expand All @@ -422,7 +418,7 @@
"1 1 0.345283 1.010940 0.064402 0.048944 0.513326 0.258317 \n",
"2 2 0.755540 1.308555 0.099312 0.127346 0.782226 0.438819 \n",
"3 3 1.101563 1.415994 0.155389 0.183362 1.098740 0.659021 \n",
"4 4 1.485405 1.499759 0.235011 0.221742 1.618124 0.993112 \n",
"4 4 1.485405 1.499759 0.235011 0.221742 1.618123 0.993112 \n",
"\n",
" 4801 4809 \n",
"0 0.170211 0.272679 \n",
Expand All @@ -432,7 +428,7 @@
"4 0.632635 1.268810 "
]
},
"execution_count": 10,
"execution_count": 20,
"metadata": {},
"output_type": "execute_result"
}
Expand Down

0 comments on commit 4d21d78

Please sign in to comment.