notebooks/optimization_functions.ipynb

{"nbformat":4,"nbformat_minor":0,"metadata":{"colab":{"name":"optimization_functions.ipynb","provenance":[],"private_outputs":true,"collapsed_sections":[],"toc_visible":true},"kernelspec":{"name":"python3","display_name":"Python 3"}},"cells":[{"cell_type":"markdown","metadata":{"id":"Fbd7yuUGJ7-B"},"source":[""]},{"cell_type":"markdown","metadata":{"id":"c_IZXKICfV2u"},"source":["**First clone the github repository**"]},{"cell_type":"code","metadata":{"id":"kwM7YJ-RJ6lz"},"source":["#%cd /content\n","#! git clone https://github.com/Tikquuss/lwd\n","#%cd lwd/scripts"],"execution_count":null,"outputs":[]},{"cell_type":"code","metadata":{"id":"rvQSW7sNpvP_"},"source":["#from google.colab import drive\n","#drive.mount('/content/drive')"],"execution_count":null,"outputs":[]},{"cell_type":"code","metadata":{"id":"nMR_g89Wpr85"},"source":["#main_path = \"/content/drive/My Drive/Pascal Notsawo Summer 2020 WL Research Internship/Project 2: Learning with Derivatives/week 6/Styblinski-Tang\"\n","main_path = \"/content\"\n","dump_path = None # folder in which the models will be stored (left to None/False/0/\"\" if we don't want to save the models)"],"execution_count":null,"outputs":[]},{"cell_type":"markdown","metadata":{"id":"hDYl-5mLN6yv"},"source":["# **Set-up** \n","*Once this section is configured, you can comment (optionally) the previous cell and run everything at once (Runtime -> Run all).*  \n","**Once everything is executed, a csv file will be automatically created containing all the losses on the test data for each model.**"]},{"cell_type":"markdown","metadata":{"id":"YW63hnDOJW3L"},"source":["* Uncomment the line corresponding to your function, and choose the values of the bounds of the intervals in which the learning and test data will be generated (uniformly) : we have at the end of each line the corresponding values taken from the [\"Sobolev Training\"](https://arxiv.org/abs/1706.04859) original paper (page 14-15)\n","* The steps are used for the curves: they can therefore be chosen according to the desired precision."]},{"cell_type":"code","metadata":{"id":"AXcYoqZtH31b"},"source":["f_name = \"Styblinski-Tang\" # https://www.sfu.ca/~ssurjano/stybtang.html (-5, 5), (-5, 5)\n","#f_name = \"Ackley\" # http://www.sfu.ca/~ssurjano/ackley.html (-5, 5), (-5, 5)\n","#f_name = \"Beale\" # https://www.sfu.ca/~ssurjano/beale.html (-4.5, 4.5), (-4.5, 4.5)\n","#f_name = \"Booth\" # https://www.sfu.ca/~ssurjano/booth.html (-10, 10), (-10, 10)\n","#f_name = \"Bukin\" # https://www.sfu.ca/~ssurjano/bukin6.html (-15, -5), (-3, 3)\n","#f_name = \"McCormick\" # https://www.sfu.ca/~ssurjano/mccorm.html (-1.5, 4), (-3, 4)\n","#f_name = \"Rosenbrock\" # https://www.sfu.ca/~ssurjano/rosen.html (-2, 2), (-2, 2)\n","\n","(min_x, max_x), (min_y, max_y) = (-5, 5), (-5, 5)\n","step_x, step_y = 0.25, 0.25"],"execution_count":null,"outputs":[]},{"cell_type":"code","metadata":{"id":"2d-rdlazooRS"},"source":["try:\n","    %tensorflow_version 1.x\n","    %matplotlib inline\n","except Exception:\n","    pass\n","\n","import tensorflow as tf\n","import torch\n","import torch.nn.functional as F\n","import numpy as np\n","import matplotlib.pyplot as plt\n","import random\n","import os\n","\n","from utils import gradient, genData, plotFunction, plotGrad, get_data_loader, plot_stat, global_stat, reshape, to_csv\n","from utils import forward, backprop, MLP, Siren, train, test\n","from twin_net_tf import graph, Generator, test as twin_net_tf_test \n","from functions import * # Styblinski-Tang (ST), Ackley, Beale, Booth, Bukin, McCormick, Rosenbrock"],"execution_count":null,"outputs":[]},{"cell_type":"code","metadata":{"id":"TM_wHdPGpKxF"},"source":["# Global\n","max_epoch = 2 # maximun number of epoch\n","#batch_sizes = [20, 100, 1000, 1024, 1024] # batch_size\n","#nTrains = [20, 100, 1000, 10000, 100000] # number of training examples\n","#nTests = [10000]*5 # number of test examples\n","\n","batch_sizes = [10] # batch_size\n","nTrains = [10] # number of training examples\n","nTests = [10] # number of test examples\n","\n","train_seed, test_seed = 0, 1 # for reproducibility\n","\n","learning_rate = 3e-5 # learning rate\n","\n","learning_rate_schedule = [(0.0, 1.0e-8), (0.2, 0.1), (0.6, 0.01), (0.9, 1.0e-6), (1.0, 1.0e-8)]\n","\n","improving_limit = float(\"inf\") # Stop training if the training loss does not decrease n times (no limit here)\n","twin_net_tf_improving_limit = float(\"inf\") # tf\n","INPUT_DIM = 2 \n","HIDDEN_DIM = 256\n","N_HIDDEN = 4 # number of hidden layers\n","OUTPUT_DIM = 1\n","# To initialize the model parameters and to make sure that the models are initialized with the same values.\n","params_seed = 0 \n","init_weights = True\n","tf_config = {\"init_weights\" : init_weights, \"input_dim\" : OUTPUT_DIM}\n","\n","# hyperparameters in the different loss functions to express a tradeoff between y loss and dydx loss\n","# Leave None and None instead of 1 and 1, this will be managed automatically.\n","loss_config = {'alpha': None, \"beta\" : None} # loss = alpha * loss_y + beta * loss_dydx\n","\n","# MLP\n","import math\n","a = math.sqrt(2/math.pi)\n","b = 0.044715\n","part = lambda x : 1 + torch.tanh(a*(x + b*(x**3)))\n","dpart = lambda x : (a*(1 + 3*b*(x)**2))*(1 - torch.tanh(a*(x + b*(x**3)))**2)\n","g = F.gelu # or lambda x : x * part(x) / 2\n","dg = lambda x : part(x) / 2 + x * dpart(x) / 2\n","activation_function, deriv_activation_function = g, dg\n","\n","# twin_net tf \n","tf_config.update({\"activation_function\" : tf.nn.softplus, \"deriv_activation_function\" : tf.nn.sigmoid})\n","\n","mlp_model_kwargs = {\"in_features\" : INPUT_DIM, # depends on the function\n","                    \"hidden_features\" : HIDDEN_DIM, \n","                    \"hidden_layers\" : N_HIDDEN, \n","                    \"out_features\": OUTPUT_DIM, \n","                    \"activation_function\" : activation_function, \n","                    \"deriv_activation_function\" : deriv_activation_function,\n","                    \"init_weights\" : init_weights,\n","                    \"params_seed\" : params_seed\n","                   }\n","# Siren\n","first_omega_0 = 30.\n","hidden_omega_0 = 30.\n","outermost_linear = True\n","\n","siren_model_kwargs = {\"in_features\" : INPUT_DIM, \n","                      \"hidden_features\" : HIDDEN_DIM, \n","                      \"hidden_layers\" : N_HIDDEN, \n","                      \"out_features\": OUTPUT_DIM, \n","                      \"outermost_linear\" : outermost_linear, \n","                      \"first_omega_0\" : first_omega_0, \n","                      \"hidden_omega_0\" : hidden_omega_0,\n","                      \"init_weights\" : init_weights,\n","                      \"params_seed\" : params_seed\n","                      }\n","\n","# twin_net tf                \n","generator_kwargs = {\"hidden_units\" : HIDDEN_DIM, \"hidden_layers\" : N_HIDDEN}"],"execution_count":null,"outputs":[]},{"cell_type":"markdown","metadata":{"id":"ziEks-FxpPkD"},"source":["# **To avoid repeating the same code too much**"]},{"cell_type":"code","metadata":{"id":"cKjcjjYEBQbo"},"source":["if f_name == \"Styblinski-Tang\" :\n","    callable_function = STFunction\n","    callable_function_deriv = STDeriv\n","    function = \"Styblinski-Tang Function\"\n","elif f_name == \"Ackley\" :\n","    callable_function = AckleyFunction\n","    callable_function_deriv = AckleyDeriv\n","    function = \"Ackley Function\"\n","elif f_name == \"Beale\" :\n","    callable_function = BealeFunction\n","    callable_function_deriv = BealeDeriv\n","    function = \"Beale Function\"\n","elif f_name == \"Booth\" :\n","    callable_function = BoothFunction\n","    callable_function_deriv = BoothDeriv\n","    function = \"Booth Function\"\n","elif f_name == \"Bukin\" :\n","    callable_function = BukinFunction\n","    callable_function_deriv = BukinDeriv\n","    function = \"Bukin Function\"\n","elif f_name == \"McCormick\" :\n","    callable_function = McCormickFunction\n","    callable_function_deriv = McCormickDeriv\n","    function = \"McCormick Function\"\n","elif f_name == \"Rosenbrock\" :\n","    callable_function = RosenbrockFunction\n","    callable_function_deriv = RosenbrockDeriv\n","    function = \"Rosenbrock Function\"\n","\n","csv_path = os.path.join(main_path, f_name + \".csv\")\n","import os\n","if not os.path.exists(main_path):\n","    os.makedirs(main_path)\n","\n","grad = \"Gradient Field of %s\" % function"],"execution_count":null,"outputs":[]},{"cell_type":"code","metadata":{"id":"byiZp_sUmzhN"},"source":["def run_train(name, model_class, model_kwargs, with_derivative, name_function, name_grad,  \n","              normalize = False, learning_rate_schedule = None):\n","    global callable_function, callable_function_deriv\n","    global nTrains, nTests, batch_sizes, train_seed, test_seed\n","    global learning_rate, max_epoch, improving_limit\n","    global min_x, max_x, step_x, min_y, max_y, step_y\n","    global loss_config\n","    global f_name, main_path, dump_path\n","\n","    model_list, loss_list, stat_list = [], [], []\n","    \n","    for nTrain, nTest, batch_size in zip(nTrains, nTests, batch_sizes) : \n","\n","        print(\"========== nTrain %d ===========\" % nTrain)\n","        \n","        batch_samples = genData(function = callable_function, deriv_function = callable_function_deriv, dim_x = INPUT_DIM, min_x = min_x, max_x = max_x, num_samples = nTrain, random_seed = train_seed)\n","        x_train, y_train, dydx_train = zip(*batch_samples)\n","\n","        batch_samples = genData(function = callable_function, deriv_function = callable_function_deriv, dim_x = INPUT_DIM, min_x = min_x, max_x = max_x, num_samples = nTest, random_seed = test_seed)\n","        x_test, y_test, dydx_test = zip(*batch_samples)\n","\n","        if with_derivative :\n","            train_dataloader, config = get_data_loader(x = x_train, y = y_train,  dydx = dydx_train, \n","                                                      batch_size = batch_size, normalize = normalize)\n","            test_dataloader, _ = get_data_loader(x = x_test, y = y_test,  dydx = dydx_test, batch_size = batch_size)\n","        else :\n","            train_dataloader, config = get_data_loader(x = x_train, y = y_train,  dydx = None, \n","                                                      batch_size = batch_size, normalize = normalize)\n","            test_dataloader, _ = get_data_loader(x = x_test, y = y_test, dydx = None, batch_size = batch_size)\n","\n","        config[\"learning_rate_schedule\"] = learning_rate_schedule\n","        config.update({key : value for key, value in loss_config.items() if value})\n","        config[\"dump_path\"] = dump_path\n","        config[\"function_name\"] = f_name\n","        model_name = name # 'net', 'twin_net'\n","        if name == \"net\" :\n","            model_name = \"normal\" if not with_derivative else \"sobolev\"\n","        model_name += \"-norm\" if normalize else \"\"\n","        model_name += \"-lrs\" if learning_rate_schedule else \"\"\n","        config[\"model_name\"] = model_name\n","        config[\"nTrain\"] = nTrain\n","        config[\"batch_size\"] = batch_size\n","                           \n","        model = model_class(**model_kwargs)\n","        criterion = torch.nn.MSELoss()\n","        optimizer = torch.optim.Adam(model.parameters(), lr = learning_rate)\n","        \n","        model, stats, best_loss = train(name, model, train_dataloader, optimizer, criterion, config, \n","                                        with_derivative, max_epoch = max_epoch, improving_limit = improving_limit)\n","        \n","        plot_stat(stats, with_derivative = with_derivative)\n","\n","        (test_loss, r_y, r_dydx), (x_list, y_list, dydx_list, y_pred_list, dydx_pred_list) = test(\n","            name, model, test_dataloader, criterion, config, with_derivative\n","        )\n","\n","        x_mean, x_std = config.get(\"x_mean\", 0.), config.get(\"x_std\", 1.)\n","        y_mean, y_std = config.get(\"y_mean\", 0.), config.get(\"y_std\", 1.)\n","\n","        def function(x):\n","            x = torch.tensor(x)\n","            x_scaled = (x-x_mean) / x_std\n","            y_pred_scaled = model(x = x_scaled.float())\n","            y_pred = y_mean + y_std * y_pred_scaled\n","            y_pred = y_pred.detach().squeeze().numpy()\n","            return y_pred\n","\n","        def deriv_function(index):\n","            def f(x) :\n","                x = torch.tensor(x, requires_grad = True)\n","                x_scaled = (x-x_mean) / x_std\n","                if name == \"net\" :\n","                    y_pred_scaled = model(x = x_scaled.float()) \n","                    dydx_pred_scaled = gradient(y_pred_scaled, x_scaled)\n","                elif name == \"twin_net\" :\n","                    y_pred_scaled, zs = forward(net = model.net, x = x_scaled.float(), return_layers = True)\n","                    dydx_pred_scaled = backprop(net = model.net, y = y_pred_scaled, zs = zs)\n","                dydx_pred = y_std / x_std * dydx_pred_scaled\n","                dydx_pred = dydx_pred.detach().squeeze().numpy()\n","                return dydx_pred[index]\n","            return f\n","\n","        plotFunction(name = name_function, function = function, \n","                    min_x = min_x, max_x = max_x, step_x = step_x, \n","                    min_y = min_y, max_y = max_y, step_y = step_y)\n","\n","        plotGrad(name = name_grad, deriv_function = deriv_function, \n","                min_x = min_x, max_x = max_x, step_x = step_x, \n","                min_y = min_y, max_y = max_y, step_y = step_y)\n","        \n","        model_list.append(model)\n","        loss_list.append((r_y if r_y else test_loss, r_dydx, test_loss if r_y else None))\n","        stat_list.append(stats)\n","\n","    return model_list, loss_list, stat_list\n","\n","def run_diffML_train(name, generator, generator_kwargs, name_function, name_grad, siren = False, \n","                     normalize = True, learning_rate_schedule = None):\n","    global nTrains, nTests, batch_sizes, train_seed, test_seed, learning_rate\n","    global min_x, max_x, step_x, min_y, max_y, step_y\n","    global max_epoch, twin_net_tf_improving_limit\n","    global first_omega_0, hidden_omega_0, outermost_linear\n","    global tf_config, loss_config\n","    global main_path, f_name, dump_path\n","  \n","    config = {}\n","    config[\"learning_rate_schedule\"] = learning_rate_schedule\n","    config[\"learning_rate\"] = learning_rate\n","    config.update({key : value for key, value in loss_config.items() if value})\n","    config.update(tf_config)\n","    config[\"dump_path\"] = dump_path\n","    config[\"function_name\"] = f_name  \n","    model_name = \"\"\n","    model_name += \"-norm\" if normalize else \"\"\n","    model_name += \"-lrs\" if learning_rate_schedule else \"\"\n","    config[\"model_name\"] = model_name\n","\n","    regressor_list, loss_list = [], []\n","    ass = {}\n","    ass[\"normal\"], ass[\"differential\"] = [], []\n","\n","    for nTrain, nTest, batch_size in zip(nTrains, nTests, batch_sizes) :\n","        print(\"========== nTrain %d ===========\" % nTrain)\n","\n","        config[\"nTrain\"] = nTrain\n","        config[\"batch_size\"] = batch_size\n","\n","        if siren :\n","            config.update({\"first_omega_0\" : first_omega_0, \n","                           \"hidden_omega_0\": hidden_omega_0, \n","                           \"outermost_linear\" : outermost_linear})\n","            \n","            config[\"activation_function\"] = tf.math.sin\n","            config[\"deriv_activation_function\"] = tf.math.cos\n","            \n","        dic_loss, regressor, dtrain, dtest, dydxTest, values, deltas = twin_net_tf_test(\n","                  generator, [nTrain], \n","                  nTrain, nTest, \n","                  trainSeed = train_seed, testSeed = test_seed, weightSeed = 0, \n","                  deltidx=0,\n","                  generator_kwargs = generator_kwargs,\n","                  epochs = max_epoch,\n","                  normalize = normalize,\n","                  improving_limit = twin_net_tf_improving_limit, \n","                  min_batch_size = batch_size,\n","                  config = config\n","              )\n","        \n","        plot_stat(regressor.stats[\"normal\"], with_derivative = True)\n","        plot_stat(regressor.stats[\"differential\"], with_derivative = True)\n","\n","        plotFunction(name = name_function, function =  lambda x : regressor.predict_values([x])[0][0], \n","                    min_x = min_x, max_x = max_x, step_x = step_x, \n","                    min_y = min_y, max_y = max_y, step_y = step_y)\n","\n","        plotGrad(name = name_grad, \n","                deriv_function = lambda index : lambda x : regressor.predict_values_and_derivs([x])[1][0][index], \n","                min_x = min_x, max_x = max_x, step_x = step_x, \n","                min_y = min_y, max_y = max_y, step_y = step_y)\n","\n","        # show_graph_per_axis\n","        yTest = dtest[1]\n","        for i in range(2) :\n","            xAxis  = np.array([[x[i]] for x in dtest[0]])\n","            # show predicitions\n","            graph(\"%s x%d vs y\" % (name, (i+1)), values, xAxis, \"\", \"values\", yTest, [nTrain], True)\n","            # show deltas\n","            graph(\"%s x%d vs dxdy\" % (name, (i+1)), deltas, xAxis, \"\", \"deltas\", dydxTest, [nTrain], True)\n","      \n","        a = dic_loss['standard_loss'][\"yloss\"][-1]\n","        b = dic_loss['standard_loss'][\"dyloss\"][-1]\n","        normal = (a, b, a+b)\n","        a = dic_loss['differential_loss'][\"yloss\"][-1]\n","        b = dic_loss['differential_loss'][\"dyloss\"][-1]\n","        differential = (a, b, a+b)\n","\n","        ass[\"normal\"].append(normal)\n","        ass[\"differential\"].append(differential)\n","\n","        regressor_list.append(regressor) \n","\n","    return regressor_list, ass"],"execution_count":null,"outputs":[]},{"cell_type":"markdown","metadata":{"id":"rydMp8bIQ12w"},"source":["# **Ground Truth**"]},{"cell_type":"code","metadata":{"id":"BlUpRgS_qHVY"},"source":["plotFunction(name = function, function = callable_function, \n","             min_x = min_x, max_x = max_x, step_x = step_x, \n","             min_y = min_y, max_y = max_y, step_y = step_y)\n","\n","plotGrad(name = grad, deriv_function = callable_function_deriv, \n","         min_x = min_x, max_x = max_x, step_x = step_x, \n","         min_y = min_y, max_y = max_y, step_y = step_y)"],"execution_count":null,"outputs":[]},{"cell_type":"markdown","metadata":{"id":"sjqGDGePRLvq"},"source":["# **Data**"]},{"cell_type":"code","metadata":{"id":"zwxu299jqZBT"},"source":["stats_dic = {}\n","tests_loss = {}"],"execution_count":null,"outputs":[]},{"cell_type":"markdown","metadata":{"id":"NlVTE5aSrVJ3"},"source":["# **1) Normal Training**"]},{"cell_type":"code","metadata":{"id":"vOkvHGOUrUKm"},"source":["name = \"net\"\n","with_derivative = False\n","key1 = \"normal_training\"\n","name_function = '%s %s' % (function, key1)\n","name_grad = '%s %s' % (grad, key1)\n","stats_dic[key1] = {}\n","tests_loss[key1] = {}"],"execution_count":null,"outputs":[]},{"cell_type":"markdown","metadata":{"id":"W79TbRImrfUa"},"source":["## **1.1) with MLP**"]},{"cell_type":"code","metadata":{"id":"ScISnEo6sRgh"},"source":["model_class = MLP\n","key2 = \"mlp\"\n","stats_dic[key1][key2] = {}\n","tests_loss[key1][key2] = {}"],"execution_count":null,"outputs":[]},{"cell_type":"markdown","metadata":{"id":"bF0giGFJr23P"},"source":["### **1.1.1) normalize = False**"]},{"cell_type":"code","metadata":{"id":"RwPHNTFvtLpv"},"source":["key3 = \"no_normalize\"\n","stats_dic[key1][key2][key3] = [None, None]\n","tests_loss[key1][key2][key3] = [None, None]"],"execution_count":null,"outputs":[]},{"cell_type":"code","metadata":{"id":"Ky5fTYERTtq3"},"source":["model, tests_loss[key1][key2][key3][0], stats_dic[key1][key2][key3][0] = run_train(name = name, \n","          model_class = model_class, \n","          model_kwargs = mlp_model_kwargs, \n","          with_derivative = with_derivative, \n","          name_function = name_function, \n","          name_grad = name_grad,  \n","          normalize = False,\n","          learning_rate_schedule = None\n","        )"],"execution_count":null,"outputs":[]},{"cell_type":"code","metadata":{"id":"ZrLygkv4rwyA"},"source":["model, tests_loss[key1][key2][key3][1], stats_dic[key1][key2][key3][1] = run_train(name = name, \n","          model_class = model_class, \n","          model_kwargs = mlp_model_kwargs, \n","          with_derivative = with_derivative, \n","          name_function = name_function, \n","          name_grad = name_grad,  \n","          normalize = False,\n","          learning_rate_schedule = learning_rate_schedule\n","        )"],"execution_count":null,"outputs":[]},{"cell_type":"markdown","metadata":{"id":"PLDpJNaBDWvp"},"source":["### **1.1.2) normalize = True**"]},{"cell_type":"code","metadata":{"id":"gCZHdDlPDmrp"},"source":["key3 = \"normalize\"\n","stats_dic[key1][key2][key3] = [None, None]\n","tests_loss[key1][key2][key3] = [None, None]"],"execution_count":null,"outputs":[]},{"cell_type":"code","metadata":{"id":"LEMZnDHYDcJk"},"source":["model, tests_loss[key1][key2][key3][0], stats_dic[key1][key2][key3][0] = run_train(name = name, \n","          model_class = model_class, \n","          model_kwargs = mlp_model_kwargs, \n","          with_derivative = with_derivative, \n","          name_function = name_function, \n","          name_grad = name_grad,  \n","          normalize = True,\n","          learning_rate_schedule = None\n","        )"],"execution_count":null,"outputs":[]},{"cell_type":"code","metadata":{"id":"SkAhfV5oVaGw"},"source":["model, tests_loss[key1][key2][key3][1], stats_dic[key1][key2][key3][1] = run_train(name = name, \n","          model_class = model_class, \n","          model_kwargs = mlp_model_kwargs, \n","          with_derivative = with_derivative, \n","          name_function = name_function, \n","          name_grad = name_grad,  \n","          normalize = True,\n","          learning_rate_schedule = learning_rate_schedule\n","        )"],"execution_count":null,"outputs":[]},{"cell_type":"markdown","metadata":{"id":"FGEFvNBSDy8d"},"source":["## **1.2) with Siren**"]},{"cell_type":"code","metadata":{"id":"BcP5oqbqSCmp"},"source":["\"\"\"\n","model_class = Siren\n","key2 = \"siren\"\n","stats_dic[key1][key2] = {}\n","tests_loss[key1][key2] = {}\n","\"\"\""],"execution_count":null,"outputs":[]},{"cell_type":"markdown","metadata":{"id":"sw-iNr_ESFPr"},"source":["### **1.2.1) normalize = False**"]},{"cell_type":"code","metadata":{"id":"AWDooS8ySH0D"},"source":["\"\"\"\n","key3 = \"no_normalize\"\n","stats_dic[key1][key2][key3] = [None, None]\n","tests_loss[key1][key2][key3] = [None, None]\n","\"\"\""],"execution_count":null,"outputs":[]},{"cell_type":"code","metadata":{"id":"Zq3rYAhLSMsX"},"source":["\"\"\"\n","model, tests_loss[key1][key2][key3][0], stats_dic[key1][key2][key3][0] = run_train(name = name, \n","          model_class = model_class, \n","          model_kwargs = siren_model_kwargs, \n","          with_derivative = with_derivative, \n","          name_function = name_function, \n","          name_grad = name_grad,  \n","          normalize = False,\n","          learning_rate_schedule = None\n","        )\n","\"\"\""],"execution_count":null,"outputs":[]},{"cell_type":"code","metadata":{"id":"Ehij4YJBSLBk"},"source":["\"\"\"\n","model, tests_loss[key1][key2][key3][1], stats_dic[key1][key2][key3][1] = run_train(name = name, \n","          model_class = model_class, \n","          model_kwargs = siren_model_kwargs, \n","          with_derivative = with_derivative, \n","          name_function = name_function, \n","          name_grad = name_grad,  \n","          normalize = False,\n","          learning_rate_schedule = learning_rate_schedule\n","        )\n","\"\"\""],"execution_count":null,"outputs":[]},{"cell_type":"markdown","metadata":{"id":"En4b8kNhSXv9"},"source":["### **1.2.2) normalize = True**"]},{"cell_type":"code","metadata":{"id":"spWEjiKGSZXy"},"source":["\"\"\"\n","key3 = \"normalize\"\n","stats_dic[key1][key2][key3] = [None, None]\n","tests_loss[key1][key2][key3] = [None, None]\n","\"\"\""],"execution_count":null,"outputs":[]},{"cell_type":"code","metadata":{"id":"oEfO5uvrSbDQ"},"source":["\"\"\"\n","model, tests_loss[key1][key2][key3][0], stats_dic[key1][key2][key3][0] = run_train(name = name, \n","          model_class = model_class, \n","          model_kwargs = siren_model_kwargs, \n","          with_derivative = with_derivative, \n","          name_function = name_function, \n","          name_grad = name_grad,  \n","          normalize = True,\n","          learning_rate_schedule = None\n","        )\n","\"\"\""],"execution_count":null,"outputs":[]},{"cell_type":"code","metadata":{"id":"Wo11YGMUDyi0"},"source":["\"\"\"\n","model, tests_loss[key1][key2][key3][1], stats_dic[key1][key2][key3][1] = run_train(name = name, \n","          model_class = model_class, \n","          model_kwargs = siren_model_kwargs, \n","          with_derivative = with_derivative, \n","          name_function = name_function, \n","          name_grad = name_grad,  \n","          normalize = True,\n","          learning_rate_schedule = learning_rate_schedule\n","        )\n","\"\"\""],"execution_count":null,"outputs":[]},{"cell_type":"markdown","metadata":{"id":"Z9OyKof_F_S7"},"source":["# **2) Sobolev Training**"]},{"cell_type":"code","metadata":{"id":"1JoELafyIH8a"},"source":["name = \"net\"\n","with_derivative = True\n","key1 = \"sobolev_training\"\n","name_function = '%s %s' % (function, key1)\n","name_grad = '%s %s' % (grad, key1)\n","stats_dic[key1] = {}\n","tests_loss[key1] = {}"],"execution_count":null,"outputs":[]},{"cell_type":"markdown","metadata":{"id":"bAdDoiChSipE"},"source":["## **2.1) with MLP**"]},{"cell_type":"code","metadata":{"id":"-ZLFXf7RSlLW"},"source":["model_class = MLP\n","key2 = \"mlp\"\n","stats_dic[key1][key2] = {}\n","tests_loss[key1][key2] = {}"],"execution_count":null,"outputs":[]},{"cell_type":"markdown","metadata":{"id":"JGMdVCHlSnSO"},"source":["### **2.1.1) normalize = False**"]},{"cell_type":"code","metadata":{"id":"hmnAWeWOSqxd"},"source":["key3 = \"no_normalize\"\n","stats_dic[key1][key2][key3] = [None, None]\n","tests_loss[key1][key2][key3] = [None, None]"],"execution_count":null,"outputs":[]},{"cell_type":"code","metadata":{"id":"XBuzawHPSuAe"},"source":["model, tests_loss[key1][key2][key3][0], stats_dic[key1][key2][key3][0] = run_train(name = name, \n","          model_class = model_class, \n","          model_kwargs = mlp_model_kwargs, \n","          with_derivative = with_derivative, \n","          name_function = name_function, \n","          name_grad = name_grad,  \n","          normalize = False,\n","          learning_rate_schedule = None\n","        )"],"execution_count":null,"outputs":[]},{"cell_type":"code","metadata":{"id":"ci2q0-yUSsfl"},"source":["model, tests_loss[key1][key2][key3][1], stats_dic[key1][key2][key3][1] = run_train(name = name, \n","          model_class = model_class, \n","          model_kwargs = mlp_model_kwargs, \n","          with_derivative = with_derivative, \n","          name_function = name_function, \n","          name_grad = name_grad,  \n","          normalize = False,\n","          learning_rate_schedule = learning_rate_schedule\n","        )"],"execution_count":null,"outputs":[]},{"cell_type":"markdown","metadata":{"id":"V8NDjhBLS62x"},"source":["### **2.1.2) normalize = True**"]},{"cell_type":"code","metadata":{"id":"M1r2J7FJS8Rb"},"source":["key3 = \"normalize\"\n","stats_dic[key1][key2][key3] = [None, None]\n","tests_loss[key1][key2][key3] = [None, None]"],"execution_count":null,"outputs":[]},{"cell_type":"code","metadata":{"id":"mL2cJRmhTAZP"},"source":["model, tests_loss[key1][key2][key3][0], stats_dic[key1][key2][key3][0] = run_train(\n","          name = name, \n","          model_class = model_class, \n","          model_kwargs = mlp_model_kwargs, \n","          with_derivative = with_derivative, \n","          name_function = name_function, \n","          name_grad = name_grad,  \n","          normalize = True,\n","          learning_rate_schedule = None\n","        )"],"execution_count":null,"outputs":[]},{"cell_type":"code","metadata":{"id":"cNr48d4gTDSS"},"source":["model, tests_loss[key1][key2][key3][1], stats_dic[key1][key2][key3][1] = run_train(\n","          name = name, \n","          model_class = model_class, \n","          model_kwargs = mlp_model_kwargs, \n","          with_derivative = with_derivative, \n","          name_function = name_function, \n","          name_grad = name_grad,  \n","          normalize = True,\n","          learning_rate_schedule = learning_rate_schedule\n","        )"],"execution_count":null,"outputs":[]},{"cell_type":"markdown","metadata":{"id":"keONZfgHTKqd"},"source":["## **2.2) with Siren**"]},{"cell_type":"code","metadata":{"id":"zzv9MhKVTMeU"},"source":["\"\"\"\n","model_class = Siren\n","key2 = \"siren\"\n","stats_dic[key1][key2] = {}\n","tests_loss[key1][key2] = {}\n","\"\"\""],"execution_count":null,"outputs":[]},{"cell_type":"markdown","metadata":{"id":"x3OPceZ6TOEl"},"source":["### **2.2.1) normalize = False**"]},{"cell_type":"code","metadata":{"id":"Tv1BLS0QTPYP"},"source":["\"\"\"\n","key3 = \"no_normalize\"\n","stats_dic[key1][key2][key3] = [None, None]\n","tests_loss[key1][key2][key3] = [None, None]\n","\"\"\""],"execution_count":null,"outputs":[]},{"cell_type":"code","metadata":{"id":"l9EfCa0jTR8g"},"source":["\"\"\"\n","model, tests_loss[key1][key2][key3][0], stats_dic[key1][key2][key3][0] = run_train(name = name, \n","          model_class = model_class, \n","          model_kwargs = siren_model_kwargs, \n","          with_derivative = with_derivative, \n","          name_function = name_function, \n","          name_grad = name_grad,  \n","          normalize = False,\n","          learning_rate_schedule = None\n","        )\n","\"\"\""],"execution_count":null,"outputs":[]},{"cell_type":"code","metadata":{"id":"uLIwRfWcTWEE"},"source":["\"\"\"\n","model, tests_loss[key1][key2][key3][1], stats_dic[key1][key2][key3][1] = run_train(name = name, \n","          model_class = model_class, \n","          model_kwargs = siren_model_kwargs, \n","          with_derivative = True, \n","          name_function = name_function, \n","          name_grad = name_grad,  \n","          normalize = False,\n","          learning_rate_schedule = learning_rate_schedule\n","        )\n","\"\"\""],"execution_count":null,"outputs":[]},{"cell_type":"markdown","metadata":{"id":"LeT0Ad0OTXrG"},"source":["### **2.2.2) normalize = True**"]},{"cell_type":"code","metadata":{"id":"qtXkjZqDTaFK"},"source":["\"\"\"\n","key3 = \"normalize\"\n","stats_dic[key1][key2][key3] = [None, None]\n","tests_loss[key1][key2][key3] = [None, None]\n","\"\"\""],"execution_count":null,"outputs":[]},{"cell_type":"code","metadata":{"id":"PJpFkH1YTb0S"},"source":["\"\"\"\n","model, tests_loss[key1][key2][key3][0], stats_dic[key1][key2][key3][0] = run_train(name = name, \n","          model_class = model_class, \n","          model_kwargs = siren_model_kwargs, \n","          with_derivative = with_derivative, \n","          name_function = name_function, \n","          name_grad = name_grad,  \n","          normalize = True,\n","          learning_rate_schedule = None\n","        )\n","\"\"\""],"execution_count":null,"outputs":[]},{"cell_type":"code","metadata":{"id":"H59np3EDGE_W"},"source":["\"\"\"\n","model, tests_loss[key1][key2][key3][1], stats_dic[key1][key2][key3][1] = run_train(name = name, \n","          model_class = model_class, \n","          model_kwargs = siren_model_kwargs, \n","          with_derivative = with_derivative, \n","          name_function = name_function, \n","          name_grad = name_grad,  \n","          normalize = True,\n","          learning_rate_schedule = learning_rate_schedule\n","        )\n","\"\"\""],"execution_count":null,"outputs":[]},{"cell_type":"markdown","metadata":{"id":"o6L6HhyOI_tN"},"source":["# **3) twin_net tensorflow**"]},{"cell_type":"code","metadata":{"id":"jltzwWMYGGEi"},"source":["generator = Generator(callable_function = callable_function, \n","                      callable_function_deriv = callable_function_deriv, \n","                      dim_x = INPUT_DIM,\n","                      min_x = min_x, max_x = max_x)\n","\n","key1 = \"twin_net_tf\"\n","name_function = '%s %s' % (function, key1)\n","name_grad = '%s %s' % (grad, key1)\n","key1_1 = \"%s_normal\" % key1\n","key1_2 = \"%s_differential\" % key1\n","stats_dic[key1_1] = {}\n","stats_dic[key1_2] = {}\n","tests_loss[key1_1] = {}\n","tests_loss[key1_2] = {}"],"execution_count":null,"outputs":[]},{"cell_type":"markdown","metadata":{"id":"mrRSskeeLHJt"},"source":["## **3.1) with MLP**"]},{"cell_type":"code","metadata":{"id":"HbbhgYoVprVe"},"source":["key2 = \"mlp\"\n","stats_dic[key1_1][key2] = {}\n","stats_dic[key1_2][key2] = {}\n","tests_loss[key1_1][key2] = {}\n","tests_loss[key1_2][key2] = {}"],"execution_count":null,"outputs":[]},{"cell_type":"markdown","metadata":{"id":"laU7_gOjRfkK"},"source":["\n","### **3.1.1) normalize = False**"]},{"cell_type":"code","metadata":{"id":"nAYv3p5DWfcq"},"source":["key3 = \"no_normalize\"\n","stats_dic[key1_1][key2][key3] = [None, None]\n","stats_dic[key1_2][key2][key3] = [None, None]\n","tests_loss[key1_1][key2][key3] = [None, None]\n","tests_loss[key1_2][key2][key3] = [None, None]"],"execution_count":null,"outputs":[]},{"cell_type":"code","metadata":{"id":"Mvg7bMMXpvVE"},"source":["graph_name = \"%s %s %s\" % (name_function, key2, key3)\n","regressor, loss = run_diffML_train(graph_name, generator, generator_kwargs, name_function, name_grad, \n","                                   normalize = False)\n","\n","stats_dic[key1_1][key2][key3][0] = [r.stats[\"normal\"] for r in regressor]\n","stats_dic[key1_2][key2][key3][0] = [r.stats[\"differential\"] for r in regressor]\n","tests_loss[key1_1][key2][key3][0] = loss[\"normal\"]\n","tests_loss[key1_2][key2][key3][0] = loss[\"differential\"]"],"execution_count":null,"outputs":[]},{"cell_type":"code","metadata":{"id":"eWNvpd5dWg_Y"},"source":["graph_name = \"%s %s %s\" % (name_function, key2, key3)\n","regressor, loss = run_diffML_train(graph_name, generator, generator_kwargs, name_function, name_grad, \n","                                   normalize = False, \n","                                   learning_rate_schedule = learning_rate_schedule \n","                                   )\n","\n","stats_dic[key1_1][key2][key3][1] = [r.stats[\"normal\"] for r in regressor]\n","stats_dic[key1_2][key2][key3][1] = [r.stats[\"differential\"] for r in regressor]\n","tests_loss[key1_1][key2][key3][1] = loss[\"normal\"]\n","tests_loss[key1_2][key2][key3][1] = loss[\"differential\"]"],"execution_count":null,"outputs":[]},{"cell_type":"markdown","metadata":{"id":"AfHhpB_tRjz_"},"source":["### **3.1.2) normalize = True**"]},{"cell_type":"code","metadata":{"id":"Ecbp509vXqaO"},"source":["key3 = \"normalize\"\n","stats_dic[key1_1][key2][key3] = [None, None]\n","stats_dic[key1_2][key2][key3] = [None, None]\n","tests_loss[key1_1][key2][key3] = [None, None]\n","tests_loss[key1_2][key2][key3] = [None, None]"],"execution_count":null,"outputs":[]},{"cell_type":"code","metadata":{"id":"1RXz24Wes2ce"},"source":["graph_name = \"%s %s %s\" % (name_function, key2, key3)\n","regressor, loss = run_diffML_train(graph_name, generator, generator_kwargs, name_function, name_grad, \n","                                   normalize = True)\n","\n","stats_dic[key1_1][key2][key3][0] = [r.stats[\"normal\"] for r in regressor]\n","stats_dic[key1_2][key2][key3][0] = [r.stats[\"differential\"] for r in regressor]\n","tests_loss[key1_1][key2][key3][0] = loss[\"normal\"]\n","tests_loss[key1_2][key2][key3][0] = loss[\"differential\"]"],"execution_count":null,"outputs":[]},{"cell_type":"code","metadata":{"id":"nGaijVvZXxcN"},"source":["graph_name = \"%s %s %s\" % (name_function, key2, key3)\n","regressor, loss = run_diffML_train(graph_name, generator, generator_kwargs, name_function, name_grad, \n","                                   normalize = True,\n","                                   learning_rate_schedule = learning_rate_schedule)\n","\n","stats_dic[key1_1][key2][key3][1] = [r.stats[\"normal\"] for r in regressor]\n","stats_dic[key1_2][key2][key3][1] = [r.stats[\"differential\"] for r in regressor]\n","tests_loss[key1_1][key2][key3][1] = loss[\"normal\"]\n","tests_loss[key1_2][key2][key3][1] = loss[\"differential\"]"],"execution_count":null,"outputs":[]},{"cell_type":"markdown","metadata":{"id":"I5iVjWyOLC9X"},"source":["## **3.2) with Siren**"]},{"cell_type":"code","metadata":{"id":"_XxEM3AytYHD"},"source":["\"\"\"\n","key2 = \"siren\"\n","stats_dic[key1_1][key2] = {}\n","stats_dic[key1_2][key2] = {}\n","tests_loss[key1_1][key2] = {}\n","tests_loss[key1_2][key2] = {}\n","\"\"\""],"execution_count":null,"outputs":[]},{"cell_type":"markdown","metadata":{"id":"fkjbt94CRnro"},"source":["### **3.2.1) normalize = False**"]},{"cell_type":"code","metadata":{"id":"bnohx9mDYrQm"},"source":["\"\"\"\n","key3 = \"no_normalize\"\n","stats_dic[key1_1][key2][key3] = [None, None]\n","stats_dic[key1_2][key2][key3] = [None, None]\n","tests_loss[key1_1][key2][key3] = [None, None]\n","tests_loss[key1_2][key2][key3] = [None, None]\n","\"\"\""],"execution_count":null,"outputs":[]},{"cell_type":"code","metadata":{"id":"l6NHOxIXLBeC"},"source":["\"\"\"\n","graph_name = \"%s %s %s\" % (name_function, key2, key3)\n","regressor, loss = run_diffML_train(graph_name, generator, generator_kwargs, name_function, name_grad, \n","                                   siren = True, normalize = False)\n","\n","stats_dic[key1_1][key2][key3][0] = [r.stats[\"normal\"] for r in regressor]\n","stats_dic[key1_2][key2][key3][0] = [r.stats[\"differential\"] for r in regressor]\n","tests_loss[key1_1][key2][key3][0] = loss[\"normal\"]\n","tests_loss[key1_2][key2][key3][0] = loss[\"differential\"]\n","\"\"\""],"execution_count":null,"outputs":[]},{"cell_type":"code","metadata":{"id":"IAZjYAvZYt1H"},"source":["\"\"\"\n","graph_name = \"%s %s %s\" % (name_function, key2, key3)\n","regressor, loss = run_diffML_train(graph_name, generator, generator_kwargs, name_function, name_grad, \n","                                   siren = True, normalize = False,\n","                                   learning_rate_schedule = learning_rate_schedule)\n","\n","stats_dic[key1_1][key2][key3][1] = [r.stats[\"normal\"] for r in regressor]\n","stats_dic[key1_2][key2][key3][1] = [r.stats[\"differential\"] for r in regressor]\n","tests_loss[key1_1][key2][key3][1] = loss[\"normal\"]\n","tests_loss[key1_2][key2][key3][1] = loss[\"differential\"]\n","\"\"\""],"execution_count":null,"outputs":[]},{"cell_type":"markdown","metadata":{"id":"LPgtWQpvRshe"},"source":["### **3.2.2) normalize = True**"]},{"cell_type":"code","metadata":{"id":"pL3mCTKfZDzO"},"source":["\"\"\"\n","key3 = \"normalize\"\n","stats_dic[key1_1][key2][key3] = [None, None]\n","stats_dic[key1_2][key2][key3] = [None, None]\n","tests_loss[key1_1][key2][key3] = [None, None]\n","tests_loss[key1_2][key2][key3] = [None, None]\n","\"\"\""],"execution_count":null,"outputs":[]},{"cell_type":"code","metadata":{"id":"GpSBDcKjZFEi"},"source":["\"\"\"\n","graph_name = \"%s %s %s\" % (name_function, key2, key3)\n","regressor, loss = run_diffML_train(graph_name, generator, generator_kwargs, name_function, name_grad, \n","                                   siren = True, normalize = True)\n","\n","stats_dic[key1_1][key2][key3][0] = [r.stats[\"normal\"] for r in regressor]\n","stats_dic[key1_2][key2][key3][0] = [r.stats[\"differential\"] for r in regressor]\n","tests_loss[key1_1][key2][key3][0] = loss[\"normal\"]\n","tests_loss[key1_2][key2][key3][0] = loss[\"differential\"]\n","\"\"\""],"execution_count":null,"outputs":[]},{"cell_type":"code","metadata":{"id":"Ic_VMAeYtn1o"},"source":["\"\"\"\n","graph_name = \"%s %s %s\" % (name_function, key2, key3)\n","regressor, loss = run_diffML_train(graph_name, generator, generator_kwargs, name_function, name_grad, \n","                                   siren = True, normalize = True, learning_rate_schedule = learning_rate_schedule)\n","\n","stats_dic[key1_1][key2][key3][1] = [r.stats[\"normal\"] for r in regressor]\n","stats_dic[key1_2][key2][key3][1] = [r.stats[\"differential\"] for r in regressor]\n","tests_loss[key1_1][key2][key3][1] = loss[\"normal\"]\n","tests_loss[key1_2][key2][key3][1] = loss[\"differential\"]\n","\"\"\""],"execution_count":null,"outputs":[]},{"cell_type":"markdown","metadata":{"id":"A-8OyX9xJHJg"},"source":["# **4) twin_net pytorch**"]},{"cell_type":"code","metadata":{"id":"vhNYuqVSPlBU"},"source":["name = \"twin_net\"\n","with_derivative = True\n","key1 = \"twin_net_pytorch\"\n","name_function = '%s %s' % (function, key1)\n","name_grad = '%s %s' % (grad, key1)\n","stats_dic[key1] = {}\n","tests_loss[key1] = {}"],"execution_count":null,"outputs":[]},{"cell_type":"markdown","metadata":{"id":"_buVdSyGPpRa"},"source":["## **4.1) with MLP**"]},{"cell_type":"code","metadata":{"id":"A6-36JjePrQD"},"source":["model_class = MLP\n","key2 = \"mlp\"\n","stats_dic[key1][key2] = {}\n","tests_loss[key1][key2] = {}"],"execution_count":null,"outputs":[]},{"cell_type":"markdown","metadata":{"id":"MwYtIPKfPt-V"},"source":["### **4.1.1) normalize = False**"]},{"cell_type":"code","metadata":{"id":"NKPA3le_P_4S"},"source":["key3 = \"no_normalize\"\n","stats_dic[key1][key2][key3] = [None, None]\n","tests_loss[key1][key2][key3] = [None, None]"],"execution_count":null,"outputs":[]},{"cell_type":"code","metadata":{"id":"ougkYZSMQBWd"},"source":["model, tests_loss[key1][key2][key3][0], stats_dic[key1][key2][key3][0] = run_train(\n","          name = name, \n","          model_class = model_class, \n","          model_kwargs = mlp_model_kwargs, \n","          with_derivative = True, \n","          name_function = name_function, \n","          name_grad = name_grad,  \n","          normalize = False,\n","          learning_rate_schedule = None\n","        )"],"execution_count":null,"outputs":[]},{"cell_type":"code","metadata":{"id":"KZl058M9P1o7"},"source":["model, tests_loss[key1][key2][key3][1], stats_dic[key1][key2][key3][1] = run_train(\n","          name = name, \n","          model_class = model_class, \n","          model_kwargs = mlp_model_kwargs, \n","          with_derivative = True, \n","          name_function = name_function, \n","          name_grad = name_grad,  \n","          normalize = False,\n","          learning_rate_schedule = learning_rate_schedule\n","        )"],"execution_count":null,"outputs":[]},{"cell_type":"markdown","metadata":{"id":"0yMnkCibQF2W"},"source":["### **4.1.2) normalize = True**"]},{"cell_type":"code","metadata":{"id":"1FpRCKfcQhhP"},"source":["key3 = \"normalize\"\n","stats_dic[key1][key2][key3] = [None, None]\n","tests_loss[key1][key2][key3] = [None, None]"],"execution_count":null,"outputs":[]},{"cell_type":"code","metadata":{"id":"r7Rmy9iEQI63"},"source":["model, tests_loss[key1][key2][key3][0], stats_dic[key1][key2][key3][0] = run_train(\n","          name = name, \n","          model_class = model_class, \n","          model_kwargs = mlp_model_kwargs, \n","          with_derivative = with_derivative, \n","          name_function = name_function, \n","          name_grad = name_grad,  \n","          normalize = True,\n","          learning_rate_schedule = None\n","        )"],"execution_count":null,"outputs":[]},{"cell_type":"code","metadata":{"id":"12UcwT4HQkMa"},"source":["model, tests_loss[key1][key2][key3][1], stats_dic[key1][key2][key3][1] = run_train(name = name, \n","          model_class = model_class, \n","          model_kwargs = mlp_model_kwargs, \n","          with_derivative = with_derivative, \n","          name_function = name_function, \n","          name_grad = name_grad,  \n","          normalize = True,\n","          learning_rate_schedule = learning_rate_schedule\n","        )"],"execution_count":null,"outputs":[]},{"cell_type":"markdown","metadata":{"id":"EolCPZyjQn0d"},"source":["## **4.2) with Siren**"]},{"cell_type":"code","metadata":{"id":"06BeHvCHQp0R"},"source":["\"\"\"\n","model_class = Siren\n","key2 = \"siren\"\n","stats_dic[key1][key2] = {}\n","tests_loss[key1][key2] = {}\n","\"\"\""],"execution_count":null,"outputs":[]},{"cell_type":"markdown","metadata":{"id":"PMOHVsaWQuGH"},"source":["### **4.2.1) normalize = False**"]},{"cell_type":"code","metadata":{"id":"J7OcnkPoQwjY"},"source":["\"\"\"\n","key3 = \"no_normalize\"\n","stats_dic[key1][key2][key3] = [None, None]\n","tests_loss[key1][key2][key3] = [None, None]\n","\"\"\""],"execution_count":null,"outputs":[]},{"cell_type":"code","metadata":{"id":"gNQ3vVKLQy4f"},"source":["\"\"\"\n","model, tests_loss[key1][key2][key3][0], stats_dic[key1][key2][key3][0] = run_train(name = name, \n","          model_class = model_class, \n","          model_kwargs = siren_model_kwargs, \n","          with_derivative = True, \n","          name_function = name_function, \n","          name_grad = name_grad,  \n","          normalize = False,\n","          learning_rate_schedule = None\n","        )\n","\"\"\""],"execution_count":null,"outputs":[]},{"cell_type":"code","metadata":{"id":"CiZJlEpjQ0vh"},"source":["\"\"\"\n","model, tests_loss[key1][key2][key3][1], stats_dic[key1][key2][key3][1] = run_train(name = name, \n","          model_class = model_class, \n","          model_kwargs = siren_model_kwargs, \n","          with_derivative = True, \n","          name_function = name_function, \n","          name_grad = name_grad,  \n","          normalize = False,\n","          learning_rate_schedule = learning_rate_schedule\n","        )\n","\"\"\""],"execution_count":null,"outputs":[]},{"cell_type":"markdown","metadata":{"id":"ZnKJGZ_8Q4FT"},"source":["### **4.2.2) normalize = True**"]},{"cell_type":"code","metadata":{"id":"QmJP8njRQ5ay"},"source":["\"\"\"\n","key3 = \"normalize\"\n","stats_dic[key1][key2][key3] = [None, None]\n","tests_loss[key1][key2][key3] = [None, None]\n","\"\"\""],"execution_count":null,"outputs":[]},{"cell_type":"code","metadata":{"id":"OORCxhRfQ8Lk"},"source":["\"\"\"\n","model, tests_loss[key1][key2][key3][0], stats_dic[key1][key2][key3][0] = run_train(name = name, \n","          model_class = model_class, \n","          model_kwargs = siren_model_kwargs, \n","          with_derivative = with_derivative, \n","          name_function = name_function, \n","          name_grad = name_grad,  \n","          normalize = True,\n","          learning_rate_schedule = None\n","        )\n","\"\"\""],"execution_count":null,"outputs":[]},{"cell_type":"code","metadata":{"id":"BF2-k9n6OU29"},"source":["\"\"\"\n","model, tests_loss[key1][key2][key3][1], stats_dic[key1][key2][key3][1] = run_train(name = name, \n","          model_class = model_class, \n","          model_kwargs = siren_model_kwargs, \n","          with_derivative = with_derivative, \n","          name_function = name_function, \n","          name_grad = name_grad,  \n","          normalize = True,\n","          learning_rate_schedule = learning_rate_schedule\n","        )\n","\"\"\""],"execution_count":null,"outputs":[]},{"cell_type":"markdown","metadata":{"id":"Qwgzx397rtv1"},"source":["# **5) Global Stats**"]},{"cell_type":"code","metadata":{"id":"CvvWWfHPgK2-"},"source":["import pickle, os\n","\n","for nTrain, sd in reshape(dic = stats_dic, nTrains = nTrains).items() :\n","    \n","    print(\"nTrain %d\" % nTrain)\n","\n","    global_stat(stats_dic = sd, suptitle = function, n_samples = str(nTrain))\n","\n","    file_path = os.path.join(main_path, str(nTrain) + \".pkl\")\n","    pickle.dump(sd, open(file_path, 'wb'))\n","    #stats_dic = pickle.load(open(file_path, 'rb'))"],"execution_count":null,"outputs":[]},{"cell_type":"code","metadata":{"id":"jwve_s3uhTRf"},"source":["for nTrain, tl in reshape(dic = tests_loss, nTrains = nTrains).items() :\n","    print(\"nTrain %d\" % nTrain)\n","    rows, result = to_csv(dico = tl, csv_path = csv_path, n_samples = str(nTrain), mode='a+')"],"execution_count":null,"outputs":[]}]}