clean

examples/airfoil_self_noise/main.cpp

@@ -58,11 +58,11 @@ int main()
 
         //training_strategy.set_optimization_method(TrainingStrategy::OptimizationMethod::QUASI_NEWTON_METHOD);
         //training_strategy.set_optimization_method(TrainingStrategy::OptimizationMethod::CONJUGATE_GRADIENT);
-        //training_strategy.set_optimization_method(TrainingStrategy::OptimizationMethod::LEVENBERG_MARQUARDT_ALGORITHM); //Fail-Mean Squared error / Doesnt work with MINKOWSKI_ERROR / is not implemented yet with weighted squared error
+        training_strategy.set_optimization_method(TrainingStrategy::OptimizationMethod::LEVENBERG_MARQUARDT_ALGORITHM); //Fail-Mean Squared error / Doesnt work with MINKOWSKI_ERROR / is not implemented yet with weighted squared error
         //training_strategy.set_optimization_method(TrainingStrategy::OptimizationMethod::STOCHASTIC_GRADIENT_DESCENT);
-        training_strategy.set_optimization_method(TrainingStrategy::OptimizationMethod::ADAPTIVE_MOMENT_ESTIMATION);
+        //training_strategy.set_optimization_method(TrainingStrategy::OptimizationMethod::ADAPTIVE_MOMENT_ESTIMATION);
 
-        training_strategy.get_adaptive_moment_estimation()->set_maximum_epochs_number(1000);
+        training_strategy.get_Levenberg_Marquardt_algorithm()->set_maximum_epochs_number(1);
 
         //training_strategy.set_maximum_epochs_number(10000);
 

opennn/cross_entropy_error_3d.cpp

@@ -67,7 +67,7 @@ void CrossEntropyError3D::calculate_error(const Batch& batch,
 
     const Tensor<type, 0> mask_sum = mask.cast<type>().sum();
 
-    #pragma omp parallel for
+    #pragma omp parallel for collapse(2)
 
     for(Index i = 0; i < batch_samples_number; i++)
         for(Index j = 0; j < outputs_number; j++)

opennn/data_set.cpp

@@ -6,9 +6,6 @@
 //   Artificial Intelligence Techniques SL
 //   [email protected]
 
-#include <cassert>
-
-
 #include "data_set.h"
 #include "statistics.h"
 #include "correlations.h"

opennn/levenberg_marquardt_algorithm.cpp

@@ -244,11 +244,12 @@ TrainingResults LevenbergMarquardtAlgorithm::perform_training()
     type elapsed_time = type(0);
 
     LevenbergMarquardtAlgorithmData optimization_data(this);
-
+    
     // Main loop
 
     for(Index epoch = 0; epoch <= maximum_epochs_number; epoch++)
     {
+
         if(display && epoch%display_period == 0) cout << "Epoch: " << epoch << endl;
 
         optimization_data.epoch = epoch;
@@ -258,18 +259,21 @@ TrainingResults LevenbergMarquardtAlgorithm::perform_training()
         neural_network->forward_propagate(training_batch.get_input_pairs(),
                                           training_forward_propagation,
                                           is_training);
-
+        cout << "Epoch " << epoch << endl;
+        training_forward_propagation.print();
+
         // Loss index
 
         loss_index->back_propagate_lm(training_batch,
                                       training_forward_propagation,
                                       training_back_propagation_lm);
 
-        results.training_error_history(epoch) = training_back_propagation_lm.error();
+        training_back_propagation_lm.print();
 
+        results.training_error_history(epoch) = training_back_propagation_lm.error();
+
         if(has_selection)
         {
-/*
             neural_network->forward_propagate(selection_batch.get_input_pairs(),
                                               selection_forward_propagation,
                                               is_training);
@@ -290,7 +294,6 @@ TrainingResults LevenbergMarquardtAlgorithm::perform_training()
 
             if(epoch != 0 && results.selection_error_history(epoch) > results.selection_error_history(epoch-1)) 
                 selection_failures++;
-*/
         }
 
         elapsed_time = get_elapsed_time(beginning_time);
@@ -349,12 +352,14 @@ TrainingResults LevenbergMarquardtAlgorithm::perform_training()
             break;
         }
 
-        if(epoch != 0 && epoch%save_period == 0) neural_network->save(neural_network_file_name);
+        if(epoch != 0 && epoch%save_period == 0) 
+            neural_network->save(neural_network_file_name);
 
         update_parameters(training_batch,
                           training_forward_propagation,
                           training_back_propagation_lm,
                           optimization_data);
+
     }
 
     set_unscaling();
@@ -370,6 +375,7 @@ void LevenbergMarquardtAlgorithm::update_parameters(const Batch& batch,
                                                     BackPropagationLM& back_propagation_lm,
                                                     LevenbergMarquardtAlgorithmData& optimization_data)
 {
+
     const type regularization_weight = loss_index->get_regularization_weight();
 
     NeuralNetwork* neural_network = loss_index->get_neural_network();
@@ -394,19 +400,19 @@ void LevenbergMarquardtAlgorithm::update_parameters(const Batch& batch,
         sum_diagonal(hessian, damping_parameter);
 
         parameters_increment = perform_Householder_QR_decomposition(hessian, type(-1)*gradient);
-
+        
         potential_parameters.device(*thread_pool_device) = parameters + parameters_increment;
 
         neural_network->forward_propagate(batch.get_input_pairs(),
                                           potential_parameters,
                                           forward_propagation);
-
+        
         loss_index->calculate_errors_lm(batch, forward_propagation, back_propagation_lm);
-
+        
         loss_index->calculate_squared_errors_lm(batch, forward_propagation, back_propagation_lm);
-
+        
         loss_index->calculate_error_lm(batch, forward_propagation, back_propagation_lm);
-
+        
         type new_loss;
 
         try
@@ -436,9 +442,8 @@ void LevenbergMarquardtAlgorithm::update_parameters(const Batch& batch,
 
             set_damping_parameter(damping_parameter*damping_parameter_factor);
         }
-
     }while(damping_parameter < maximum_damping_parameter);
-
+    
     if(!success)
     {
         const type epsilon = numeric_limits<type>::epsilon();
@@ -453,7 +458,7 @@ void LevenbergMarquardtAlgorithm::update_parameters(const Batch& batch,
             }
             else
             {
-                parameters_increment(i) = (gradient(i) > type(0))
+                parameters_increment(i) = gradient(i) > type(0)
                     ? -epsilon
                     : epsilon;
 
@@ -462,8 +467,6 @@ void LevenbergMarquardtAlgorithm::update_parameters(const Batch& batch,
         }
     }
 
-    // Set parameters
-
     neural_network->set_parameters(parameters);
 }
 

opennn/loss_index.cpp

@@ -286,7 +286,7 @@ void LossIndex::calculate_layers_squared_errors_jacobian_lm(const Batch& batch,
         layers[i]->insert_squared_errors_Jacobian_lm(back_propagation_lm.neural_network.layers[i],
                                                      index,
                                                      back_propagation_lm.squared_errors_jacobian);
-        // @todo crashes here
+
         index += layer_parameter_numbers[i] * batch_samples_number;
     }
 

opennn/mean_squared_error.cpp

@@ -62,6 +62,7 @@ void MeanSquaredError::calculate_error_lm(const Batch& batch,
                                           const ForwardPropagation&,
                                           BackPropagationLM& back_propagation) const
 {
+
     const Index outputs_number = neural_network->get_outputs_number();
 
     const Index batch_samples_number = batch.get_batch_samples_number();
@@ -73,6 +74,7 @@ void MeanSquaredError::calculate_error_lm(const Batch& batch,
     error.device(*thread_pool_device) = squared_errors.square().sum() / type(batch_samples_number * outputs_number);
 
     if(isnan(error())) throw runtime_error("\nError is NAN.");
+
 }
 
 
@@ -94,9 +96,7 @@ void MeanSquaredError::calculate_output_delta(const Batch& batch,
 
      TensorMap<Tensor<type, 2>> output_deltas = tensor_map_2(output_deltas_pair);
 
-     const type coefficient = type(2.0) / type(outputs_number * batch_samples_number);
-
-     output_deltas.device(*thread_pool_device) = coefficient*errors;
+     output_deltas.device(*thread_pool_device) = errors / type(0.5 * outputs_number * batch_samples_number);
 }
 
 

opennn/pch.h

@@ -11,6 +11,7 @@
 
 #include <algorithm>
 
+#include <cassert>
 #include <cmath>
 #include <ctime>
 #include <codecvt>

opennn/perceptron_layer.cpp

@@ -387,32 +387,27 @@ void PerceptronLayer::back_propagate_lm(const vector<pair<type*, dimensions>>& i
     const bool& is_first_layer = perceptron_layer_back_propagation_lm->is_first_layer;
 
     Tensor<type, 2>& input_derivatives = perceptron_layer_back_propagation_lm->input_derivatives;
-
-    // Parameters derivatives
 
     combination_derivatives.device(*thread_pool_device) = deltas * activation_derivatives;
 
     Index synaptic_weight_index = 0;
 
     for(Index neuron_index = 0; neuron_index < outputs_number; neuron_index++)
     {
-        const TensorMap<Tensor<type, 1>> combinations_derivatives_neuron = tensor_map(combination_derivatives, neuron_index);
+        const TensorMap<Tensor<type, 1>> combinations_derivatives_neuron 
+            = tensor_map(combination_derivatives, neuron_index);
 
         for(Index input_index = 0; input_index < inputs_number; input_index++)
         {
             const TensorMap<Tensor<type, 1>> input = tensor_map(inputs, input_index);
 
             TensorMap<Tensor<type, 1>> squared_errors_jacobian_synaptic_weight 
-                = tensor_map(squared_errors_Jacobian, synaptic_weight_index);
+                = tensor_map(squared_errors_Jacobian, synaptic_weight_index++);
 
             squared_errors_jacobian_synaptic_weight.device(*thread_pool_device) 
                 = combinations_derivatives_neuron * input;
-
-            synaptic_weight_index++;
         }
 
-        // bias
-
         const Index bias_index = synaptic_weights_number + neuron_index;
 
         TensorMap<Tensor<type, 1>> squared_errors_jacobian_bias 
@@ -421,10 +416,9 @@ void PerceptronLayer::back_propagate_lm(const vector<pair<type*, dimensions>>& i
         squared_errors_jacobian_bias.device(*thread_pool_device) = combinations_derivatives_neuron;
     }
 
-    // Input derivatives
-
     if(!is_first_layer)
-        input_derivatives.device(*thread_pool_device) = combination_derivatives.contract(synaptic_weights, A_BT);
+        input_derivatives.device(*thread_pool_device) 
+        = combination_derivatives.contract(synaptic_weights, A_BT);
 }
 
 
@@ -535,7 +529,8 @@ void PerceptronLayer::from_XML(const XMLDocument& document)
     set_input_dimensions({ read_xml_index(perceptron_layer_element, "InputsNumber") });
     set_output_dimensions({ read_xml_index(perceptron_layer_element, "NeuronsNumber") });
     set_activation_function(read_xml_string(perceptron_layer_element, "ActivationFunction"));
-    set_parameters(to_type_vector(read_xml_string(perceptron_layer_element, "Parameters"), " "));
+    // @todo chech this
+    //set_parameters(to_type_vector(read_xml_string(perceptron_layer_element, "Parameters"), " "));
 }
 
 
@@ -700,10 +695,10 @@ void PerceptronLayerBackPropagationLM::set(const Index &new_batch_samples_number
     const Index inputs_number = layer->get_input_dimensions()[0];
     const Index parameters_number = layer->get_parameters_number();
 
-    squared_errors_Jacobian.resize(batch_samples_number, parameters_number);
-
     combination_derivatives.resize(batch_samples_number, outputs_number);
 
+    squared_errors_Jacobian.resize(batch_samples_number, parameters_number);
+
     input_derivatives.resize(batch_samples_number, inputs_number);
 }
 
@@ -718,8 +713,12 @@ vector<pair<type*, dimensions>> PerceptronLayerBackPropagationLM::get_input_deri
 
 void PerceptronLayerBackPropagationLM::print() const
 {
+    cout << "Combination derivatives: " << endl
+        << combination_derivatives << endl;
     cout << "Squared errors Jacobian: " << endl
-         << squared_errors_Jacobian << endl;
+        << squared_errors_Jacobian << endl;
+    cout << "Input derivatives: " << endl
+        << input_derivatives << endl;
 }
 
 } // namespace opennn

opennn/perceptron_layer.h

@@ -61,7 +61,7 @@ class PerceptronLayer : public Layer
     void set_input_dimensions(const dimensions&) override;
     void set_output_dimensions(const dimensions&) override;
 
-    void set_parameters(const Tensor<type, 1>&, const Index& index = 0) override;
+    void set_parameters(const Tensor<type, 1>&, const Index&) override;
     void set_parameters_constant(const type&) override;
     void set_parameters_random() override;
 

opennn/perceptron_layer_3d.h

@@ -29,9 +29,9 @@ class PerceptronLayer3D : public Layer
                                   RectifiedLinear};
 
    PerceptronLayer3D(const Index& = 0,
-                              const Index& = 0,
-                              const Index& = 0,
-                              const ActivationFunction& = PerceptronLayer3D::ActivationFunction::HyperbolicTangent);
+                     const Index& = 0,
+                     const Index& = 0,
+                     const ActivationFunction& = PerceptronLayer3D::ActivationFunction::HyperbolicTangent);
 
    Index get_inputs_number_xxx() const;
    Index get_inputs_depth() const;

tests/data_set_test.cpp

@@ -62,13 +62,12 @@ TEST(DataSetTest, VariableDescriptives)
     const vector<Descriptives> variable_descriptives = data_set.calculate_variable_descriptives();
 
     EXPECT_EQ(variable_descriptives.size(), 3);
-//    EXPECT_NEAR(variable_descriptives[0].minimum, type(1000), NUMERIC_LIMITS_MIN);
-//    EXPECT_NEAR(variable_descriptives[1].minimum, type(2), NUMERIC_LIMITS_MIN);
-//    EXPECT_NEAR(variable_descriptives[2].minimum, 0, NUMERIC_LIMITS_MIN);
-//    EXPECT_NEAR(variable_descriptives[0].maximum, - type(1), NUMERIC_LIMITS_MIN);
-//    EXPECT_NEAR(variable_descriptives[1].maximum, - type(4), NUMERIC_LIMITS_MIN);
-//    EXPECT_NEAR(variable_descriptives[2].maximum, - type(2), NUMERIC_LIMITS_MIN);
-
+    EXPECT_NEAR(variable_descriptives[0].minimum, type(-1000), NUMERIC_LIMITS_MIN);
+    EXPECT_NEAR(variable_descriptives[1].minimum, type(2), NUMERIC_LIMITS_MIN);
+    EXPECT_NEAR(variable_descriptives[2].minimum, 0, NUMERIC_LIMITS_MIN);
+    EXPECT_NEAR(variable_descriptives[0].maximum, type(1), NUMERIC_LIMITS_MIN);
+    EXPECT_NEAR(variable_descriptives[1].maximum, type(4), NUMERIC_LIMITS_MIN);
+    EXPECT_NEAR(variable_descriptives[2].maximum, type(2), NUMERIC_LIMITS_MIN);
 }
 
 
@@ -86,16 +85,16 @@ TEST(DataSetTest, RawVariableDistributions)
 
     const vector<Histogram> histograms = data_set.calculate_raw_variable_distributions(2);
 
-//    EXPECT_EQ(histograms.size(), 3);
-/*
-    EXPECT_NEAR(histograms(0).frequencies(0), 2, NUMERIC_LIMITS_MIN);
-    EXPECT_NEAR(histograms(1).frequencies(0), 1, NUMERIC_LIMITS_MIN);
-    EXPECT_NEAR(histograms(2).frequencies(0), 2, NUMERIC_LIMITS_MIN);
+    EXPECT_EQ(histograms.size(), 3);
+
+    EXPECT_NEAR(histograms[0].frequencies(0), 2, NUMERIC_LIMITS_MIN);
+    EXPECT_NEAR(histograms[1].frequencies(0), 1, NUMERIC_LIMITS_MIN);
+    EXPECT_NEAR(histograms[2].frequencies(0), 2, NUMERIC_LIMITS_MIN);
+
+    EXPECT_NEAR(histograms[0].centers(0), 1, NUMERIC_LIMITS_MIN);
+    EXPECT_NEAR(histograms[1].centers(0), 1, NUMERIC_LIMITS_MIN);
+    EXPECT_NEAR(histograms[2].centers(0), 1, NUMERIC_LIMITS_MIN);
 
-    EXPECT_NEAR(histograms(0).centers(0), 1, NUMERIC_LIMITS_MIN);
-    EXPECT_NEAR(histograms(1).centers(0), 1, NUMERIC_LIMITS_MIN);
-    EXPECT_NEAR(histograms(2).centers(0), 1, NUMERIC_LIMITS_MIN);
-*/
 }