#951 plotting script implemented

necessary plot.yml uploaded in NAS already

atom_batch_execution/scripts/plot_graphs

@@ -0,0 +1,268 @@
+#!/usr/bin/env python3
+"""
+Plots processed results from a batch given a yaml file for configuration
+"""
+import argparse
+import glob
+import os
+import shutil
+from os.path import isfile
+import yaml
+from pprint import pprint
+import csv
+
+from colorama import Fore, Style
+import matplotlib.pyplot as plt
+
+from atom_core.utilities import atomWarn
+
+
+
+def computeLambdaConditions(line_dict,args):
+    '''
+    This function will be used to filter which files to take data from
+    '''
+
+    def addConditionToLambda(lambda_expression,condition,operator = "and"):
+
+        if lambda_expression == "":
+            lambda_expression = f"lambda x : {condition}"
+        else:
+            lambda_expression += f" {operator} {condition}"
+
+        return lambda_expression
+
+
+    lambda_expression = ""
+
+    for key,value in line_dict.items():
+
+        # These two keys determine which field to plot, it's not relevant here
+        if key == "ydata" or key == "xfield" or key == "legend":
+            continue
+
+        # Lists determine a range of values to plot
+        if isinstance(value,list):
+             lambda_expression = addConditionToLambda(lambda_expression,f"x['{key}'] >= {value[0]}","and")
+             lambda_expression = addConditionToLambda(lambda_expression,f"x['{key}'] <= {value[1]}","and")
+
+        elif isinstance(value,int) or isinstance(value,float):
+             lambda_expression = addConditionToLambda(lambda_expression,f"x['{key}'] == {value}","and")
+
+        # A bool determines whether to include all available values of a field or not
+        # Being false is equal to not being declared
+        elif isinstance(value,bool):
+            # Retrieve the value, if the key doesn't exist .get() returns None
+            if value : 
+                lambda_expression = addConditionToLambda(lambda_expression,f"x.get('{key}')","and")
+            else:
+                lambda_expression = addConditionToLambda(lambda_expression,f"not x.get('{key}')","and")
+
+        # Any other key that isn't listed as a bool or list should be added to the blackblist as well
+        else:
+                lambda_expression = addConditionToLambda(lambda_expression,f"not x.get('{key}')","and")
+
+    if args["verbose"]:
+        print(f"Printing lambda used in filtering files {Fore.BLUE}{lambda_expression}{Style.RESET_ALL}")
+
+
+    lambda_function = eval(lambda_expression)
+
+    return lambda_function
+
+
+
+
+def main():
+
+    ap = argparse.ArgumentParser()  # Parse command line arguments
+    ap.add_argument("-v", "--verbose", help="Prints the stdout_data of each command to the terminal.",
+                    action='store_true', default=False)
+    ap.add_argument("-ow", "--overwrite", help="Overwrites output folder.",
+                    action='store_true', default=False)
+    ap.add_argument("-rf", "--results_folder", help="Folder containing the processed results",
+                    required=True, type=str)
+    ap.add_argument("-of", "--output_folder", help="Folder where to store the plots' images",
+                    required=True, type=str)
+    ap.add_argument("-df", "--data_filename", help="Yaml containing variables used to compute the plots.",
+                    required=True, type=str)
+    ap.add_argument("-sp", "--show_plots", help="Shows plots",
+                    required=False, action='store_true', default=False)
+
+    args = vars(ap.parse_args())
+
+    # Load plot configs
+    with open(args["data_filename"], "r") as file:
+        plots_configs = yaml.safe_load(file)
+
+    # Get results paths
+    results_processed_folders = next(os.walk(args["results_folder"]))[1]
+    data_file_paths = glob.glob(f'{args["results_folder"]}/*/*[!_settings.yml]')
+    settings_file_paths = glob.glob(f'{args["results_folder"]}/*/*_settings.yml')
+
+    # Get all possible experiment settings
+    with open(settings_file_paths[0], "r") as file:
+        settings_file_fields = yaml.safe_load(file)
+
+    settings_file_fields = list(settings_file_fields.keys())
+    settings_file_fields.remove("name")
+
+
+    # Grab data for each plot
+    print(f"Grabbing data from {Fore.BLUE}{args['results_folder']}{Style.RESET_ALL}")
+    for plot_figure in plots_configs.keys():
+
+        x_data_files_names = []
+        x_data = []
+        y_data = []
+
+        for plot_line in plots_configs[plot_figure]['data']:
+            filtering_function = computeLambdaConditions(plot_line,args)
+
+            # Get x data
+            for settings_file_path in settings_file_paths:
+                with open(settings_file_path, "r") as file:
+                    settings = yaml.safe_load(file)
+
+                if filtering_function(settings):
+                    x_data_files_names.append(settings["name"])
+                    x_data.append(settings[plot_line['xfield']])
+
+            # Get y data
+
+            # I don't like this method, but the "0," on the top row of all the processed csv's is making pandas a 
+            # nightmare to work with
+            # Get y data
+            for name in x_data_files_names:
+                file_path_to_get_data_fom = glob.glob(f'{args["results_folder"]}/{name}/{plot_line["ydata"]["file"]}')[0]
+                # Method with pandas, not working
+                # df = df.T
+                # df.drop(0)
+                # print(df)
+                # for nam in df.columns:
+                #     print(nam)
+                # # element = df[plot_line["ydata"]["field"]].iloc[0]
+                # element = df['front_left_camera [px]'].iloc[0]
+                # print(element)
+
+                with open(file_path_to_get_data_fom, mode ='r')as file:
+                  ydata_raw = list(csv.reader(file))
+
+                for line in ydata_raw:
+                    if line[0] == plot_line["ydata"]["field"]:
+                        y_data.append(round(float(line[1]),4))
+
+            # Sorting data --> https://stackoverflow.com/questions/9764298/given-parallel-lists-how-can-i-sort-one-while-permuting-rearranging-the-other
+            x_data_sorted,y_data_sorted = zip(*sorted(zip(x_data,y_data)))
+
+            plot_line['x_values'] = x_data_sorted
+            plot_line['y_values'] = y_data_sorted
+
+    # ----------- End of loop over entire config to grab xy data
+
+    if args["verbose"]:
+        print(f"Printing all {Fore.BLUE}plot configurations{Style.RESET_ALL}, along with xy data")
+        pprint(plots_configs)
+
+    # Change directory to save files in the results folder
+    cwd = os.getcwd()
+
+    if not os.path.exists(args['output_folder']):  # create stdout_data folder if it does not exist.
+        os.mkdir(args['output_folder'])  # Create the new folder
+    elif os.path.exists(args['output_folder']) and args['overwrite']:
+        atomWarn(f"Found existing {Fore.YELLOW}{args['output_folder']}{Style.RESET_ALL}, overwritting entire diretory because overwrite is {Fore.YELLOW}True{Style.RESET_ALL}")
+        shutil.rmtree(args['output_folder'])  # Create the new folder
+        os.mkdir(args['output_folder'])  # Create the new folder
+
+
+    os.chdir(f"./{args['output_folder']}")
+
+    #Generate plots
+    print(f"Generating {Fore.BLUE}plots{Style.RESET_ALL}")
+    for plot_figure in plots_configs.keys():
+
+        print(f"Storing plot options for {Fore.BLUE}{plot_figure}{Style.RESET_ALL}.")
+        plot_options = plots_configs[plot_figure]["options"]
+        number_of_lines = len(plots_configs[plot_figure]['data'])
+
+        title       = plot_options['title']
+        xlabel      = plot_options['xlabel']
+        ylabel      = plot_options['ylabel']
+        colors      = plot_options['colors']
+        linestyles  = plot_options['linestyles']
+        linewidths  = plot_options['linewidths']
+        grid        = plot_options['grid']
+        markersizes = plot_options['markersizes']
+        markers     = plot_options['markers']
+
+        # Normalizing options to allow them to be defined as lists or individual str/int
+        #Strs
+        if isinstance(colors,str):
+            print(f"Colors {Fore.BLUE}{colors}{Style.RESET_ALL} set as a single {Fore.BLUE}str{Style.RESET_ALL}, expanding to all plots in this figure.")
+            colors = [colors for _ in range(number_of_lines)]
+
+        if isinstance(linestyles,str):
+            print(f"Linestyles {Fore.BLUE}{linestyles}{Style.RESET_ALL} set as a single {Fore.BLUE}str{Style.RESET_ALL}, expanding to all plots in this figure.")
+            linestyles = [linestyles for _ in range(number_of_lines)]
+
+        if isinstance(markers,str):
+            print(f"Markers {Fore.BLUE}{markers}{Style.RESET_ALL} set as a single {Fore.BLUE}str{Style.RESET_ALL}, expanding to all plots in this figure.")
+            markers = [markers for _ in range(number_of_lines)]
+
+        #Ints
+        if isinstance(markersizes,int):
+            print(f"Markersizes {Fore.BLUE}{markersizes}{Style.RESET_ALL} set as a single {Fore.BLUE}int{Style.RESET_ALL}, expanding to all plots in this figure.")
+            markersizes = [markersizes for _ in range(number_of_lines)]
+
+        if isinstance(linewidths,int):
+            print(f"Linewidths {Fore.BLUE}{linewidths}{Style.RESET_ALL} set as a single {Fore.BLUE}int{Style.RESET_ALL}, expanding to all plots in this figure.")
+            linewidths = [linewidths for _ in range(number_of_lines)]
+
+
+        plt.figure()
+        for idx,plot_line in enumerate(plots_configs[plot_figure]['data']):
+
+            x_values = plot_line['x_values']
+            y_values = plot_line['y_values']
+            legend   = plot_line['legend'] if plot_line['legend'] else None
+
+            # Plotting
+            plt.plot(x_values, y_values, color=colors[idx], linestyle=linestyles[idx], linewidth=linewidths[idx],
+                     markersize = linewidths[idx],marker = markers[idx],label = legend)
+            plt.title(title)
+            plt.xlabel(xlabel)
+            plt.ylabel(ylabel)
+            plt.grid(grid)
+            if legend is not None:
+                plt.legend()
+
+        # Save figure in output folder
+
+        image_file_name = f'{title.replace(" ", "_")}.png'
+
+        if os.path.isfile(image_file_name) and not args['overwrite']:
+            atomWarn(f'Plot with name {Fore.YELLOW}{image_file_name}{Style.RESET_ALL} found in {Fore.YELLOW}{args["output_folder"]}{Style.RESET_ALL}, not saving')
+        else:
+            print(f'Saving plot {Fore.BLUE}{title}{Style.RESET_ALL} as {Fore.BLUE}{image_file_name}{Style.RESET_ALL} in {Fore.BLUE}{args["output_folder"]}{Style.RESET_ALL}')
+            plt.savefig(image_file_name,dpi=600)
+
+        if args["show_plots"]:
+            plt.show()
+
+    # Getting back to cwd, to prevent confusion if this script is further modified
+    os.chdir(cwd)
+
+
+
+
+
+
+
+
+
+
+
+
+
+if __name__ == "__main__":
+    main()

atom_batch_execution/scripts/process_results

@@ -119,7 +119,7 @@ def main():
         settings_file_path_to = f'{args["output_folder"]}/{experiment}/{settings_file_name}'
 
         resolved_file, _, _ = uriReader(settings_file_path_from)
-        print('Saving settings to ' + Fore.BLUE + resolved_file + Style.RESET_ALL)
+        print('Saving settings to ' + Fore.BLUE + settings_file_path_to + Style.RESET_ALL)
         execute('cp ' + resolved_file + ' ' + settings_file_path_to , verbose=False)