From 5e747c50ebf3e994ba6042fa50b0672e003c42d4 Mon Sep 17 00:00:00 2001
From: Sergei <spodshivalov@scmarinetech.com>
Date: Fri, 13 Oct 2023 15:27:15 -0700
Subject: [PATCH] Bug fix in CSV generation. Add timestamp to the plots

---
 scripts/plot_instr.py | 30 ++++++++++++++++++++++++------
 ydn2csv/main.cpp      |  6 +-----
 2 files changed, 25 insertions(+), 11 deletions(-)

diff --git a/scripts/plot_instr.py b/scripts/plot_instr.py
index 70f08c7..9842af5 100644
--- a/scripts/plot_instr.py
+++ b/scripts/plot_instr.py
@@ -1,7 +1,8 @@
 import argparse
-
-import matplotlib.pyplot as plt
+import datetime as dt
 import numpy as np
+import matplotlib.pyplot as plt
+import matplotlib.dates as md
 
 
 def proces_logs(args):
@@ -18,7 +19,7 @@ def proces_logs(args):
 
     # Heading data
     process_sources(['heading-Precision-9', 'heading-ZG100'],
-                    ['mag', 'mag-cog'],
+                    ['mag', 'cog', 'mag-cog'],
                     args.work_dir)
 
 
@@ -33,13 +34,23 @@ def plot_values(sources, values, vals):
         for source in sources:
             label = source
             val = np.array(vals[source][value])
+            timestamps = np.array(vals[source][value + '-time']) / 1000.0
+            dates = [dt.datetime.fromtimestamp(ts) for ts in timestamps]
+
             if np.isnan(val).all():
                 continue
+
+            plt.xticks(rotation=25)
+            ax = plt.gca()
+            xfmt = md.DateFormatter('%Y-%m-%d %H:%M:%S')
+            ax.xaxis.set_major_formatter(xfmt)
+
             if '-' in value:
                 mean = np.nanmean(val)
                 std = np.nanstd(val)
                 label += f' {mean:.2f} +/- {std:.2f}'
-            plt.plot(val, label=label)
+            plt.plot(dates, val, label=label)
+
         plt.legend()
         plt.title(value)
         plt.grid(True)
@@ -50,8 +61,10 @@ def read_values(sources, values, work_dir):
     for source in sources:
 
         vals[source] = {}
+        vals[source]['time'] = []
         for value in values:
             vals[source][value] = []
+            vals[source][value + '-time'] = []
 
         csv_file_name = work_dir + '/' + source + '-instr.csv'
         with open(csv_file_name, 'r') as f:
@@ -59,8 +72,9 @@ def read_values(sources, values, work_dir):
             for line in f:
                 t = line.split(',')
                 for value in values:
+                    time_stamp_ms = int(t[0])
                     if '-' in value:
-                        exp = value.split('-')
+                        exp = value.split('-')  # e.g. mag-cog compute difference between two values
                         op1 = exp[0]
                         op2 = exp[1]
                     else:
@@ -70,11 +84,11 @@ def read_values(sources, values, work_dir):
                         v = t[i + 1]
                         if t[i] == value:
                             val = float(v) if len(v) > 0 else np.nan
-                            vals[source][value].append(val)
                         if op1 is not None and t[i] == op1:
                             val1 = float(v) if len(v) > 0 else np.nan
                         if op2 is not None and t[i] == op2:
                             val2 = float(v) if len(v) > 0 else np.nan
+
                     if op1 is not None and op2 is not None:
                         diff = val1 - val2
                         if diff > 180:
@@ -82,6 +96,10 @@ def read_values(sources, values, work_dir):
                         elif diff < -180:
                             diff += 360
                         vals[source][value].append(diff)
+                    else:
+                        vals[source][value].append(val)
+
+                    vals[source][value + '-time'].append(time_stamp_ms)
 
             print(f'Processed {len(vals[source][values[0]])} values')
     return vals
diff --git a/ydn2csv/main.cpp b/ydn2csv/main.cpp
index 6f6b5fd..0c32b3e 100644
--- a/ydn2csv/main.cpp
+++ b/ydn2csv/main.cpp
@@ -35,11 +35,6 @@ void makeNewPgnCsvFile(const std::string &oldPgnCsvFile, const std::string &newP
         if ( tokens.size() > 3 ){
             uint32_t pgn = std::stoul(tokens[0]);
             std::string desc = tokens[1];
-            std::vector<std::string> srcs;
-            uint32_t srcIdx = std::stoul(tokens[2]);
-            for (int i = 3; i < tokens.size(); i++){
-                srcs.push_back(tokens[i]);
-            }
             if( pgn == pgnToReplace ){
                 // Replace with the line that has this device only
                 fout << pgn << "," << desc << "," << 0 << "," << deviceName << std::endl;
@@ -73,6 +68,7 @@ int main(int argc, char** argv) {
     std::string stYdvrDir = result["ydvr-dir"].as<std::string>();
     std::string stCacheDir = result["cache-dir"].as<std::string>();
     std::string stPgnCsvFile = result["pgn-src"].as<std::string>();
+    std::filesystem::create_directories(stCacheDir);
 
     Source sources[] = {
         // Speed over water