From 977e8ba4358a4992627448972052bceb42099fac Mon Sep 17 00:00:00 2001
From: Rob Carver <robcarver17@gmail.com>
Date: Mon, 4 Dec 2023 10:28:14 +0000
Subject: [PATCH] couple more new pandas issues

---
 syscore/pandas/pdutils.py                      | 15 +++++++++++----
 sysdata/sim/futures_sim_data_with_data_blob.py |  6 ++++++
 2 files changed, 17 insertions(+), 4 deletions(-)

diff --git a/syscore/pandas/pdutils.py b/syscore/pandas/pdutils.py
index 582bc112ca..24f06f81fe 100755
--- a/syscore/pandas/pdutils.py
+++ b/syscore/pandas/pdutils.py
@@ -13,6 +13,7 @@
 DEFAULT_DATE_FORMAT_FOR_CSV = "%Y-%m-%d %H:%M:%S"
 EXPECTED_LENGTH_OF_DATE = 19
 
+FALLBACK_DATE_FORMAT_FOR_CSV = "%Y-%m-%d"
 
 def rolling_pairwise_correlation(
     x: pd.DataFrame, periods: int, min_periods: int = 3
@@ -85,6 +86,7 @@ def pd_readcsv(
     filename: str,
     date_index_name: str = "DATETIME",
     date_format: str = DEFAULT_DATE_FORMAT_FOR_CSV,
+    fallback_date_format: str = FALLBACK_DATE_FORMAT_FOR_CSV,
     input_column_mapping: Union[dict, named_object] = arg_not_supplied,
     skiprows: int = 0,
     skipfooter: int = 0,
@@ -106,9 +108,14 @@ def pd_readcsv(
     df = pd.read_csv(filename, skiprows=skiprows, skipfooter=skipfooter)
 
     ## Add time index as index
-    df = add_datetime_index(
-        df=df, date_index_name=date_index_name, date_format=date_format
-    )
+    try:
+        df = add_datetime_index(
+            df=df, date_index_name=date_index_name, date_format=date_format
+        )
+    except:
+        df =add_datetime_index(
+            df=df, date_index_name=date_index_name, date_format=fallback_date_format
+        )
 
     if input_column_mapping is not arg_not_supplied:
         df = remap_columns_in_pd(df, input_column_mapping)
@@ -128,7 +135,7 @@ def add_datetime_index(
     def left(x: str, n):
         return x[:n]
 
-    date_index = date_index.apply(left, n=EXPECTED_LENGTH_OF_DATE)
+    date_index = date_index.apply(left, n=expected_length_of_date)
     df.index = pd.to_datetime(date_index, format=date_format).values
     del df[date_index_name]
     df.index.name = None
diff --git a/sysdata/sim/futures_sim_data_with_data_blob.py b/sysdata/sim/futures_sim_data_with_data_blob.py
index e5a557b485..dbff988846 100644
--- a/sysdata/sim/futures_sim_data_with_data_blob.py
+++ b/sysdata/sim/futures_sim_data_with_data_blob.py
@@ -13,6 +13,7 @@
     assetClassesAndInstruments,
     futuresInstrumentWithMetaData,
 )
+from syscore.exceptions import missingData
 from sysobjects.spot_fx_prices import fxPrices
 from sysobjects.adjusted_prices import futuresAdjustedPrices
 from sysobjects.multiple_prices import futuresMultiplePrices
@@ -70,6 +71,11 @@ def get_multiple_prices_from_start_date(
         self, instrument_code: str, start_date
     ) -> futuresMultiplePrices:
         data = self.db_futures_multiple_prices_data.get_multiple_prices(instrument_code)
+        if len(data)==0:
+            raise missingData(
+                "Data for %s not found! Remove from instrument list, or add to config.ignore_instruments"
+                % instrument_code
+            )
 
         return data[start_date:]