Skip to content

Commit

Permalink
blacked
Browse files Browse the repository at this point in the history
  • Loading branch information
Rob Carver committed Nov 27, 2023
1 parent 198023a commit cbea961
Show file tree
Hide file tree
Showing 5 changed files with 22 additions and 13 deletions.
19 changes: 15 additions & 4 deletions syscore/pandas/pdutils.py
Original file line number Diff line number Diff line change
Expand Up @@ -11,7 +11,8 @@
from syscore.constants import named_object, arg_not_supplied

DEFAULT_DATE_FORMAT_FOR_CSV = "%Y-%m-%d %H:%M:%S"
EXPECTED_LENGTH_OF_DATE = 19
EXPECTED_LENGTH_OF_DATE = 19


def rolling_pairwise_correlation(
x: pd.DataFrame, periods: int, min_periods: int = 3
Expand Down Expand Up @@ -105,26 +106,36 @@ def pd_readcsv(
df = pd.read_csv(filename, skiprows=skiprows, skipfooter=skipfooter)

## Add time index as index
df = add_datetime_index(df=df, date_index_name=date_index_name, date_format=date_format)
df = add_datetime_index(
df=df, date_index_name=date_index_name, date_format=date_format
)

if input_column_mapping is not arg_not_supplied:
df = remap_columns_in_pd(df, input_column_mapping)

return df


def add_datetime_index(df: pd.DataFrame, date_index_name: str, date_format: str = DEFAULT_DATE_FORMAT_FOR_CSV, expected_length_of_date: int = EXPECTED_LENGTH_OF_DATE) -> pd.DataFrame:
def add_datetime_index(
df: pd.DataFrame,
date_index_name: str,
date_format: str = DEFAULT_DATE_FORMAT_FOR_CSV,
expected_length_of_date: int = EXPECTED_LENGTH_OF_DATE,
) -> pd.DataFrame:
date_index = df[date_index_name]
date_index = date_index.astype(str)
def left(x:str, n):

def left(x: str, n):
return x[:n]

date_index = date_index.apply(left, n=EXPECTED_LENGTH_OF_DATE)
df.index = pd.to_datetime(date_index, format=date_format).values
del df[date_index_name]
df.index.name = None

return df


def remap_columns_in_pd(df: pd.DataFrame, input_column_mapping: dict) -> pd.DataFrame:
"""
Returns the bool for columns of slice_data for which we have at least one non nan value
Expand Down
8 changes: 2 additions & 6 deletions sysproduction/backup_parquet_data_to_remote.py
Original file line number Diff line number Diff line change
@@ -1,11 +1,7 @@
import os
from sysdata.config.production_config import get_production_config

from sysproduction.data.directories import (

get_parquet_backup_directory

)
from sysproduction.data.directories import get_parquet_backup_directory

from sysdata.data_blob import dataBlob

Expand All @@ -18,7 +14,6 @@ def backup_parquet_data_to_remote():
return None



def get_parquet_directory(data):
return data.parquet_root_directory

Expand All @@ -33,6 +28,7 @@ def backup_parquet(self):
log.debug("Copying data to backup destination")
backup_parquet_data_to_remote_with_data(data)


def backup_parquet_data_to_remote_with_data(data):
source_path = get_parquet_directory(data)
destination_path = get_parquet_backup_directory()
Expand Down
2 changes: 1 addition & 1 deletion sysproduction/data/directories.py
Original file line number Diff line number Diff line change
Expand Up @@ -7,7 +7,6 @@
production_config = get_production_config()



def get_main_backup_directory():
ans = production_config.get_element("offsystem_backup_directory")
return get_resolved_pathname(ans)
Expand All @@ -19,6 +18,7 @@ def get_csv_backup_directory():

return ans


def get_parquet_backup_directory():
main_backup = get_main_backup_directory()
ans = os.path.join(main_backup, "parquet")
Expand Down
4 changes: 3 additions & 1 deletion sysproduction/reporting/data/pricechanges.py
Original file line number Diff line number Diff line change
Expand Up @@ -209,7 +209,9 @@ def get_percentage_change_from_series_for_period(
price_series_for_period = price_series[start_date:end_date]
if len(price_series_for_period) == 0:
return np.nan
return 100 * ((price_series_for_period.iloc[-1] / price_series_for_period.iloc[0]) - 1)
return 100 * (
(price_series_for_period.iloc[-1] / price_series_for_period.iloc[0]) - 1
)


def get_stdev_at_start_date_for_instrument(
Expand Down
2 changes: 1 addition & 1 deletion sysproduction/run_backups.py
Original file line number Diff line number Diff line change
Expand Up @@ -29,7 +29,7 @@ def get_list_of_timer_functions_for_backup():
("backup_db_to_csv", db_backup_object),
("backup_mongo_data_as_dump", mongodump_backup_object),
("backup_files", statefile_backup_object),
("backup_parquet", parquet_backup_object)
("backup_parquet", parquet_backup_object),
]

return list_of_timer_names_and_functions
Expand Down

0 comments on commit cbea961

Please sign in to comment.