From 43e1fe5fd4c53421c3ac76c9aa8c9f4856c9500c Mon Sep 17 00:00:00 2001 From: scientes <34819304+scientes@users.noreply.github.com> Date: Mon, 13 Dec 2021 15:06:46 +0100 Subject: [PATCH 01/45] calculate prices from csv updated year changed db format from float to str changed tablename format --- src/book.py | 31 +++++++++++++++++++++++++++++++ src/config.py | 2 +- src/main.py | 2 +- src/price_data.py | 32 ++++++++++++++++++++++++++------ 4 files changed, 59 insertions(+), 8 deletions(-) diff --git a/src/book.py b/src/book.py index 61541bc2..ab5e8f94 100644 --- a/src/book.py +++ b/src/book.py @@ -696,6 +696,37 @@ def detect_exchange(self, file_path: Path) -> Optional[str]: return None + def get_price_from_csv(self): + print("hi") + for platform, operations_a in misc.group_by( + self.operations, "platform" + ).items(): + for timestamp, operations_b in misc.group_by( + operations_a, "utc_time" + ).items(): + if len(operations_b) > 1: + buytransaction = selltransaction = None + for operation in operations_b: + if isinstance(operation, tr.Buy): + buytransaction = operation + elif isinstance(operation, tr.Sell): + selltransaction = operation + if buytransaction is not None and selltransaction is not None: + price = decimal.Decimal( + selltransaction.change / buytransaction.change + ) + logging.debug( + f"Added price from csv: {selltransaction.coin}/{buytransaction.coin} price: {price}" + ) + self.price_data.set_price_db( + platform, + selltransaction.coin, + buytransaction.coin, + timestamp, + price, + ) + break + def read_file(self, file_path: Path) -> None: """Import transactions form an account statement. diff --git a/src/config.py b/src/config.py index bba8ced3..eec24964 100644 --- a/src/config.py +++ b/src/config.py @@ -23,7 +23,7 @@ # User specific constants. COUNTRY = core.Country.GERMANY -TAX_YEAR = 2020 +TAX_YEAR = 2021 # Country specific constants. if COUNTRY == core.Country.GERMANY: diff --git a/src/main.py b/src/main.py index e6d13374..9640e9f6 100644 --- a/src/main.py +++ b/src/main.py @@ -34,7 +34,7 @@ def main() -> None: if not status: log.warning("Stopping CoinTaxman.") return - + book.get_price_from_csv() taxman.evaluate_taxation() taxman.export_evaluation_as_csv() taxman.print_evaluation() diff --git a/src/price_data.py b/src/price_data.py index da491708..505898e1 100644 --- a/src/price_data.py +++ b/src/price_data.py @@ -459,7 +459,7 @@ def __set_price_db( create_query = ( f"CREATE TABLE `{tablename}`" "(utc_time DATETIME PRIMARY KEY, " - "price FLOAT NOT NULL);" + "price STR NOT NULL);" ) cur.execute(create_query) cur.execute(query, (utc_time, str(price))) @@ -489,13 +489,19 @@ def set_price_db( price (decimal.Decimal): [description] """ assert coin != reference_coin + if coin < reference_coin: + coin_a = coin + coin_b = reference_coin + else: + coin_a = reference_coin + coin_b = coin db_path = self.get_db_path(platform) - tablename = self.get_tablename(coin, reference_coin) + tablename = self.get_tablename(coin_a, coin_b) try: self.__set_price_db(db_path, tablename, utc_time, price) except sqlite3.IntegrityError as e: if str(e) == f"UNIQUE constraint failed: {tablename}.utc_time": - price_db = self.get_price(platform, coin, utc_time, reference_coin) + price_db = self.get_price(platform, coin_a, utc_time, coin_b) if price != price_db: log.warning( "Tried to write price to database, " @@ -536,11 +542,22 @@ def get_price( return decimal.Decimal("1") db_path = self.get_db_path(platform) - tablename = self.get_tablename(coin, reference_coin) + if coin < reference_coin: + coin_a = coin + coin_b = reference_coin + inverted = False + else: + coin_a = reference_coin + coin_b = coin + inverted = True + tablename = self.get_tablename(coin_a, coin_b) # Check if price exists already in our database. if (price := self.__get_price_db(db_path, tablename, utc_time)) is not None: - return price + if inverted: + return decimal.Decimal(1 / price) + else: + return price try: get_price = getattr(self, f"_get_price_{platform}") @@ -549,7 +566,10 @@ def get_price( price = get_price(coin, utc_time, reference_coin, **kwargs) self.__set_price_db(db_path, tablename, utc_time, price) - return price + if inverted: + return decimal.Decimal(1 / price) + else: + return price def get_cost( self, From f2729c7883addc3793127671c6753d6db611ba09 Mon Sep 17 00:00:00 2001 From: scientes <34819304+scientes@users.noreply.github.com> Date: Mon, 13 Dec 2021 15:21:35 +0100 Subject: [PATCH 02/45] fixed linting except line to long --- src/book.py | 18 ++++++++---------- src/price_data.py | 8 ++++---- 2 files changed, 12 insertions(+), 14 deletions(-) diff --git a/src/book.py b/src/book.py index 8881605d..63116a7d 100644 --- a/src/book.py +++ b/src/book.py @@ -843,23 +843,21 @@ def get_price_from_csv(self): operations_a, "utc_time" ).items(): if len(operations_b) > 1: - buytransaction = selltransaction = None + buytr = selltr = None for operation in operations_b: if isinstance(operation, tr.Buy): - buytransaction = operation + buytr = operation elif isinstance(operation, tr.Sell): - selltransaction = operation - if buytransaction is not None and selltransaction is not None: - price = decimal.Decimal( - selltransaction.change / buytransaction.change - ) + selltr = operation + if buytr is not None and selltr is not None: + price = decimal.Decimal(selltr.change / buytr.change) logging.debug( - f"Added price from csv: {selltransaction.coin}/{buytransaction.coin} price: {price}" + f"Added price from csv: {selltr.coin}/{buytr.coin} price: {price}" ) self.price_data.set_price_db( platform, - selltransaction.coin, - buytransaction.coin, + selltr.coin, + buytr.coin, timestamp, price, ) diff --git a/src/price_data.py b/src/price_data.py index b7567b1f..a3340396 100644 --- a/src/price_data.py +++ b/src/price_data.py @@ -658,11 +658,11 @@ def get_price( # transaction and estimate the price. # Do not save price in database. price = self.__mean_price_db(db_path, tablename, utc_time) - + if inverted: - return decimal.Decimal(1 / price) - else: - return price + return decimal.Decimal(1 / price) + else: + return price def get_cost( self, From 13fa82eb0835cacedf4e8a5f664619f6134550c6 Mon Sep 17 00:00:00 2001 From: scientes <34819304+scientes@users.noreply.github.com> Date: Sun, 19 Dec 2021 11:38:49 +0100 Subject: [PATCH 03/45] fixed for multiple coins fixed reciprocal (both not finished) --- src/book.py | 30 ++++++++++++++++-------------- src/price_data.py | 4 +++- 2 files changed, 19 insertions(+), 15 deletions(-) diff --git a/src/book.py b/src/book.py index 63116a7d..77033dde 100644 --- a/src/book.py +++ b/src/book.py @@ -835,7 +835,6 @@ def detect_exchange(self, file_path: Path) -> Optional[str]: return None def get_price_from_csv(self): - print("hi") for platform, operations_a in misc.group_by( self.operations, "platform" ).items(): @@ -844,24 +843,27 @@ def get_price_from_csv(self): ).items(): if len(operations_b) > 1: buytr = selltr = None + buycount=sellcount=0 for operation in operations_b: if isinstance(operation, tr.Buy): buytr = operation + buycount+=1 elif isinstance(operation, tr.Sell): selltr = operation - if buytr is not None and selltr is not None: - price = decimal.Decimal(selltr.change / buytr.change) - logging.debug( - f"Added price from csv: {selltr.coin}/{buytr.coin} price: {price}" - ) - self.price_data.set_price_db( - platform, - selltr.coin, - buytr.coin, - timestamp, - price, - ) - break + sellcount+=1 + + if buycount==1 and sellcount==1: + price = decimal.Decimal(selltr.change / buytr.change) + logging.debug( + f"Added price from csv: {selltr.coin}/{buytr.coin} price: {price}" + ) + self.price_data.set_price_db( + platform, + selltr.coin, + buytr.coin, + timestamp, + price, + ) def read_file(self, file_path: Path) -> None: """Import transactions form an account statement. diff --git a/src/price_data.py b/src/price_data.py index a3340396..bc291add 100644 --- a/src/price_data.py +++ b/src/price_data.py @@ -660,7 +660,7 @@ def get_price( price = self.__mean_price_db(db_path, tablename, utc_time) if inverted: - return decimal.Decimal(1 / price) + return misc.reciprocal(price) else: return price @@ -700,6 +700,8 @@ def check_database(self): tablenames = (result[0] for result in cur.fetchall()) for tablename in tablenames: base_asset, quote_asset = tablename.split("/") + if base_asset>quote_asset: + query=f"Select" query = f"SELECT utc_time FROM `{tablename}` WHERE price<=0.0;" cur = conn.execute(query) From 9afb70a9a13c0ab5e689c4d80347781630e77f74 Mon Sep 17 00:00:00 2001 From: scientes <34819304+scientes@users.noreply.github.com> Date: Sat, 25 Dec 2021 14:37:53 +0100 Subject: [PATCH 04/45] added sql migration and extra function --- src/book.py | 10 +++++----- src/price_data.py | 50 +++++++++++++++++++++++++++++++---------------- 2 files changed, 38 insertions(+), 22 deletions(-) diff --git a/src/book.py b/src/book.py index 77033dde..32ed90af 100644 --- a/src/book.py +++ b/src/book.py @@ -843,16 +843,16 @@ def get_price_from_csv(self): ).items(): if len(operations_b) > 1: buytr = selltr = None - buycount=sellcount=0 + buycount = sellcount = 0 for operation in operations_b: if isinstance(operation, tr.Buy): buytr = operation - buycount+=1 + buycount += 1 elif isinstance(operation, tr.Sell): selltr = operation - sellcount+=1 - - if buycount==1 and sellcount==1: + sellcount += 1 + + if buycount == 1 and sellcount == 1: price = decimal.Decimal(selltr.change / buytr.change) logging.debug( f"Added price from csv: {selltr.coin}/{buytr.coin} price: {price}" diff --git a/src/price_data.py b/src/price_data.py index bc291add..f7e19418 100644 --- a/src/price_data.py +++ b/src/price_data.py @@ -580,14 +580,11 @@ def set_price_db( price (decimal.Decimal): [description] """ assert coin != reference_coin - if coin < reference_coin: - coin_a = coin - coin_b = reference_coin - else: - coin_a = reference_coin - coin_b = coin + coin_a, coin_b, reciprocal = self._sort_pair(coin, reference_coin) db_path = self.get_db_path(platform) tablename = self.get_tablename(coin_a, coin_b) + if reciprocal: + price = misc.reciprocal(price) try: self.__set_price_db(db_path, tablename, utc_time, price) except sqlite3.IntegrityError as e: @@ -602,6 +599,15 @@ def set_price_db( else: raise e + def _sort_pair(self, coin: str, reference_coin: str) -> list[str, str, bool]: + if reciprocal := coin > reference_coin: + coin_a = reference_coin + coin_b = coin + else: + coin_a = coin + coin_b = reference_coin + return coin_a, coin_b, reciprocal + def get_price( self, platform: str, @@ -633,14 +639,7 @@ def get_price( return decimal.Decimal("1") db_path = self.get_db_path(platform) - if coin < reference_coin: - coin_a = coin - coin_b = reference_coin - inverted = False - else: - coin_a = reference_coin - coin_b = coin - inverted = True + coin_a, coin_b, reciprocal = self._sort_pair(coin, reference_coin) tablename = self.get_tablename(coin_a, coin_b) # Check if price exists already in our database. @@ -659,7 +658,7 @@ def get_price( # Do not save price in database. price = self.__mean_price_db(db_path, tablename, utc_time) - if inverted: + if reciprocal: return misc.reciprocal(price) else: return price @@ -700,8 +699,25 @@ def check_database(self): tablenames = (result[0] for result in cur.fetchall()) for tablename in tablenames: base_asset, quote_asset = tablename.split("/") - if base_asset>quote_asset: - query=f"Select" + if base_asset > quote_asset: + query = f"Select utc_time,price FROM `{tablename}`" + cur = conn.execute(query) + + for row in cur.fetchall(): + utc_time = datetime.datetime.strptime( + row[0], "%Y-%m-%d %H:%M:%S%z" + ) + price = misc.reciprocal(decimal.Decimal(row[1])) + self.set_price_db(platform, quote_asset, + base_asset, utc_time, price) + query = f"DROP TABLE `{tablename}`" + cur = conn.execute(query) + + query = "SELECT name FROM sqlite_master WHERE type='table'" + cur = conn.execute(query) + tablenames = (result[0] for result in cur.fetchall()) + for tablename in tablenames: + base_asset, quote_asset = tablename.split("/") query = f"SELECT utc_time FROM `{tablename}` WHERE price<=0.0;" cur = conn.execute(query) From 8e21d8a7673c6952620d823720a90612e932fb15 Mon Sep 17 00:00:00 2001 From: scientes <34819304+scientes@users.noreply.github.com> Date: Sat, 25 Dec 2021 15:17:59 +0100 Subject: [PATCH 05/45] bugfix (prices were inverted db) and migration fix --- src/book.py | 4 ++-- src/price_data.py | 15 ++++++++++++--- 2 files changed, 14 insertions(+), 5 deletions(-) diff --git a/src/book.py b/src/book.py index 32ed90af..ed5aeacf 100644 --- a/src/book.py +++ b/src/book.py @@ -853,9 +853,9 @@ def get_price_from_csv(self): sellcount += 1 if buycount == 1 and sellcount == 1: - price = decimal.Decimal(selltr.change / buytr.change) + price = decimal.Decimal(buytr.change / selltr.change) logging.debug( - f"Added price from csv: {selltr.coin}/{buytr.coin} price: {price}" + f"Added price from csv: {selltr.coin}/{buytr.coin} price: {price} timestamp: {timestamp}" ) self.price_data.set_price_db( platform, diff --git a/src/price_data.py b/src/price_data.py index f7e19418..7a2f6c5f 100644 --- a/src/price_data.py +++ b/src/price_data.py @@ -694,10 +694,19 @@ def check_database(self): ) with sqlite3.connect(db_path) as conn: - query = "SELECT name FROM sqlite_master WHERE type='table'" + query = "SELECT name,sql FROM sqlite_master WHERE type='table'" cur = conn.execute(query) - tablenames = (result[0] for result in cur.fetchall()) - for tablename in tablenames: + for tablename,sql in cur.fetchall(): + if not sql.lower().contains("price str"): + query=f""" + CREATE TABLE "sql_temp_table" ( + "utc_time" DATETIME PRIMARY KEY, + "price" STR NOT NULL + ); + INSERT INTO "sql_temp_table" ("price","utc_time") SELECT "price","utc_time" FROM "{tablename}"; + DROP TABLE "{tablename}"; + ALTER TABLE "sql_temp_table" "{tablename}"; + """ base_asset, quote_asset = tablename.split("/") if base_asset > quote_asset: query = f"Select utc_time,price FROM `{tablename}`" From 9c52dd83b6e6d6a78a6600f8897e90f9884414cc Mon Sep 17 00:00:00 2001 From: scientes <34819304+scientes@users.noreply.github.com> Date: Fri, 31 Dec 2021 11:44:50 +0100 Subject: [PATCH 06/45] fixed missing inversion --- src/price_data.py | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/src/price_data.py b/src/price_data.py index 7a2f6c5f..2c2204f6 100644 --- a/src/price_data.py +++ b/src/price_data.py @@ -649,6 +649,9 @@ def get_price( except AttributeError: raise NotImplementedError("Unable to read data from %s", platform) price = get_price(coin, utc_time, reference_coin, **kwargs) + if reciprocal: + price = misc.reciprocal(price) + reciprocal = False assert isinstance(price, decimal.Decimal) self.__set_price_db(db_path, tablename, utc_time, price) @@ -696,9 +699,9 @@ def check_database(self): with sqlite3.connect(db_path) as conn: query = "SELECT name,sql FROM sqlite_master WHERE type='table'" cur = conn.execute(query) - for tablename,sql in cur.fetchall(): + for tablename, sql in cur.fetchall(): if not sql.lower().contains("price str"): - query=f""" + query = f""" CREATE TABLE "sql_temp_table" ( "utc_time" DATETIME PRIMARY KEY, "price" STR NOT NULL @@ -717,8 +720,9 @@ def check_database(self): row[0], "%Y-%m-%d %H:%M:%S%z" ) price = misc.reciprocal(decimal.Decimal(row[1])) - self.set_price_db(platform, quote_asset, - base_asset, utc_time, price) + self.set_price_db( + platform, quote_asset, base_asset, utc_time, price + ) query = f"DROP TABLE `{tablename}`" cur = conn.execute(query) From f74327ddbb22af94ee2644a476f75ba675e59a37 Mon Sep 17 00:00:00 2001 From: scientes <34819304+scientes@users.noreply.github.com> Date: Fri, 31 Dec 2021 11:59:57 +0100 Subject: [PATCH 07/45] formatting --- src/price_data.py | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/src/price_data.py b/src/price_data.py index 2c2204f6..b90c5b49 100644 --- a/src/price_data.py +++ b/src/price_data.py @@ -599,14 +599,14 @@ def set_price_db( else: raise e - def _sort_pair(self, coin: str, reference_coin: str) -> list[str, str, bool]: + def _sort_pair(self, coin: str, reference_coin: str) -> list[Union[str, str, bool]]: if reciprocal := coin > reference_coin: coin_a = reference_coin coin_b = coin else: coin_a = coin coin_b = reference_coin - return coin_a, coin_b, reciprocal + return [coin_a, coin_b, reciprocal] def get_price( self, @@ -703,10 +703,11 @@ def check_database(self): if not sql.lower().contains("price str"): query = f""" CREATE TABLE "sql_temp_table" ( - "utc_time" DATETIME PRIMARY KEY, - "price" STR NOT NULL + "utc_time" DATETIME PRIMARY KEY, + "price" STR NOT NULL ); - INSERT INTO "sql_temp_table" ("price","utc_time") SELECT "price","utc_time" FROM "{tablename}"; + INSERT INTO "sql_temp_table" ("price","utc_time") + SELECT "price","utc_time" FROM "{tablename}"; DROP TABLE "{tablename}"; ALTER TABLE "sql_temp_table" "{tablename}"; """ From 5112a14045039ebcd606a0f77986a25782f334d9 Mon Sep 17 00:00:00 2001 From: Griffsano <18743559+Griffsano@users.noreply.github.com> Date: Fri, 31 Dec 2021 17:46:53 +0100 Subject: [PATCH 08/45] Prices from csv (#3) * reciprocal price for API price data, note for CSV price data, address flake8 and mypy errors * remove duplicate reciprocal * remove set reciprocal False --- src/book.py | 4 +++- src/price_data.py | 9 ++++----- 2 files changed, 7 insertions(+), 6 deletions(-) diff --git a/src/book.py b/src/book.py index ed5aeacf..df87e40b 100644 --- a/src/book.py +++ b/src/book.py @@ -853,9 +853,11 @@ def get_price_from_csv(self): sellcount += 1 if buycount == 1 and sellcount == 1: + # price definition example: BTCEUR = traded EUR / traded BTC price = decimal.Decimal(buytr.change / selltr.change) logging.debug( - f"Added price from csv: {selltr.coin}/{buytr.coin} price: {price} timestamp: {timestamp}" + f"Added {selltr.coin}/{buytr.coin} price from CSV: " + f"{price} for {platform} at {timestamp}" ) self.price_data.set_price_db( platform, diff --git a/src/price_data.py b/src/price_data.py index b90c5b49..51a794fa 100644 --- a/src/price_data.py +++ b/src/price_data.py @@ -22,7 +22,7 @@ import sqlite3 import time from pathlib import Path -from typing import Any, Optional, Union +from typing import Any, Optional, Tuple, Union import requests @@ -599,14 +599,14 @@ def set_price_db( else: raise e - def _sort_pair(self, coin: str, reference_coin: str) -> list[Union[str, str, bool]]: + def _sort_pair(self, coin: str, reference_coin: str) -> Tuple[str, str, bool]: if reciprocal := coin > reference_coin: coin_a = reference_coin coin_b = coin else: coin_a = coin coin_b = reference_coin - return [coin_a, coin_b, reciprocal] + return coin_a, coin_b, reciprocal def get_price( self, @@ -651,7 +651,6 @@ def get_price( price = get_price(coin, utc_time, reference_coin, **kwargs) if reciprocal: price = misc.reciprocal(price) - reciprocal = False assert isinstance(price, decimal.Decimal) self.__set_price_db(db_path, tablename, utc_time, price) @@ -703,7 +702,7 @@ def check_database(self): if not sql.lower().contains("price str"): query = f""" CREATE TABLE "sql_temp_table" ( - "utc_time" DATETIME PRIMARY KEY, + "utc_time" DATETIME PRIMARY KEY, "price" STR NOT NULL ); INSERT INTO "sql_temp_table" ("price","utc_time") From 45b6a39a020d144602c0819a72213dd9e523e7f5 Mon Sep 17 00:00:00 2001 From: Jeppy Date: Sun, 2 Jan 2022 19:54:03 +0100 Subject: [PATCH 09/45] FIX type annotation of group_by Not all values in the list are strings --- src/misc.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/misc.py b/src/misc.py index cb4846ca..32f80cdb 100644 --- a/src/misc.py +++ b/src/misc.py @@ -170,7 +170,7 @@ def parse_iso_timestamp_to_decimal_timestamp(d: str) -> decimal.Decimal: return to_decimal_timestamp(datetime.datetime.fromisoformat(d)) -def group_by(lst: L, key: str) -> dict[str, L]: +def group_by(lst: L, key: str) -> dict[Any, L]: """Group a list of objects by `key`. Args: @@ -178,7 +178,7 @@ def group_by(lst: L, key: str) -> dict[str, L]: key (str) Returns: - dict[str, list]: Dict with different `key`as keys. + dict[Any, list]: Dict with different `key`as keys. """ d = collections.defaultdict(list) for e in lst: From 7c0f9248eac0f24d505bfa6d624678817493e6d6 Mon Sep 17 00:00:00 2001 From: Jeppy Date: Sun, 2 Jan 2022 19:54:16 +0100 Subject: [PATCH 10/45] REFACTOR add newline --- src/main.py | 1 + 1 file changed, 1 insertion(+) diff --git a/src/main.py b/src/main.py index 9640e9f6..85fc15bf 100644 --- a/src/main.py +++ b/src/main.py @@ -34,6 +34,7 @@ def main() -> None: if not status: log.warning("Stopping CoinTaxman.") return + book.get_price_from_csv() taxman.evaluate_taxation() taxman.export_evaluation_as_csv() From dc852eea258a89e899477aa364bafaae4acea5f4 Mon Sep 17 00:00:00 2001 From: Jeppy Date: Sun, 2 Jan 2022 19:55:49 +0100 Subject: [PATCH 11/45] REFACTOR get_price_from_csv - Add comments to get_price_from_csv - Rename operations variables - CHANGE Calculated price to buy/sell instead of sell/buy --- src/book.py | 84 +++++++++++++++++++++++++++++++++++------------------ 1 file changed, 55 insertions(+), 29 deletions(-) diff --git a/src/book.py b/src/book.py index df87e40b..2860fe70 100644 --- a/src/book.py +++ b/src/book.py @@ -834,38 +834,64 @@ def detect_exchange(self, file_path: Path) -> Optional[str]: return None - def get_price_from_csv(self): - for platform, operations_a in misc.group_by( + def get_price_from_csv(self) -> None: + """Calculate coin prices from buy/sell operations in CSV files. + + When exactly one buy and sell happend at the exact same time, + these two operations might belong together and we can calculate + the paid price for this transaction. + """ + # Group operations by platform. + for platform, platform_operations in misc.group_by( self.operations, "platform" ).items(): - for timestamp, operations_b in misc.group_by( - operations_a, "utc_time" + # Group operations by time. + # Look at all operations which happend at the same time. + for timestamp, time_operations in misc.group_by( + platform_operations, "utc_time" ).items(): - if len(operations_b) > 1: - buytr = selltr = None - buycount = sellcount = 0 - for operation in operations_b: - if isinstance(operation, tr.Buy): - buytr = operation - buycount += 1 - elif isinstance(operation, tr.Sell): - selltr = operation - sellcount += 1 - - if buycount == 1 and sellcount == 1: - # price definition example: BTCEUR = traded EUR / traded BTC - price = decimal.Decimal(buytr.change / selltr.change) - logging.debug( - f"Added {selltr.coin}/{buytr.coin} price from CSV: " - f"{price} for {platform} at {timestamp}" - ) - self.price_data.set_price_db( - platform, - selltr.coin, - buytr.coin, - timestamp, - price, - ) + buytr = selltr = None + buycount = sellcount = 0 + + # Extract the buy and sell operation. + for operation in time_operations: + if isinstance(operation, tr.Buy): + buytr = operation + buycount += 1 + elif isinstance(operation, tr.Sell): + selltr = operation + sellcount += 1 + + # Skip the operations of this timestamp when there aren't + # exactly one buy and one sell operation. + # We can only match the buy and sell operations, when there + # are exactly one buy and one sell operation. + if not (buycount == 1 and sellcount == 1): + continue + + assert isinstance(timestamp, datetime.datetime) + assert isinstance(buytr, tr.Buy) + assert isinstance(selltr, tr.Sell) + + # Price definition example for buying BTC with EUR: + # Symbol: BTCEUR + # coin: BTC (buytr.coin) + # reference coin: EUR (selltr.coin) + # price = traded EUR / traded BTC + price = decimal.Decimal(selltr.change / buytr.change) + + logging.debug( + f"Added {buytr.coin}/{selltr.coin} price from CSV: " + f"{price} for {platform} at {timestamp}" + ) + + self.price_data.set_price_db( + platform, + buytr.coin, + selltr.coin, + timestamp, + price, + ) def read_file(self, file_path: Path) -> None: """Import transactions form an account statement. From 62b861c268e19bf39e70f8709e9834fc42f9ef11 Mon Sep 17 00:00:00 2001 From: Jeppy Date: Sun, 2 Jan 2022 22:10:29 +0100 Subject: [PATCH 12/45] ADD docstring to _sort_pair --- src/price_data.py | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/src/price_data.py b/src/price_data.py index 51a794fa..cc1ec81d 100644 --- a/src/price_data.py +++ b/src/price_data.py @@ -600,7 +600,16 @@ def set_price_db( raise e def _sort_pair(self, coin: str, reference_coin: str) -> Tuple[str, str, bool]: - if reciprocal := coin > reference_coin: + """Sort the coin pair in alphanumerical order. + + Args: + coin (str) + reference_coin (str) + + Returns: + Tuple[str, str, bool]: First coin, second coin, inverted + """ + if inverted := coin > reference_coin: coin_a = reference_coin coin_b = coin else: From 5124ebe8805eb19516013b42f176726779384ece Mon Sep 17 00:00:00 2001 From: Jeppy Date: Sun, 2 Jan 2022 22:10:56 +0100 Subject: [PATCH 13/45] RENAME reciprocal bool to inverted --- src/price_data.py | 15 +++++++-------- 1 file changed, 7 insertions(+), 8 deletions(-) diff --git a/src/price_data.py b/src/price_data.py index cc1ec81d..e57f8c65 100644 --- a/src/price_data.py +++ b/src/price_data.py @@ -580,10 +580,10 @@ def set_price_db( price (decimal.Decimal): [description] """ assert coin != reference_coin - coin_a, coin_b, reciprocal = self._sort_pair(coin, reference_coin) + coin_a, coin_b, inverted = self._sort_pair(coin, reference_coin) db_path = self.get_db_path(platform) tablename = self.get_tablename(coin_a, coin_b) - if reciprocal: + if inverted: price = misc.reciprocal(price) try: self.__set_price_db(db_path, tablename, utc_time, price) @@ -615,7 +615,7 @@ def _sort_pair(self, coin: str, reference_coin: str) -> Tuple[str, str, bool]: else: coin_a = coin coin_b = reference_coin - return coin_a, coin_b, reciprocal + return coin_a, coin_b, inverted def get_price( self, @@ -648,7 +648,7 @@ def get_price( return decimal.Decimal("1") db_path = self.get_db_path(platform) - coin_a, coin_b, reciprocal = self._sort_pair(coin, reference_coin) + coin_a, coin_b, inverted = self._sort_pair(coin, reference_coin) tablename = self.get_tablename(coin_a, coin_b) # Check if price exists already in our database. @@ -658,7 +658,7 @@ def get_price( except AttributeError: raise NotImplementedError("Unable to read data from %s", platform) price = get_price(coin, utc_time, reference_coin, **kwargs) - if reciprocal: + if inverted: price = misc.reciprocal(price) assert isinstance(price, decimal.Decimal) self.__set_price_db(db_path, tablename, utc_time, price) @@ -669,9 +669,8 @@ def get_price( # Do not save price in database. price = self.__mean_price_db(db_path, tablename, utc_time) - if reciprocal: - return misc.reciprocal(price) - else: + if inverted: + price = misc.reciprocal(price) return price def get_cost( From 3d10f3de1c3e3f7477b9c9130308d4a4380d38f0 Mon Sep 17 00:00:00 2001 From: Jeppy Date: Sun, 2 Jan 2022 22:12:47 +0100 Subject: [PATCH 14/45] ADD comment in get_price when price is missing --- src/price_data.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/price_data.py b/src/price_data.py index e57f8c65..d45d23ba 100644 --- a/src/price_data.py +++ b/src/price_data.py @@ -653,6 +653,8 @@ def get_price( # Check if price exists already in our database. if (price := self.__get_price_db(db_path, tablename, utc_time)) is None: + # The price does not exist in our database. + # Gather the price from a platform specific function. try: get_price = getattr(self, f"_get_price_{platform}") except AttributeError: From 30fe87689b5d98833666fc97784b3dad2ea52356 Mon Sep 17 00:00:00 2001 From: Jeppy Date: Sun, 2 Jan 2022 22:15:17 +0100 Subject: [PATCH 15/45] TODO ADD rudimentary patch functions --- src/book.py | 2 + src/main.py | 3 + src/patch_database.py | 174 ++++++++++++++++++++++++++++++++++++++++++ src/price_data.py | 35 +-------- 4 files changed, 183 insertions(+), 31 deletions(-) create mode 100644 src/patch_database.py diff --git a/src/book.py b/src/book.py index 2860fe70..353c413a 100644 --- a/src/book.py +++ b/src/book.py @@ -905,6 +905,8 @@ def read_file(self, file_path: Path) -> None: assert file_path.is_file() if exchange := self.detect_exchange(file_path): + # TODO check that database file exists. if missing, add file with + # highest version number (highest patch number) try: read_file = getattr(self, f"_read_{exchange}") except AttributeError: diff --git a/src/main.py b/src/main.py index 85fc15bf..842f1da7 100644 --- a/src/main.py +++ b/src/main.py @@ -18,6 +18,7 @@ import log_config # noqa: F401 from book import Book +from patch_database import patch_databases from price_data import PriceData from taxman import Taxman @@ -25,6 +26,8 @@ def main() -> None: + patch_databases() + price_data = PriceData() book = Book(price_data) taxman = Taxman(book, price_data) diff --git a/src/patch_database.py b/src/patch_database.py new file mode 100644 index 00000000..b5f2e6d8 --- /dev/null +++ b/src/patch_database.py @@ -0,0 +1,174 @@ +# CoinTaxman +# Copyright (C) 2021 Carsten Docktor + +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as +# published by the Free Software Foundation, either version 3 of the +# License, or (at your option) any later version. + +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. + +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see . + +import datetime +import decimal +import logging +import sqlite3 +from pathlib import Path + +import config +import misc + +FUNC_PREFIX = "__patch_" +log = logging.getLogger(__name__) + + +def get_version(db_path: Path) -> int: + """Get database version from a database file. + + If the version table is missing, one is created. + + Args: + db_path (str): Path to database file. + + Raises: + RuntimeError: The database version is ambiguous. + + Returns: + int: Version of database file. + """ + with sqlite3.connect(db_path) as conn: + cur = conn.cursor() + try: + cur.execute("SELECT version FROM §version;") + versions = [int(v[0]) for v in cur.fetchall()] + except sqlite3.OperationalError as e: + if str(e) == "no such table: §version": + # The §version table doesn't exist. Create one. + cur.execute("CREATE TABLE §version(version INT);") + cur.execute("INSERT INTO §version (version) VALUES (0);") + return 0 + else: + raise e + + if len(versions) == 1: + version = versions[0] + return version + else: + raise RuntimeError( + f"The database version of the file `{db_path.name}` is ambigious. " + f"The table `§version` should have one entry, but has {len(versions)}." + ) + + +def update_version(db_path: Path, version: int) -> None: + with sqlite3.connect(db_path) as conn: + cur = conn.cursor() + cur.execute("TRUNCATE §version;") + assert isinstance(version, int) + cur.execute(f"INSERT INTO §version (version) VALUES ({version});") + + +def get_patch_func_version(func_name: str) -> int: + assert func_name.startswith( + FUNC_PREFIX + ), f"Patch function `{func_name}` should start with {FUNC_PREFIX}." + len_func_prefix = len(FUNC_PREFIX) + version_str = func_name[len_func_prefix:] + version = int(version_str) + return version + + +def get_tablenames(cur: sqlite3.Cursor) -> list[str]: + cur.execute("SELECT name FROM sqlite_master WHERE type='table';") + tablenames = [result[0] for result in cur.fetchall()] + return tablenames + + +def __patch_001(db_path: Path) -> None: + """Convert prices from float to string + + Args: + db_path (Path): [description] + """ + raise NotImplementedError + + +def __patch_002(db_path: Path) -> None: + """Group tablenames, so that the symbols are alphanumerical. + + Args: + db_path (Path) + """ + with sqlite3.connect(db_path) as conn: + cur = conn.cursor() + tablenames = get_tablenames(cur) + + # Iterate over all tables. + for tablename in tablenames: + base_asset, quote_asset = tablename.split("/") + + # Adjust the order, when the symbols aren't ordered alphanumerical. + if base_asset > quote_asset: + + # Query all prices from the table. + cur = conn.execute(f"Select utc_time, price FROM `{tablename}`;") + + new_values = [] + for _utc_time, _price in cur.fetchall(): + # Convert the data. + utc_time = datetime.datetime.strptime( + _utc_time, "%Y-%m-%d %H:%M:%S%z" + ) + price = decimal.Decimal(_price) + + # Calculate the price of the inverse symbol. + oth_price = misc.reciprocal(price) + new_values.append(utc_time, oth_price) + + assert quote_asset < base_asset + # TODO Refactor code, so that a DatabaseHandle/Class exists, + # which presents basic functions to work with the database. + # e.g. get_tablename function from price_data + new_tablename = f"{quote_asset}/{base_asset}" + # TODO bulk insert new values in table. + # TODO Make sure, that no duplicates exists in the new table. + + # Remove the old table. + cur = conn.execute(f"DROP TABLE `{tablename}`;") + + +def patch_databases() -> None: + # Check if any database paths exist. + database_paths = [p for p in Path(config.DATA_PATH).glob("*.db") if p.is_file()] + if not database_paths: + return + + # Patch all databases separatly. + for db_path in database_paths: + # Read version from database. + current_version = get_version(db_path) + + # Determine all necessary patch functions. + patch_func_names = [ + func + for func in dir() + if func.startswith(FUNC_PREFIX) + if get_patch_func_version(func) > current_version + ] + + # Sort patch functions chronological. + patch_func_names.sort(key=get_patch_func_version) + + # Run the patch functions. + for patch_func_name in patch_func_names: + patch_func = eval(patch_func_name) + patch_func(db_path) + + # Update version. + new_version = get_patch_func_version(patch_func_name) + update_version(db_path, new_version) diff --git a/src/price_data.py b/src/price_data.py index d45d23ba..80582d35 100644 --- a/src/price_data.py +++ b/src/price_data.py @@ -540,6 +540,9 @@ def __set_price_db( utc_time (datetime.datetime) price (decimal.Decimal) """ + # TODO if db_path doesn't exists. Create db with §version table and + # newest version number. It would be nicer, if this could be done + # as a preprocessing step. see book.py with sqlite3.connect(db_path) as conn: cur = conn.cursor() query = f"INSERT INTO `{tablename}`" "('utc_time', 'price') VALUES (?, ?);" @@ -673,7 +676,7 @@ def get_price( if inverted: price = misc.reciprocal(price) - return price + return price def get_cost( self, @@ -706,36 +709,6 @@ def check_database(self): ) with sqlite3.connect(db_path) as conn: - query = "SELECT name,sql FROM sqlite_master WHERE type='table'" - cur = conn.execute(query) - for tablename, sql in cur.fetchall(): - if not sql.lower().contains("price str"): - query = f""" - CREATE TABLE "sql_temp_table" ( - "utc_time" DATETIME PRIMARY KEY, - "price" STR NOT NULL - ); - INSERT INTO "sql_temp_table" ("price","utc_time") - SELECT "price","utc_time" FROM "{tablename}"; - DROP TABLE "{tablename}"; - ALTER TABLE "sql_temp_table" "{tablename}"; - """ - base_asset, quote_asset = tablename.split("/") - if base_asset > quote_asset: - query = f"Select utc_time,price FROM `{tablename}`" - cur = conn.execute(query) - - for row in cur.fetchall(): - utc_time = datetime.datetime.strptime( - row[0], "%Y-%m-%d %H:%M:%S%z" - ) - price = misc.reciprocal(decimal.Decimal(row[1])) - self.set_price_db( - platform, quote_asset, base_asset, utc_time, price - ) - query = f"DROP TABLE `{tablename}`" - cur = conn.execute(query) - query = "SELECT name FROM sqlite_master WHERE type='table'" cur = conn.execute(query) tablenames = (result[0] for result in cur.fetchall()) From 9ea78221f1fdd4bbe261141785248aaea27b904b Mon Sep 17 00:00:00 2001 From: scientes <34819304+scientes@users.noreply.github.com> Date: Fri, 7 Jan 2022 17:54:06 +0100 Subject: [PATCH 16/45] boilerplate --- src/database.py | 213 ++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 213 insertions(+) create mode 100644 src/database.py diff --git a/src/database.py b/src/database.py new file mode 100644 index 00000000..db9d6f7a --- /dev/null +++ b/src/database.py @@ -0,0 +1,213 @@ +import datetime +import decimal +import logging +import sqlite3 +from pathlib import Path +import misc +from typing import Optional + +import config + +log = logging.getLogger(__name__) + + +class Database: + def get_version(self, db_path: Path) -> int: + """Get database version from a database file. + + If the version table is missing, one is created. + + Args: + db_path (str): Path to database file. + + Raises: + RuntimeError: The database version is ambiguous. + + Returns: + int: Version of database file. + """ + with sqlite3.connect(db_path) as conn: + cur = conn.cursor() + try: + cur.execute("SELECT version FROM §version;") + versions = [int(v[0]) for v in cur.fetchall()] + except sqlite3.OperationalError as e: + if str(e) == "no such table: §version": + # The §version table doesn't exist. Create one. + cur.execute("CREATE TABLE §version(version INT);") + cur.execute("INSERT INTO §version (version) VALUES (0);") + return 0 + else: + raise e + + if len(versions) == 1: + version = versions[0] + return version + else: + raise RuntimeError( + f"The database version of the file `{db_path.name}` is ambigious. " + f"The table `§version` should have one entry, but has {len(versions)}." + ) + + def get_price( + self, + db_path: Path, + tablename: str, + utc_time: datetime.datetime, + ) -> Optional[decimal.Decimal]: + """Try to retrieve the price from our local database. + + Args: + db_path (Path) + tablename (str) + utc_time (datetime.datetime) + + Returns: + Optional[decimal.Decimal]: Price. + """ + if db_path.is_file(): + with sqlite3.connect(db_path) as conn: + cur = conn.cursor() + query = f"SELECT price FROM `{tablename}` WHERE utc_time=?;" + + try: + cur.execute(query, (utc_time,)) + except sqlite3.OperationalError as e: + if str(e) == f"no such table: {tablename}": + return None + raise e + + if prices := cur.fetchone(): + return misc.force_decimal(prices[0]) + + return None + + def mean_price( + self, + db_path: Path, + tablename: str, + utc_time: datetime.datetime, + ) -> decimal.Decimal: + """Try to retrieve the price right before and after `utc_time` + from our local database. + + Return 0 if the price could not be estimated. + The function does not check, if a price for `utc_time` exists. + + Args: + db_path (Path) + tablename (str) + utc_time (datetime.datetime) + + Returns: + decimal.Decimal: Price. + """ + if db_path.is_file(): + with sqlite3.connect(db_path) as conn: + cur = conn.cursor() + + before_query = ( + f"SELECT utc_time, price FROM `{tablename}` " + f"WHERE utc_time 0 " + "ORDER BY utc_time DESC " + "LIMIT 1" + ) + try: + cur.execute(before_query, (utc_time,)) + except sqlite3.OperationalError as e: + if str(e) == f"no such table: {tablename}": + return decimal.Decimal() + raise e + if result := cur.fetchone(): + before_time = misc.parse_iso_timestamp_to_decimal_timestamp( + result[0] + ) + before_price = misc.force_decimal(result[1]) + else: + return decimal.Decimal() + + after_query = ( + f"SELECT utc_time, price FROM `{tablename}` " + f"WHERE utc_time>? AND price > 0 " + "ORDER BY utc_time ASC " + "LIMIT 1" + ) + try: + cur.execute(after_query, (utc_time,)) + except sqlite3.OperationalError as e: + if str(e) == f"no such table: {tablename}": + return decimal.Decimal() + raise e + if result := cur.fetchone(): + after_time = misc.parse_iso_timestamp_to_decimal_timestamp( + result[0] + ) + after_price = misc.force_decimal(result[1]) + else: + return decimal.Decimal() + + if before_price and after_price: + d_utc_time = misc.to_decimal_timestamp(utc_time) + # Linear gradiant between the neighbored transactions. + m = (after_price - before_price) / (after_time - before_time) + price = before_price + (d_utc_time - before_time) * m + return price + + return decimal.Decimal() + + def set_price_db( + self, + db_path: Path, + tablename: str, + utc_time: datetime.datetime, + price: decimal.Decimal, + ) -> None: + """Write price to database. + + Create database/table if necessary. + + Args: + db_path (Path) + tablename (str) + utc_time (datetime.datetime) + price (decimal.Decimal) + """ + # TODO if db_path doesn't exists. Create db with §version table and + # newest version number. It would be nicer, if this could be done + # as a preprocessing step. see book.py + with sqlite3.connect(db_path) as conn: + cur = conn.cursor() + query = f"INSERT INTO `{tablename}`" "('utc_time', 'price') VALUES (?, ?);" + try: + cur.execute(query, (utc_time, str(price))) + except sqlite3.OperationalError as e: + if str(e) == f"no such table: {tablename}": + create_query = ( + f"CREATE TABLE `{tablename}`" + "(utc_time DATETIME PRIMARY KEY, " + "price STR NOT NULL);" + ) + cur.execute(create_query) + cur.execute(query, (utc_time, str(price))) + else: + raise e + conn.commit() + + def get_tablename(self, coin: str, reference_coin: str) -> str: + return f"{coin}/{reference_coin}" + + def get_tablenames_from_db(self, cur: sqlite3.Cursor) -> list[str]: + cur.execute("SELECT name FROM sqlite_master WHERE type='table';") + tablenames = [result[0] for result in cur.fetchall()] + return tablenames + + +class Databases: + def __init__(self) -> None: + platforms = self.get_all_dbs() + + def get_all_dbs(): + pass + + def get_db_path(self, platform: str) -> Path: + return Path(config.DATA_PATH, f"{platform}.db") From e711b4734befe287ebc851768f362835d5891aa2 Mon Sep 17 00:00:00 2001 From: scientes <34819304+scientes@users.noreply.github.com> Date: Sun, 9 Jan 2022 11:41:48 +0100 Subject: [PATCH 17/45] removed classes and replaced get_price occurrences --- src/database.py | 364 ++++++++++++++++++++++------------------------ src/price_data.py | 274 +--------------------------------- 2 files changed, 184 insertions(+), 454 deletions(-) diff --git a/src/database.py b/src/database.py index db9d6f7a..b8295741 100644 --- a/src/database.py +++ b/src/database.py @@ -11,203 +11,193 @@ log = logging.getLogger(__name__) -class Database: - def get_version(self, db_path: Path) -> int: - """Get database version from a database file. +def get_version(db_path: Path) -> int: + """Get database version from a database file. + + If the version table is missing, one is created. + + Args: + db_path (str): Path to database file. + + Raises: + RuntimeError: The database version is ambiguous. + + Returns: + int: Version of database file. + """ + with sqlite3.connect(db_path) as conn: + cur = conn.cursor() + try: + cur.execute("SELECT version FROM §version;") + versions = [int(v[0]) for v in cur.fetchall()] + except sqlite3.OperationalError as e: + if str(e) == "no such table: §version": + # The §version table doesn't exist. Create one. + cur.execute("CREATE TABLE §version(version INT);") + cur.execute("INSERT INTO §version (version) VALUES (0);") + return 0 + else: + raise e + + if len(versions) == 1: + version = versions[0] + return version + else: + raise RuntimeError( + f"The database version of the file `{db_path.name}` is ambigious. " + f"The table `§version` should have one entry, but has {len(versions)}." + ) + + +def get_price( + db_path: Path, + tablename: str, + utc_time: datetime.datetime, +) -> Optional[decimal.Decimal]: + """Try to retrieve the price from our local database. + + Args: + db_path (Path) + tablename (str) + utc_time (datetime.datetime) + + Returns: + Optional[decimal.Decimal]: Price. + """ + if db_path.is_file(): + with sqlite3.connect(db_path) as conn: + cur = conn.cursor() + query = f"SELECT price FROM `{tablename}` WHERE utc_time=?;" + + try: + cur.execute(query, (utc_time,)) + except sqlite3.OperationalError as e: + if str(e) == f"no such table: {tablename}": + return None + raise e + + if prices := cur.fetchone(): + return misc.force_decimal(prices[0]) + + return None - If the version table is missing, one is created. - Args: - db_path (str): Path to database file. +def mean_price( + db_path: Path, + tablename: str, + utc_time: datetime.datetime, +) -> decimal.Decimal: + """Try to retrieve the price right before and after `utc_time` + from our local database. - Raises: - RuntimeError: The database version is ambiguous. + Return 0 if the price could not be estimated. + The function does not check, if a price for `utc_time` exists. - Returns: - int: Version of database file. - """ + Args: + db_path (Path) + tablename (str) + utc_time (datetime.datetime) + + Returns: + decimal.Decimal: Price. + """ + if db_path.is_file(): with sqlite3.connect(db_path) as conn: cur = conn.cursor() + + before_query = ( + f"SELECT utc_time, price FROM `{tablename}` " + f"WHERE utc_time 0 " + "ORDER BY utc_time DESC " + "LIMIT 1" + ) try: - cur.execute("SELECT version FROM §version;") - versions = [int(v[0]) for v in cur.fetchall()] + cur.execute(before_query, (utc_time,)) except sqlite3.OperationalError as e: - if str(e) == "no such table: §version": - # The §version table doesn't exist. Create one. - cur.execute("CREATE TABLE §version(version INT);") - cur.execute("INSERT INTO §version (version) VALUES (0);") - return 0 - else: - raise e - - if len(versions) == 1: - version = versions[0] - return version + if str(e) == f"no such table: {tablename}": + return decimal.Decimal() + raise e + if result := cur.fetchone(): + before_time = misc.parse_iso_timestamp_to_decimal_timestamp(result[0]) + before_price = misc.force_decimal(result[1]) else: - raise RuntimeError( - f"The database version of the file `{db_path.name}` is ambigious. " - f"The table `§version` should have one entry, but has {len(versions)}." + return decimal.Decimal() + + after_query = ( + f"SELECT utc_time, price FROM `{tablename}` " + f"WHERE utc_time>? AND price > 0 " + "ORDER BY utc_time ASC " + "LIMIT 1" + ) + try: + cur.execute(after_query, (utc_time,)) + except sqlite3.OperationalError as e: + if str(e) == f"no such table: {tablename}": + return decimal.Decimal() + raise e + if result := cur.fetchone(): + after_time = misc.parse_iso_timestamp_to_decimal_timestamp(result[0]) + after_price = misc.force_decimal(result[1]) + else: + return decimal.Decimal() + + if before_price and after_price: + d_utc_time = misc.to_decimal_timestamp(utc_time) + # Linear gradiant between the neighbored transactions. + m = (after_price - before_price) / (after_time - before_time) + price = before_price + (d_utc_time - before_time) * m + return price + + return decimal.Decimal() + + +def __set_price_db( + db_path: Path, + tablename: str, + utc_time: datetime.datetime, + price: decimal.Decimal, +) -> None: + """Write price to database. + + Create database/table if necessary. + + Args: + db_path (Path) + tablename (str) + utc_time (datetime.datetime) + price (decimal.Decimal) + """ + # TODO if db_path doesn't exists. Create db with §version table and + # newest version number. It would be nicer, if this could be done + # as a preprocessing step. see book.py + with sqlite3.connect(db_path) as conn: + cur = conn.cursor() + query = f"INSERT INTO `{tablename}`" "('utc_time', 'price') VALUES (?, ?);" + try: + cur.execute(query, (utc_time, str(price))) + except sqlite3.OperationalError as e: + if str(e) == f"no such table: {tablename}": + create_query = ( + f"CREATE TABLE `{tablename}`" + "(utc_time DATETIME PRIMARY KEY, " + "price STR NOT NULL);" ) + cur.execute(create_query) + cur.execute(query, (utc_time, str(price))) + else: + raise e + conn.commit() - def get_price( - self, - db_path: Path, - tablename: str, - utc_time: datetime.datetime, - ) -> Optional[decimal.Decimal]: - """Try to retrieve the price from our local database. - - Args: - db_path (Path) - tablename (str) - utc_time (datetime.datetime) - - Returns: - Optional[decimal.Decimal]: Price. - """ - if db_path.is_file(): - with sqlite3.connect(db_path) as conn: - cur = conn.cursor() - query = f"SELECT price FROM `{tablename}` WHERE utc_time=?;" - - try: - cur.execute(query, (utc_time,)) - except sqlite3.OperationalError as e: - if str(e) == f"no such table: {tablename}": - return None - raise e - - if prices := cur.fetchone(): - return misc.force_decimal(prices[0]) - - return None - - def mean_price( - self, - db_path: Path, - tablename: str, - utc_time: datetime.datetime, - ) -> decimal.Decimal: - """Try to retrieve the price right before and after `utc_time` - from our local database. - - Return 0 if the price could not be estimated. - The function does not check, if a price for `utc_time` exists. - - Args: - db_path (Path) - tablename (str) - utc_time (datetime.datetime) - - Returns: - decimal.Decimal: Price. - """ - if db_path.is_file(): - with sqlite3.connect(db_path) as conn: - cur = conn.cursor() - - before_query = ( - f"SELECT utc_time, price FROM `{tablename}` " - f"WHERE utc_time 0 " - "ORDER BY utc_time DESC " - "LIMIT 1" - ) - try: - cur.execute(before_query, (utc_time,)) - except sqlite3.OperationalError as e: - if str(e) == f"no such table: {tablename}": - return decimal.Decimal() - raise e - if result := cur.fetchone(): - before_time = misc.parse_iso_timestamp_to_decimal_timestamp( - result[0] - ) - before_price = misc.force_decimal(result[1]) - else: - return decimal.Decimal() - after_query = ( - f"SELECT utc_time, price FROM `{tablename}` " - f"WHERE utc_time>? AND price > 0 " - "ORDER BY utc_time ASC " - "LIMIT 1" - ) - try: - cur.execute(after_query, (utc_time,)) - except sqlite3.OperationalError as e: - if str(e) == f"no such table: {tablename}": - return decimal.Decimal() - raise e - if result := cur.fetchone(): - after_time = misc.parse_iso_timestamp_to_decimal_timestamp( - result[0] - ) - after_price = misc.force_decimal(result[1]) - else: - return decimal.Decimal() +def get_tablename(coin: str, reference_coin: str) -> str: + return f"{coin}/{reference_coin}" - if before_price and after_price: - d_utc_time = misc.to_decimal_timestamp(utc_time) - # Linear gradiant between the neighbored transactions. - m = (after_price - before_price) / (after_time - before_time) - price = before_price + (d_utc_time - before_time) * m - return price - - return decimal.Decimal() - - def set_price_db( - self, - db_path: Path, - tablename: str, - utc_time: datetime.datetime, - price: decimal.Decimal, - ) -> None: - """Write price to database. - - Create database/table if necessary. - - Args: - db_path (Path) - tablename (str) - utc_time (datetime.datetime) - price (decimal.Decimal) - """ - # TODO if db_path doesn't exists. Create db with §version table and - # newest version number. It would be nicer, if this could be done - # as a preprocessing step. see book.py - with sqlite3.connect(db_path) as conn: - cur = conn.cursor() - query = f"INSERT INTO `{tablename}`" "('utc_time', 'price') VALUES (?, ?);" - try: - cur.execute(query, (utc_time, str(price))) - except sqlite3.OperationalError as e: - if str(e) == f"no such table: {tablename}": - create_query = ( - f"CREATE TABLE `{tablename}`" - "(utc_time DATETIME PRIMARY KEY, " - "price STR NOT NULL);" - ) - cur.execute(create_query) - cur.execute(query, (utc_time, str(price))) - else: - raise e - conn.commit() - - def get_tablename(self, coin: str, reference_coin: str) -> str: - return f"{coin}/{reference_coin}" - - def get_tablenames_from_db(self, cur: sqlite3.Cursor) -> list[str]: - cur.execute("SELECT name FROM sqlite_master WHERE type='table';") - tablenames = [result[0] for result in cur.fetchall()] - return tablenames - - -class Databases: - def __init__(self) -> None: - platforms = self.get_all_dbs() - - def get_all_dbs(): - pass - - def get_db_path(self, platform: str) -> Path: - return Path(config.DATA_PATH, f"{platform}.db") + +def get_tablenames_from_db(cur: sqlite3.Cursor) -> list[str]: + cur.execute("SELECT name FROM sqlite_master WHERE type='table';") + tablenames = [result[0] for result in cur.fetchall()] + return tablenames + + +def get_db_path(platform: str) -> Path: + return Path(config.DATA_PATH, f"{platform}.db") diff --git a/src/price_data.py b/src/price_data.py index 80582d35..7d5ef017 100644 --- a/src/price_data.py +++ b/src/price_data.py @@ -29,6 +29,7 @@ import config import misc import transaction +from database import get_price from core import kraken_pair_map log = logging.getLogger(__name__) @@ -107,13 +108,13 @@ def _get_price_binance( raise RuntimeError(f"Can not retrieve {symbol=} from binance") # Changing the order of the assets require to # invert the price. - price = self.get_price( + price = get_price( "binance", quote_asset, utc_time, base_asset, swapped_symbols=True ) return misc.reciprocal(price) - btc = self.get_price("binance", base_asset, utc_time, "BTC") - quote = self.get_price("binance", "BTC", utc_time, quote_asset) + btc = get_price("binance", base_asset, utc_time, "BTC") + quote = get_price("binance", "BTC", utc_time, quote_asset) return btc * quote response.raise_for_status() @@ -128,10 +129,10 @@ def _get_price_binance( ) if quote_asset == "USDT": return decimal.Decimal() - quote = self.get_price("binance", quote_asset, utc_time, "USDT") + quote = get_price("binance", quote_asset, utc_time, "USDT") if quote == 0.0: return quote - usdt = self.get_price("binance", base_asset, utc_time, "USDT") + usdt = get_price("binance", base_asset, utc_time, "USDT") return usdt / quote # Calculate average price. @@ -417,274 +418,13 @@ def _get_price_kraken( ) return decimal.Decimal() - def __get_price_db( - self, - db_path: Path, - tablename: str, - utc_time: datetime.datetime, - ) -> Optional[decimal.Decimal]: - """Try to retrieve the price from our local database. - - Args: - db_path (Path) - tablename (str) - utc_time (datetime.datetime) - - Returns: - Optional[decimal.Decimal]: Price. - """ - if db_path.is_file(): - with sqlite3.connect(db_path) as conn: - cur = conn.cursor() - query = f"SELECT price FROM `{tablename}` WHERE utc_time=?;" - - try: - cur.execute(query, (utc_time,)) - except sqlite3.OperationalError as e: - if str(e) == f"no such table: {tablename}": - return None - raise e - - if prices := cur.fetchone(): - return misc.force_decimal(prices[0]) - - return None - - def __mean_price_db( - self, - db_path: Path, - tablename: str, - utc_time: datetime.datetime, - ) -> decimal.Decimal: - """Try to retrieve the price right before and after `utc_time` - from our local database. - - Return 0 if the price could not be estimated. - The function does not check, if a price for `utc_time` exists. - - Args: - db_path (Path) - tablename (str) - utc_time (datetime.datetime) - - Returns: - decimal.Decimal: Price. - """ - if db_path.is_file(): - with sqlite3.connect(db_path) as conn: - cur = conn.cursor() - - before_query = ( - f"SELECT utc_time, price FROM `{tablename}` " - f"WHERE utc_time 0 " - "ORDER BY utc_time DESC " - "LIMIT 1" - ) - try: - cur.execute(before_query, (utc_time,)) - except sqlite3.OperationalError as e: - if str(e) == f"no such table: {tablename}": - return decimal.Decimal() - raise e - if result := cur.fetchone(): - before_time = misc.parse_iso_timestamp_to_decimal_timestamp( - result[0] - ) - before_price = misc.force_decimal(result[1]) - else: - return decimal.Decimal() - - after_query = ( - f"SELECT utc_time, price FROM `{tablename}` " - f"WHERE utc_time>? AND price > 0 " - "ORDER BY utc_time ASC " - "LIMIT 1" - ) - try: - cur.execute(after_query, (utc_time,)) - except sqlite3.OperationalError as e: - if str(e) == f"no such table: {tablename}": - return decimal.Decimal() - raise e - if result := cur.fetchone(): - after_time = misc.parse_iso_timestamp_to_decimal_timestamp( - result[0] - ) - after_price = misc.force_decimal(result[1]) - else: - return decimal.Decimal() - - if before_price and after_price: - d_utc_time = misc.to_decimal_timestamp(utc_time) - # Linear gradiant between the neighbored transactions. - m = (after_price - before_price) / (after_time - before_time) - price = before_price + (d_utc_time - before_time) * m - return price - - return decimal.Decimal() - - def __set_price_db( - self, - db_path: Path, - tablename: str, - utc_time: datetime.datetime, - price: decimal.Decimal, - ) -> None: - """Write price to database. - - Create database/table if necessary. - - Args: - db_path (Path) - tablename (str) - utc_time (datetime.datetime) - price (decimal.Decimal) - """ - # TODO if db_path doesn't exists. Create db with §version table and - # newest version number. It would be nicer, if this could be done - # as a preprocessing step. see book.py - with sqlite3.connect(db_path) as conn: - cur = conn.cursor() - query = f"INSERT INTO `{tablename}`" "('utc_time', 'price') VALUES (?, ?);" - try: - cur.execute(query, (utc_time, str(price))) - except sqlite3.OperationalError as e: - if str(e) == f"no such table: {tablename}": - create_query = ( - f"CREATE TABLE `{tablename}`" - "(utc_time DATETIME PRIMARY KEY, " - "price STR NOT NULL);" - ) - cur.execute(create_query) - cur.execute(query, (utc_time, str(price))) - else: - raise e - conn.commit() - - def set_price_db( - self, - platform: str, - coin: str, - reference_coin: str, - utc_time: datetime.datetime, - price: decimal.Decimal, - ) -> None: - """Write price to database. - - Tries to insert a historical price into the local database. - - A warning will be raised, if there is already a different price. - - Args: - platform (str): [description] - coin (str): [description] - reference_coin (str): [description] - utc_time (datetime.datetime): [description] - price (decimal.Decimal): [description] - """ - assert coin != reference_coin - coin_a, coin_b, inverted = self._sort_pair(coin, reference_coin) - db_path = self.get_db_path(platform) - tablename = self.get_tablename(coin_a, coin_b) - if inverted: - price = misc.reciprocal(price) - try: - self.__set_price_db(db_path, tablename, utc_time, price) - except sqlite3.IntegrityError as e: - if str(e) == f"UNIQUE constraint failed: {tablename}.utc_time": - price_db = self.get_price(platform, coin_a, utc_time, coin_b) - if price != price_db: - log.warning( - "Tried to write price to database, " - "but a different price exists already." - f"({platform=}, {tablename=}, {utc_time=}, {price=})" - ) - else: - raise e - - def _sort_pair(self, coin: str, reference_coin: str) -> Tuple[str, str, bool]: - """Sort the coin pair in alphanumerical order. - - Args: - coin (str) - reference_coin (str) - - Returns: - Tuple[str, str, bool]: First coin, second coin, inverted - """ - if inverted := coin > reference_coin: - coin_a = reference_coin - coin_b = coin - else: - coin_a = coin - coin_b = reference_coin - return coin_a, coin_b, inverted - - def get_price( - self, - platform: str, - coin: str, - utc_time: datetime.datetime, - reference_coin: str = config.FIAT, - **kwargs: Any, - ) -> decimal.Decimal: - """Get the price of a coin pair from a specific `platform` at `utc_time`. - - The function tries to retrieve the price from the local database first. - If the price does not exist, its gathered from a platform specific - function and saved to our local database for future access. - - Args: - platform (str) - coin (str) - utc_time (datetime.datetime) - reference_coin (str, optional): Defaults to config.FIAT. - - Raises: - NotImplementedError: Platform specific GET function is not - implemented. - - Returns: - decimal.Decimal: Price of the coin pair. - """ - if coin == reference_coin: - return decimal.Decimal("1") - - db_path = self.get_db_path(platform) - coin_a, coin_b, inverted = self._sort_pair(coin, reference_coin) - tablename = self.get_tablename(coin_a, coin_b) - - # Check if price exists already in our database. - if (price := self.__get_price_db(db_path, tablename, utc_time)) is None: - # The price does not exist in our database. - # Gather the price from a platform specific function. - try: - get_price = getattr(self, f"_get_price_{platform}") - except AttributeError: - raise NotImplementedError("Unable to read data from %s", platform) - price = get_price(coin, utc_time, reference_coin, **kwargs) - if inverted: - price = misc.reciprocal(price) - assert isinstance(price, decimal.Decimal) - self.__set_price_db(db_path, tablename, utc_time, price) - - if config.MEAN_MISSING_PRICES and price <= 0.0: - # The price is missing. Check for prices before and after the - # transaction and estimate the price. - # Do not save price in database. - price = self.__mean_price_db(db_path, tablename, utc_time) - - if inverted: - price = misc.reciprocal(price) - return price - def get_cost( self, tr: Union[transaction.Operation, transaction.SoldCoin], reference_coin: str = config.FIAT, ) -> decimal.Decimal: op = tr if isinstance(tr, transaction.Operation) else tr.op - price = self.get_price(op.platform, op.coin, op.utc_time, reference_coin) + price = get_price(op.platform, op.coin, op.utc_time, reference_coin) if isinstance(tr, transaction.Operation): return price * tr.change if isinstance(tr, transaction.SoldCoin): From 0dd01323b9261fa25d69413593edece499ed4cd1 Mon Sep 17 00:00:00 2001 From: scientes <34819304+scientes@users.noreply.github.com> Date: Sun, 9 Jan 2022 12:04:59 +0100 Subject: [PATCH 18/45] refractored db functions --- src/book.py | 11 ++++---- src/database.py | 66 ++++++++++++++++++++++++++++++++++++++++++--- src/main.py | 2 +- src/price_data.py | 68 ++++++++++++++++++++++++++++++++++++++++++----- 4 files changed, 131 insertions(+), 16 deletions(-) diff --git a/src/book.py b/src/book.py index 353c413a..1770d1eb 100644 --- a/src/book.py +++ b/src/book.py @@ -27,6 +27,7 @@ import transaction as tr from core import kraken_asset_map from price_data import PriceData +from database import set_price_db log = logging.getLogger(__name__) @@ -284,7 +285,7 @@ def _read_coinbase(self, file_path: Path) -> None: assert _currency_spot == "EUR" # Save price in our local database for later. - self.price_data.set_price_db(platform, coin, "EUR", utc_time, eur_spot) + set_price_db(platform, coin, "EUR", utc_time, eur_spot) if operation == "Convert": # Parse change + coin from remark, which is @@ -317,7 +318,7 @@ def _read_coinbase(self, file_path: Path) -> None: ) # Save convert price in local database, too. - self.price_data.set_price_db( + set_price_db( platform, convert_coin, "EUR", utc_time, convert_eur_spot ) else: @@ -721,9 +722,9 @@ def _read_bitpanda_pro_trades(self, file_path: Path) -> None: # Save price in our local database for later. price = misc.force_decimal(_price) - self.price_data.set_price_db(platform, coin, "EUR", utc_time, price) + set_price_db(platform, coin, "EUR", utc_time, price) if best_price: - self.price_data.set_price_db( + set_price_db( platform, "BEST", "EUR", @@ -885,7 +886,7 @@ def get_price_from_csv(self) -> None: f"{price} for {platform} at {timestamp}" ) - self.price_data.set_price_db( + set_price_db( platform, buytr.coin, selltr.coin, diff --git a/src/database.py b/src/database.py index b8295741..54bbb168 100644 --- a/src/database.py +++ b/src/database.py @@ -4,7 +4,7 @@ import sqlite3 from pathlib import Path import misc -from typing import Optional +from typing import Optional, Any, Tuple import config @@ -49,7 +49,7 @@ def get_version(db_path: Path) -> int: ) -def get_price( +def get_price_db( db_path: Path, tablename: str, utc_time: datetime.datetime, @@ -82,7 +82,7 @@ def get_price( return None -def mean_price( +def mean_price_db( db_path: Path, tablename: str, utc_time: datetime.datetime, @@ -189,6 +189,66 @@ def __set_price_db( conn.commit() +def set_price_db( + platform: str, + coin: str, + reference_coin: str, + utc_time: datetime.datetime, + price: decimal.Decimal, +) -> None: + """Write price to database. + + Tries to insert a historical price into the local database. + + A warning will be raised, if there is already a different price. + + Args: + platform (str): [description] + coin (str): [description] + reference_coin (str): [description] + utc_time (datetime.datetime): [description] + price (decimal.Decimal): [description] + """ + assert coin != reference_coin + coin_a, coin_b, inverted = _sort_pair(coin, reference_coin) + db_path = get_db_path(platform) + tablename = get_tablename(coin_a, coin_b) + if inverted: + price = misc.reciprocal(price) + try: + __set_price_db(db_path, tablename, utc_time, price) + except sqlite3.IntegrityError as e: + if str(e) == f"UNIQUE constraint failed: {tablename}.utc_time": + price_db = get_price_db(db_path, tablename, utc_time) + if price != price_db: + log.warning( + "Tried to write price to database, " + "but a different price exists already." + f"({platform=}, {tablename=}, {utc_time=}, {price=})" + ) + else: + raise e + + +def _sort_pair(coin: str, reference_coin: str) -> Tuple[str, str, bool]: + """Sort the coin pair in alphanumerical order. + + Args: + coin (str) + reference_coin (str) + + Returns: + Tuple[str, str, bool]: First coin, second coin, inverted + """ + if inverted := coin > reference_coin: + coin_a = reference_coin + coin_b = coin + else: + coin_a = coin + coin_b = reference_coin + return coin_a, coin_b, inverted + + def get_tablename(coin: str, reference_coin: str) -> str: return f"{coin}/{reference_coin}" diff --git a/src/main.py b/src/main.py index 842f1da7..b2f3e005 100644 --- a/src/main.py +++ b/src/main.py @@ -26,7 +26,7 @@ def main() -> None: - patch_databases() + # patch_databases() price_data = PriceData() book = Book(price_data) diff --git a/src/price_data.py b/src/price_data.py index 7d5ef017..481813bf 100644 --- a/src/price_data.py +++ b/src/price_data.py @@ -29,7 +29,13 @@ import config import misc import transaction -from database import get_price +from database import ( + set_price_db, + mean_price_db, + get_price_db, + get_tablename, + get_db_path, +) from core import kraken_pair_map log = logging.getLogger(__name__) @@ -108,13 +114,13 @@ def _get_price_binance( raise RuntimeError(f"Can not retrieve {symbol=} from binance") # Changing the order of the assets require to # invert the price. - price = get_price( + price = self.get_price( "binance", quote_asset, utc_time, base_asset, swapped_symbols=True ) return misc.reciprocal(price) - btc = get_price("binance", base_asset, utc_time, "BTC") - quote = get_price("binance", "BTC", utc_time, quote_asset) + btc = self.get_price("binance", base_asset, utc_time, "BTC") + quote = self.get_price("binance", "BTC", utc_time, quote_asset) return btc * quote response.raise_for_status() @@ -129,10 +135,10 @@ def _get_price_binance( ) if quote_asset == "USDT": return decimal.Decimal() - quote = get_price("binance", quote_asset, utc_time, "USDT") + quote = self.get_price("binance", quote_asset, utc_time, "USDT") if quote == 0.0: return quote - usdt = get_price("binance", base_asset, utc_time, "USDT") + usdt = self.get_price("binance", base_asset, utc_time, "USDT") return usdt / quote # Calculate average price. @@ -418,13 +424,61 @@ def _get_price_kraken( ) return decimal.Decimal() + def get_price( + self, + platform: str, + coin: str, + utc_time: datetime.datetime, + reference_coin: str = config.FIAT, + **kwargs: Any, + ) -> decimal.Decimal: + """Get the price of a coin pair from a specific `platform` at `utc_time`. + The function tries to retrieve the price from the local database first. + If the price does not exist, its gathered from a platform specific + function and saved to our local database for future access. + Args: + platform (str) + coin (str) + utc_time (datetime.datetime) + reference_coin (str, optional): Defaults to config.FIAT. + Raises: + NotImplementedError: Platform specific GET function is not + implemented. + Returns: + decimal.Decimal: Price of the coin pair. + """ + if coin == reference_coin: + return decimal.Decimal("1") + + db_path = get_db_path(platform) + tablename = get_tablename(coin, reference_coin) + + # Check if price exists already in our database. + if (price := get_price_db(db_path, tablename, utc_time)) is None: + try: + get_price = getattr(self, f"_get_price_{platform}") + except AttributeError: + raise NotImplementedError("Unable to read data from %s", platform) + + price = get_price(coin, utc_time, reference_coin, **kwargs) + assert isinstance(price, decimal.Decimal) + set_price_db(db_path, tablename, utc_time, price) + + if config.MEAN_MISSING_PRICES and price <= 0.0: + # The price is missing. Check for prices before and after the + # transaction and estimate the price. + # Do not save price in database. + price = mean_price_db(db_path, tablename, utc_time) + + return price + def get_cost( self, tr: Union[transaction.Operation, transaction.SoldCoin], reference_coin: str = config.FIAT, ) -> decimal.Decimal: op = tr if isinstance(tr, transaction.Operation) else tr.op - price = get_price(op.platform, op.coin, op.utc_time, reference_coin) + price = self.get_price(op.platform, op.coin, op.utc_time, reference_coin) if isinstance(tr, transaction.Operation): return price * tr.change if isinstance(tr, transaction.SoldCoin): From 33876f289a6c6f10e04fbb84adb134d52975479a Mon Sep 17 00:00:00 2001 From: scientes <34819304+scientes@users.noreply.github.com> Date: Sun, 9 Jan 2022 15:14:13 +0100 Subject: [PATCH 19/45] fixed patch function --- src/main.py | 2 +- src/patch_database.py | 4 +++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/src/main.py b/src/main.py index b2f3e005..842f1da7 100644 --- a/src/main.py +++ b/src/main.py @@ -26,7 +26,7 @@ def main() -> None: - # patch_databases() + patch_databases() price_data = PriceData() book = Book(price_data) diff --git a/src/patch_database.py b/src/patch_database.py index b5f2e6d8..9e064739 100644 --- a/src/patch_database.py +++ b/src/patch_database.py @@ -22,6 +22,8 @@ import config import misc +from inspect import getmembers, isfunction +import sys FUNC_PREFIX = "__patch_" log = logging.getLogger(__name__) @@ -156,7 +158,7 @@ def patch_databases() -> None: # Determine all necessary patch functions. patch_func_names = [ func - for func in dir() + for func in str(getmembers(sys.modules[__name__], isfunction)[0]) if func.startswith(FUNC_PREFIX) if get_patch_func_version(func) > current_version ] From ab15eaa6a59add4b665071bcbfc044bac6e3ac72 Mon Sep 17 00:00:00 2001 From: scientes <34819304+scientes@users.noreply.github.com> Date: Mon, 10 Jan 2022 10:10:09 +0100 Subject: [PATCH 20/45] fixed imports and flake added patch 001 --- src/database.py | 8 ++++--- src/patch_database.py | 51 ++++++++++++++++++++++--------------------- src/price_data.py | 4 ++-- 3 files changed, 33 insertions(+), 30 deletions(-) diff --git a/src/database.py b/src/database.py index 54bbb168..671dc26b 100644 --- a/src/database.py +++ b/src/database.py @@ -3,10 +3,10 @@ import logging import sqlite3 from pathlib import Path -import misc -from typing import Optional, Any, Tuple +from typing import Optional, Tuple import config +import misc log = logging.getLogger(__name__) @@ -195,6 +195,7 @@ def set_price_db( reference_coin: str, utc_time: datetime.datetime, price: decimal.Decimal, + db_path: Path = None, ) -> None: """Write price to database. @@ -211,7 +212,8 @@ def set_price_db( """ assert coin != reference_coin coin_a, coin_b, inverted = _sort_pair(coin, reference_coin) - db_path = get_db_path(platform) + if db_path is None and platform == "": + db_path = get_db_path(platform) tablename = get_tablename(coin_a, coin_b) if inverted: price = misc.reciprocal(price) diff --git a/src/patch_database.py b/src/patch_database.py index 9e064739..b85b4c57 100644 --- a/src/patch_database.py +++ b/src/patch_database.py @@ -18,12 +18,12 @@ import decimal import logging import sqlite3 +import sys +from inspect import getmembers, isfunction from pathlib import Path import config -import misc -from inspect import getmembers, isfunction -import sys +from database import set_price_db FUNC_PREFIX = "__patch_" log = logging.getLogger(__name__) @@ -97,7 +97,22 @@ def __patch_001(db_path: Path) -> None: Args: db_path (Path): [description] """ - raise NotImplementedError + logging.info("applying patch 001") + with sqlite3.connect(db_path) as conn: + query = "SELECT name,sql FROM sqlite_master WHERE type='table'" + cur = conn.execute(query) + for tablename, sql in cur.fetchall(): + if not sql.lower().contains("price str"): + query = f""" + CREATE TABLE "sql_temp_table" ( + "utc_time" DATETIME PRIMARY KEY, + "price" STR NOT NULL + ); + INSERT INTO "sql_temp_table" ("price","utc_time") + SELECT "price","utc_time" FROM "{tablename}"; + DROP TABLE "{tablename}"; + ALTER TABLE "sql_temp_table" "{tablename}"; + """ def __patch_002(db_path: Path) -> None: @@ -106,10 +121,10 @@ def __patch_002(db_path: Path) -> None: Args: db_path (Path) """ + logging.info("applying patch 002") with sqlite3.connect(db_path) as conn: cur = conn.cursor() tablenames = get_tablenames(cur) - # Iterate over all tables. for tablename in tablenames: base_asset, quote_asset = tablename.split("/") @@ -120,27 +135,13 @@ def __patch_002(db_path: Path) -> None: # Query all prices from the table. cur = conn.execute(f"Select utc_time, price FROM `{tablename}`;") - new_values = [] - for _utc_time, _price in cur.fetchall(): + for _utc_time, _price in list(cur.fetchall()): # Convert the data. utc_time = datetime.datetime.strptime( _utc_time, "%Y-%m-%d %H:%M:%S%z" ) price = decimal.Decimal(_price) - - # Calculate the price of the inverse symbol. - oth_price = misc.reciprocal(price) - new_values.append(utc_time, oth_price) - - assert quote_asset < base_asset - # TODO Refactor code, so that a DatabaseHandle/Class exists, - # which presents basic functions to work with the database. - # e.g. get_tablename function from price_data - new_tablename = f"{quote_asset}/{base_asset}" - # TODO bulk insert new values in table. - # TODO Make sure, that no duplicates exists in the new table. - - # Remove the old table. + set_price_db("", base_asset, quote_asset, utc_time, price, db_path) cur = conn.execute(f"DROP TABLE `{tablename}`;") @@ -157,10 +158,10 @@ def patch_databases() -> None: # Determine all necessary patch functions. patch_func_names = [ - func - for func in str(getmembers(sys.modules[__name__], isfunction)[0]) - if func.startswith(FUNC_PREFIX) - if get_patch_func_version(func) > current_version + func[0] + for func in getmembers(sys.modules[__name__], isfunction) + if func[0].startswith(FUNC_PREFIX) + if get_patch_func_version(func[0]) > current_version ] # Sort patch functions chronological. diff --git a/src/price_data.py b/src/price_data.py index 481813bf..91dd70b6 100644 --- a/src/price_data.py +++ b/src/price_data.py @@ -22,7 +22,7 @@ import sqlite3 import time from pathlib import Path -from typing import Any, Optional, Tuple, Union +from typing import Any, Union import requests @@ -462,7 +462,7 @@ def get_price( price = get_price(coin, utc_time, reference_coin, **kwargs) assert isinstance(price, decimal.Decimal) - set_price_db(db_path, tablename, utc_time, price) + set_price_db("", coin, reference_coin, utc_time, price, db_path) if config.MEAN_MISSING_PRICES and price <= 0.0: # The price is missing. Check for prices before and after the From 7265ee744d0fd67c5aa0e251ba70172c7f389044 Mon Sep 17 00:00:00 2001 From: scientes <34819304+scientes@users.noreply.github.com> Date: Mon, 10 Jan 2022 10:13:45 +0100 Subject: [PATCH 21/45] fix iosort --- src/book.py | 2 +- src/price_data.py | 8 ++++---- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/src/book.py b/src/book.py index 1770d1eb..4840349f 100644 --- a/src/book.py +++ b/src/book.py @@ -26,8 +26,8 @@ import misc import transaction as tr from core import kraken_asset_map -from price_data import PriceData from database import set_price_db +from price_data import PriceData log = logging.getLogger(__name__) diff --git a/src/price_data.py b/src/price_data.py index 91dd70b6..2e89f2fa 100644 --- a/src/price_data.py +++ b/src/price_data.py @@ -29,14 +29,14 @@ import config import misc import transaction +from core import kraken_pair_map from database import ( - set_price_db, - mean_price_db, + get_db_path, get_price_db, get_tablename, - get_db_path, + mean_price_db, + set_price_db, ) -from core import kraken_pair_map log = logging.getLogger(__name__) From 7bb7bd8eff964fba9be5dac1d410fe5eec6aa61b Mon Sep 17 00:00:00 2001 From: Griffsano <18743559+Griffsano@users.noreply.github.com> Date: Fri, 28 Jan 2022 10:09:20 +0100 Subject: [PATCH 22/45] Prices from csv (#5) * barebones config * RM debug print * ADD Comment to CALCULATE VIRTUAL SELL config * fixes in DB patch functions Co-authored-by: scientes <34819304+scientes@users.noreply.github.com> Co-authored-by: Jeppy --- config.ini | 17 +++++++++++++++++ src/config.py | 23 ++++++++--------------- src/patch_database.py | 23 ++++++++++++++++------- 3 files changed, 41 insertions(+), 22 deletions(-) create mode 100644 config.ini diff --git a/config.ini b/config.ini new file mode 100644 index 00000000..d44dd177 --- /dev/null +++ b/config.ini @@ -0,0 +1,17 @@ +[BASE] +# User specific constants (might be overwritten by environmental variables). +COUNTRY = GERMANY +TAX_YEAR = 2021 +# If the price for a coin is missing, check if there are known prices before +# and after the specific transaction and use linear regression to estimate +# the price inbetween. +# Important: The code must be run twice for this option to take effect. +MEAN_MISSING_PRICES = False +# Calculate the (taxed) gains, if the left over coins would be sold right now. +# This will fetch the current prices and therefore slow down repetitive runs. +# This will not have an effect, if the TAX_YEAR is not the current year. +CALCULATE_VIRTUAL_SELL = True +# Evaluate taxes for each depot/platform separately. This may reduce your +# taxable gains. Make sure, that this method is accepted by your tax +# authority. +MULTI_DEPOT = True diff --git a/src/config.py b/src/config.py index 3690ac6c..5677f0e3 100644 --- a/src/config.py +++ b/src/config.py @@ -14,6 +14,7 @@ # You should have received a copy of the GNU Affero General Public License # along with this program. If not, see . +import configparser from datetime import datetime from os import environ from pathlib import Path @@ -22,21 +23,13 @@ import core -# User specific constants (might be overwritten by environmental variables). -COUNTRY = core.Country.GERMANY -TAX_YEAR = 2021 -# If the price for a coin is missing, check if there are known prices before -# and after the specific transaction and use linear regression to estimate -# the price inbetween. -# Important: The code must be run twice for this option to take effect. -MEAN_MISSING_PRICES = False -# Calculate the (taxed) gains, if the left over coins would be sold right now. -# This will fetch the current prices and therefore slow down repetitive runs. -CALCULATE_VIRTUAL_SELL = True -# Evaluate taxes for each depot/platform separately. This may reduce your -# taxable gains. Make sure, that this method is accepted by your tax -# authority. -MULTI_DEPOT = True +config = configparser.ConfigParser() +config.read("config.ini") +COUNTRY = core.Country[config["BASE"].get("COUNTRY", "GERMANY")] +TAX_YEAR = int(config["BASE"].get("TAX_YEAR", "2021")) +MEAN_MISSING_PRICES = config["BASE"].getboolean("MEAN_MISSING_PRICES") +CALCULATE_VIRTUAL_SELL = config["BASE"].getboolean("CALCULATE_VIRTUAL_SELL") +MULTI_DEPOT = config["BASE"].getboolean("MULTI_DEPOT") # Read in environmental variables. if _env_country := environ.get("COUNTRY"): diff --git a/src/patch_database.py b/src/patch_database.py index b85b4c57..81449a7d 100644 --- a/src/patch_database.py +++ b/src/patch_database.py @@ -70,7 +70,7 @@ def get_version(db_path: Path) -> int: def update_version(db_path: Path, version: int) -> None: with sqlite3.connect(db_path) as conn: cur = conn.cursor() - cur.execute("TRUNCATE §version;") + cur.execute("DELETE FROM §version;") assert isinstance(version, int) cur.execute(f"INSERT INTO §version (version) VALUES ({version});") @@ -102,7 +102,7 @@ def __patch_001(db_path: Path) -> None: query = "SELECT name,sql FROM sqlite_master WHERE type='table'" cur = conn.execute(query) for tablename, sql in cur.fetchall(): - if not sql.lower().contains("price str"): + if "price str" not in sql.lower(): query = f""" CREATE TABLE "sql_temp_table" ( "utc_time" DATETIME PRIMARY KEY, @@ -127,6 +127,8 @@ def __patch_002(db_path: Path) -> None: tablenames = get_tablenames(cur) # Iterate over all tables. for tablename in tablenames: + if tablename == "§version": + continue base_asset, quote_asset = tablename.split("/") # Adjust the order, when the symbols aren't ordered alphanumerical. @@ -137,9 +139,15 @@ def __patch_002(db_path: Path) -> None: for _utc_time, _price in list(cur.fetchall()): # Convert the data. - utc_time = datetime.datetime.strptime( - _utc_time, "%Y-%m-%d %H:%M:%S%z" - ) + # Try non-fractional seconds first, then fractional seconds + try: + utc_time = datetime.datetime.strptime( + _utc_time, "%Y-%m-%d %H:%M:%S%z" + ) + except ValueError: + utc_time = datetime.datetime.strptime( + _utc_time, "%Y-%m-%d %H:%M:%S.%f%z" + ) price = decimal.Decimal(_price) set_price_db("", base_asset, quote_asset, utc_time, price, db_path) cur = conn.execute(f"DROP TABLE `{tablename}`;") @@ -173,5 +181,6 @@ def patch_databases() -> None: patch_func(db_path) # Update version. - new_version = get_patch_func_version(patch_func_name) - update_version(db_path, new_version) + if patch_func_names: + new_version = get_patch_func_version(patch_func_name) + update_version(db_path, new_version) From 179bb05bd742797ec2a4ab3f0c9863d20d874699 Mon Sep 17 00:00:00 2001 From: Jeppy Date: Sat, 5 Feb 2022 16:59:46 +0100 Subject: [PATCH 23/45] Refactor logging patching info --- src/patch_database.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/patch_database.py b/src/patch_database.py index 81449a7d..3665da6e 100644 --- a/src/patch_database.py +++ b/src/patch_database.py @@ -97,7 +97,6 @@ def __patch_001(db_path: Path) -> None: Args: db_path (Path): [description] """ - logging.info("applying patch 001") with sqlite3.connect(db_path) as conn: query = "SELECT name,sql FROM sqlite_master WHERE type='table'" cur = conn.execute(query) @@ -121,7 +120,6 @@ def __patch_002(db_path: Path) -> None: Args: db_path (Path) """ - logging.info("applying patch 002") with sqlite3.connect(db_path) as conn: cur = conn.cursor() tablenames = get_tablenames(cur) @@ -177,6 +175,7 @@ def patch_databases() -> None: # Run the patch functions. for patch_func_name in patch_func_names: + logging.info("applying patch %s", patch_func_name.removeprefix(FUNC_PREFIX)) patch_func = eval(patch_func_name) patch_func(db_path) From acc215b71d3ee96dfbaa880e2c2f4aa5cb52eab2 Mon Sep 17 00:00:00 2001 From: Jeppy Date: Sat, 5 Feb 2022 17:00:25 +0100 Subject: [PATCH 24/45] =?UTF-8?q?CHANGE=20get=5Ftablenames=20does=20not=20?= =?UTF-8?q?query=20=C2=A7version=20table=20per=20default?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/patch_database.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/src/patch_database.py b/src/patch_database.py index 3665da6e..a5f1bf98 100644 --- a/src/patch_database.py +++ b/src/patch_database.py @@ -85,8 +85,11 @@ def get_patch_func_version(func_name: str) -> int: return version -def get_tablenames(cur: sqlite3.Cursor) -> list[str]: - cur.execute("SELECT name FROM sqlite_master WHERE type='table';") +def get_tablenames(cur: sqlite3.Cursor, ignore_version_table: bool = True) -> list[str]: + query = "SELECT name FROM sqlite_master WHERE type='table'" + if ignore_version_table: + query += " AND name != '§version'" + cur.execute(f"{query};") tablenames = [result[0] for result in cur.fetchall()] return tablenames @@ -125,8 +128,6 @@ def __patch_002(db_path: Path) -> None: tablenames = get_tablenames(cur) # Iterate over all tables. for tablename in tablenames: - if tablename == "§version": - continue base_asset, quote_asset = tablename.split("/") # Adjust the order, when the symbols aren't ordered alphanumerical. From 1419746b1b541908665cddf6154303cb269b7d91 Mon Sep 17 00:00:00 2001 From: Jeppy Date: Sat, 5 Feb 2022 17:01:04 +0100 Subject: [PATCH 25/45] UPDATE transfer platform parameter --- src/price_data.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/price_data.py b/src/price_data.py index 2e89f2fa..7bf8b7a1 100644 --- a/src/price_data.py +++ b/src/price_data.py @@ -462,7 +462,9 @@ def get_price( price = get_price(coin, utc_time, reference_coin, **kwargs) assert isinstance(price, decimal.Decimal) - set_price_db("", coin, reference_coin, utc_time, price, db_path) + set_price_db( + platform, coin, reference_coin, utc_time, price, db_path=db_path + ) if config.MEAN_MISSING_PRICES and price <= 0.0: # The price is missing. Check for prices before and after the From d70559fb2b302ecb06c483f356058d3b676d9554 Mon Sep 17 00:00:00 2001 From: Jeppy Date: Sat, 5 Feb 2022 17:01:56 +0100 Subject: [PATCH 26/45] REFACTOR function docstrings --- src/database.py | 10 +++++----- src/patch_database.py | 2 +- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/src/database.py b/src/database.py index 671dc26b..7ea107d6 100644 --- a/src/database.py +++ b/src/database.py @@ -204,11 +204,11 @@ def set_price_db( A warning will be raised, if there is already a different price. Args: - platform (str): [description] - coin (str): [description] - reference_coin (str): [description] - utc_time (datetime.datetime): [description] - price (decimal.Decimal): [description] + platform (str) + coin (str) + reference_coin (str) + utc_time (datetime.datetime) + price (decimal.Decimal) """ assert coin != reference_coin coin_a, coin_b, inverted = _sort_pair(coin, reference_coin) diff --git a/src/patch_database.py b/src/patch_database.py index a5f1bf98..2cd3b5bf 100644 --- a/src/patch_database.py +++ b/src/patch_database.py @@ -98,7 +98,7 @@ def __patch_001(db_path: Path) -> None: """Convert prices from float to string Args: - db_path (Path): [description] + db_path (Path) """ with sqlite3.connect(db_path) as conn: query = "SELECT name,sql FROM sqlite_master WHERE type='table'" From 09a81e59de2827cd902b0afa899ee9ab2d0e67a2 Mon Sep 17 00:00:00 2001 From: Jeppy Date: Sat, 5 Feb 2022 17:02:12 +0100 Subject: [PATCH 27/45] FIX mypy linting error --- src/database.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/database.py b/src/database.py index 7ea107d6..6bf11bef 100644 --- a/src/database.py +++ b/src/database.py @@ -195,7 +195,7 @@ def set_price_db( reference_coin: str, utc_time: datetime.datetime, price: decimal.Decimal, - db_path: Path = None, + db_path: Optional[Path] = None, ) -> None: """Write price to database. From a06f73512d3063094e27fe17c18bc2e2e5a9d5cb Mon Sep 17 00:00:00 2001 From: Jeppy Date: Sat, 5 Feb 2022 17:02:34 +0100 Subject: [PATCH 28/45] Reorder preamble of set_price_db --- src/database.py | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/src/database.py b/src/database.py index 6bf11bef..86597644 100644 --- a/src/database.py +++ b/src/database.py @@ -211,12 +211,18 @@ def set_price_db( price (decimal.Decimal) """ assert coin != reference_coin + coin_a, coin_b, inverted = _sort_pair(coin, reference_coin) - if db_path is None and platform == "": - db_path = get_db_path(platform) tablename = get_tablename(coin_a, coin_b) + if inverted: price = misc.reciprocal(price) + + if db_path is None and platform: + db_path = get_db_path(platform) + + assert isinstance(db_path, Path), "no db path given" + try: __set_price_db(db_path, tablename, utc_time, price) except sqlite3.IntegrityError as e: From e99a7aa65b021111eb0015fbe65c54d88738e5a5 Mon Sep 17 00:00:00 2001 From: Jeppy Date: Sat, 5 Feb 2022 17:03:05 +0100 Subject: [PATCH 29/45] UPDATE assert db exists before setting new price --- src/database.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/src/database.py b/src/database.py index 86597644..7ba47510 100644 --- a/src/database.py +++ b/src/database.py @@ -167,9 +167,8 @@ def __set_price_db( utc_time (datetime.datetime) price (decimal.Decimal) """ - # TODO if db_path doesn't exists. Create db with §version table and - # newest version number. It would be nicer, if this could be done - # as a preprocessing step. see book.py + assert db_path.exists(), f"db doesn't exist: {db_path}" + with sqlite3.connect(db_path) as conn: cur = conn.cursor() query = f"INSERT INTO `{tablename}`" "('utc_time', 'price') VALUES (?, ?);" From cd33058f60d7a8255867cced21a022e8d03d0de9 Mon Sep 17 00:00:00 2001 From: Jeppy Date: Sat, 5 Feb 2022 17:04:15 +0100 Subject: [PATCH 30/45] ADD helper functions to get patch func names/versions --- src/patch_database.py | 43 ++++++++++++++++++++++++++++++++----------- 1 file changed, 32 insertions(+), 11 deletions(-) diff --git a/src/patch_database.py b/src/patch_database.py index 2cd3b5bf..c519d0cb 100644 --- a/src/patch_database.py +++ b/src/patch_database.py @@ -21,6 +21,7 @@ import sys from inspect import getmembers, isfunction from pathlib import Path +from typing import Iterator, Optional import config from database import set_price_db @@ -152,6 +153,36 @@ def __patch_002(db_path: Path) -> None: cur = conn.execute(f"DROP TABLE `{tablename}`;") +def _get_patch_func_names() -> Iterator[str]: + func_names = ( + f[0] + for f in getmembers(sys.modules[__name__], isfunction) + if f[0].startswith(FUNC_PREFIX) + ) + return func_names + + +def _get_patch_func_versions() -> Iterator[int]: + func_names = _get_patch_func_names() + func_version = map(get_patch_func_version, func_names) + return func_version + + +def get_sorted_patch_func_names(current_version: Optional[int] = None) -> list[str]: + func_names = ( + f + for f in _get_patch_func_names() + if current_version is None or get_patch_func_version(f) > current_version + ) + # Sort patch functions chronological. + return sorted(func_names, key=get_patch_func_version) + + +def get_latest_version() -> int: + func_versions = _get_patch_func_versions() + return max(func_versions) + + def patch_databases() -> None: # Check if any database paths exist. database_paths = [p for p in Path(config.DATA_PATH).glob("*.db") if p.is_file()] @@ -162,17 +193,7 @@ def patch_databases() -> None: for db_path in database_paths: # Read version from database. current_version = get_version(db_path) - - # Determine all necessary patch functions. - patch_func_names = [ - func[0] - for func in getmembers(sys.modules[__name__], isfunction) - if func[0].startswith(FUNC_PREFIX) - if get_patch_func_version(func[0]) > current_version - ] - - # Sort patch functions chronological. - patch_func_names.sort(key=get_patch_func_version) + patch_func_names = get_sorted_patch_func_names(current_version=current_version) # Run the patch functions. for patch_func_name in patch_func_names: From b54c3664541ba08c157c4286e8acbc7d9384643d Mon Sep 17 00:00:00 2001 From: Jeppy Date: Sat, 5 Feb 2022 17:04:29 +0100 Subject: [PATCH 31/45] REFACTOR patch databases --- src/patch_database.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/patch_database.py b/src/patch_database.py index c519d0cb..94a8679c 100644 --- a/src/patch_database.py +++ b/src/patch_database.py @@ -193,7 +193,10 @@ def patch_databases() -> None: for db_path in database_paths: # Read version from database. current_version = get_version(db_path) + patch_func_names = get_sorted_patch_func_names(current_version=current_version) + if not patch_func_names: + continue # Run the patch functions. for patch_func_name in patch_func_names: @@ -202,6 +205,5 @@ def patch_databases() -> None: patch_func(db_path) # Update version. - if patch_func_names: new_version = get_patch_func_version(patch_func_name) update_version(db_path, new_version) From 63b3959a80d56d861e36c5eb31be82b71a488b69 Mon Sep 17 00:00:00 2001 From: Jeppy Date: Sat, 5 Feb 2022 17:04:49 +0100 Subject: [PATCH 32/45] ADD create database in book.py when missing --- src/book.py | 6 +++--- src/database.py | 7 +++++++ src/patch_database.py | 14 ++++++++++---- 3 files changed, 20 insertions(+), 7 deletions(-) diff --git a/src/book.py b/src/book.py index 4840349f..a9d4d9e6 100644 --- a/src/book.py +++ b/src/book.py @@ -26,7 +26,7 @@ import misc import transaction as tr from core import kraken_asset_map -from database import set_price_db +from database import check_database_or_create, set_price_db from price_data import PriceData log = logging.getLogger(__name__) @@ -906,8 +906,8 @@ def read_file(self, file_path: Path) -> None: assert file_path.is_file() if exchange := self.detect_exchange(file_path): - # TODO check that database file exists. if missing, add file with - # highest version number (highest patch number) + check_database_or_create(exchange) + try: read_file = getattr(self, f"_read_{exchange}") except AttributeError: diff --git a/src/database.py b/src/database.py index 7ba47510..0249eb38 100644 --- a/src/database.py +++ b/src/database.py @@ -7,6 +7,7 @@ import config import misc +from patch_database import create_new_database log = logging.getLogger(__name__) @@ -268,3 +269,9 @@ def get_tablenames_from_db(cur: sqlite3.Cursor) -> list[str]: def get_db_path(platform: str) -> Path: return Path(config.DATA_PATH, f"{platform}.db") + + +def check_database_or_create(platform: str) -> None: + db_path = get_db_path(platform) + if not db_path.exists(): + create_new_database(db_path) diff --git a/src/patch_database.py b/src/patch_database.py index 94a8679c..02b3f4e6 100644 --- a/src/patch_database.py +++ b/src/patch_database.py @@ -76,6 +76,12 @@ def update_version(db_path: Path, version: int) -> None: cur.execute(f"INSERT INTO §version (version) VALUES ({version});") +def create_new_database(db_path: Path) -> None: + assert not db_path.exists() + version = get_latest_version() + update_version(db_path, version) + + def get_patch_func_version(func_name: str) -> int: assert func_name.startswith( FUNC_PREFIX @@ -108,8 +114,8 @@ def __patch_001(db_path: Path) -> None: if "price str" not in sql.lower(): query = f""" CREATE TABLE "sql_temp_table" ( - "utc_time" DATETIME PRIMARY KEY, - "price" STR NOT NULL + "utc_time" DATETIME PRIMARY KEY, + "price" STR NOT NULL ); INSERT INTO "sql_temp_table" ("price","utc_time") SELECT "price","utc_time" FROM "{tablename}"; @@ -205,5 +211,5 @@ def patch_databases() -> None: patch_func(db_path) # Update version. - new_version = get_patch_func_version(patch_func_name) - update_version(db_path, new_version) + new_version = get_patch_func_version(patch_func_name) + update_version(db_path, new_version) From c400a4c0c0d981a5bb1ad46ae37c8b59519cccd9 Mon Sep 17 00:00:00 2001 From: Jeppy Date: Sat, 5 Feb 2022 17:06:30 +0100 Subject: [PATCH 33/45] AUTOFORMAT database.py --- src/database.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/database.py b/src/database.py index 0249eb38..3b1b9571 100644 --- a/src/database.py +++ b/src/database.py @@ -169,7 +169,7 @@ def __set_price_db( price (decimal.Decimal) """ assert db_path.exists(), f"db doesn't exist: {db_path}" - + with sqlite3.connect(db_path) as conn: cur = conn.cursor() query = f"INSERT INTO `{tablename}`" "('utc_time', 'price') VALUES (?, ?);" From 78046f49938916284c9ec76ae790ec43f6f74471 Mon Sep 17 00:00:00 2001 From: Jeppy Date: Sat, 5 Feb 2022 17:13:55 +0100 Subject: [PATCH 34/45] Avoid circular import --- src/database.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/database.py b/src/database.py index 3b1b9571..fa479a2f 100644 --- a/src/database.py +++ b/src/database.py @@ -7,7 +7,6 @@ import config import misc -from patch_database import create_new_database log = logging.getLogger(__name__) @@ -272,6 +271,8 @@ def get_db_path(platform: str) -> Path: def check_database_or_create(platform: str) -> None: + from patch_database import create_new_database + db_path = get_db_path(platform) if not db_path.exists(): create_new_database(db_path) From 7b81aa92ff96e86fe1ea6f785dd1076081f4351a Mon Sep 17 00:00:00 2001 From: Jeppy Date: Sat, 5 Feb 2022 17:14:17 +0100 Subject: [PATCH 35/45] =?UTF-8?q?CHANGE=20update=5Fversion=20create=20?= =?UTF-8?q?=C2=A7version=20table=20when=20missing?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/patch_database.py | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/src/patch_database.py b/src/patch_database.py index 02b3f4e6..bbf3918a 100644 --- a/src/patch_database.py +++ b/src/patch_database.py @@ -52,8 +52,7 @@ def get_version(db_path: Path) -> int: except sqlite3.OperationalError as e: if str(e) == "no such table: §version": # The §version table doesn't exist. Create one. - cur.execute("CREATE TABLE §version(version INT);") - cur.execute("INSERT INTO §version (version) VALUES (0);") + update_version(db_path, 0) return 0 else: raise e @@ -71,7 +70,15 @@ def get_version(db_path: Path) -> int: def update_version(db_path: Path, version: int) -> None: with sqlite3.connect(db_path) as conn: cur = conn.cursor() - cur.execute("DELETE FROM §version;") + + try: + cur.execute("DELETE FROM §version;") + except sqlite3.OperationalError as e: + if str(e) == "no such table: §version": + cur.execute("CREATE TABLE §version(version INT);") + else: + raise e + assert isinstance(version, int) cur.execute(f"INSERT INTO §version (version) VALUES ({version});") From 0f2e7fb8b6b30c71ff0fbb9ba25d782e42b61a14 Mon Sep 17 00:00:00 2001 From: Jeppy Date: Sat, 5 Feb 2022 17:24:31 +0100 Subject: [PATCH 36/45] UPDATE format of warning message --- src/database.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/database.py b/src/database.py index fa479a2f..ae91ccdf 100644 --- a/src/database.py +++ b/src/database.py @@ -230,8 +230,8 @@ def set_price_db( if price != price_db: log.warning( "Tried to write price to database, " - "but a different price exists already." - f"({platform=}, {tablename=}, {utc_time=}, {price=})" + "but a different price exists already: " + f"{platform=}, {tablename=}, {utc_time=}, {price=}" ) else: raise e From a6fc0eb9f9c4084cba0248a751c43d94a6f438c6 Mon Sep 17 00:00:00 2001 From: Jeppy Date: Sat, 5 Feb 2022 17:30:07 +0100 Subject: [PATCH 37/45] UPDATE reword debug message --- src/book.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/book.py b/src/book.py index cece3e3d..7a4c195f 100644 --- a/src/book.py +++ b/src/book.py @@ -1096,7 +1096,7 @@ def get_price_from_csv(self) -> None: price = decimal.Decimal(selltr.change / buytr.change) logging.debug( - f"Added {buytr.coin}/{selltr.coin} price from CSV: " + f"Adding {buytr.coin}/{selltr.coin} price from CSV: " f"{price} for {platform} at {timestamp}" ) From 7cae5c3437a46f35f7c6c07c6a581c250a6b6c08 Mon Sep 17 00:00:00 2001 From: Jeppy Date: Sat, 5 Feb 2022 17:30:17 +0100 Subject: [PATCH 38/45] ADD debug message when updating db version --- src/patch_database.py | 1 + 1 file changed, 1 insertion(+) diff --git a/src/patch_database.py b/src/patch_database.py index bbf3918a..35c40854 100644 --- a/src/patch_database.py +++ b/src/patch_database.py @@ -80,6 +80,7 @@ def update_version(db_path: Path, version: int) -> None: raise e assert isinstance(version, int) + log.debug(f"Updating version of {db_path} to {version}") cur.execute(f"INSERT INTO §version (version) VALUES ({version});") From f932e0c112ef98f36f89ebca262919db0c1c33c0 Mon Sep 17 00:00:00 2001 From: Jeppy Date: Sat, 5 Feb 2022 17:34:24 +0100 Subject: [PATCH 39/45] CHANGE Create db not in book, but on set_price when db is missing Unfortunatly, there is no good place in book.py to create a db as preprocessing step as the platform string is at first defined in the read_function and not beforehand --- src/book.py | 1 - src/database.py | 5 ++++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/src/book.py b/src/book.py index 7a4c195f..f93f86da 100644 --- a/src/book.py +++ b/src/book.py @@ -1120,7 +1120,6 @@ def read_file(self, file_path: Path) -> None: assert file_path.is_file() if exchange := self.detect_exchange(file_path): - check_database_or_create(exchange) try: read_file = getattr(self, f"_read_{exchange}") diff --git a/src/database.py b/src/database.py index ae91ccdf..77ec2704 100644 --- a/src/database.py +++ b/src/database.py @@ -167,7 +167,10 @@ def __set_price_db( utc_time (datetime.datetime) price (decimal.Decimal) """ - assert db_path.exists(), f"db doesn't exist: {db_path}" + if not db_path.exists(): + from patch_database import create_new_database + + create_new_database(db_path) with sqlite3.connect(db_path) as conn: cur = conn.cursor() From 0b93b3b4aa25ed0b7874fdcf75a8883f206cf50c Mon Sep 17 00:00:00 2001 From: Jeppy Date: Sat, 5 Feb 2022 18:08:12 +0100 Subject: [PATCH 40/45] UPDATE duplicate price warning, add db price for comparison --- src/database.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/database.py b/src/database.py index 77ec2704..719c0e17 100644 --- a/src/database.py +++ b/src/database.py @@ -234,7 +234,7 @@ def set_price_db( log.warning( "Tried to write price to database, " "but a different price exists already: " - f"{platform=}, {tablename=}, {utc_time=}, {price=}" + f"{platform=}, {tablename=}, {utc_time=}, {price=} != {price_db=}" ) else: raise e From 81b4e306601cf5193f01622b67a962e3b2e501f8 Mon Sep 17 00:00:00 2001 From: Jeppy Date: Sat, 5 Feb 2022 18:08:27 +0100 Subject: [PATCH 41/45] AUTOFORMAT price_data --- src/price_data.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/src/price_data.py b/src/price_data.py index d2ccb8a8..94efbaf4 100644 --- a/src/price_data.py +++ b/src/price_data.py @@ -286,8 +286,9 @@ def _get_price_bitpanda_pro( for num_offset in range(num_max_offsets): # if no trades can be found, move 30 min window to the past window_offset = num_offset * t - end = utc_time.astimezone(datetime.timezone.utc) \ - - datetime.timedelta(minutes=window_offset) + end = utc_time.astimezone(datetime.timezone.utc) - datetime.timedelta( + minutes=window_offset + ) begin = end - datetime.timedelta(minutes=t) # https://github.com/python/mypy/issues/3176 @@ -295,8 +296,8 @@ def _get_price_bitpanda_pro( "unit": "MINUTES", "period": t, # convert ISO 8601 format to RFC3339 timestamp - "from": begin.isoformat().replace('+00:00', 'Z'), - "to": end.isoformat().replace('+00:00', 'Z'), + "from": begin.isoformat().replace("+00:00", "Z"), + "to": end.isoformat().replace("+00:00", "Z"), } if num_offset: log.debug( From 53753d6a68efa729ccca389d7f5da33b42787498 Mon Sep 17 00:00:00 2001 From: Jeppy Date: Sat, 5 Feb 2022 18:08:43 +0100 Subject: [PATCH 42/45] ADD comment to price_data when querying new price --- src/price_data.py | 1 + 1 file changed, 1 insertion(+) diff --git a/src/price_data.py b/src/price_data.py index 94efbaf4..a7b52daf 100644 --- a/src/price_data.py +++ b/src/price_data.py @@ -496,6 +496,7 @@ def get_price( # Check if price exists already in our database. if (price := get_price_db(db_path, tablename, utc_time)) is None: + # Price doesn't exists. Fetch price from platform. try: get_price = getattr(self, f"_get_price_{platform}") except AttributeError: From 16dabab61389f0c74c3e18c4dabfe580a0b99b33 Mon Sep 17 00:00:00 2001 From: Jeppy Date: Sat, 5 Feb 2022 18:09:00 +0100 Subject: [PATCH 43/45] FIX database type for price --- src/database.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/database.py b/src/database.py index 719c0e17..934752e1 100644 --- a/src/database.py +++ b/src/database.py @@ -182,7 +182,7 @@ def __set_price_db( create_query = ( f"CREATE TABLE `{tablename}`" "(utc_time DATETIME PRIMARY KEY, " - "price STR NOT NULL);" + "price VARCHAR(255) NOT NULL);" ) cur.execute(create_query) cur.execute(query, (utc_time, str(price))) From c1538cfe634cfef1ef99184386f9a3f8e687a4cd Mon Sep 17 00:00:00 2001 From: Jeppy Date: Sat, 5 Feb 2022 18:22:45 +0100 Subject: [PATCH 44/45] ADD set_price_db parameter to overwrite already existing prices --- src/book.py | 1 + src/database.py | 44 ++++++++++++++++++++++++++++++++++++++------ 2 files changed, 39 insertions(+), 6 deletions(-) diff --git a/src/book.py b/src/book.py index f93f86da..081047a4 100644 --- a/src/book.py +++ b/src/book.py @@ -1106,6 +1106,7 @@ def get_price_from_csv(self) -> None: selltr.coin, timestamp, price, + overwrite=True, ) def read_file(self, file_path: Path) -> None: diff --git a/src/database.py b/src/database.py index 934752e1..92b13fd5 100644 --- a/src/database.py +++ b/src/database.py @@ -151,6 +151,26 @@ def mean_price_db( return decimal.Decimal() +def __delete_price_db( + db_path: Path, + tablename: str, + utc_time: datetime.datetime, +) -> None: + """Delete price from database + + Args: + db_path (Path) + tablename (str) + utc_time (datetime.datetime) + """ + + with sqlite3.connect(db_path) as conn: + cur = conn.cursor() + query = f"DELETE FROM `{tablename}` WHERE utc_time=?;" + cur.execute(query, (utc_time,)) + conn.commit() + + def __set_price_db( db_path: Path, tablename: str, @@ -198,6 +218,7 @@ def set_price_db( utc_time: datetime.datetime, price: decimal.Decimal, db_path: Optional[Path] = None, + overwrite: bool = False, ) -> None: """Write price to database. @@ -229,13 +250,24 @@ def set_price_db( __set_price_db(db_path, tablename, utc_time, price) except sqlite3.IntegrityError as e: if str(e) == f"UNIQUE constraint failed: {tablename}.utc_time": - price_db = get_price_db(db_path, tablename, utc_time) - if price != price_db: - log.warning( - "Tried to write price to database, " - "but a different price exists already: " - f"{platform=}, {tablename=}, {utc_time=}, {price=} != {price_db=}" + # Trying to add an already existing price in db. + if overwrite: + # Overwrite price. + log.debug( + "Overwriting price information for " + f"{platform=}, {tablename=} at {utc_time=}" ) + __delete_price_db(db_path, tablename, utc_time) + __set_price_db(db_path, tablename, utc_time, price) + else: + # Check price from db and issue warning, if prices do not match. + price_db = get_price_db(db_path, tablename, utc_time) + if price != price_db: + log.warning( + "Tried to write price to database, " + "but a different price exists already: " + f"{platform=}, {tablename=}, {utc_time=}, {price=} != {price_db=}" + ) else: raise e From 9f906b172bbf0e8e03cd9807c3d7e2ecbbc2f9d9 Mon Sep 17 00:00:00 2001 From: Jeppy Date: Sat, 5 Feb 2022 18:27:49 +0100 Subject: [PATCH 45/45] FIX linting errors --- src/book.py | 6 ++---- src/database.py | 3 ++- 2 files changed, 4 insertions(+), 5 deletions(-) diff --git a/src/book.py b/src/book.py index 081047a4..79dc8391 100644 --- a/src/book.py +++ b/src/book.py @@ -26,7 +26,7 @@ import misc import transaction as tr from core import kraken_asset_map -from database import check_database_or_create, set_price_db +from database import set_price_db from price_data import PriceData log = logging.getLogger(__name__) @@ -863,9 +863,7 @@ def _read_bitpanda(self, file_path: Path) -> None: change_fiat = misc.force_decimal(amount_fiat) # Save price in our local database for later. price = misc.force_decimal(asset_price) - self.price_data.set_price_db( - platform, asset, config.FIAT.upper(), utc_time, price - ) + set_price_db(platform, asset, config.FIAT.upper(), utc_time, price) if change < 0: log.error( diff --git a/src/database.py b/src/database.py index 92b13fd5..211b57d5 100644 --- a/src/database.py +++ b/src/database.py @@ -266,7 +266,8 @@ def set_price_db( log.warning( "Tried to write price to database, " "but a different price exists already: " - f"{platform=}, {tablename=}, {utc_time=}, {price=} != {price_db=}" + f"{platform=}, {tablename=}, {utc_time=}, " + f"{price=} != {price_db=}" ) else: raise e