Concat (#83)

* added slice property & tests

* changed toml

* created branch concat

* first commit

* works for flat pflines, nested not yet

* draft of concat function done, need more testing though

* created error_case tests for concat

* wrote test_func for concat_pflines, improved the code for concat

* added test cases for concat pfstates, requires approval(!)

* changed test_function

* change general to take iterable as argument

---------

Co-authored-by: Alina Voilova <[email protected]>

dev_scripts/checks.py

@@ -0,0 +1,42 @@
+import pandas as pd
+import portfolyo as pf
+from portfolyo.core.shared import concat
+
+
+def get_idx(
+    startdate: str, starttime: str, tz: str, freq: str, enddate: str
+) -> pd.DatetimeIndex:
+    # Empty index.
+    if startdate is None:
+        return pd.DatetimeIndex([], freq=freq, tz=tz)
+    # Normal index.
+    ts_start = pd.Timestamp(f"{startdate} {starttime}", tz=tz)
+    ts_end = pd.Timestamp(f"{enddate} {starttime}", tz=tz)
+    return pd.date_range(ts_start, ts_end, freq=freq, inclusive="left")
+
+
+index = pd.date_range("2020", "2024", freq="QS", inclusive="left")
+# index2 = pd.date_range("2023", "2025", freq="QS", inclusive="left")
+# pfl = pf.dev.get_flatpfline(index)
+# pfl2 = pf.dev.get_flatpfline(index2)
+# print(pfl)
+# print(pfl2)
+
+# pfs = pf.dev.get_pfstate(index)
+
+# pfs2 = pf.dev.get_pfstate(index2)
+# pfl3 = concat.general(pfl, pfl2)
+# print(pfl3)
+
+# print(index)
+# print(index2)
+
+whole_pfl = pf.dev.get_nestedpfline(index)
+pfl_a = whole_pfl.slice[:"2021"]
+
+pfl_b = whole_pfl.slice["2021":"2022"]
+pfl_c = whole_pfl.slice["2022":]
+result = concat.concat_pflines(pfl_a, pfl_b, pfl_c)
+result2 = concat.concat_pflines(pfl_b, pfl_c, pfl_a)
+print(result)
+print(result2)

docs/core/pfline.rst

@@ -270,6 +270,29 @@ Another slicing method is implemented with the ``.slice[]`` property. The improv
 
 
 
+Concatenation
+=============
+
+Portfolio lines can be concatenated with the ``portfolio.concat()`` function. This only works if the input portfolio lines have contain compatible information (the same frequency, timezone, start-of-day, kind, etc) and, crucially, their indices are gapless and without overlap. To remove any overlap, use the ``.slice[]`` property.
+
+.. exec_code::
+
+   # --- hide: start ---
+   import portfolyo as pf, pandas as pd
+   index = pd.date_range('2024', freq='AS', periods=3)
+   input_df = pd.DataFrame({'w':[200, 220, 300], 'p': [100, 150, 200]}, index)
+   pfl = pf.PfLine(input_df)
+   # --- hide: stop ---
+   # continuation of previous code example
+   index2 = pd.date_range('2025', freq='AS', periods=3)  # 2 years' overlap with pfl
+   pfl2 = pf.PfLine(pd.DataFrame({'w':[22, 30, 40], 'p': [15, 20, 21]}, index))
+   # first two datapoints (until/excl 2026) from pfl, last two datapoints (from/incl 2026) from pfl2 
+   pf.concat([pfl.slice[:'2026'], pfl2.slice['2026':]]) 
+   # --- hide: start ---
+   print(pf.concat([pfl.slice[:'2026'], pfl2.slice['2026':]]))
+   # --- hide: stop ---
+
+
 Volume-only, price-only or revenue-only
 =======================================
 

portfolyo/__init__.py

@@ -1,10 +1,9 @@
 """Package to analyse and manipulate timeseries related to power and gas offtake portfolios."""
 
-
 from . import _version, dev, tools
 from .core import extendpandas  # extend functionalty of pandas
 from .core import suppresswarnings
-from .core.mixins.plot import plot_pfstates
+from .core.shared.plot import plot_pfstates
 from .core.pfline import Kind, PfLine, Structure, create
 from .core.pfstate import PfState
 from .prices.hedge import hedge
@@ -15,6 +14,9 @@
 from .tools.tzone import force_agnostic, force_aware
 from .tools.unit import Q_, ureg, Unit
 from .tools.wavg import general as wavg
+from .core.shared.concat import general as concat
+
+# from .core.shared.concat import general as concat
 
 VOLUME = Kind.VOLUME
 PRICE = Kind.PRICE

portfolyo/core/pfline/classes.py

@@ -8,7 +8,7 @@
 import pandas as pd
 
 from ... import tools
-from ..mixins import ExcelClipboardOutput, PfLinePlot, PfLineText
+from ..shared import ExcelClipboardOutput, PfLinePlot, PfLineText
 from ..ndframelike import NDFrameLike
 from . import (
     create,

portfolyo/core/pfstate/pfstate.py

@@ -12,7 +12,7 @@
 import pandas as pd
 
 from ... import tools
-from ..mixins import ExcelClipboardOutput, PfStatePlot, PfStateText
+from ..shared import ExcelClipboardOutput, PfStatePlot, PfStateText
 from ..ndframelike import NDFrameLike
 from ..pfline import PfLine, create
 from . import pfstate_helper

portfolyo/core/shared/concat.py

@@ -0,0 +1,149 @@
+# import pandas as pd
+# import portfolyo as pf
+from __future__ import annotations
+from typing import Iterable
+import pandas as pd
+from portfolyo import tools
+
+from ..pfstate import PfState
+from ..pfline.enums import Structure
+
+from ..pfline import PfLine, create
+from .. import pfstate
+
+
+def general(pfl_or_pfs: Iterable[PfLine | PfState]) -> None:
+    """
+    Based on passed parameters calls either concat_pflines() or concat_pfstates().
+
+    Parameters
+    ----------
+    pfl_or_pfs: Iterable[PfLine | PfState]
+        The input values. Can be either a list of Pflines or PfStates to concatenate.
+
+    Returns
+    -------
+    None
+
+    Notes
+    -----
+    Input portfolio lines must contain compatible information, i.e., same frequency,
+    timezone, start-of-day, and kind. Their indices must be gapless and without overlap.
+
+    For nested pflines, the number and names of their children must match; concatenation
+    is done on a name-by-name basis.
+
+    Concatenation returns the same result regardless of input order.
+
+    """
+    if all(isinstance(item, PfLine) for item in pfl_or_pfs):
+        return concat_pflines(pfl_or_pfs)
+    elif all(isinstance(item, PfState) for item in pfl_or_pfs):
+        return concat_pfstates(pfl_or_pfs)
+    else:
+        raise NotImplementedError(
+            "Concatenation is implemented only for PfState or PfLine."
+        )
+
+
+def concat_pflines(pfls: Iterable[PfLine]) -> PfLine:
+    """
+    Concatenate porfolyo lines along their index.
+
+    Parameters
+    ----------
+    pfls: Iterable[PfLine]
+        The input values.
+
+    Returns
+    -------
+    PfLine
+        Concatenated version of PfLines.
+
+    Notes
+    -----
+    Input portfolio lines must contain compatible information, i.e., same frequency,
+    timezone, start-of-day, and kind. Their indices must be gapless and without overlap.
+
+    For nested pflines, the number and names of their children must match; concatenation
+    is done on a name-by-name basis.
+
+    Concatenation returns the same result regardless of input order.
+    """
+    if len(pfls) < 2:
+        raise NotImplementedError(
+            "Cannot perform operation with less than 2 portfolio lines."
+        )
+    if len({pfl.kind for pfl in pfls}) != 1:
+        raise TypeError("Not possible to concatenate PfLines of different kinds.")
+    if len({pfl.index.freq for pfl in pfls}) != 1:
+        raise TypeError("Not possible to concatenate PfLines of different frequencies.")
+    if len({pfl.index.tz for pfl in pfls}) != 1:
+        raise TypeError("Not possible to concatenate PfLines of different time zones.")
+    if len({tools.startofday.get(pfl.index, "str") for pfl in pfls}) != 1:
+        raise TypeError(
+            "Not possible to concatenate PfLines of different start_of_day."
+        )
+    # we can concatenate only pflines of the same type: nested of flat
+    # with this test and check whether pfls are the same types and they have the same number of children
+    if len({pfl.structure for pfl in pfls}) != 1:
+        raise TypeError("Not possible to concatenate PfLines of different structures.")
+    if pfls[0].structure is Structure.NESTED:
+        child_names = pfls[0].children.keys()
+        for pfl in pfls:
+            diffs = set(child_names) ^ set(pfl.children.keys())
+            if len(diffs) != 0:
+                raise TypeError(
+                    "Not possible to concatenate PfLines with different children names."
+                )
+    # If we reach here, all pfls have same kind, same number and names of children.
+
+    # concat(a,b) and concat(b,a) should give the same result:
+    sorted_pfls = sorted(pfls, key=lambda pfl: pfl.index[0])
+    if pfls[0].structure is Structure.FLAT:
+        # create flat dataframe of parent
+        dataframes_flat = [pfl.df for pfl in sorted_pfls]
+        # concatenate dataframes into one
+        concat_data = pd.concat(dataframes_flat, axis=0)
+        try:
+            # Call create.flatpfline() and catch any ValueError
+            return create.flatpfline(concat_data)
+        except ValueError as e:
+            # Handle the error
+            raise ValueError(
+                "Error by creating PfLine. PfLine is either not gapless or has overlaps"
+            ) from e
+    child_data = {}
+    child_names = pfls[0].children.keys()
+    for cname in child_names:
+        # for every name in children need to concatenate elements
+        child_values = [pfl.children[cname] for pfl in sorted_pfls]
+        child_data[cname] = concat_pflines(child_values)
+
+    # create pfline from dataframes: ->
+    # call the constructor of pfl to check check gaplesnes and overplap
+    return create.nestedpfline(child_data)
+
+
+def concat_pfstates(pfss: Iterable[PfState]) -> PfState:
+    """
+    Concatenate porfolyo states along their index.
+
+    Parameters
+    ----------
+    pfss: Iterable[PfState]
+         The input values.
+
+    Returns
+    -------
+     PfState
+         Concatenated version of PfStates.
+
+    """
+    if len(pfss) < 2:
+        print("Concatenate needs at least two elements.")
+        return
+    offtakevolume = concat_pflines([pfs.offtakevolume for pfs in pfss])
+    sourced = concat_pflines([pfs.sourced for pfs in pfss])
+    unsourcedprice = concat_pflines([pfs.unsourcedprice for pfs in pfss])
+    return pfstate.PfState(offtakevolume, unsourcedprice, sourced)