From ee92612696016a6783c7d66588221502dc2f7039 Mon Sep 17 00:00:00 2001 From: jinningwang Date: Fri, 27 Sep 2024 16:30:42 -0400 Subject: [PATCH 01/27] Fix find_idx to allow multiple matches --- andes/core/model/modeldata.py | 49 +++++++++++++++++++++++++++++------ andes/models/group.py | 48 ++++++++++++++++++++-------------- 2 files changed, 70 insertions(+), 27 deletions(-) diff --git a/andes/core/model/modeldata.py b/andes/core/model/modeldata.py index 29b21308f..0822ad7da 100644 --- a/andes/core/model/modeldata.py +++ b/andes/core/model/modeldata.py @@ -11,6 +11,7 @@ from andes.core.param import (BaseParam, DataParam, IdxParam, NumParam, TimerParam) from andes.shared import pd +from andes.utils.func import list_flatten logger = logging.getLogger(__name__) @@ -277,7 +278,8 @@ def find_param(self, prop): return out - def find_idx(self, keys, values, allow_none=False, default=False): + def find_idx(self, keys, values, allow_none=False, default=False, + no_flatten=False): """ Find `idx` of devices whose values match the given pattern. @@ -292,11 +294,30 @@ def find_idx(self, keys, values, allow_none=False, default=False): Allow key, value to be not found. Used by groups. default : bool Default idx to return if not found (missing) + no_flatten : bool + If True, return the non-flattened list of idxes. Otherwise, flatten the list. Returns ------- list indices of devices + + Examples + -------- + >>> # Use example case of IEEE 14-bus system with PVD1 + >>> ss = andes.load(andes.get_case('ieee14/ieee14_pvd1.xlsx')) + + >>> # To find the idx of `PVD1` with `name` of 'PVD1_1' and 'PVD1_2' + >>> ss.find_idx(keys='name', values=['PVD1_1', 'PVD1_2']) + [1, 2] + + >>> # To find the idx of `PVD1` with `gammap` equals to 0.1 + >>> ss.find_idx(keys='gammap', values=0.1) + [1, 2, 3, 4, 5, 6, 7, 8, 9, 10] + + >>> # To find the idx of `PVD1` with `gammap` equals to 0.1 and `name` of 'PVD1_1' + >>> ss.find_idx(keys=['gammap', 'name'], values=[[0.1], ['PVD1_1']]) + [1] """ if isinstance(keys, str): keys = (keys,) @@ -316,21 +337,33 @@ def find_idx(self, keys, values, allow_none=False, default=False): if len(keys) != len(values): raise ValueError("keys and values must have the same length") + # check if all elements in values have the same length + iterator = iter(values) + try: + first_length = len(next(iterator)) + ok_len = all(len(element) == first_length for element in iterator) + except StopIteration: # empty iterable + ok_len = True + if not ok_len: + raise ValueError("All elements in values must have the same length") + v_attrs = [self.__dict__[key].v for key in keys] idxes = [] for v_search in zip(*values): - v_idx = None + v_idx_list = [] for pos, v_attr in enumerate(zip(*v_attrs)): if all([i == j for i, j in zip(v_search, v_attr)]): - v_idx = self.idx.v[pos] - break - if v_idx is None: + v_idx_list.append(self.idx.v[pos]) + if not v_idx_list: if allow_none is False: raise IndexError(f'{list(keys)}={v_search} not found in {self.class_name}') else: - v_idx = default + v_idx_list.append(default) - idxes.append(v_idx) + idxes.append(v_idx_list) - return idxes + if not no_flatten: + return list_flatten(idxes) + else: + return idxes diff --git a/andes/models/group.py b/andes/models/group.py index 44480bac3..250413f32 100644 --- a/andes/models/group.py +++ b/andes/models/group.py @@ -243,31 +243,41 @@ def set(self, src: str, idx, attr, value): return True - def find_idx(self, keys, values, allow_none=False, default=None): + def find_idx(self, keys, values, allow_none=False, default=None, + no_flatten=False): """ Find indices of devices that satisfy the given `key=value` condition. This method iterates over all models in this group. """ - indices_found = [] - # `indices_found` contains found indices returned from all models of this group + idx_mdls = [] for model in self.models.values(): - indices_found.append(model.find_idx(keys, values, allow_none=True, default=default)) - - out = [] - for idx, idx_found in enumerate(zip(*indices_found)): - if not allow_none: - if idx_found.count(None) == len(idx_found): - missing_values = [item[idx] for item in values] - raise IndexError(f'{list(keys)} = {missing_values} not found in {self.class_name}') - - real_idx = default - for item in idx_found: - if item is not None: - real_idx = item - break - out.append(real_idx) - return out + idx_mdls.append(model.find_idx(keys, values, allow_none=True, default=default, + no_flatten=True)) + + # `indices_found` contains found indices returned from all models of this group + # NOTE: if the idx returned to [default] across all models, it means there is + # no such idx in this group. If so, return default or raise an key error. + indices_found = [] + uid_missing = [] + for uid, col in enumerate(zip(*idx_mdls)): + if all(item == [default] for item in col): + if allow_none: + indices_found.append([default]) + else: + uid_missing.append(uid) + else: + col_filter = [item for item in col if item != [default]] + indices_found.append(list_flatten(col_filter)) + + if uid_missing: + miss_str = f'{keys}={[v[u] for v in values for u in uid_missing]}' + raise IndexError(f'Group <{self.class_name}> does not contain device with {miss_str}') + + if not no_flatten: + return list_flatten(indices_found) + else: + return indices_found def _check_src(self, src: str): """ From 9d3ea4308dd4292330119c5ab2a743f9f939bc24 Mon Sep 17 00:00:00 2001 From: jinningwang Date: Fri, 27 Sep 2024 16:30:56 -0400 Subject: [PATCH 02/27] Add more test on find_idx --- tests/test_model_set.py | 49 +++++++++++++++++++++++++++++++++++++++++ 1 file changed, 49 insertions(+) diff --git a/tests/test_model_set.py b/tests/test_model_set.py index e3ef8b5ed..3d062879c 100644 --- a/tests/test_model_set.py +++ b/tests/test_model_set.py @@ -54,3 +54,52 @@ def test_model_set(self): ss.GENROU.set("M", np.array(["GENROU_4"]), "v", 6.0) np.testing.assert_equal(ss.GENROU.M.v[3], 6.0) self.assertEqual(ss.TDS.Teye[omega_addr[3], omega_addr[3]], 6.0) + + def test_find_idx(self): + ss = andes.load(andes.get_case('ieee14/ieee14_pvd1.xlsx')) + mdl = ss.PVD1 + + # multiple values + self.assertEqual(mdl.find_idx(keys='name', values=['PVD1_1', 'PVD1_2'], + allow_none=False, default=False, no_flatten=False), + [1, 2]) + # multiple values, no flatten + self.assertEqual(mdl.find_idx(keys='name', values=['PVD1_1', 'PVD1_2'], + allow_none=False, default=False, no_flatten=True), + [[1], [2]]) + # non-existing value + self.assertEqual(mdl.find_idx(keys='name', values=['PVD1_999'], + allow_none=True, default=False, no_flatten=False), + [False]) + + # non-existing value is not allowed + with self.assertRaises(IndexError): + mdl.find_idx(keys='name', values=['PVD1_999'], + allow_none=False, default=False, no_flatten=False) + + # multiple keys + self.assertEqual(mdl.find_idx(keys=['gammap', 'name'], + values=[[0.1, 0.1], ['PVD1_1', 'PVD1_2']]), + [1, 2]) + + # multiple keys, with non-existing values + self.assertEqual(mdl.find_idx(keys=['gammap', 'name'], + values=[[0.1, 0.1], ['PVD1_1', 'PVD1_999']], + allow_none=True, default='CURENT'), + [1, 'CURENT']) + + # multiple keys, with non-existing values not allowed + with self.assertRaises(IndexError): + mdl.find_idx(keys=['gammap', 'name'], + values=[[0.1, 0.1], ['PVD1_1', 'PVD1_999']], + allow_none=False, default=999) + + # multiple keys, values are not iterable + with self.assertRaises(ValueError): + mdl.find_idx(keys=['gammap', 'name'], + values=[0.1, 0.1]) + + # multiple keys, items length are inconsistent in values + with self.assertRaises(ValueError): + mdl.find_idx(keys=['gammap', 'name'], + values=[[0.1, 0.1], ['PVD1_1']]) From 6da66ea98e6a329fd493e401fb9ffb3b224991c6 Mon Sep 17 00:00:00 2001 From: jinningwang Date: Fri, 27 Sep 2024 16:36:18 -0400 Subject: [PATCH 03/27] Update release-notes --- docs/source/release-notes.rst | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/source/release-notes.rst b/docs/source/release-notes.rst index 9b053e084..bc690e2f4 100644 --- a/docs/source/release-notes.rst +++ b/docs/source/release-notes.rst @@ -19,6 +19,7 @@ v1.9.3 (2024-04-XX) - Adjust `BusFreq.Tw.default` to 0.1. - Add parameter from_csv=None in TDS.run() to allow loading data from CSV files at TDS begining. - Fix `TDS.init()` and `TDS._csv_step()` to fit loading from CSV when `Output` exists. +- Fix `ModelData.find_idx()` to return all matches v1.9.2 (2024-03-25) ------------------- From 1e71bf231ac1ab4de73943634d140f59a62f9402 Mon Sep 17 00:00:00 2001 From: jinningwang Date: Fri, 27 Sep 2024 16:52:59 -0400 Subject: [PATCH 04/27] Minor fix --- tests/test_model_set.py | 32 ++++++++++++++++---------------- 1 file changed, 16 insertions(+), 16 deletions(-) diff --git a/tests/test_model_set.py b/tests/test_model_set.py index 3d062879c..d9e6b878f 100644 --- a/tests/test_model_set.py +++ b/tests/test_model_set.py @@ -60,17 +60,17 @@ def test_find_idx(self): mdl = ss.PVD1 # multiple values - self.assertEqual(mdl.find_idx(keys='name', values=['PVD1_1', 'PVD1_2'], - allow_none=False, default=False, no_flatten=False), - [1, 2]) + self.assertListEqual(mdl.find_idx(keys='name', values=['PVD1_1', 'PVD1_2'], + allow_none=False, default=False, no_flatten=False), + [1, 2]) # multiple values, no flatten - self.assertEqual(mdl.find_idx(keys='name', values=['PVD1_1', 'PVD1_2'], - allow_none=False, default=False, no_flatten=True), - [[1], [2]]) + self.assertListEqual(mdl.find_idx(keys='name', values=['PVD1_1', 'PVD1_2'], + allow_none=False, default=False, no_flatten=True), + [[1], [2]]) # non-existing value - self.assertEqual(mdl.find_idx(keys='name', values=['PVD1_999'], - allow_none=True, default=False, no_flatten=False), - [False]) + self.assertListEqual(mdl.find_idx(keys='name', values=['PVD1_999'], + allow_none=True, default=False, no_flatten=False), + [False]) # non-existing value is not allowed with self.assertRaises(IndexError): @@ -78,15 +78,15 @@ def test_find_idx(self): allow_none=False, default=False, no_flatten=False) # multiple keys - self.assertEqual(mdl.find_idx(keys=['gammap', 'name'], - values=[[0.1, 0.1], ['PVD1_1', 'PVD1_2']]), - [1, 2]) + self.assertListEqual(mdl.find_idx(keys=['gammap', 'name'], + values=[[0.1, 0.1], ['PVD1_1', 'PVD1_2']]), + [1, 2]) # multiple keys, with non-existing values - self.assertEqual(mdl.find_idx(keys=['gammap', 'name'], - values=[[0.1, 0.1], ['PVD1_1', 'PVD1_999']], - allow_none=True, default='CURENT'), - [1, 'CURENT']) + self.assertListEqual(mdl.find_idx(keys=['gammap', 'name'], + values=[[0.1, 0.1], ['PVD1_1', 'PVD1_999']], + allow_none=True, default='CURENT'), + [1, 'CURENT']) # multiple keys, with non-existing values not allowed with self.assertRaises(IndexError): From 4d69304a69e79370a6d335b9952804697a002104 Mon Sep 17 00:00:00 2001 From: jinningwang Date: Fri, 27 Sep 2024 16:53:22 -0400 Subject: [PATCH 05/27] Add test on GroupBase.finx_idx --- tests/test_group.py | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/tests/test_group.py b/tests/test_group.py index 9f7a21bad..4dd33d894 100644 --- a/tests/test_group.py +++ b/tests/test_group.py @@ -71,6 +71,7 @@ def test_group_access(self): [6, 7, 8, 1]) # --- find_idx --- + # same Model self.assertListEqual(ss.DG.find_idx('name', ['PVD1_1', 'PVD1_2']), ss.PVD1.find_idx('name', ['PVD1_1', 'PVD1_2']), ) @@ -82,6 +83,22 @@ def test_group_access(self): [('PVD1_1', 'PVD1_2'), (1.0, 1.0)])) + # cross Model, given results + self.assertListEqual(ss.StaticGen.find_idx(keys='bus', + values=[1, 2, 3, 4]), + [1, 2, 3, 6]) + + self.assertListEqual(ss.StaticGen.find_idx(keys='bus', + values=[1, 2, 3, 4], + no_flatten=True), + [[1], [2], [3], [6]]) + + self.assertListEqual(ss.StaticGen.find_idx(keys='bus', + values=[1, 2, 3, 4, 2024], + allow_none=True, + default=2011), + [1, 2, 3, 6, 2011]) + # --- get_field --- ff = ss.DG.get_field('f', list(ss.DG._idx2model.keys()), 'v_code') self.assertTrue(any([item == 'y' for item in ff])) From 1ae1f4effcfdd361b631e461a4aeb34b8acbb091 Mon Sep 17 00:00:00 2001 From: jinningwang Date: Fri, 27 Sep 2024 17:00:32 -0400 Subject: [PATCH 06/27] Fix deprecateion of np.in1d --- andes/models/misc/output.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/andes/models/misc/output.py b/andes/models/misc/output.py index 1f01f3c92..8dfb0b4bc 100644 --- a/andes/models/misc/output.py +++ b/andes/models/misc/output.py @@ -50,9 +50,9 @@ def in1d(self, addr, v_code): """ if v_code == 'x': - return np.in1d(self.xidx, addr) + return np.isin(self.xidx, addr) if v_code == 'y': - return np.in1d(self.yidx, addr) + return np.isin(self.yidx, addr) raise NotImplementedError("v_code <%s> not recognized" % v_code) From 0443b3818d05595c02c6992c66a1cec300fa0139 Mon Sep 17 00:00:00 2001 From: jinningwang Date: Sun, 29 Sep 2024 15:47:27 -0400 Subject: [PATCH 07/27] [WIP] Try to fix docker error in github actions --- .github/workflows/pythonapp.yml | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/.github/workflows/pythonapp.yml b/.github/workflows/pythonapp.yml index c5797d0a5..0bf8548dd 100644 --- a/.github/workflows/pythonapp.yml +++ b/.github/workflows/pythonapp.yml @@ -8,6 +8,16 @@ jobs: runs-on: ubuntu-latest steps: - uses: actions/checkout@v4 + + - name: Log in to Docker Hub + run: echo "${{ secrets.DOCKER_PASSWORD }}" | docker login -u "${{ secrets.DOCKER_USERNAME }}" --password-stdin + + - name: Build Docker image + run: docker build -t andes:latest . + + - name: Push Docker image + run: docker push andes:latest + - uses: conda-incubator/setup-miniconda@v3 with: python-version: 3.11 From eb7d23fa545d1f3eb363a5abf251cb9d237efe5c Mon Sep 17 00:00:00 2001 From: jinningwang Date: Sun, 29 Sep 2024 15:49:14 -0400 Subject: [PATCH 08/27] [WIP] Undo changes to pythonapp.yml --- .github/workflows/pythonapp.yml | 10 ---------- 1 file changed, 10 deletions(-) diff --git a/.github/workflows/pythonapp.yml b/.github/workflows/pythonapp.yml index 0bf8548dd..c5797d0a5 100644 --- a/.github/workflows/pythonapp.yml +++ b/.github/workflows/pythonapp.yml @@ -8,16 +8,6 @@ jobs: runs-on: ubuntu-latest steps: - uses: actions/checkout@v4 - - - name: Log in to Docker Hub - run: echo "${{ secrets.DOCKER_PASSWORD }}" | docker login -u "${{ secrets.DOCKER_USERNAME }}" --password-stdin - - - name: Build Docker image - run: docker build -t andes:latest . - - - name: Push Docker image - run: docker push andes:latest - - uses: conda-incubator/setup-miniconda@v3 with: python-version: 3.11 From e099c9be768ed066757681e6ea4c434bfde4c880 Mon Sep 17 00:00:00 2001 From: jinningwang Date: Sun, 29 Sep 2024 15:53:46 -0400 Subject: [PATCH 09/27] [WIP] Fix github action error, add a step to install mamba --- .github/workflows/pythonapp.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.github/workflows/pythonapp.yml b/.github/workflows/pythonapp.yml index c5797d0a5..ffbf102d4 100644 --- a/.github/workflows/pythonapp.yml +++ b/.github/workflows/pythonapp.yml @@ -16,6 +16,10 @@ jobs: channels: conda-forge,defaults channel-priority: true activate-environment: anaconda-client-env + - shell: bash -el {0} + name: Install mamba + run: | + conda install -n base -c conda-forge mamba - shell: bash -el {0} name: Install dependencies run: | From 71cc0e1d761ee4b7421a32188ec791d7befa3380 Mon Sep 17 00:00:00 2001 From: jinningwang Date: Sun, 29 Sep 2024 16:00:39 -0400 Subject: [PATCH 10/27] [WIP] Fix github action error --- .github/workflows/pythonapp.yml | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/.github/workflows/pythonapp.yml b/.github/workflows/pythonapp.yml index ffbf102d4..496eff4fe 100644 --- a/.github/workflows/pythonapp.yml +++ b/.github/workflows/pythonapp.yml @@ -15,11 +15,15 @@ jobs: miniforge-version: "latest" channels: conda-forge,defaults channel-priority: true - activate-environment: anaconda-client-env - shell: bash -el {0} name: Install mamba run: | conda install -n base -c conda-forge mamba + - shell: bash -el {0} + name: Create and activate environment + run: | + mamba create -n anaconda-client-env + echo "conda activate anaconda-client-env" >> $GITHUB_ENV - shell: bash -el {0} name: Install dependencies run: | From 45f86d0c97ee70f2eda1136b6b1de91f67688ae5 Mon Sep 17 00:00:00 2001 From: jinningwang Date: Sun, 29 Sep 2024 16:07:09 -0400 Subject: [PATCH 11/27] [WIP] Try to fix github action error, reset .yml --- .github/workflows/pythonapp.yml | 10 +--------- 1 file changed, 1 insertion(+), 9 deletions(-) diff --git a/.github/workflows/pythonapp.yml b/.github/workflows/pythonapp.yml index 496eff4fe..c5797d0a5 100644 --- a/.github/workflows/pythonapp.yml +++ b/.github/workflows/pythonapp.yml @@ -15,15 +15,7 @@ jobs: miniforge-version: "latest" channels: conda-forge,defaults channel-priority: true - - shell: bash -el {0} - name: Install mamba - run: | - conda install -n base -c conda-forge mamba - - shell: bash -el {0} - name: Create and activate environment - run: | - mamba create -n anaconda-client-env - echo "conda activate anaconda-client-env" >> $GITHUB_ENV + activate-environment: anaconda-client-env - shell: bash -el {0} name: Install dependencies run: | From 307bfae688e3f9a4967c29e4b6d3c00592f6f6cc Mon Sep 17 00:00:00 2001 From: jinningwang Date: Sun, 29 Sep 2024 16:09:51 -0400 Subject: [PATCH 12/27] [WIP] Try to fix github action error, use classic solver instead of libmambapy --- .github/workflows/pythonapp.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/pythonapp.yml b/.github/workflows/pythonapp.yml index c5797d0a5..4629658ae 100644 --- a/.github/workflows/pythonapp.yml +++ b/.github/workflows/pythonapp.yml @@ -16,6 +16,8 @@ jobs: channels: conda-forge,defaults channel-priority: true activate-environment: anaconda-client-env + - name: Configure conda to use default solver + run: conda config --set solver classic - shell: bash -el {0} name: Install dependencies run: | From f6ef1e286de4aba6671a62f0442e023316299541 Mon Sep 17 00:00:00 2001 From: jinningwang Date: Sun, 29 Sep 2024 16:13:43 -0400 Subject: [PATCH 13/27] [WIP] Try to fix github action error --- .github/workflows/pythonapp.yml | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/.github/workflows/pythonapp.yml b/.github/workflows/pythonapp.yml index 4629658ae..9b0eaf8fb 100644 --- a/.github/workflows/pythonapp.yml +++ b/.github/workflows/pythonapp.yml @@ -16,8 +16,14 @@ jobs: channels: conda-forge,defaults channel-priority: true activate-environment: anaconda-client-env - - name: Configure conda to use default solver - run: conda config --set solver classic + - shell: bash -el {0} + name: Configure conda to use default solver + run: | + conda config --set solver classic + - shell: bash -el {0} + name: Install mamba + run: | + conda install -n base -c conda-forge mamba - shell: bash -el {0} name: Install dependencies run: | From e5e19d4715982ffb529ca41d11e4eb8641860c72 Mon Sep 17 00:00:00 2001 From: jinningwang Date: Tue, 1 Oct 2024 09:10:23 -0400 Subject: [PATCH 14/27] Remove no_flatten in find_idx --- andes/core/model/modeldata.py | 23 +++++++------------- andes/models/group.py | 41 ++++++++++++++++++++++++++++------- 2 files changed, 41 insertions(+), 23 deletions(-) diff --git a/andes/core/model/modeldata.py b/andes/core/model/modeldata.py index 0822ad7da..84cf94b9e 100644 --- a/andes/core/model/modeldata.py +++ b/andes/core/model/modeldata.py @@ -11,7 +11,6 @@ from andes.core.param import (BaseParam, DataParam, IdxParam, NumParam, TimerParam) from andes.shared import pd -from andes.utils.func import list_flatten logger = logging.getLogger(__name__) @@ -278,8 +277,7 @@ def find_param(self, prop): return out - def find_idx(self, keys, values, allow_none=False, default=False, - no_flatten=False): + def find_idx(self, keys, values, allow_none=False, default=False): """ Find `idx` of devices whose values match the given pattern. @@ -294,8 +292,6 @@ def find_idx(self, keys, values, allow_none=False, default=False, Allow key, value to be not found. Used by groups. default : bool Default idx to return if not found (missing) - no_flatten : bool - If True, return the non-flattened list of idxes. Otherwise, flatten the list. Returns ------- @@ -308,16 +304,16 @@ def find_idx(self, keys, values, allow_none=False, default=False, >>> ss = andes.load(andes.get_case('ieee14/ieee14_pvd1.xlsx')) >>> # To find the idx of `PVD1` with `name` of 'PVD1_1' and 'PVD1_2' - >>> ss.find_idx(keys='name', values=['PVD1_1', 'PVD1_2']) - [1, 2] + >>> ss.PVD1.find_idx(keys='name', values=['PVD1_1', 'PVD1_2']) + [[1], [2]] >>> # To find the idx of `PVD1` with `gammap` equals to 0.1 - >>> ss.find_idx(keys='gammap', values=0.1) - [1, 2, 3, 4, 5, 6, 7, 8, 9, 10] + >>> ss.PVD1.find_idx(keys='gammap', values=[0.1]) + ss.PVD1.find_idx(keys='gammap', values=[0.1]) >>> # To find the idx of `PVD1` with `gammap` equals to 0.1 and `name` of 'PVD1_1' - >>> ss.find_idx(keys=['gammap', 'name'], values=[[0.1], ['PVD1_1']]) - [1] + >>> ss.PVD1.find_idx(keys=['gammap', 'name'], values=[[0.1], ['PVD1_1']]) + [[1]] """ if isinstance(keys, str): keys = (keys,) @@ -363,7 +359,4 @@ def find_idx(self, keys, values, allow_none=False, default=False, idxes.append(v_idx_list) - if not no_flatten: - return list_flatten(idxes) - else: - return idxes + return idxes diff --git a/andes/models/group.py b/andes/models/group.py index 250413f32..7b1bd6a18 100644 --- a/andes/models/group.py +++ b/andes/models/group.py @@ -243,17 +243,45 @@ def set(self, src: str, idx, attr, value): return True - def find_idx(self, keys, values, allow_none=False, default=None, - no_flatten=False): + def find_idx(self, keys, values, allow_none=False, default=None): """ Find indices of devices that satisfy the given `key=value` condition. This method iterates over all models in this group. + + Parameters + ---------- + keys : str, array-like, Sized + A string or an array-like of strings containing the names of parameters for the search criteria + values : array, array of arrays, Sized + Values for the corresponding key to search for. If keys is a str, values should be an array of + elements. If keys is a list, values should be an array of arrays, each corresponds to the key. + allow_none : bool, Sized + Allow key, value to be not found. Used by groups. + default : bool + Default idx to return if not found (missing) + + Returns + ------- + list + indices of devices + + Examples + -------- + >>> # Use example case of IEEE 14-bus system with PVD1 + >>> ss = andes.load(andes.get_case('ieee14/ieee14_pvd1.xlsx')) + + >>> # To find the idx of `DG` connected to `Bus` 4 and 5 + >>> ss.DG.find_idx(keys='bus', values=[4, 5], allow_none=True) + [[1, 2, 3, 4, 5, 6, 7, 8, 9, 10], [None]] + + >>> # To find the idx of `StaticGen` connected to `Bus` 2, 3, and 4 + >>> ss.StaticGen.find_idx(keys='bus', values=[2, 3, 4]) + [[2], [3], [6]] """ idx_mdls = [] for model in self.models.values(): - idx_mdls.append(model.find_idx(keys, values, allow_none=True, default=default, - no_flatten=True)) + idx_mdls.append(model.find_idx(keys, values, allow_none=True, default=default)) # `indices_found` contains found indices returned from all models of this group # NOTE: if the idx returned to [default] across all models, it means there is @@ -274,10 +302,7 @@ def find_idx(self, keys, values, allow_none=False, default=None, miss_str = f'{keys}={[v[u] for v in values for u in uid_missing]}' raise IndexError(f'Group <{self.class_name}> does not contain device with {miss_str}') - if not no_flatten: - return list_flatten(indices_found) - else: - return indices_found + return indices_found def _check_src(self, src: str): """ From 6abf2b4a7f1132c91af67627291575b5fca2c917 Mon Sep 17 00:00:00 2001 From: jinningwang Date: Tue, 1 Oct 2024 09:13:09 -0400 Subject: [PATCH 15/27] Fix tests for find_idx --- tests/test_group.py | 7 +------ tests/test_model_set.py | 16 ++++++---------- 2 files changed, 7 insertions(+), 16 deletions(-) diff --git a/tests/test_group.py b/tests/test_group.py index 4dd33d894..e70e345eb 100644 --- a/tests/test_group.py +++ b/tests/test_group.py @@ -86,18 +86,13 @@ def test_group_access(self): # cross Model, given results self.assertListEqual(ss.StaticGen.find_idx(keys='bus', values=[1, 2, 3, 4]), - [1, 2, 3, 6]) - - self.assertListEqual(ss.StaticGen.find_idx(keys='bus', - values=[1, 2, 3, 4], - no_flatten=True), [[1], [2], [3], [6]]) self.assertListEqual(ss.StaticGen.find_idx(keys='bus', values=[1, 2, 3, 4, 2024], allow_none=True, default=2011), - [1, 2, 3, 6, 2011]) + [[1], [2], [3], [6], [2011]]) # --- get_field --- ff = ss.DG.get_field('f', list(ss.DG._idx2model.keys()), 'v_code') diff --git a/tests/test_model_set.py b/tests/test_model_set.py index d9e6b878f..a8b1ca174 100644 --- a/tests/test_model_set.py +++ b/tests/test_model_set.py @@ -61,32 +61,28 @@ def test_find_idx(self): # multiple values self.assertListEqual(mdl.find_idx(keys='name', values=['PVD1_1', 'PVD1_2'], - allow_none=False, default=False, no_flatten=False), - [1, 2]) - # multiple values, no flatten - self.assertListEqual(mdl.find_idx(keys='name', values=['PVD1_1', 'PVD1_2'], - allow_none=False, default=False, no_flatten=True), + allow_none=False, default=False), [[1], [2]]) # non-existing value self.assertListEqual(mdl.find_idx(keys='name', values=['PVD1_999'], - allow_none=True, default=False, no_flatten=False), - [False]) + allow_none=True, default=False), + [[False]]) # non-existing value is not allowed with self.assertRaises(IndexError): mdl.find_idx(keys='name', values=['PVD1_999'], - allow_none=False, default=False, no_flatten=False) + allow_none=False, default=False) # multiple keys self.assertListEqual(mdl.find_idx(keys=['gammap', 'name'], values=[[0.1, 0.1], ['PVD1_1', 'PVD1_2']]), - [1, 2]) + [[1], [2]]) # multiple keys, with non-existing values self.assertListEqual(mdl.find_idx(keys=['gammap', 'name'], values=[[0.1, 0.1], ['PVD1_1', 'PVD1_999']], allow_none=True, default='CURENT'), - [1, 'CURENT']) + [[1], ['CURENT']]) # multiple keys, with non-existing values not allowed with self.assertRaises(IndexError): From 1c489df17ab14d17352df2e19988e985b6b8f581 Mon Sep 17 00:00:00 2001 From: jinningwang Date: Tue, 1 Oct 2024 09:14:15 -0400 Subject: [PATCH 16/27] Typo --- andes/core/model/modeldata.py | 2 +- andes/models/group.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/andes/core/model/modeldata.py b/andes/core/model/modeldata.py index 84cf94b9e..ffdf7eeb0 100644 --- a/andes/core/model/modeldata.py +++ b/andes/core/model/modeldata.py @@ -296,7 +296,7 @@ def find_idx(self, keys, values, allow_none=False, default=False): Returns ------- list - indices of devices + A list of lists containing the indices of devices Examples -------- diff --git a/andes/models/group.py b/andes/models/group.py index 7b1bd6a18..2d4177fe9 100644 --- a/andes/models/group.py +++ b/andes/models/group.py @@ -264,7 +264,7 @@ def find_idx(self, keys, values, allow_none=False, default=None): Returns ------- list - indices of devices + A list of lists containing the indices of devices Examples -------- From bccc0801715fe5ed3c98c053f3924c14919112ec Mon Sep 17 00:00:00 2001 From: jinningwang Date: Tue, 1 Oct 2024 10:13:39 -0400 Subject: [PATCH 17/27] [WIP] Fix find_idx, revert changes --- andes/core/model/modeldata.py | 40 ++++---------------- andes/models/group.py | 71 +++++++++-------------------------- 2 files changed, 25 insertions(+), 86 deletions(-) diff --git a/andes/core/model/modeldata.py b/andes/core/model/modeldata.py index ffdf7eeb0..29b21308f 100644 --- a/andes/core/model/modeldata.py +++ b/andes/core/model/modeldata.py @@ -296,24 +296,7 @@ def find_idx(self, keys, values, allow_none=False, default=False): Returns ------- list - A list of lists containing the indices of devices - - Examples - -------- - >>> # Use example case of IEEE 14-bus system with PVD1 - >>> ss = andes.load(andes.get_case('ieee14/ieee14_pvd1.xlsx')) - - >>> # To find the idx of `PVD1` with `name` of 'PVD1_1' and 'PVD1_2' - >>> ss.PVD1.find_idx(keys='name', values=['PVD1_1', 'PVD1_2']) - [[1], [2]] - - >>> # To find the idx of `PVD1` with `gammap` equals to 0.1 - >>> ss.PVD1.find_idx(keys='gammap', values=[0.1]) - ss.PVD1.find_idx(keys='gammap', values=[0.1]) - - >>> # To find the idx of `PVD1` with `gammap` equals to 0.1 and `name` of 'PVD1_1' - >>> ss.PVD1.find_idx(keys=['gammap', 'name'], values=[[0.1], ['PVD1_1']]) - [[1]] + indices of devices """ if isinstance(keys, str): keys = (keys,) @@ -333,30 +316,21 @@ def find_idx(self, keys, values, allow_none=False, default=False): if len(keys) != len(values): raise ValueError("keys and values must have the same length") - # check if all elements in values have the same length - iterator = iter(values) - try: - first_length = len(next(iterator)) - ok_len = all(len(element) == first_length for element in iterator) - except StopIteration: # empty iterable - ok_len = True - if not ok_len: - raise ValueError("All elements in values must have the same length") - v_attrs = [self.__dict__[key].v for key in keys] idxes = [] for v_search in zip(*values): - v_idx_list = [] + v_idx = None for pos, v_attr in enumerate(zip(*v_attrs)): if all([i == j for i, j in zip(v_search, v_attr)]): - v_idx_list.append(self.idx.v[pos]) - if not v_idx_list: + v_idx = self.idx.v[pos] + break + if v_idx is None: if allow_none is False: raise IndexError(f'{list(keys)}={v_search} not found in {self.class_name}') else: - v_idx_list.append(default) + v_idx = default - idxes.append(v_idx_list) + idxes.append(v_idx) return idxes diff --git a/andes/models/group.py b/andes/models/group.py index 2d4177fe9..44480bac3 100644 --- a/andes/models/group.py +++ b/andes/models/group.py @@ -248,61 +248,26 @@ def find_idx(self, keys, values, allow_none=False, default=None): Find indices of devices that satisfy the given `key=value` condition. This method iterates over all models in this group. - - Parameters - ---------- - keys : str, array-like, Sized - A string or an array-like of strings containing the names of parameters for the search criteria - values : array, array of arrays, Sized - Values for the corresponding key to search for. If keys is a str, values should be an array of - elements. If keys is a list, values should be an array of arrays, each corresponds to the key. - allow_none : bool, Sized - Allow key, value to be not found. Used by groups. - default : bool - Default idx to return if not found (missing) - - Returns - ------- - list - A list of lists containing the indices of devices - - Examples - -------- - >>> # Use example case of IEEE 14-bus system with PVD1 - >>> ss = andes.load(andes.get_case('ieee14/ieee14_pvd1.xlsx')) - - >>> # To find the idx of `DG` connected to `Bus` 4 and 5 - >>> ss.DG.find_idx(keys='bus', values=[4, 5], allow_none=True) - [[1, 2, 3, 4, 5, 6, 7, 8, 9, 10], [None]] - - >>> # To find the idx of `StaticGen` connected to `Bus` 2, 3, and 4 - >>> ss.StaticGen.find_idx(keys='bus', values=[2, 3, 4]) - [[2], [3], [6]] """ - idx_mdls = [] - for model in self.models.values(): - idx_mdls.append(model.find_idx(keys, values, allow_none=True, default=default)) - - # `indices_found` contains found indices returned from all models of this group - # NOTE: if the idx returned to [default] across all models, it means there is - # no such idx in this group. If so, return default or raise an key error. indices_found = [] - uid_missing = [] - for uid, col in enumerate(zip(*idx_mdls)): - if all(item == [default] for item in col): - if allow_none: - indices_found.append([default]) - else: - uid_missing.append(uid) - else: - col_filter = [item for item in col if item != [default]] - indices_found.append(list_flatten(col_filter)) - - if uid_missing: - miss_str = f'{keys}={[v[u] for v in values for u in uid_missing]}' - raise IndexError(f'Group <{self.class_name}> does not contain device with {miss_str}') - - return indices_found + # `indices_found` contains found indices returned from all models of this group + for model in self.models.values(): + indices_found.append(model.find_idx(keys, values, allow_none=True, default=default)) + + out = [] + for idx, idx_found in enumerate(zip(*indices_found)): + if not allow_none: + if idx_found.count(None) == len(idx_found): + missing_values = [item[idx] for item in values] + raise IndexError(f'{list(keys)} = {missing_values} not found in {self.class_name}') + + real_idx = default + for item in idx_found: + if item is not None: + real_idx = item + break + out.append(real_idx) + return out def _check_src(self, src: str): """ From 919c8a4bbc807cf33d9a5590de078bd8200f9c05 Mon Sep 17 00:00:00 2001 From: jinningwang Date: Tue, 1 Oct 2024 10:51:38 -0400 Subject: [PATCH 18/27] [WIP] Fix find_idx, add parameter allow_all=False to ModelData.find_idx() --- andes/core/model/modeldata.py | 47 +++++++++++++++++++++++++++++------ 1 file changed, 40 insertions(+), 7 deletions(-) diff --git a/andes/core/model/modeldata.py b/andes/core/model/modeldata.py index 29b21308f..d12646d12 100644 --- a/andes/core/model/modeldata.py +++ b/andes/core/model/modeldata.py @@ -277,7 +277,7 @@ def find_param(self, prop): return out - def find_idx(self, keys, values, allow_none=False, default=False): + def find_idx(self, keys, values, allow_none=False, default=False, allow_all=False): """ Find `idx` of devices whose values match the given pattern. @@ -292,12 +292,43 @@ def find_idx(self, keys, values, allow_none=False, default=False): Allow key, value to be not found. Used by groups. default : bool Default idx to return if not found (missing) + allow_all : bool + Return all matches if set to True Returns ------- list indices of devices + + Notes + ----- + - Only the first match is returned by default. + - If all matches are needed, set `allow_all` to True. + - When `allow_all` is set to True, the function returns a list of lists where each nested list contains + all the matches for the corresponding search criteria. + + Examples + -------- + >>> # Use example case of IEEE 14-bus system with PVD1 + >>> ss = andes.load(andes.get_case('ieee14/ieee14_pvd1.xlsx')) + + >>> # To find the idx of `PVD1` with `name` of 'PVD1_1' and 'PVD1_2' + >>> ss.PVD1.find_idx(keys='name', values=['PVD1_1', 'PVD1_2']) + [1, 2] + + >>> # To find the idx of `PVD1` connected to bus 4 + >>> ss.PVD1.find_idx(keys='bus', values=[4]) + [1] + + >>> # To find ALL the idx of `PVD1` with `gammap` equals to 0.1 + >>> ss.PVD1.find_idx(keys='gammap', values=[0.1], allow_all=True) + [[1, 2, 3, 4, 5, 6, 7, 8, 9, 10]] + + >>> # To find the idx of `PVD1` with `gammap` equals to 0.1 and `name` of 'PVD1_1' + >>> ss.PVD1.find_idx(keys=['gammap', 'name'], values=[[0.1], ['PVD1_1']]) + [1] """ + if isinstance(keys, str): keys = (keys,) if not isinstance(values, (int, float, str, np.floating)) and not isinstance(values, Iterable): @@ -320,17 +351,19 @@ def find_idx(self, keys, values, allow_none=False, default=False): idxes = [] for v_search in zip(*values): - v_idx = None + v_idx = [] for pos, v_attr in enumerate(zip(*v_attrs)): if all([i == j for i, j in zip(v_search, v_attr)]): - v_idx = self.idx.v[pos] - break - if v_idx is None: + v_idx.append(self.idx.v[pos]) + if not v_idx: if allow_none is False: raise IndexError(f'{list(keys)}={v_search} not found in {self.class_name}') else: - v_idx = default + v_idx = [default] - idxes.append(v_idx) + if allow_all: + idxes.append(v_idx) + else: + idxes.append(v_idx[0]) return idxes From b01fe4d8e116b0df9d0d37b5cfc832a2ffd28262 Mon Sep 17 00:00:00 2001 From: jinningwang Date: Thu, 3 Oct 2024 09:14:39 -0400 Subject: [PATCH 19/27] [WIP] Fix find_idx, add inner lists length check --- andes/core/model/modeldata.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/andes/core/model/modeldata.py b/andes/core/model/modeldata.py index d12646d12..26c8c6f3c 100644 --- a/andes/core/model/modeldata.py +++ b/andes/core/model/modeldata.py @@ -347,6 +347,10 @@ def find_idx(self, keys, values, allow_none=False, default=False, allow_all=Fals if len(keys) != len(values): raise ValueError("keys and values must have the same length") + if isinstance(values[0], Iterable): + if not all([len(val) == len(values[0]) for val in values]): + raise ValueError("All items in values must have the same length") + v_attrs = [self.__dict__[key].v for key in keys] idxes = [] From e1422b2651686574b347b2661062118294fdfd53 Mon Sep 17 00:00:00 2001 From: jinningwang Date: Thu, 3 Oct 2024 09:20:57 -0400 Subject: [PATCH 20/27] [WIP] Fix find_idx, refactor input check --- andes/core/model/modeldata.py | 75 ++++++++++++++++++++++++----------- 1 file changed, 51 insertions(+), 24 deletions(-) diff --git a/andes/core/model/modeldata.py b/andes/core/model/modeldata.py index 26c8c6f3c..fd4432be9 100644 --- a/andes/core/model/modeldata.py +++ b/andes/core/model/modeldata.py @@ -288,11 +288,11 @@ def find_idx(self, keys, values, allow_none=False, default=False, allow_all=Fals values : array, array of arrays, Sized Values for the corresponding key to search for. If keys is a str, values should be an array of elements. If keys is a list, values should be an array of arrays, each corresponds to the key. - allow_none : bool, Sized + allow_none : bool, Sized, optional Allow key, value to be not found. Used by groups. - default : bool + default : bool, optional Default idx to return if not found (missing) - allow_all : bool + allow_all : bool, optional Return all matches if set to True Returns @@ -329,27 +329,7 @@ def find_idx(self, keys, values, allow_none=False, default=False, allow_all=Fals [1] """ - if isinstance(keys, str): - keys = (keys,) - if not isinstance(values, (int, float, str, np.floating)) and not isinstance(values, Iterable): - raise ValueError(f"value must be a string, scalar or an iterable, got {values}") - - if len(values) > 0 and not isinstance(values[0], (list, tuple, np.ndarray)): - values = (values,) - - elif isinstance(keys, Sized): - if not isinstance(values, Iterable): - raise ValueError(f"value must be an iterable, got {values}") - - if len(values) > 0 and not isinstance(values[0], Iterable): - raise ValueError(f"if keys is an iterable, values must be an iterable of iterables. got {values}") - - if len(keys) != len(values): - raise ValueError("keys and values must have the same length") - - if isinstance(values[0], Iterable): - if not all([len(val) == len(values[0]) for val in values]): - raise ValueError("All items in values must have the same length") + keys, values = _validate_keys_values(keys, values) v_attrs = [self.__dict__[key].v for key in keys] @@ -371,3 +351,50 @@ def find_idx(self, keys, values, allow_none=False, default=False, allow_all=Fals idxes.append(v_idx[0]) return idxes + + +def _validate_keys_values(keys, values): + """ + Validate the inputs for the func `find_idx`. + + Parameters + ---------- + keys : str, array-like, Sized + A string or an array-like of strings containing the names of parameters for the search criteria. + values : array, array of arrays, Sized + Values for the corresponding key to search for. If keys is a str, values should be an array of + elements. If keys is a list, values should be an array of arrays, each corresponds to the key. + + Returns + ------- + tuple + Sanitized keys and values + + Raises + ------ + ValueError + If the inputs are not valid. + """ + if isinstance(keys, str): + keys = (keys,) + if not isinstance(values, (int, float, str, np.floating)) and not isinstance(values, Iterable): + raise ValueError(f"value must be a string, scalar or an iterable, got {values}") + + if len(values) > 0 and not isinstance(values[0], (list, tuple, np.ndarray)): + values = (values,) + + elif isinstance(keys, Sized): + if not isinstance(values, Iterable): + raise ValueError(f"value must be an iterable, got {values}") + + if len(values) > 0 and not isinstance(values[0], Iterable): + raise ValueError(f"if keys is an iterable, values must be an iterable of iterables. got {values}") + + if len(keys) != len(values): + raise ValueError("keys and values must have the same length") + + if isinstance(values[0], Iterable): + if not all([len(val) == len(values[0]) for val in values]): + raise ValueError("All items in values must have the same length") + + return keys, values From fdad124dc967f68b45b37c74bc8fd8f3fd942b65 Mon Sep 17 00:00:00 2001 From: jinningwang Date: Thu, 3 Oct 2024 09:39:58 -0400 Subject: [PATCH 21/27] [WIP] Fix find_idx, move input check from modeldata to utils.func --- andes/core/model/modeldata.py | 51 ++--------------------------------- andes/utils/func.py | 48 +++++++++++++++++++++++++++++++++ 2 files changed, 50 insertions(+), 49 deletions(-) diff --git a/andes/core/model/modeldata.py b/andes/core/model/modeldata.py index fd4432be9..76ded8a23 100644 --- a/andes/core/model/modeldata.py +++ b/andes/core/model/modeldata.py @@ -4,13 +4,13 @@ import logging from collections import OrderedDict -from typing import Iterable, Sized import numpy as np from andes.core.model.modelcache import ModelCache from andes.core.param import (BaseParam, DataParam, IdxParam, NumParam, TimerParam) from andes.shared import pd +from andes.utils.func import validate_keys_values logger = logging.getLogger(__name__) @@ -329,7 +329,7 @@ def find_idx(self, keys, values, allow_none=False, default=False, allow_all=Fals [1] """ - keys, values = _validate_keys_values(keys, values) + keys, values = validate_keys_values(keys, values) v_attrs = [self.__dict__[key].v for key in keys] @@ -351,50 +351,3 @@ def find_idx(self, keys, values, allow_none=False, default=False, allow_all=Fals idxes.append(v_idx[0]) return idxes - - -def _validate_keys_values(keys, values): - """ - Validate the inputs for the func `find_idx`. - - Parameters - ---------- - keys : str, array-like, Sized - A string or an array-like of strings containing the names of parameters for the search criteria. - values : array, array of arrays, Sized - Values for the corresponding key to search for. If keys is a str, values should be an array of - elements. If keys is a list, values should be an array of arrays, each corresponds to the key. - - Returns - ------- - tuple - Sanitized keys and values - - Raises - ------ - ValueError - If the inputs are not valid. - """ - if isinstance(keys, str): - keys = (keys,) - if not isinstance(values, (int, float, str, np.floating)) and not isinstance(values, Iterable): - raise ValueError(f"value must be a string, scalar or an iterable, got {values}") - - if len(values) > 0 and not isinstance(values[0], (list, tuple, np.ndarray)): - values = (values,) - - elif isinstance(keys, Sized): - if not isinstance(values, Iterable): - raise ValueError(f"value must be an iterable, got {values}") - - if len(values) > 0 and not isinstance(values[0], Iterable): - raise ValueError(f"if keys is an iterable, values must be an iterable of iterables. got {values}") - - if len(keys) != len(values): - raise ValueError("keys and values must have the same length") - - if isinstance(values[0], Iterable): - if not all([len(val) == len(values[0]) for val in values]): - raise ValueError("All items in values must have the same length") - - return keys, values diff --git a/andes/utils/func.py b/andes/utils/func.py index d9e615a36..d0b172456 100644 --- a/andes/utils/func.py +++ b/andes/utils/func.py @@ -1,5 +1,6 @@ import functools import operator +from typing import Iterable, Sized from andes.shared import np @@ -36,3 +37,50 @@ def interp_n2(t, x, y): """ return y[:, 0] + (t - x[0]) * (y[:, 1] - y[:, 0]) / (x[1] - x[0]) + + +def validate_keys_values(keys, values): + """ + Validate the inputs for the func `find_idx`. + + Parameters + ---------- + keys : str, array-like, Sized + A string or an array-like of strings containing the names of parameters for the search criteria. + values : array, array of arrays, Sized + Values for the corresponding key to search for. If keys is a str, values should be an array of + elements. If keys is a list, values should be an array of arrays, each corresponds to the key. + + Returns + ------- + tuple + Sanitized keys and values + + Raises + ------ + ValueError + If the inputs are not valid. + """ + if isinstance(keys, str): + keys = (keys,) + if not isinstance(values, (int, float, str, np.floating)) and not isinstance(values, Iterable): + raise ValueError(f"value must be a string, scalar or an iterable, got {values}") + + if len(values) > 0 and not isinstance(values[0], (list, tuple, np.ndarray)): + values = (values,) + + elif isinstance(keys, Sized): + if not isinstance(values, Iterable): + raise ValueError(f"value must be an iterable, got {values}") + + if len(values) > 0 and not isinstance(values[0], Iterable): + raise ValueError(f"if keys is an iterable, values must be an iterable of iterables. got {values}") + + if len(keys) != len(values): + raise ValueError("keys and values must have the same length") + + if isinstance(values[0], Iterable): + if not all([len(val) == len(values[0]) for val in values]): + raise ValueError("All items in values must have the same length") + + return keys, values From 47e935247214e20d224d9f01a07e9423ba2ee5d2 Mon Sep 17 00:00:00 2001 From: jinningwang Date: Thu, 3 Oct 2024 11:17:35 -0400 Subject: [PATCH 22/27] [WIP] Fix find_idx --- andes/models/group.py | 73 +++++++++++++++++++++++++++++++++---------- 1 file changed, 57 insertions(+), 16 deletions(-) diff --git a/andes/models/group.py b/andes/models/group.py index 44480bac3..eff858d8e 100644 --- a/andes/models/group.py +++ b/andes/models/group.py @@ -5,7 +5,7 @@ import numpy as np from andes.core.service import BackRef -from andes.utils.func import list_flatten +from andes.utils.func import list_flatten, validate_keys_values logger = logging.getLogger(__name__) @@ -243,30 +243,71 @@ def set(self, src: str, idx, attr, value): return True - def find_idx(self, keys, values, allow_none=False, default=None): + def find_idx(self, keys, values, allow_none=False, default=None, allow_all=False): """ Find indices of devices that satisfy the given `key=value` condition. This method iterates over all models in this group. + + Parameters + ---------- + keys : str, array-like, Sized + A string or an array-like of strings containing the names of parameters for the search criteria. + values : array, array of arrays, Sized + Values for the corresponding key to search for. If keys is a str, values should be an array of + elements. If keys is a list, values should be an array of arrays, each corresponding to the key. + allow_none : bool, optional + Allow key, value to be not found. Used by groups. Default is False. + default : bool, optional + Default idx to return if not found (missing). Default is None. + allow_all : bool, optional + Return all matches if set to True. Default is False. + + Returns + ------- + list + Indices of devices. """ + + keys, values = validate_keys_values(keys, values) + + n_val, n_pair = len(values), len(values[0]) + indices_found = [] # `indices_found` contains found indices returned from all models of this group for model in self.models.values(): - indices_found.append(model.find_idx(keys, values, allow_none=True, default=default)) - - out = [] - for idx, idx_found in enumerate(zip(*indices_found)): - if not allow_none: - if idx_found.count(None) == len(idx_found): - missing_values = [item[idx] for item in values] - raise IndexError(f'{list(keys)} = {missing_values} not found in {self.class_name}') - - real_idx = default - for item in idx_found: - if item is not None: - real_idx = item + indices_found.append(model.find_idx(keys, values, allow_none=True, default=default, allow_all=True)) + + # --- find missing pairs --- + i_val_miss = [] + for i in range(n_pair): + idx_cross_mdls = [indices_found[j][i] for j in range(n_val)] + if all(item == [default] for item in idx_cross_mdls): + i_val_miss.append(i) + + if (not allow_none) and i_val_miss: + miss_pairs = [] + for i in i_val_miss: + miss_pairs.append([values[j][i] for j in range(len(keys))]) + raise IndexError(f'{keys} = {miss_pairs} not found in {self.class_name}') + + # --- output --- + out_pre = [] + for i in range(n_pair): + idx_cross_mdls = [indices_found[j][i] for j in range(n_val)] + if all(item == [default] for item in idx_cross_mdls): + out_pre.append([default]) + continue + for item in idx_cross_mdls: + if item != [default]: + out_pre.append(item) break - out.append(real_idx) + + if allow_all: + out = out_pre + else: + out = [item[0] for item in out_pre] + return out def _check_src(self, src: str): From ef344882688e4bd56e201a59eae1da462e0135bd Mon Sep 17 00:00:00 2001 From: jinningwang Date: Thu, 3 Oct 2024 15:43:35 -0400 Subject: [PATCH 23/27] [WIP] Fix find_idx, minor fix --- andes/models/group.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/andes/models/group.py b/andes/models/group.py index eff858d8e..0c33a49af 100644 --- a/andes/models/group.py +++ b/andes/models/group.py @@ -271,7 +271,7 @@ def find_idx(self, keys, values, allow_none=False, default=None, allow_all=False keys, values = validate_keys_values(keys, values) - n_val, n_pair = len(values), len(values[0]) + n_mdl, n_pair = len(self.models), len(values[0]) indices_found = [] # `indices_found` contains found indices returned from all models of this group @@ -281,7 +281,7 @@ def find_idx(self, keys, values, allow_none=False, default=None, allow_all=False # --- find missing pairs --- i_val_miss = [] for i in range(n_pair): - idx_cross_mdls = [indices_found[j][i] for j in range(n_val)] + idx_cross_mdls = [indices_found[j][i] for j in range(n_mdl)] if all(item == [default] for item in idx_cross_mdls): i_val_miss.append(i) @@ -294,7 +294,7 @@ def find_idx(self, keys, values, allow_none=False, default=None, allow_all=False # --- output --- out_pre = [] for i in range(n_pair): - idx_cross_mdls = [indices_found[j][i] for j in range(n_val)] + idx_cross_mdls = [indices_found[j][i] for j in range(n_mdl)] if all(item == [default] for item in idx_cross_mdls): out_pre.append([default]) continue From 33c7ee9d799f2625477c8b78912fb2f3c393dcb7 Mon Sep 17 00:00:00 2001 From: jinningwang Date: Thu, 3 Oct 2024 16:22:40 -0400 Subject: [PATCH 24/27] [WIP] Fix find_idx, fix invovled tests --- tests/test_group.py | 7 ++++++- tests/test_model_set.py | 16 ++++++++++++---- 2 files changed, 18 insertions(+), 5 deletions(-) diff --git a/tests/test_group.py b/tests/test_group.py index e70e345eb..dcb9e97fb 100644 --- a/tests/test_group.py +++ b/tests/test_group.py @@ -86,12 +86,17 @@ def test_group_access(self): # cross Model, given results self.assertListEqual(ss.StaticGen.find_idx(keys='bus', values=[1, 2, 3, 4]), + [1, 2, 3, 6]) + self.assertListEqual(ss.StaticGen.find_idx(keys='bus', + values=[1, 2, 3, 4], + allow_all=True), [[1], [2], [3], [6]]) self.assertListEqual(ss.StaticGen.find_idx(keys='bus', values=[1, 2, 3, 4, 2024], allow_none=True, - default=2011), + default=2011, + allow_all=True), [[1], [2], [3], [6], [2011]]) # --- get_field --- diff --git a/tests/test_model_set.py b/tests/test_model_set.py index a8b1ca174..df29b54d6 100644 --- a/tests/test_model_set.py +++ b/tests/test_model_set.py @@ -59,14 +59,22 @@ def test_find_idx(self): ss = andes.load(andes.get_case('ieee14/ieee14_pvd1.xlsx')) mdl = ss.PVD1 + # not allow all matches + self.assertListEqual(mdl.find_idx(keys='gammap', values=[0.1], allow_all=False), + [1]) + + # allow all matches + self.assertListEqual(mdl.find_idx(keys='gammap', values=[0.1], allow_all=True), + [[1, 2, 3, 4, 5, 6, 7, 8, 9, 10]]) + # multiple values self.assertListEqual(mdl.find_idx(keys='name', values=['PVD1_1', 'PVD1_2'], allow_none=False, default=False), - [[1], [2]]) + [1, 2]) # non-existing value self.assertListEqual(mdl.find_idx(keys='name', values=['PVD1_999'], allow_none=True, default=False), - [[False]]) + [False]) # non-existing value is not allowed with self.assertRaises(IndexError): @@ -76,13 +84,13 @@ def test_find_idx(self): # multiple keys self.assertListEqual(mdl.find_idx(keys=['gammap', 'name'], values=[[0.1, 0.1], ['PVD1_1', 'PVD1_2']]), - [[1], [2]]) + [1, 2]) # multiple keys, with non-existing values self.assertListEqual(mdl.find_idx(keys=['gammap', 'name'], values=[[0.1, 0.1], ['PVD1_1', 'PVD1_999']], allow_none=True, default='CURENT'), - [[1], ['CURENT']]) + [1, 'CURENT']) # multiple keys, with non-existing values not allowed with self.assertRaises(IndexError): From 22eceeb6080ccf77527b0229fa0706124914d959 Mon Sep 17 00:00:00 2001 From: jinningwang Date: Thu, 3 Oct 2024 16:24:09 -0400 Subject: [PATCH 25/27] Restore pythonapp workflow --- .github/workflows/pythonapp.yml | 8 -------- 1 file changed, 8 deletions(-) diff --git a/.github/workflows/pythonapp.yml b/.github/workflows/pythonapp.yml index 9b0eaf8fb..c5797d0a5 100644 --- a/.github/workflows/pythonapp.yml +++ b/.github/workflows/pythonapp.yml @@ -16,14 +16,6 @@ jobs: channels: conda-forge,defaults channel-priority: true activate-environment: anaconda-client-env - - shell: bash -el {0} - name: Configure conda to use default solver - run: | - conda config --set solver classic - - shell: bash -el {0} - name: Install mamba - run: | - conda install -n base -c conda-forge mamba - shell: bash -el {0} name: Install dependencies run: | From 9a675f5773ebccfe930c2daef4565d694322660d Mon Sep 17 00:00:00 2001 From: jinningwang Date: Thu, 3 Oct 2024 16:41:33 -0400 Subject: [PATCH 26/27] Update release notes --- docs/source/release-notes.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/source/release-notes.rst b/docs/source/release-notes.rst index bc690e2f4..dfef9bd87 100644 --- a/docs/source/release-notes.rst +++ b/docs/source/release-notes.rst @@ -19,7 +19,7 @@ v1.9.3 (2024-04-XX) - Adjust `BusFreq.Tw.default` to 0.1. - Add parameter from_csv=None in TDS.run() to allow loading data from CSV files at TDS begining. - Fix `TDS.init()` and `TDS._csv_step()` to fit loading from CSV when `Output` exists. -- Fix `ModelData.find_idx()` to return all matches +- Add parameter `allow_all=False` to `ModelData.find_idx()` `GroupBase.find_idx()` to allow searching all matches. v1.9.2 (2024-03-25) ------------------- From 3adc1109f7fec126e44cc66818f81ff21a94ac7a Mon Sep 17 00:00:00 2001 From: jinningwang Date: Thu, 3 Oct 2024 21:56:05 -0400 Subject: [PATCH 27/27] Format --- andes/core/model/modeldata.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/andes/core/model/modeldata.py b/andes/core/model/modeldata.py index 76ded8a23..a66f2cb8d 100644 --- a/andes/core/model/modeldata.py +++ b/andes/core/model/modeldata.py @@ -293,7 +293,8 @@ def find_idx(self, keys, values, allow_none=False, default=False, allow_all=Fals default : bool, optional Default idx to return if not found (missing) allow_all : bool, optional - Return all matches if set to True + If True, returns a list of lists where each nested list contains all the matches for the + corresponding search criteria. Returns ------- @@ -304,8 +305,6 @@ def find_idx(self, keys, values, allow_none=False, default=False, allow_all=Fals ----- - Only the first match is returned by default. - If all matches are needed, set `allow_all` to True. - - When `allow_all` is set to True, the function returns a list of lists where each nested list contains - all the matches for the corresponding search criteria. Examples --------