fix ci

dbbs-lab · Feb 13, 2024 · e304130 · e304130
1 parent d4e90c1
commit e304130
Show file tree

Hide file tree

Showing 6 changed files with 79 additions and 109 deletions.
diff --git a/.github/workflows/main.yaml b/.github/workflows/main.yaml
@@ -1,4 +1,4 @@
-name: Build and test BSB JSON
+name: Test BSB unit testing tools
 
 on: [push, pull_request]
 
@@ -10,10 +10,10 @@ jobs:
       matrix:
         python-version: ["3.9", "3.10", "3.11", "3.12"]
     steps:
-    - uses: actions/checkout@v3.5.0
+    - uses: actions/checkout@v4
 
     - name: Set up Python ${{ matrix.python-version }}
-      uses: actions/setup-python@v4.3.0
+      uses: actions/setup-python@v5
       with:
         python-version: ${{ matrix.python-version }}
 

diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml
@@ -0,0 +1,10 @@
+repos:
+  - repo: https://github.com/psf/black-pre-commit-mirror
+    rev: 24.1.1
+    hooks:
+      - id: black
+  - repo: https://github.com/pycqa/isort
+    rev: 5.12.0
+    hooks:
+      - id: isort
+        name: isort (python)
diff --git a/bsb_test/__init__.py b/bsb_test/__init__.py
@@ -1,27 +1,30 @@
 """
 Helpers for better and more complete tests for component developers of the BSB framework.
 """
-import unittest
-
-import requests
-from bsb.storage._files import UrlScheme
 
+import glob as _glob
+import os as _os
+import unittest
 from pathlib import Path
 
-from .configs import get_test_config, get_test_configs, list_test_configs, get_test_config_tree
-from .parallel import *
+import numpy as _np
+import requests
+from bsb.config import Configuration as _Configuration
 from bsb.core import Scaffold as _Scaffold
+from bsb.exceptions import FixtureError
 from bsb.morphologies import parse_morphology_file
-from bsb.storage import (
-    Storage as _Storage,
-    get_engine_node as _get_engine_node,
-    Chunk as _Chunk,
+from bsb.storage import Chunk as _Chunk
+from bsb.storage import Storage as _Storage
+from bsb.storage import get_engine_node as _get_engine_node
+from bsb.storage._files import UrlScheme
+
+from .configs import (
+    get_test_config,
+    get_test_config_tree,
+    get_test_configs,
+    list_test_configs,
 )
-from bsb.config import Configuration as _Configuration
-from bsb.exceptions import FixtureError
-import numpy as _np
-import glob as _glob
-import os as _os
+from .parallel import *
 
 __version__ = "0.0.0b7"
 

diff --git a/bsb_test/engines.py b/bsb_test/engines.py
@@ -14,30 +14,29 @@ class TestMyStorage(TestStorage, unittest.TestCase, engine_name="my_storage"):
 engine is available as "my_storage" (e.g. `bsb-hdf5` would use `engine_name="hdf5"`).
 """
 
-from bsb.exceptions import DatasetNotFoundError, DatasetExistsError
-from bsb.core import Scaffold
+import time
+from collections import defaultdict
+
+import numpy as np
 from bsb.cell_types import CellType
 from bsb.config import Configuration
+from bsb.core import Scaffold
+from bsb.exceptions import DatasetExistsError, DatasetNotFoundError
 from bsb.morphologies import Morphology, MorphologySet
-from bsb.storage import Storage, Chunk
 from bsb.services import MPI
+from bsb.storage import Chunk, Storage
+
 from . import (
-    NumpyTestCase,
     FixedPosConfigFixture,
+    NumpyTestCase,
     RandomStorageFixture,
-    timeout,
-    single_process_test,
     get_all_morphology_paths,
     get_morphology_path,
+    single_process_test,
+    timeout,
 )
-import time
-import numpy as np
-from collections import defaultdict
 
-
-cfg = Configuration.default(
-    cell_types=dict(test_cell=dict(spatial=dict(radius=2, density=1e-3)))
-)
+cfg = Configuration.default(cell_types=dict(test_cell=dict(spatial=dict(radius=2, density=1e-3))))
 
 
 class _ScaffoldDummy:
@@ -145,9 +144,7 @@ def setUp(self):
         self.network = Scaffold(storage=self.storage)
 
 
-class TestPlacementSet(
-    FixedPosConfigFixture, RandomStorageFixture, NumpyTestCase, engine_name=None
-):
+class TestPlacementSet(FixedPosConfigFixture, RandomStorageFixture, NumpyTestCase, engine_name=None):
     def setUp(self):
         super().setUp()
         self.network = Scaffold(self.cfg, self.storage)
@@ -158,9 +155,7 @@ def test_init(self):
         self.assertEqual(ct, ps.cell_type, "cell type incorrect")
         self.assertEqual(ct.name, ps.tag, "tag incorrect")
         ct2 = CellType(name="boo", spatial=dict(radius=2, density=1e-3))
-        with self.assertRaises(
-            DatasetNotFoundError, msg="should raise `DatasetNotFoundError` for unknown PS"
-        ):
+        with self.assertRaises(DatasetNotFoundError, msg="should raise `DatasetNotFoundError` for unknown PS"):
             self.network.get_placement_set(ct2)
 
     def test_create(self):
@@ -174,9 +169,7 @@ def test_create(self):
             ps = self.storage._PlacementSet(self.storage._engine, ct)
         self.assertEqual("hehe", ps.tag, "tag should be cell type name")
         self.assertEqual(0, len(ps), "new ps should be empty")
-        with self.assertRaises(
-            DatasetExistsError, msg="creating existing PS should error"
-        ):
+        with self.assertRaises(DatasetExistsError, msg="creating existing PS should error"):
             ps = self.storage._PlacementSet.create(self.storage._engine, ct)
 
     def test_exists(self):
@@ -218,9 +211,7 @@ def test_clear(self):
     def test_get_all_chunks(self):
         self.network.compile()
         ps = self.network.get_placement_set("test_cell")
-        self.assertEqual(
-            sorted(self.chunks), sorted(ps.get_all_chunks()), "populated chunks incorrect"
-        )
+        self.assertEqual(sorted(self.chunks), sorted(ps.get_all_chunks()), "populated chunks incorrect")
 
     def test_load_ids(self):
         self.network.compile()
@@ -253,9 +244,7 @@ def test_load_no_morphologies(self):
             ps.load_morphologies()
 
     def test_load_morphologies(self):
-        self.network.cell_types.test_cell.spatial.morphologies.append(
-            dict(names=["test_cell_A", "test_cell_B"])
-        )
+        self.network.cell_types.test_cell.spatial.morphologies.append(dict(names=["test_cell_A", "test_cell_B"]))
         mA = Morphology.from_swc(get_morphology_path("2branch.swc"))
         mB = Morphology.from_swc(get_morphology_path("2comp.swc"))
         self.network.morphologies.save("test_cell_A", mA, overwrite=True)
@@ -276,20 +265,14 @@ def test_load_morphologies(self):
                 "It seems not all morphologies occur in random morphology distr."
                 + "\nShould find: 'test_cell_A', 'test_cell_B'"
                 + "\nFound: "
-                + ", ".join(
-                    f"'{m.meta['name']}'" for m in ms.iter_morphologies(unique=True)
-                )
+                + ", ".join(f"'{m.meta['name']}'" for m in ms.iter_morphologies(unique=True))
                 + "\nAll data:\n["
-                + ", ".join(
-                    f"'{m.meta['name']}'" for m in ms.iter_morphologies(hard_cache=True)
-                )
+                + ", ".join(f"'{m.meta['name']}'" for m in ms.iter_morphologies(hard_cache=True))
                 + "]"
             )
 
     def test_load_no_rotations(self):
-        self.network.cell_types.test_cell.spatial.morphologies.append(
-            dict(names=["test_cell_A", "test_cell_B"])
-        )
+        self.network.cell_types.test_cell.spatial.morphologies.append(dict(names=["test_cell_A", "test_cell_B"]))
         mA = Morphology.from_swc(get_morphology_path("2branch.swc"))
         mB = Morphology.from_swc(get_morphology_path("2comp.swc"))
         self.network.morphologies.save("test_cell_A", mA, overwrite=True)
@@ -304,9 +287,7 @@ def test_load_no_rotations(self):
         self.assertEqual(len(ps), len(rot), "expected equal amounts of rotations")
 
     def test_load_rotations(self):
-        self.network.cell_types.test_cell.spatial.morphologies.append(
-            dict(names=["test_cell_A", "test_cell_B"])
-        )
+        self.network.cell_types.test_cell.spatial.morphologies.append(dict(names=["test_cell_A", "test_cell_B"]))
         self.network.placement.ch4_c25.distribute.rotations = dict(strategy="random")
         mA = Morphology.from_swc(get_morphology_path("2branch.swc"))
         mB = Morphology.from_swc(get_morphology_path("2comp.swc"))
@@ -345,8 +326,7 @@ def test_4chunks_25cells(self):
         self.assertClose(
             pos_sort,
             pspos_sort,
-            "Network was compiled with FixedPositions,"
-            + " but different positions were found.",
+            "Network was compiled with FixedPositions," + " but different positions were found.",
         )
 
     @single_process_test
@@ -386,9 +366,7 @@ def test_list_input(self):
                 "PlacementSet failed to append `list` typed data. PlacementSets should"
                 + " allow this short form to work: `.append_data([0, 0, 0], [[1,1,1]])`"
             )
-        self.assertEqual(
-            1, len(ps), f"PlacementSet placement {len(ps)} after 1 list type input"
-        )
+        self.assertEqual(1, len(ps), f"PlacementSet placement {len(ps)} after 1 list type input")
 
     def test_label(self):
         self.network.compile()
@@ -454,9 +432,7 @@ def test_swc_saveload(self):
                 m = Morphology.from_swc(path)
                 self.mr.save("X", m, overwrite=True)
                 lm = self.mr.load("X")
-                self.assertEqual(
-                    len(m.branches), len(lm.branches), "num branches changed"
-                )
+                self.assertEqual(len(m.branches), len(lm.branches), "num branches changed")
                 self.assertEqual(
                     m.points.shape,
                     lm.points.shape,
@@ -482,9 +458,7 @@ def test_swc_ldc_mdc(self):
                 self.assertIn("ldc", m.meta, "missing ldc in loaded morphology")
 
 
-class TestConnectivitySet(
-    FixedPosConfigFixture, RandomStorageFixture, NumpyTestCase, engine_name=None
-):
+class TestConnectivitySet(FixedPosConfigFixture, RandomStorageFixture, NumpyTestCase, engine_name=None):
     def setUp(self):
         super().setUp()
         self.cfg.connectivity.add(
@@ -499,24 +473,16 @@ def setUp(self):
         self.network.compile(clear=True)
 
     def test_require(self):
-        ct = self.network.cell_types.add(
-            "new_cell", dict(spatial=dict(radius=2, density=1e-3))
-        )
-        self.network.require_connectivity_set(
-            ct, self.network.cell_types.test_cell, "test"
-        )
+        ct = self.network.cell_types.add("new_cell", dict(spatial=dict(radius=2, density=1e-3)))
+        self.network.require_connectivity_set(ct, self.network.cell_types.test_cell, "test")
         self.assertTrue(
             self.storage._ConnectivitySet.exists(self.storage._engine, "test"),
             "must exist after require",
         )
 
     def test_attrs(self):
-        ct = self.network.cell_types.add(
-            "new_cell", dict(spatial=dict(radius=2, density=1e-3))
-        )
-        self.network.require_connectivity_set(
-            ct, self.network.cell_types.test_cell, "test"
-        )
+        ct = self.network.cell_types.add("new_cell", dict(spatial=dict(radius=2, density=1e-3)))
+        self.network.require_connectivity_set(ct, self.network.cell_types.test_cell, "test")
         cs = self.storage._ConnectivitySet(self.storage._engine, "test")
         for attr in ("tag", "pre_type_name", "post_type_name"):
             with self.subTest(attr=attr):
@@ -562,9 +528,7 @@ def test_local(self):
 
     @single_process_test
     def test_connect_connect(self):
-        ct = self.network.cell_types.add(
-            "new_cell", dict(spatial=dict(radius=2, density=1e-3))
-        )
+        ct = self.network.cell_types.add("new_cell", dict(spatial=dict(radius=2, density=1e-3)))
         self.network.place_cells(ct, [[0, 0, 0], [1, 1, 1], [2, 2, 2]], chunk=[0, 0, 0])
         self.network.place_cells(ct, [[3, 3, 3], [4, 4, 4]], chunk=[0, 0, 1])
         ps0 = self.network.get_placement_set(ct, [[0, 0, 0]])
@@ -578,9 +542,7 @@ def test_connect_connect(self):
             "After connecting empty data, the ConnectivitySet should remain empty.",
         )
         cs.connect(ps0, ps1, [[1, -1, -1]], [[1, -1, -1]])
-        self.assertEqual(
-            1, len(cs), "After making 1 connection, the ConnectivitySet should be len 1."
-        )
+        self.assertEqual(1, len(cs), "After making 1 connection, the ConnectivitySet should be len 1.")
         data = [*cs.flat_iter_connections("out")]
         self.assertEqual(
             1,
@@ -643,9 +605,7 @@ def test_nested_iter(self):
                     try:
                         locals_, globals_ = data
                     except TypeError:
-                        self.fail(
-                            "`nested_iter_connections` return value should be unpackable"
-                        )
+                        self.fail("`nested_iter_connections` return value should be unpackable")
                     except ValueError:
                         self.fail("`nested_iter_connections` should return 2 data values")
                     self.assertClose(625, len(locals_), "expected 625 local locs")
@@ -700,28 +660,20 @@ def check_a2a_flat_iter(self, itr, dirs, lcount, gcount):
             f"expected {', '.join(dirs)} blocks",
         )
         for dir in dirs:
-            self.assertEqual(
-                perdir, spies["dirs"][dir], f"expected {perdir} {dir} blocks"
-            )
+            self.assertEqual(perdir, spies["dirs"][dir], f"expected {perdir} {dir} blocks")
         local_counts = dict(spies["lchunks"].items())
-        self.assertEqual(
-            lcount, len(list(local_counts.keys())), f"expected {lcount} local chunks"
-        )
+        self.assertEqual(lcount, len(list(local_counts.keys())), f"expected {lcount} local chunks")
         self.assertClose(
             dircount * gcount,
             list(local_counts.values()),
-            "expected each local chunk to occur"
-            f" {dircount} x {gcount} times: {local_counts}",
+            "expected each local chunk to occur" f" {dircount} x {gcount} times: {local_counts}",
         )
         global_counts = dict(spies["gchunks"].items())
-        self.assertEqual(
-            gcount, len(list(global_counts.keys())), f"expected {gcount} global chunks"
-        )
+        self.assertEqual(gcount, len(list(global_counts.keys())), f"expected {gcount} global chunks")
         self.assertClose(
             dircount * lcount,
             list(global_counts.values()),
-            "expected each global chunk to occur"
-            f" {dircount} x {lcount} times: {global_counts}",
+            "expected each global chunk to occur" f" {dircount} x {lcount} times: {global_counts}",
         )
         self.assertClose(
             2,

diff --git a/bsb_test/parallel.py b/bsb_test/parallel.py
@@ -1,9 +1,9 @@
-import unittest as _unittest
+import http.client as _http
 import inspect as _inspect
 import threading as _threading
-import http.client as _http
-from bsb.services import MPI
+import unittest as _unittest
 
+from bsb.services import MPI
 
 _mpi_size = MPI.get_size()
 
@@ -90,7 +90,9 @@ def timed_f(*args, **kwargs):
                     raise e
             except Exception as e:
                 if MPI.get_size() > 1:
-                    import traceback, sys
+                    import sys
+                    import traceback
+
                     errlines = traceback.format_exception(type(e), e, e.__traceback__)
                     print(
                         "--- EXCEPTION UNDER MPI (ABORTING) ---\n",

diff --git a/pyproject.toml b/pyproject.toml
@@ -12,10 +12,13 @@ dynamic = ["version", "description"]
 dependencies = ["bsb-core>=4.0.0b0,<=4.0.0b9999"]
 
 [project.optional-dependencies]
-test = ["bsb-core>=4.0.0b0,<=4.0.0b9999", "bsb-hdf5>=1.0.0b0"]
+test = ["bsb-core>=4.0.0b0,<=4.0.0b9999", "bsb-hdf5>=1.0.0b0", "coverage~=7.0"]
 
 [tool.flit.module]
 name = "bsb_test"
 
 [tool.black]
-line-length=120
+line-length=120
+
+[tool.isort]
+profile = "black"