feat: parallel

automated_test.py

@@ -19,9 +19,10 @@ def gt_c2f(gt):
   f_gt[ f_gt == mx ] = 3
   return f_gt
 
+@pytest.mark.parametrize("parallel", (1,2))
 @pytest.mark.parametrize("connectivity", (4, 6, 8, 18, 26))
 @pytest.mark.parametrize("dtype", TEST_TYPES)
-def test_2d_square(dtype, connectivity):
+def test_2d_square(dtype, connectivity, parallel):
   def test(order, ground_truth):
     input_labels = np.zeros( (16,16), dtype=dtype, order=order )
     input_labels[:8,:8] = 8
@@ -30,7 +31,7 @@ def test(order, ground_truth):
     input_labels[8:,8:] = 11
 
     output_labels = cc3d.connected_components(
-      input_labels, connectivity=connectivity
+      input_labels, connectivity=connectivity, parallel=parallel
     ).astype(dtype)
 
     print(output_labels)
@@ -229,13 +230,16 @@ def test(order, ground_truth):
   test("C", ground_truth)
   test("F", gt_c2f(ground_truth))
 
+@pytest.mark.parametrize("parallel", (1,2))
 @pytest.mark.parametrize("order", ('C', 'F'))
 @pytest.mark.parametrize("connectivity", (6,18,26))
-def test_3d_all_different(order, connectivity):
+def test_3d_all_different(order, connectivity, parallel):
   input_labels = np.arange(0, 100 * 99 * 98).astype(np.uint32) + 1
   input_labels = input_labels.reshape((100,99,98), order=order)
 
-  output_labels = cc3d.connected_components(input_labels, connectivity=connectivity)
+  output_labels = cc3d.connected_components(
+    input_labels, connectivity=connectivity, parallel=parallel
+  )
 
   assert np.unique(output_labels).shape[0] == 100*99*98
   assert output_labels.shape == (100, 99, 98)
@@ -442,7 +446,8 @@ def test_max_labels_nonsensical():
   assert np.all(real_labels == zero_labels)
   assert np.all(real_labels == negative_labels)
 
-def test_compare_scipy_26():
+@pytest.mark.parametrize("parallel", (1,2))
+def test_compare_scipy_26(parallel):
   import scipy.ndimage.measurements
 
   sx, sy, sz = 128, 128, 128
@@ -454,7 +459,9 @@ def test_compare_scipy_26():
     [[1,1,1], [1,1,1], [1,1,1]]
   ]
 
-  cc3d_labels, Ncc3d = cc3d.connected_components(labels, connectivity=26, return_N=True)
+  cc3d_labels, Ncc3d = cc3d.connected_components(
+    labels, connectivity=26, return_N=True, parallel=parallel
+  )
   scipy_labels, Nscipy = scipy.ndimage.measurements.label(labels, structure=structure)
 
   print(cc3d_labels)

benchmarks/benchmarks.py

@@ -5,6 +5,35 @@
 import fastremap
 import time
 
+def prettyprint(tm, voxels):
+  print("{:.2f} sec. {:.2f} MVx/sec".format(tm, voxels / tm / (1000*1000)))
+
+def summary(times, voxels):
+  times = times[1:]
+  mean = sum(times) / float(len(times))
+
+  print("mean:")
+  prettyprint(mean, voxels)
+  print("stddev: %.2f" % np.std(times))
+
+structures = {
+  6: [
+    [[0,0,0], [0,1,0], [0,0,0]],
+    [[0,1,0], [1,1,1], [0,1,0]],
+    [[0,0,0], [0,1,0], [0,0,0]]
+  ],
+  18: [
+    [[0,1,0], [1,1,1], [0,1,0]],
+    [[1,1,1], [1,1,1], [1,1,1]],
+    [[0,1,0], [1,1,1], [0,1,0]]
+  ],
+  26: [
+    [[1,1,1], [1,1,1], [1,1,1]],
+    [[1,1,1], [1,1,1], [1,1,1]],
+    [[1,1,1], [1,1,1], [1,1,1]]
+  ]
+}
+
 def cc3d_mutlilabel_extraction(labels):
   res, N = cc3d.connected_components(labels, return_N=True)
   for label, extracted in tqdm(cc3d.each(labels, binary=True, in_place=True), total=N):
@@ -26,40 +55,37 @@ def ndimage_mutlilabel_extraction(labels):
     for ccid in tqdm(range(1,N+1)):
       extracted = (res == ccid)
 
-def pinky40subvol():
-  from cloudvolume import CloudVolume
-  return CloudVolume('gs://neuroglancer/wms/skeletonization/pinky40subvol', 
-    mip=2, progress=True, cache=True)[:,:,:][...,0]
+def test_binary_image_processing(labels, connectivity=26):
+  voxels = labels.size
+  times = []
 
+  for label, img in tqdm(cc3d.each(labels, binary=True, in_place=True)):
+    s = time.time()
+    cc3d.connected_components(img)
+    # scipy.ndimage.measurements.label(labels, structure=structures[connectivity])
+    e = time.time()
+    dt = e-s
+    times.append(dt)
+    prettyprint(dt, voxels)
+  summary(times, voxels)
 
+def test_multilabel_speed(labels, connectivity=26):
+  voxels = labels.size
+  times = []
 
-labels = pinky40subvol()
-cc3d_mutlilabel_extraction(labels)
-# ndimage_mutlilabel_extraction(labels)
+  for i in range(10):
+    s = time.time()
+    cc3d.connected_components(labels, connectivity=connectivity)
+    e = time.time()
+    dt = e-s
+    times.append(dt)
+    prettyprint(dt, voxels)
+  summary(times, voxels)
 
-# voxels = labels.size
-# times = []
-# # for label, img in tqdm(cc3d.each(labels, binary=False, in_place=True)):
-# #   s = time.time()
-# #   cc3d.connected_components(img)
-# #   # scipy.ndimage.measurements.label(labels, structure=structure)
-# #   e = time.time()
-# #   dt = e-s
-# #   times.append(dt)
-# #   prettyprint(dt, voxels)
+labels = np.load("connectomics.npy")
+# cc3d_mutlilabel_extraction(labels)
+# ndimage_mutlilabel_extraction(labels)
 
-# for i in range(10):
-#   s = time.time()
-#   cc3d.connected_components(labels)#, connectivity=26)
-#   # scipy.ndimage.measurements.label(labels)
-#   e = time.time()
-#   dt = e-s
-#   times.append(dt)
-#   prettyprint(dt, voxels)
+test_multilabel_speed(labels)
 
-# times = times[1:]
-# mean = sum(times) / float(len(times))
 
-# print("mean:")
-# prettyprint(mean, voxels)
-# print("stddev: %.2f" % np.std(times))