Dev

.gitignore

@@ -161,4 +161,5 @@ cython_debug/
 data
 .snakemake
 analysed
-.history
+.history
+results

.vscode/launch.json

@@ -0,0 +1,35 @@
+{
+    "version": "0.2.0",
+    "configurations": [
+      {
+        "name": "Snakemake Debug",
+        "type": "python",
+        "request": "launch",
+        "module": "snakemake",
+        // "console": "integratedTerminal",
+        "args": [
+          "--snakefile",
+          "${workspaceFolder}/Snakefile",
+        //   "--configfile",
+        //   "${workspaceFolder}/config.yaml",
+          "--cores",
+          "1",
+          "--dry-run",
+          "--printshellcmds",
+          "--debug-dag",
+          "--reason",
+          "--forceall",
+          "--nocolor",
+          "--nolock"
+        ],
+        "env": {"PYTHONIOENCODING": "utf-8"},
+        // "stopOnEntry": false,  
+        "console": "integratedTerminal",
+        "justMyCode": false,
+        "cwd": "${fileDirname}",
+        // "python": "${command:python.defaultInterpreterPath}"
+        // "python": "${command:python.interpreterPath}"
+      }
+    ]
+  }
+

.vscode/settings.json

@@ -0,0 +1,3 @@
+{
+    "python.formatting.provider": "black"
+}

Snakefile

@@ -1,14 +1,10 @@
 # report: "report/workflow.rst"
-from snakemake.remote.zenodo import RemoteProvider
+from snakemake.remote.zenodo import RemoteProvider,
 import os
 
 from snakemake.remote import AUTO
 
 # from snakemake.remote.FTP import RemoteProvider as FTPRemoteProvider
-import pandas as pd
-import os
-import dask.dataframe as dd
-import pandas as pd
 
 
 import os
@@ -20,10 +16,10 @@ envvars:
     "BIOSTUDIES_FTP_SERVER",
 
 
-access_token = os.environ["ZENODO_ACCESS_TOKEN"]
-
+zenodo_access_token = os.environ["ZENODO_ACCESS_TOKEN"]
+deposition_id = 7267108
 
-zenodo = RemoteProvider(access_token=access_token)
+zenodo = RemoteProvider(deposition=deposition_id, access_token=zenodo_access_token)
 
 
 CSV_VARIANTS = ["FilteredNuclei", "Image"]
@@ -71,8 +67,7 @@ def aggregate_input_stardist(wildcards):
         "data/cellesce_2d/{images}/projection_XY_16_bit.tif"
     )
     return expand(
-        "analysed/stardist_inference/{images}/labels.png", images_in=images_glob
-    )
+        "analysed/stardist_inference/{images}/labels.png", images_in=images_glob)
 
 
 def aggregate_decompress_images(wildcards):
@@ -84,7 +79,7 @@ def aggregate_decompress_images(wildcards):
     # checkpoints.move_data.get(images_raw=images_raw,images=images,**wildcards)
     return expand(
         "analysed/data/images/temp/{images_raw}/projection_XY_16_bit.chkpt",
-        images_raw=images_raw,
+images_raw=images_raw,
     )
 
 
@@ -161,13 +156,15 @@ checkpoint move_data:
         folder=IMAGES_IN_DIR,
         file="data/cellesce_2d/{images_raw}/projection_XY_16_bit.tif",
     output:
-        checkpoint="analysed/data/images/temp/{images_raw}/projection_XY_16_bit.chkpt",
+        checkpoint_dir = directory("analysed/data/images/temp/{images_raw}/"),
+        checkpoint=touch("analysed/data/images/temp/{images_raw}/projection_XY_16_bit.chkpt"),
     params:
         file_name=lambda wildcards: "analysed/data/images/"
         + (wildcards.images_raw).replace("/", "_").replace(" ", "_")
         + ".tif",
     shell:
         """
+        mkdir -p '{params.checkpoint_dir}'
         cp -n '{input.file}' '{params.file_name}'
         touch '{output.checkpoint}'
         """
@@ -344,12 +341,17 @@ rule upload:
             feature_inclusions=FEATURE_INCLUSIONS,
             csv_variants=CSV_VARIANTS,
         ),
+    params:
+        # access_token=zenodo_access_token,
+        # deposition_id=deposition_id, # replace with your actual deposition ID
     output:
-        # zip_file="results_csv.zip",
+        zip_file=temp("data/results_csv.zip"),
         remote=zenodo.remote("results_csv.zip"),
+        # remote=zenodo.remote("zenodo://{access_token}/{deposition_id}/results_csv.zip"),
     shell:
         """
-        zip {output.remote} {input}
+        zip {output.zip_file} {input}
+        cp {output.remote} {output.zip_file}
         """
 
 

config/config.yaml

@@ -0,0 +1,35 @@
+data_in: data/cellesce_2d
+data_out: results/cellesce_2d
+# images_in: "{config['data_in']}/{images}/projection_XY_16_bit.tif"
+
+ext: projection_XY_16_bit.tif
+
+csv_variants:
+  - FilteredNuclei
+  - Image
+feature_inclusions:
+  - all
+  - objects
+
+cellprofiler_files:
+  - all_Experiment
+  - all_FilteredNuclei
+  - all_Image
+  - all_IdentifySecondaryObjects
+  - all_nuclei_objects
+  - objects_Experiment
+  - objects_FilteredNuclei
+  - objects_Image
+  - objects_IdentifySecondaryObjects
+  - objects_nuclei_objects
+
+models:
+  # - stardist
+  - splinedist
+  - unet
+
+
+inference_segmentation_cpconfig:
+  splinedist: cellprofiler/instance_cp4.cpproj
+  stardist: cellprofiler/instance_cp4.cpproj
+  unet: cellprofiler/unet_cp4_3_class.cpproj

splinedist/environment.yaml

@@ -0,0 +1,47 @@
+name: splinedist
+channels:
+  - anaconda
+  - conda-forge
+  - bioconda
+  - defaults
+  # - nvidia
+dependencies:
+  - conda
+  - python=3.8
+  - cudatoolkit
+  - pip=21
+  - pandas
+  # - mamba
+  - pytest
+  - tensorflow
+  - tensorflow-gpu
+  - tifffile
+  - nbmake
+  - scikit-image
+  # - nb_conda_kernels
+  - ipykernel
+  - ipywidgets
+  - ipython
+  - numpy
+  - tqdm
+  - scikit-learn
+  - opencv
+  - mesa-libgl-devel-cos7-x86_64
+  - mesa-libgl-cos6-x86_64
+  - libopencv
+  # - py-opencv
+  - ipython
+  # - jupyter
+  - nbconvert
+  - matplotlib
+  # - scipy
+  - cudnn
+  # - cxx-compilern
+  # - gcc_linux-64
+  # - nvcc_linux-64
+  # - nccl
+  # - nvidia-ml
+  - pip:
+    - git+https://github.com/uhlmanngroup/splinedist.git
+    # - -e "../../splinedist/splinedist"
+    # - stardist==0.6.2

splinedist/infer.py

@@ -0,0 +1,175 @@
+from __future__ import print_function, unicode_literals, absolute_import, division
+import sys
+import numpy as np
+
+# matplotlib.rcParams["image.interpolation"] = None
+import matplotlib.pyplot as plt
+
+# %matplotlib inline
+# %config InlineBackend.figure_format = 'retina'
+from tifffile import imread
+from csbdeep.utils import Path, normalize
+from splinedist.models import SplineDist2D
+
+
+from splinedist import random_label_cmap
+
+
+# import tensorflow.compat.v1 as tf
+# tf.disable_v2_behavior()
+
+# config = tf.ConfigProto()
+# config.gpu_options.per_process_gpu_memory_fraction = 0.5
+# tf.keras.backend.set_session(tf.Session(config=config));
+
+np.random.seed(6)
+lbl_cmap = random_label_cmap()
+
+import os
+
+os.environ["CUDA_VISIBLE_DEVICES"] = "-1"
+# import os
+# os.environ['CUDA_VISIBLE_DEVICES']="0"
+# %config Completer.use_jedi = False
+
+import pandas as pd
+
+import argparse
+
+parser = argparse.ArgumentParser(description="Process some integers.")
+parser.add_argument("--image_in", "--image_in", type=str, help="images_dir")
+parser.add_argument("--figure_out", "--figure_out", type=str, help="figure_out")
+parser.add_argument(
+    "--model_path", "--model_path", default="models", type=str, help="model"
+)
+parser.add_argument(
+    "--model_name", "--model_name", default="model_name", type=str, help="model_name"
+)
+parser.add_argument("--instance", "--instance", type=str, help="instance")
+parser.add_argument(
+    "--control_points", "--control_points", type=str, help="control_points"
+)
+parser.add_argument("--raw_image", "--raw_image", type=str, help="raw_image")
+args = parser.parse_args()
+
+image_in = args.image_in
+model_path = args.model_path
+model_name = args.model_name
+figure_out = args.figure_out
+instance = args.instance
+control_points = args.control_points
+raw_image = args.raw_image
+
+print(args)
+
+X = [image_in]
+X = list(map(imread, X))
+# plt.imshow(X[0])
+# plt.show()
+
+n_channel = 1 if X[0].ndim == 2 else X[0].shape[-1]
+axis_norm = (0, 1)  # normalize channels independently
+# axis_norm = (0,1,2) # normalize channels jointly
+if n_channel > 1:
+    print(
+        "Normalizing image channels %s."
+        % ("jointly" if axis_norm is None or 2 in axis_norm else "independently")
+    )
+
+# plt.imshow(X[0])
+# plt.show()
+
+
+# show all test images
+if False:
+    fig, ax = plt.subplots(7, 8, figsize=(16, 16))
+    for i, (a, x) in enumerate(zip(ax.flat, X)):
+        a.imshow(x if x.ndim == 2 else x[..., 0], cmap="gray")
+        a.set_title(i)
+    [a.axis("off") for a in ax.flat]
+    plt.tight_layout()
+
+
+model = SplineDist2D(None, name=model_name, basedir=model_path)
+
+image_dims = (2048, 2048)
+
+im = X[0]
+pad_width = (np.subtract(image_dims, im.shape)) / 2
+pad_width_x, pad_width_y = pad_width
+pad_vector = np.rint(
+    [
+        [np.floor(pad_width_x), np.ceil(pad_width_x)],
+        [np.floor(pad_width_y), np.ceil(pad_width_y)],
+    ]
+).astype(int)
+im_padded = np.pad(
+    im, pad_width=pad_vector, mode="constant", constant_values=((0, 0), (0, 0))
+)
+
+img = normalize(im_padded, 1, 99.8, axis=axis_norm)
+labels, details = model.predict_instances(img)
+
+
+# plt.figure(figsize=(16,16))
+# plt.imshow(img if img.ndim==2 else img[...,0], clim=(0,1))
+# plt.imshow(labels, cmap=lbl_cmap, alpha=0.5)
+# plt.axis('off');
+
+
+# masks = []
+# controlpoints = []
+
+# # for i in tqdm(range(len(X))):
+# img = normalize(X[0], 1,99.8, axis=axis_norm)
+# labels, details = model.predict_instances(img)
+
+coord = details["coord"]
+
+
+# %%
+# def example(model, i, show_dist=True):
+#     img = normalize(X[i], 1,99.8, axis=axis_norm)
+#     labels, details = model.predict_instances(img)
+
+#     plt.figure(figsize=(13,10))
+#     img_show = img if img.ndim==2 else img[...,0]
+#     coord, points, prob = details['coord'], details['points'], details['prob']
+#     print(coord.shape, points.shape, prob.shape)
+#     plt.subplot(121); plt.imshow(img_show, cmap='gray'); plt.axis('off')
+#     a = plt.axis()
+#     _draw_polygons(coord, points, prob, show_dist=show_dist)
+#     plt.axis(a)
+#     plt.subplot(122); plt.imshow(img_show, cmap='gray'); plt.axis('off')
+#     plt.imshow(labels, cmap=lbl_cmap, alpha=0.5)
+#     plt.tight_layout()
+#     plt.show()
+
+# cellesce_results = [X_ids, controlpoints]
+# cellesce_results = np.array(cellesce_results, dtype = 'object')
+from skimage import io
+
+io.imsave(instance, labels)
+io.imsave(raw_image, im_padded)
+
+# print(coord)
+# print(coord.shape)
+
+# mask_num,y,x = X.shape
+# in your case
+# a,b,c = 1797, 500
+# print(pd.DataFrame.from_records(coord))
+mask_num, y, x = coord.shape
+
+df = pd.DataFrame(
+    data=coord.flatten(),
+    index=pd.MultiIndex.from_product(
+        [np.arange(0, mask_num), np.arange(0, y), np.arange(0, x)],
+        names=["mask_num", "y", "x"],
+    ),
+    columns=["Value"],
+)
+# print(df)
+
+# np.save(control_points,coord)
+df.to_csv(control_points)

splinedist/models/cellesce_M16/config.json

@@ -0,0 +1 @@
+{"n_dim": 2, "axes": "YXC", "n_channel_in": 1, "n_channel_out": 33, "train_checkpoint": "weights_best.h5", "train_checkpoint_last": "weights_last.h5", "train_checkpoint_epoch": "weights_now.h5", "n_params": 32, "grid": [2, 2], "backbone": "unet", "contoursize_max": 200, "unet_n_depth": 3, "unet_kernel_size": [3, 3], "unet_n_filter_base": 32, "unet_n_conv_per_depth": 2, "unet_pool": [2, 2], "unet_activation": "relu", "unet_last_activation": "relu", "unet_batch_norm": false, "unet_dropout": 0.0, "unet_prefix": "", "net_conv_after_unet": 128, "net_input_shape": [null, null, 1], "net_mask_shape": [null, null, 1], "train_shape_completion": false, "train_completion_crop": 32, "train_patch_size": [256, 256], "train_background_reg": 0.0001, "train_foreground_only": 0.9, "train_dist_loss": "mae", "train_loss_weights": [1, 0.2], "train_epochs": 400, "train_steps_per_epoch": 100, "train_learning_rate": 0.0003, "train_batch_size": 4, "train_n_val_patches": null, "train_tensorboard": true, "train_reduce_lr": {"factor": 0.5, "patience": 40, "min_delta": 0}, "use_gpu": false}