Skip to content

Commit

Permalink
Merge pull request #15 from pgleeson/main
Browse files Browse the repository at this point in the history 
Lots more info on cell types
  • Loading branch information
@pgleeson
pgleeson authored Aug 9, 2024
2 parents ace538e + 6262c02 commit 25cbcca
Show file tree
Hide file tree
Showing 187 changed files with 69,754 additions and 44,258 deletions.
647 changes: 500 additions & 147 deletions cect/Cells.py
View file

Large diffs are not rendered by default.

325 changes: 173 additions & 152 deletions cect/Comparison.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2,16 +2,17 @@

from cect.ConnectomeReader import analyse_connections
from cect.ConnectomeReader import check_neurons
from cect.Cells import get_cell_link
from cect.Cells import get_cell_internal_link
from cect import print_
import json

import sys
import numpy as np

import importlib

all_data = {}

quick = len(sys.argv) > 1 and eval(sys.argv[1])

reader_pages = {
"TestData": "Test_data",
Expand Down Expand Up @@ -39,24 +40,6 @@
]


readers = {
"SSData": ["cect.SpreadsheetDataReader", None],
"UpdSSData": ["cect.UpdatedSpreadsheetDataReader", None],
"UpdSSData2": ["cect.UpdatedSpreadsheetDataReader2", None],
"Varshney": ["cect.VarshneyDataReader", "Varshney_2011"],
"White_L4": ["cect.White_L4", "White_1986"],
"White_whole": ["cect.White_whole", "White_1986"],
"TestData": ["cect.TestDataReader", None],
"Cook2020": ["cect.Cook2020DataReader", "Cook_2020"],
}
if not quick:
readers["White_A"] = ["cect.White_A", "White_1986"]
readers["Witvliet1"] = ["cect.WitvlietDataReader1", "Witvliet_2021"]
readers["Witvliet2"] = ["cect.WitvlietDataReader2", "Witvliet_2021"]
readers["WormNeuroAtlas"] = ["cect.WormNeuroAtlasReader", "Randi_2023"]
readers["Cook2019Herm"] = ["cect.Cook2019HermReader", "Cook_2019"]


def shorten_neurotransmitter(nt):
return (
nt.replace("Acetylcholine", "ACh")
Expand All @@ -72,7 +55,7 @@ def _format_json(json_str):
return json.dumps(json.loads(json_str), sort_keys=True, indent=2)


def get_2d_graph_markdown(reader_name, view_name, connectome, synclass, indent=" "):
def get_2d_graph_markdown(reader_name, view, connectome, synclass, indent=" "):
view_name = view.name

fig = connectome.to_plotly_graph_fig(synclass, view)
Expand Down Expand Up @@ -130,168 +113,206 @@ def get_matrix_markdown(reader_name, view_name, connectome, synclass, indent="
)


for reader_name, reader_info in readers.items():
reader = reader_info[0]
decription_page = reader_info[1] if len(reader_info) > 1 else None
def generate_comparison_page(quick: bool):
connectomes = {}

print_("\n****** Importing dataset %s using %s ******" % (reader_name, reader))
readers = {
"SSData": ["cect.SpreadsheetDataReader", None],
"UpdSSData": ["cect.UpdatedSpreadsheetDataReader", None],
"UpdSSData2": ["cect.UpdatedSpreadsheetDataReader2", None],
"Varshney": ["cect.VarshneyDataReader", "Varshney_2011"],
"White_whole": ["cect.White_whole", "White_1986"],
"TestData": ["cect.TestDataReader", None],
"Cook2020": ["cect.Cook2020DataReader", "Cook_2020"],
}
if not quick:
readers["White_A"] = ["cect.White_A", "White_1986"]
readers["White_L4"] = ["cect.White_L4", "White_1986"]
readers["Witvliet1"] = ["cect.WitvlietDataReader1", "Witvliet_2021"]
readers["Witvliet2"] = ["cect.WitvlietDataReader2", "Witvliet_2021"]
readers["WormNeuroAtlas"] = ["cect.WormNeuroAtlasReader", "Randi_2023"]
readers["Cook2019Herm"] = ["cect.Cook2019HermReader", "Cook_2019"]

exec("from %s import read_data, read_muscle_data, READER_DESCRIPTION" % reader)
cells, neuron_conns = read_data(include_nonconnected_cells=True)
for reader_name, reader_info in readers.items():
reader = reader_info[0]
decription_page = reader_info[1] if len(reader_info) > 1 else None

preferred, not_in_preferred, missing_preferred = check_neurons(cells)
print_("\n****** Importing dataset %s using %s ******" % (reader_name, reader))

neuron_nts = {}
reader_module = importlib.import_module(reader)

connectome = None
try:
exec("from %s import get_instance" % reader)
connectome = get_instance()
print_("Adding full connectome info")
except:
print_("NOT adding full connectome info")
# exec("from %s import read_data, read_muscle_data, READER_DESCRIPTION" % reader)
cells, neuron_conns = reader_module.read_data(include_nonconnected_cells=True)

preferred, not_in_preferred, missing_preferred = check_neurons(cells)

neuron_nts = {}
connectome = None

for c in neuron_conns:
nt = c.synclass
if len(nt) == 0:
nt = "**MISSING**"
try:
connectome = reader_module.get_instance()
print_("Adding full connectome info: %s" % connectome)

if not nt in neuron_nts:
neuron_nts[nt] = 0
except:
print_("NOT adding full connectome info")
connectome = None

neuron_nts[nt] += 1
for c in neuron_conns:
nt = c.synclass
if len(nt) == 0:
nt = "**MISSING**"

nts_info = ""
for nt in sorted(neuron_nts.keys()):
nts_info += "%s (%i)<br/>" % (shorten_neurotransmitter(nt), neuron_nts[nt])
if not nt in neuron_nts:
neuron_nts[nt] = 0

neurons2muscles, muscles, muscle_conns = read_muscle_data()
neuron_nts[nt] += 1

muscle_nts = {}
for c in muscle_conns:
nt = c.synclass
if len(nt) == 0:
nt = "**MISSING**"
nts_info = ""
for nt in sorted(neuron_nts.keys()):
nts_info += "%s (%i)<br/>" % (shorten_neurotransmitter(nt), neuron_nts[nt])

if not nt in muscle_nts:
muscle_nts[nt] = 0
neurons2muscles, muscles, muscle_conns = reader_module.read_muscle_data()

muscle_nts[nt] += 1
muscle_nts = {}
for c in muscle_conns:
nt = c.synclass
if len(nt) == 0:
nt = "**MISSING**"

m_nts_info = ""
for nt in sorted(muscle_nts):
m_nts_info += "%s (%i)<br/>" % (shorten_neurotransmitter(nt), muscle_nts[nt])
if not nt in muscle_nts:
muscle_nts[nt] = 0

ref = (
"[%s](%s.md)" % (reader_name, reader_pages[reader_name])
if reader_name in reader_pages
else reader_name
)
muscle_nts[nt] += 1

if reader_name in reader_pages:
matrix_filename = "docs/%s.md" % reader_pages[reader_name]
graph_filename = "docs/%s_graph.md" % reader_pages[reader_name]
m_nts_info = ""
for nt in sorted(muscle_nts):
m_nts_info += "%s (%i)<br/>" % (
shorten_neurotransmitter(nt),
muscle_nts[nt],
)

for filename in [matrix_filename, graph_filename]:
with open(filename, "w") as f:
matrix = filename == matrix_filename
ref = (
"[%s](%s.md)" % (reader_name, reader_pages[reader_name])
if reader_name in reader_pages
else reader_name
)

f.write("## %s\n" % reader_name)
if decription_page is not None:
f.write(
"[Source publication of dataset](%s.md)\n\n" % decription_page
)
if reader_name in reader_pages:
connectomes[reader_name] = connectome

matrix_filename = "docs/%s.md" % reader_pages[reader_name]
graph_filename = "docs/%s_graph.md" % reader_pages[reader_name]

for filename in [graph_filename, matrix_filename]:
with open(filename, "w") as f:
matrix = filename == matrix_filename

f.write("%s\n\n" % READER_DESCRIPTION)
f.write("## %s\n" % reader_name)
if decription_page is not None:
f.write(
"[Source publication of dataset](%s.md)\n\n"
% decription_page
)

f.write(
"[View as matrix](%s.md){ .md-button } [View as graph](%s_graph.md){ .md-button }\n\n"
% (reader_pages[reader_name], reader_pages[reader_name])
)
f.write("%s\n\n" % reader_module.READER_DESCRIPTION)

f.write(
"[View as graph](%s_graph.md){ .md-button } [View as matrix](%s.md){ .md-button }\n\n"
% (reader_pages[reader_name], reader_pages[reader_name])
)

if connectome is not None:
from ConnectomeView import ALL_VIEWS
if connectome is not None:
from cect.ConnectomeView import ALL_VIEWS

indent = " "
indent = " "

for view in ALL_VIEWS:
cv = connectome.get_connectome_view(view)
for view in ALL_VIEWS:
cv = connectome.get_connectome_view(view)

f.write('=== "%s"\n' % view.name)
f.write('=== "%s"\n' % view.name)

for sc in view.synclass_sets:
f.write(indent + '=== "%s"\n' % sc)
for sc in view.synclass_sets:
f.write(indent + '=== "%s"\n' % sc)

if matrix:
f.write(
get_matrix_markdown(
reader_name,
view.name,
cv,
sc,
indent=indent + indent,
if matrix:
f.write(
get_matrix_markdown(
reader_name,
view.name,
cv,
sc,
indent=indent + indent,
)
)
)

else:
f.write(
get_2d_graph_markdown(
reader_name,
view,
cv,
sc,
indent=indent + indent,

else:
f.write(
get_2d_graph_markdown(
reader_name,
view,
cv,
sc,
indent=indent + indent,
)
)
)

cell_types = {
"Neurons": preferred,
"Missing neurons": missing_preferred,
"Muscles": muscles,
"Other cells": not_in_preferred,
}

for t in cell_types:
f.write("\n### %s (%i)\n" % (t, len(cell_types[t])))
if len(cell_types[t]) > 0:
f.write("<details><summary>Full list of %s</summary>\n" % t)
ss = sorted(cell_types[t])
for n in ss:
f.write("%s" % (get_cell_link(n, True)))
if n is not ss[-1]:
f.write(" | ")

f.write("\n</details>\n")

print_("Written page: %s" % filename)

all_data[ref] = [
len(preferred),
len(missing_preferred),
len(not_in_preferred),
len(muscles),
len(neuron_conns),
len(neurons2muscles),
len(muscle_conns),
nts_info,
m_nts_info,
]

print_("\nFinished loading all the data from the readers!")

import pandas as pd
import numpy as np

df_all = pd.DataFrame(all_data).transpose()
# df_all.set_index("Values")
cell_types = {
"Neurons": preferred,
"Missing neurons": missing_preferred,
"Muscles": muscles,
"Other cells": not_in_preferred,
}

for t in cell_types:
f.write("\n### %s (%i)\n" % (t, len(cell_types[t])))
if len(cell_types[t]) > 0:
f.write("<details><summary>Full list of %s</summary>\n" % t)
ss = sorted(cell_types[t])
for n in ss:
f.write("%s" % (get_cell_internal_link(n, True)))
if n is not ss[-1]:
f.write(" | ")

f.write("\n</details>\n")

print_("Written page: %s" % filename)

all_data[ref] = [
len(preferred),
len(missing_preferred),
len(not_in_preferred),
len(muscles),
len(neuron_conns),
len(neurons2muscles),
len(muscle_conns),
nts_info,
m_nts_info,
]

print_("\nFinished loading all the data from the readers!")

import pandas as pd
import numpy as np

df_all = pd.DataFrame(all_data).transpose()
# df_all.set_index("Values")

# h = HTML(df_all.to_html(escape=False, index=False))

mk = df_all.to_markdown()

filename = "docs/Comparison.md"
with open(filename, "w") as f:
f.write(mk)

print_("Written page: %s" % filename)

return connectomes

# h = HTML(df_all.to_html(escape=False, index=False))

mk = df_all.to_markdown()
if __name__ == "__main__":
quick = len(sys.argv) > 1 and eval(sys.argv[1])

filename = "docs/Comparison.md"
with open(filename, "w") as f:
f.write(mk)
connectomes = generate_comparison_page(quick)

print_("Written page: %s" % filename)
print("Finished. All loaded connectomes:\n%s" % connectomes)
Loading

0 comments on commit 25cbcca

Please sign in to comment.