Commit 3e68086d authored by Philipp Schubert's avatar Philipp Schubert
Browse files

added `dataset_specific` directory in scripts, added pylint report and badge

parent 504f4ec2
......@@ -12,6 +12,8 @@ test:
- pytest --ignore=tests/test_run.py
except:
- master
tags:
- default
test-full:
stage: test
......@@ -27,7 +29,6 @@ test-full:
build:
stage: build
# TODO: use conda environment for test stage
script:
- conda env create -f environment.yml -n pysyintegration_m python --force
- source ~/.bashrc
......@@ -45,4 +46,21 @@ cleanup:
only:
- master
tags:
- master
\ No newline at end of file
- master
pylint:
stage: test
before_script:
- pip install pylint pylint-exit anybadge
script:
- mkdir ./pylint
- pylint --ignore=ui --ignore=batchjob_scripts --ignore=cnn --output-format=text syconn | tee ./pylint/pylint.log || pylint-exit $?
- PYLINT_SCORE=$(sed -n 's/^Your code has been rated at \([-0-9.]*\)\/.*/\1/p' ./pylint/pylint.log)
- anybadge --label=Pylint --file=pylint/pylint.svg --value=$PYLINT_SCORE 2=red 4=orange 8=yellow 10=green
- echo "Pylint score is $PYLINT_SCORE"
artifacts:
paths:
- ./pylint/
tags:
- default
allow_failure: true
\ No newline at end of file
[![Documentation Status](https://readthedocs.org/projects/syconn/badge/?version=latest)](https://syconn.readthedocs.io/en/latest/?badge=latest)
[![pipeline status](https://gitlab.mpcdf.mpg.de/pschuber/SyConn/badges/master/pipeline.svg)](https://gitlab.mpcdf.mpg.de/pschuber/SyConn/commits/master)
[![coverage report](https://gitlab.mpcdf.mpg.de/pschuber/SyConn/badges/master/coverage.svg)](https://gitlab.mpcdf.mpg.de/pschuber/SyConn/commits/master)
[![pylint status](https://gitlab.mpcdf.mpg.de/pschuber/SyConn/-/jobs/artifacts/master/raw/pylint/pylint.svg?job=pylint)](https://gitlab.mpcdf.mpg.de/pschuber/SyConn/-/jobs/artifacts/master/raw/pylint/pylint.log?job=pylint)
<img align="right" width="300" src="./docs/_static/logo_SyConn.png"><br/>
......
# -*- coding: utf-8 -*-
# SyConn - Synaptic connectivity inference toolkit
#
# Copyright (c) 2016 - now
# Max-Planck-Institute of Neurobiology, Munich, Germany
# Authors: Philipp Schubert, Joergen Kornfeld
from knossos_utils import knossosdataset
knossosdataset._set_noprint(True)
import os
import time
import argparse
import networkx as nx
import re
from knossos_utils import knossosdataset
import numpy as np
from syconn.reps.segmentation import SegmentationDataset
from syconn.proc.sd_proc import dataset_analysis
from syconn.proc.ssd_proc import map_synssv_objects
from syconn.extraction import cs_processing_steps as cps
knossosdataset._set_noprint(True)
from syconn.handler.config import initialize_logging
from syconn import global_params
from syconn.exec import exec_syns, exec_multiview, exec_skeleton, exec_init
# TODO add materialize button and store current process in config.ini
# -> allows to resume interrupted processes
if __name__ == '__main__':
parser = argparse.ArgumentParser(description='SyConn example run')
parser.add_argument('--working_dir', type=str, default='',
help='Working directory of SyConn')
args = parser.parse_args()
n_folders_fs = 10000
n_folders_fs_sc = 10000
example_wd = '/ssdscratch/pschuber/songbird/j0126/areaxfs_v10_v4b_base_' \
'20180214_full_agglo_cbsplit/'
experiment_name = 'j0126'
log = initialize_logging(experiment_name, log_dir=example_wd + '/logs/')
time_stamps = [time.time()]
step_idents = ['t-0']
global_params.wd = example_wd
# get original RAG
rag_txtfname = f"{example_wd}/v4b_base_20180214.full_agglo.cbsplit.csv"
G = nx.Graph()
with open(rag_txtfname, 'r') as f:
for l in f.readlines():
edges = [int(v) for v in re.findall(r'(\d+)', l)]
G.add_edge(edges[0], edges[1])
nx.write_edgelist(G, global_params.config.init_rag_path)
start = time.time()
# Checking models
for mpath_key in ['mpath_spiness', 'mpath_syn_rfc', 'mpath_celltype',
'mpath_axoness', 'mpath_glia']:
mpath = getattr(global_params.config, mpath_key)
if not (os.path.isfile(mpath) or os.path.isdir(mpath)):
raise ValueError('Could not find model "{}". Make sure to copy the'
' "models" folder into the current working '
'directory "{}".'.format(mpath, example_wd))
# Start SyConn
# --------------------------------------------------------------------------
log.info('Data will be processed in "{}".'.format(example_wd))
time_stamps.append(time.time())
step_idents.append('Preparation')
log.info('Step 1/8 - Creating SegmentationDatasets (incl. SV meshes) and'
' the initial RAG.')
# exec_init.run_create_rag()
time_stamps.append(time.time())
step_idents.append('SD generation')
if global_params.config.prior_glia_removal:
log.info('Step 1.5/8 - Glia separation')
exec_multiview.run_glia_rendering()
exec_multiview.run_glia_prediction(e3=True)
exec_multiview.run_glia_splitting()
time_stamps.append(time.time())
step_idents.append('Glia separation')
log.info('Step 2/8 - Creating SuperSegmentationDataset')
exec_multiview.run_create_neuron_ssd()
exec_skeleton.map_myelin_global()
time_stamps.append(time.time())
step_idents.append('SSD generation')
log.info('Step 3/8 - Synapse detection')
sd_syn_ssv = SegmentationDataset(working_dir=global_params.config.working_dir,
obj_type='syn_ssv')
# # This creates an SD of type 'syn_ssv'
cps.combine_and_split_syn_old(global_params.config.working_dir,
cs_gap_nm=global_params.config['cell_objects']['cs_gap_nm'],
log=log, n_folders_fs=n_folders_fs)
cps.extract_synapse_type(sd_syn_ssv,
kd_sym_path=global_params.config.kd_sym_path,
kd_asym_path=global_params.config.kd_asym_path,
sym_label=2, asym_label=1)
log.info('Synapse objects were created.')
dataset_analysis(sd_syn_ssv, compute_meshprops=True)
log.info(f'SegmentationDataset of type "syn_ssv" was generated with {len(sd_syn_ssv.ids)} '
f'objects.')
cps.map_objects_to_synssv(global_params.config.working_dir, log=log)
log.info('Cellular organelles were mapped to "syn_ssv".')
cps.classify_synssv_objects(global_params.config.working_dir, log=log)
log.info('Synapse prediction finished.')
log.info('Collecting and writing syn-ssv objects to SSV attribute '
'dictionary.')
# This needs to be run after `classify_synssv_objects` and before
# `map_synssv_objects` if the latter uses thresholding for synaptic objects
# just collect new data: ``recompute=False``
dataset_analysis(sd_syn_ssv, compute_meshprops=False, recompute=False)
map_synssv_objects(log=log)
log.info('Finished.')
time_stamps.append(time.time())
step_idents.append('Synapse detection')
log.info('Step 4/8 - Neuron rendering')
exec_multiview.run_neuron_rendering()
time_stamps.append(time.time())
step_idents.append('Neuron rendering')
log.info('Step 5/8 - Axon prediction')
exec_multiview.run_semsegaxoness_prediction()
exec_multiview.run_semsegaxoness_mapping()
time_stamps.append(time.time())
step_idents.append('Axon prediction')
log.info('Step 6/8 - Spine prediction')
exec_multiview.run_spiness_prediction()
exec_syns.run_spinehead_volume_calc()
time_stamps.append(time.time())
step_idents.append('Spine prediction')
log.info('Step 7/9 - Morphology extraction')
exec_multiview.run_morphology_embedding()
time_stamps.append(time.time())
step_idents.append('Morphology extraction')
log.info('Step 8/9 - Celltype analysis')
exec_multiview.run_celltype_prediction()
time_stamps.append(time.time())
step_idents.append('Celltype analysis')
log.info('Step 9/9 - Matrix export')
exec_syns.run_matrix_export()
time_stamps.append(time.time())
step_idents.append('Matrix export')
time_stamps = np.array(time_stamps)
dts = time_stamps[1:] - time_stamps[:-1]
dt_tot = time_stamps[-1] - time_stamps[0]
dt_tot_str = time.strftime("%Hh:%Mmin:%Ss", time.gmtime(dt_tot))
time_summary_str = "\nEM data analysis of experiment '{}' finished after" \
" {}.\n".format(experiment_name, dt_tot_str)
n_steps = len(step_idents[1:]) - 1
for i in range(len(step_idents[1:])):
step_dt = time.strftime("%Hh:%Mmin:%Ss", time.gmtime(dts[i]))
step_dt_perc = int(dts[i] / dt_tot * 100)
step_str = "[{}/{}] {}\t\t\t{}\t\t\t{}%\n".format(
i, n_steps, step_idents[i+1], step_dt, step_dt_perc)
time_summary_str += step_str
log.info(time_summary_str)
log.info('Setting up flask server for inspection. Annotated cell reconst'
'ructions and wiring can be analyzed via the KNOSSOS-SyConn plugin'
' at `SyConn/scripts/kplugin/syconn_knossos_viewer.py`.')
fname_server = os.path.dirname(os.path.abspath(__file__)) + \
'/../kplugin/server.py'
os.system('python {} --working_dir={} --port=10002'.format(
fname_server, example_wd))
# -*- coding: utf-8 -*-
# SyConn - Synaptic connectivity inference toolkit
#
# Copyright (c) 2016 - now
# Max-Planck-Institute of Neurobiology, Munich, Germany
# Authors: Philipp Schubert, Joergen Kornfeld
from knossos_utils import knossosdataset
knossosdataset._set_noprint(True)
import os
import time
import argparse
import networkx as nx
import re
from knossos_utils import knossosdataset
import numpy as np
from syconn.reps.segmentation import SegmentationDataset
from syconn.proc.sd_proc import dataset_analysis
from syconn.proc.ssd_proc import map_synssv_objects
from syconn.extraction import cs_processing_steps as cps
knossosdataset._set_noprint(True)
from syconn.handler.config import initialize_logging
from syconn import global_params
import shutil
from syconn.exec import exec_syns, exec_multiview, exec_skeleton, exec_init
# TODO add materialize button and store current process in config.ini
# -> allows to resume interrupted processes
if __name__ == '__main__':
parser = argparse.ArgumentParser(description='SyConn example run')
parser.add_argument('--working_dir', type=str, default='',
help='Working directory of SyConn')
args = parser.parse_args()
n_folders_fs = 10000
n_folders_fs_sc = 10000
example_wd = "/ssdscratch/pschuber/songbird/j0126/assembled_core_relabeled" \
"_base_merges_relabeled_to_v4b_base_20180214_full_agglo_cbsp" \
"lit_with_reconnects_no_soma_merger_manual_edges_removed/"
experiment_name = 'j0126'
log = initialize_logging(experiment_name, log_dir=example_wd + '/logs/')
time_stamps = [time.time()]
step_idents = ['t-0']
global_params.wd = example_wd
# get original RAG
rag_txtfname = f'{example_wd}/assembled_core_relabeled_base_merges_relabeled_to_v4b_base_' \
f'20180214.full_agglo.cbsplit_with_reconnects_Nov_19_no_soma_merger.csv'
G = nx.Graph()
with open(rag_txtfname, 'r') as f:
for l in f.readlines():
edges = [int(v) for v in re.findall(r'(\d+)', l)]
G.add_edge(edges[0], edges[1])
nx.write_edgelist(G, global_params.config.init_rag_path)
exec_init.run_create_rag()
if global_params.config.prior_glia_removal:
log.info('Step 1.5/8 - Glia separation')
exec_multiview.run_glia_rendering()
exec_multiview.run_glia_prediction(e3=True)
exec_multiview.run_glia_splitting()
time_stamps.append(time.time())
step_idents.append('Glia separation')
# remove edges from original neuron rag as returned by the glia separation
shutil.move(example_wd + '/glia/neuron_rag.bz2', example_wd + '/glia/neuron_rag_ORIG.bz2')
rag = nx.read_edgelist(example_wd + '/glia/neuron_rag_ORIG.bz2', nodetype=np.uint)
with open(example_wd + '/glia/edges_to_remove.013.txt', 'r') as f:
edges = [tuple(map(int, l.replace('\n', '').split(','))) for l in f.readlines()]
# only for
with open(example_wd + '/glia/edges_to_remove.v2-1.txt', 'r') as f:
edges += [tuple(map(int, l.replace('\n', '').split(','))) for l in f.readlines()]
cnt = 0
n_edges = rag.number_of_edges()
n_nodes = rag.number_of_nodes()
for e in edges:
if e in rag.edges:
rag.remove_edge(*e)
cnt += 1
# add edges between single node connected components, so that they are saved in the edge list
cnt_singlenode = 0
for cc in nx.connected_components(rag):
if len(cc) == 1:
rag.add_edge(list(cc)[0], list(cc)[0])
cnt_singlenode += 1
log.info(f'Removed {cnt} of the {len(edges)} edges contained in the manually '
f'created edge list from the original RAG and added {cnt_singlenode} '
f'self-edges to single-node connected components (#nodes:'
f' {n_nodes}, #edges: {n_edges}).')
log.info(f'Storing modified RAG as "neuron_rag.bz2" which will be used for further '
f'processing (#nodes: {rag.number_of_nodes()}, #edges: {rag.number_of_edges()}).')
nx.write_edgelist(rag, example_wd + '/glia/neuron_rag.bz2')
log.info('Step 2/8 - Creating SuperSegmentationDataset')
# apply size threshold again because after additional edges were removed after
# glia splitting (which applies the size threshold).
exec_multiview.run_create_neuron_ssd(apply_ssv_size_threshold=True)
exec_skeleton.map_myelin_global()
time_stamps.append(time.time())
step_idents.append('SSD generation')
log.info('Step 3/8 - Synapse detection')
sd_syn_ssv = SegmentationDataset(working_dir=global_params.config.working_dir,
obj_type='syn_ssv')
# # This creates an SD of type 'syn_ssv'
cps.combine_and_split_syn_old(global_params.config.working_dir, resume_job=False,
cs_gap_nm=global_params.config['cell_objects']['cs_gap_nm'],
log=log, n_folders_fs=n_folders_fs)
cps.extract_synapse_type(sd_syn_ssv,
kd_sym_path=global_params.config.kd_sym_path,
kd_asym_path=global_params.config.kd_asym_path,
sym_label=2, asym_label=1)
log.info('Synapse objects were created.')
dataset_analysis(sd_syn_ssv, compute_meshprops=True)
log.info(f'SegmentationDataset of type "syn_ssv" was generated with {len(sd_syn_ssv.ids)} '
f'objects.')
cps.map_objects_to_synssv(global_params.config.working_dir, log=log)
log.info('Cellular organelles were mapped to "syn_ssv".')
cps.classify_synssv_objects(global_params.config.working_dir, log=log)
log.info('Synapse prediction finished.')
log.info('Collecting and writing syn-ssv objects to SSV attribute '
'dictionary.')
# This needs to be run after `classify_synssv_objects` and before
# `map_synssv_objects` if the latter uses thresholding for synaptic objects
# just collect new data: ``recompute=False``
dataset_analysis(sd_syn_ssv, compute_meshprops=False, recompute=False)
map_synssv_objects(log=log)
log.info('Finished.')
time_stamps.append(time.time())
step_idents.append('Synapse detection')
log.info('Step 4/8 - Neuron rendering')
exec_multiview.run_neuron_rendering()
time_stamps.append(time.time())
step_idents.append('Neuron rendering')
log.info('Step 5/8 - Axon prediction')
exec_multiview.run_semsegaxoness_prediction()
exec_multiview.run_semsegaxoness_mapping()
time_stamps.append(time.time())
step_idents.append('Axon prediction')
log.info('Step 6/8 - Spine prediction')
exec_multiview.run_spiness_prediction()
exec_syns.run_spinehead_volume_calc()
time_stamps.append(time.time())
step_idents.append('Spine prediction')
log.info('Step 7/9 - Morphology extraction')
exec_multiview.run_morphology_embedding()
time_stamps.append(time.time())
step_idents.append('Morphology extraction')
log.info('Step 8/9 - Celltype analysis')
exec_multiview.run_celltype_prediction()
time_stamps.append(time.time())
step_idents.append('Celltype analysis')
log.info('Step 9/9 - Matrix export')
exec_syns.run_matrix_export()
time_stamps.append(time.time())
step_idents.append('Matrix export')
time_stamps = np.array(time_stamps)
dts = time_stamps[1:] - time_stamps[:-1]
dt_tot = time_stamps[-1] - time_stamps[0]
dt_tot_str = time.strftime("%Hh:%Mmin:%Ss", time.gmtime(dt_tot))
time_summary_str = "\nEM data analysis of experiment '{}' finished after" \
" {}.\n".format(experiment_name, dt_tot_str)
n_steps = len(step_idents[1:]) - 1
for i in range(len(step_idents[1:])):
step_dt = time.strftime("%Hh:%Mmin:%Ss", time.gmtime(dts[i]))
step_dt_perc = int(dts[i] / dt_tot * 100)
step_str = "[{}/{}] {}\t\t\t{}\t\t\t{}%\n".format(
i, n_steps, step_idents[i+1], step_dt, step_dt_perc)
time_summary_str += step_str
log.info(time_summary_str)
log.info('Setting up flask server for inspection. Annotated cell reconst'
'ructions and wiring can be analyzed via the KNOSSOS-SyConn plugin'
' at `SyConn/scripts/kplugin/syconn_knossos_viewer.py`.')
fname_server = os.path.dirname(os.path.abspath(__file__)) + \
'/../kplugin/server.py'
os.system('python {} --working_dir={} --port=10002'.format(
fname_server, example_wd))
......@@ -893,9 +893,9 @@ def _combine_and_split_cs_agg_thread(args):
if n_items_for_path > n_per_voxel_path:
voxel_dc.push(cs.so_storage_path + voxel_rel_paths[cur_path_id] +
"/voxel.pkl")
"/voxel.pkl")
attr_dc.push(cs.so_storage_path + voxel_rel_paths[cur_path_id] +
"/attr_dict.pkl")
"/attr_dict.pkl")
cur_path_id += 1
n_items_for_path = 0
......@@ -910,16 +910,16 @@ def _combine_and_split_cs_agg_thread(args):
pass
voxel_dc = VoxelStorage(cs.so_storage_path + voxel_rel_paths[cur_path_id] +
"voxel.pkl", read_only=False)
"voxel.pkl", read_only=False)
attr_dc = AttributeDict(cs.so_storage_path +
voxel_rel_paths[cur_path_id] + "attr_dict.pkl",
read_only=False)
if n_items_for_path > 0:
voxel_dc.push(cs.so_storage_path + voxel_rel_paths[cur_path_id] +
"/voxel.pkl")
"/voxel.pkl")
attr_dc.push(cs.so_storage_path + voxel_rel_paths[cur_path_id] +
"/attr_dict.pkl")
"/attr_dict.pkl")
def cc_large_voxel_lists(voxel_list, cs_gap_nm, max_concurrent_nodes=5000,
......
......@@ -502,7 +502,7 @@ def map_subcell_extract_props(kd_seg_path: str, kd_organelle_paths: dict,
else:
qu.batchjob_script(multi_params, "write_props_to_sc", script_folder=None,
n_max_co_processes=global_params.config.ncore_total,
remove_jobfolder=True, n_cores=2) # TODO: change n_cores=2 to 1!
remove_jobfolder=True, n_cores=1)
for k in kd_organelle_paths:
sc_sd = segmentation.SegmentationDataset(
working_dir=global_params.config.working_dir, obj_type=k,
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment