From 25a63467ba875f7821323a37f2fc20d1399d14f6 Mon Sep 17 00:00:00 2001 From: cmaumet Date: Thu, 19 Jul 2018 10:52:13 +0200 Subject: [PATCH 01/19] Read neuroimaging soft --- nidmresults/graph.py | 91 ++++++++++++++++++++++------------ nidmresults/load.py | 2 +- nidmresults/objects/generic.py | 2 + 3 files changed, 61 insertions(+), 34 deletions(-) diff --git a/nidmresults/graph.py b/nidmresults/graph.py index 4930760..c7d2b7b 100644 --- a/nidmresults/graph.py +++ b/nidmresults/graph.py @@ -13,7 +13,7 @@ from nidmresults.objects.modelfitting import * from nidmresults.objects.contrast import * from nidmresults.objects.inference import * -from nidmresults.exporter import NIDMExporter +from nidmresults.exporter import NIDMExporter import collections # from rdflib.term import Literal @@ -29,26 +29,38 @@ class NIDMResults(): NIDM-result object containing all metadata and link to image files. """ - def __init__(self, nidm_zip=None, rdf_file=None, workaround=False, - to_replace=dict()): - self.study_name = os.path.basename(nidm_zip).replace(".nidm.zip", "") - self.zip_path = nidm_zip + def __init__(self, nidm_zip=None, rdf_file=None, json_file=None, + workaround=False, to_replace=dict()): + self.zip_path = None + self.nidm_zip = None - # Load the turtle file - with zipfile.ZipFile(self.zip_path, 'r') as z: - rdf_data = z.read('nidm.ttl') - rdf_data = rdf_data.decode() + if nidm_zip is not None: + self.study_name = os.path.basename(nidm_zip).replace(".nidm.zip", "") + self.zip_path = nidm_zip - # Exporter-version-specific fixes in the RDF - rdf_data = self.fix_for_specific_versions(rdf_data, to_replace) + # Load the turtle file + with zipfile.ZipFile(self.zip_path, 'r') as z: + rdf_data = z.read('nidm.ttl') + rdf_data = rdf_data.decode() - # Parse turtle into RDF graph - self.graph = self.parse(rdf_data) + # Exporter-version-specific fixes in the RDF + rdf_data = self.fix_for_specific_versions(rdf_data, to_replace) + + # Parse turtle into RDF graph + self.graph = self.parse(rdf_data) + + else: + self.study_name = os.path.basename(json_file).replace(".json", "") + self.json_file = json_file + with open(json_file) as json_data: + self.json = json.load(json_data) + + # TODO: add validation here of the JSON file according to JSON API self.objects = dict() self.info = None - # Query the RDF document and create the objects + # Query the RDF or JSON document and create the objects self.software = self.load_software() (self.bundle, self.exporter, self.export_act, self.export_time) = \ self.load_bundle_export() @@ -123,6 +135,12 @@ def load_from_pack(klass, nidm_zip, workaround=False, to_replace=dict()): to_replace=to_replace) return nidmr + @classmethod + def load_from_json(klass, json_file, workaround=False, to_replace=dict()): + nidmr = NIDMResults(json_file=json_file, workaround=workaround, + to_replace=to_replace) + return nidmr + def get_info(self): if self.info is None: self.info = collections.OrderedDict() @@ -442,29 +460,36 @@ def get_object(self, klass, oid=None, err_if_none=True, **kwargs): return(to_return) def load_software(self): - query = """ -prefix nidm_ModelParameterEstimation: -SELECT DISTINCT * WHERE { + if self.nidm_zip is not None: + query = """ + prefix nidm_ModelParameterEstimation: - ?ni_software_id a prov:SoftwareAgent . + SELECT DISTINCT * WHERE { - ?mpe_id a nidm_ModelParameterEstimation: ; - prov:wasAssociatedWith ?ni_software_id . -} - """ - sd = self.graph.query(query) + ?ni_software_id a prov:SoftwareAgent . - software = None - if sd: - for row in sd: - args = row.asdict() - software = self.get_object(NeuroimagingSoftware, - args['ni_software_id']) - - if software is None: - raise Exception('No results found for query:' + query) + ?mpe_id a nidm_ModelParameterEstimation: ; + prov:wasAssociatedWith ?ni_software_id . + } + """ + sd = self.graph.query(query) + + software = None + if sd: + for row in sd: + args = row.asdict() + software = self.get_object(NeuroimagingSoftware, + args['ni_software_id']) + + if software is None: + raise Exception('No results found for query:' + query) + elif self.json_file is not None: + soft_type = self.json['NeuroimagingAnalysisSoftware_type'] + version = self.json['NeuroimagingAnalysisSoftware_type'] + label = self.json.get('NeuroimagingAnalysisSoftware_label', None) + software = NeuroimagingSoftware(soft_type, version, label) return software diff --git a/nidmresults/load.py b/nidmresults/load.py index 8969073..88949a1 100644 --- a/nidmresults/load.py +++ b/nidmresults/load.py @@ -21,7 +21,7 @@ def load(filename, to_replace=dict()): raise IOException('File does not exist: %s' % filename) if filename.endswith('.json'): - raise Exception('Minimal json file: not handled yet') + nidm = NIDMResults.load_from_json(filename, to_replace=to_replace) elif filename.endswith('.nidm.zip'): nidm = NIDMResults.load_from_pack(filename, to_replace=to_replace) else: diff --git a/nidmresults/objects/generic.py b/nidmresults/objects/generic.py index cca4d5b..1c5584d 100644 --- a/nidmresults/objects/generic.py +++ b/nidmresults/objects/generic.py @@ -401,6 +401,8 @@ def __init__(self, software_type, version, label=None, feat_version=None, self.type = Identifier(software_type) elif software_type.lower() == "fsl": self.type = SCR_FSL + elif software_type.lower() == "spm": + self.type = SCR_SPM else: warnings.warn('Unrecognised software: ' + str(software_type)) self.name = str(software_type) From ecd4e1f118eb38d1a42f8ff722e0006899517635 Mon Sep 17 00:00:00 2001 From: cmaumet Date: Thu, 19 Jul 2018 11:04:07 +0200 Subject: [PATCH 02/19] Move model fitting query to the class --- nidmresults/graph.py | 71 ++--------------------------- nidmresults/objects/modelfitting.py | 69 ++++++++++++++++++++++++++++ 2 files changed, 72 insertions(+), 68 deletions(-) diff --git a/nidmresults/graph.py b/nidmresults/graph.py index c7d2b7b..78654b3 100644 --- a/nidmresults/graph.py +++ b/nidmresults/graph.py @@ -62,11 +62,11 @@ def __init__(self, nidm_zip=None, rdf_file=None, json_file=None, # Query the RDF or JSON document and create the objects self.software = self.load_software() - (self.bundle, self.exporter, self.export_act, self.export_time) = \ - self.load_bundle_export() self.model_fittings = self.load_modelfitting() self.contrasts = self.load_contrasts(workaround=workaround) self.inferences = self.load_inferences() + (self.bundle, self.exporter, self.export_act, self.export_time) = \ + self.load_bundle_export() def fix_for_specific_versions(self, rdf_data, to_replace): """ @@ -555,72 +555,7 @@ def load_bundle_export(self): return (bundle, exporter, export, export_time) def load_modelfitting(self): - query = """ -prefix nidm_DesignMatrix: -prefix nidm_hasDriftModel: -prefix nidm_Data: -prefix nidm_ErrorModel: -prefix nidm_ModelParameterEstimation: -prefix nidm_ResidualMeanSquaresMap: -prefix nidm_MaskMap: -prefix nidm_GrandMeanMap: -prefix nlx_Imaginginstrument: -prefix nlx_MagneticResonanceImagingScanner: -prefix nlx_PositronEmissionTomographyScanner: -prefix nlx_SinglePhotonEmissionComputedTomographyScanner: -prefix nlx_MagnetoencephalographyMachine: -prefix nlx_ElectroencephalographyMachine: -prefix nidm_ReselsPerVoxelMap: - -SELECT DISTINCT * WHERE { - - ?design_id a nidm_DesignMatrix: . - OPTIONAL { ?design_id dc:description ?png_id . } . - OPTIONAL { ?design_id nidm_hasDriftModel: ?drift_model_id . } . - - ?data_id a nidm_Data: ; - prov:wasAttributedTo ?machine_id . - - {?machine_id a nlx_Imaginginstrument: .} UNION - {?machine_id a nlx_MagneticResonanceImagingScanner: .} UNION - {?machine_id a nlx_PositronEmissionTomographyScanner: .} UNION - {?machine_id a nlx_SinglePhotonEmissionComputedTomographyScanner: .} UNION - {?machine_id a nlx_MagnetoencephalographyMachine: .} UNION - {?machine_id a nlx_ElectroencephalographyMachine: .} - - ?error_id a nidm_ErrorModel: . - - ?mpe_id a nidm_ModelParameterEstimation: ; - prov:used ?design_id ; - prov:used ?data_id ; - prov:used ?error_id . - - ?rms_id a nidm_ResidualMeanSquaresMap: ; - nidm_inCoordinateSpace: ?rms_coordspace_id ; - prov:wasGeneratedBy ?mpe_id . - - ?mask_id a nidm_MaskMap: ; - nidm_inCoordinateSpace: ?mask_coordspace_id ; - prov:wasGeneratedBy ?mpe_id . - - ?gm_id a nidm_GrandMeanMap: ; - nidm_inCoordinateSpace: ?gm_coordspace_id ; - prov:wasGeneratedBy ?mpe_id . - - OPTIONAL { - ?rpv_id a nidm_ReselsPerVoxelMap: ; - nidm_inCoordinateSpace: ?rpv_coordspace_id ; - prov:wasGeneratedBy ?mpe_id . - } -} - """ - sd = self.graph.query(query) + sd = self.graph.query(ModelFitting.get_query()) model_fittings = list() if sd: diff --git a/nidmresults/objects/modelfitting.py b/nidmresults/objects/modelfitting.py index 48a3b51..fa049c0 100644 --- a/nidmresults/objects/modelfitting.py +++ b/nidmresults/objects/modelfitting.py @@ -43,6 +43,75 @@ def __init__(self, activity, design_matrix, data, error_model, self.machine = machine self.subjects = subjects + @classmethod + def get_query(klass): + query = """ +prefix nidm_DesignMatrix: +prefix nidm_hasDriftModel: +prefix nidm_Data: +prefix nidm_ErrorModel: +prefix nidm_ModelParameterEstimation: +prefix nidm_ResidualMeanSquaresMap: +prefix nidm_MaskMap: +prefix nidm_GrandMeanMap: +prefix nlx_Imaginginstrument: +prefix nlx_MagneticResonanceImagingScanner: +prefix nlx_PositronEmissionTomographyScanner: +prefix nlx_SinglePhotonEmissionComputedTomographyScanner: +prefix nlx_MagnetoencephalographyMachine: +prefix nlx_ElectroencephalographyMachine: +prefix nidm_ReselsPerVoxelMap: + +SELECT DISTINCT * WHERE { + + ?design_id a nidm_DesignMatrix: . + OPTIONAL { ?design_id dc:description ?png_id . } . + OPTIONAL { ?design_id nidm_hasDriftModel: ?drift_model_id . } . + + ?data_id a nidm_Data: ; + prov:wasAttributedTo ?machine_id . + + {?machine_id a nlx_Imaginginstrument: .} UNION + {?machine_id a nlx_MagneticResonanceImagingScanner: .} UNION + {?machine_id a nlx_PositronEmissionTomographyScanner: .} UNION + {?machine_id a nlx_SinglePhotonEmissionComputedTomographyScanner: .} UNION + {?machine_id a nlx_MagnetoencephalographyMachine: .} UNION + {?machine_id a nlx_ElectroencephalographyMachine: .} + + ?error_id a nidm_ErrorModel: . + + ?mpe_id a nidm_ModelParameterEstimation: ; + prov:used ?design_id ; + prov:used ?data_id ; + prov:used ?error_id . + + ?rms_id a nidm_ResidualMeanSquaresMap: ; + nidm_inCoordinateSpace: ?rms_coordspace_id ; + prov:wasGeneratedBy ?mpe_id . + + ?mask_id a nidm_MaskMap: ; + nidm_inCoordinateSpace: ?mask_coordspace_id ; + prov:wasGeneratedBy ?mpe_id . + + ?gm_id a nidm_GrandMeanMap: ; + nidm_inCoordinateSpace: ?gm_coordspace_id ; + prov:wasGeneratedBy ?mpe_id . + + OPTIONAL { + ?rpv_id a nidm_ReselsPerVoxelMap: ; + nidm_inCoordinateSpace: ?rpv_coordspace_id ; + prov:wasGeneratedBy ?mpe_id . + } +} + """ + return query + class ImagingInstrument(NIDMObject): """ From 2ed6897f98acefaa9add17a2c5643445dac6307e Mon Sep 17 00:00:00 2001 From: cmaumet Date: Thu, 19 Jul 2018 11:35:19 +0200 Subject: [PATCH 03/19] load ModelParametersEstimation from json --- nidmresults/graph.py | 270 ++++++++++++++-------------- nidmresults/objects/generic.py | 14 ++ nidmresults/objects/modelfitting.py | 19 +- 3 files changed, 167 insertions(+), 136 deletions(-) diff --git a/nidmresults/graph.py b/nidmresults/graph.py index 78654b3..20138a0 100644 --- a/nidmresults/graph.py +++ b/nidmresults/graph.py @@ -460,7 +460,6 @@ def get_object(self, klass, oid=None, err_if_none=True, **kwargs): return(to_return) def load_software(self): - if self.nidm_zip is not None: query = """ prefix nidm_ModelParameterEstimation: -prefix nidm_hasHRFBasis: + if self.nidm_zip is not None: + sd = self.graph.query(ModelFitting.get_query()) -SELECT DISTINCT * WHERE { - <""" + args['design_id'] + """> nidm_hasHRFBasis: ?hrf_basis . -} - """ - hrf_models = None - sd_hrf = self.graph.query(query_hrf_bases) - if sd_hrf: - hrf_models = list() - # TODO: we probably can avoid the loop below - for row_hrf in sd_hrf: - args_hrf = row_hrf.asdict() - hrf_models.append( - namespace_manager.valid_qualified_name( - args_hrf['hrf_basis'])) - - if 'png_id' in args: - design_matrix_png = self.get_object(Image, args['png_id']) - else: - design_matrix_png = None + model_fittings = list() + if sd: + for row in sd: + row_num = 0 + args = row.asdict() - if 'drift_model_id' in args: - drift_model = self.get_object( - DriftModel, args['drift_model_id']) - else: - drift_model = None + # TODO: should software_id really be an input? + activity = self.get_object( + ModelParametersEstimation, args['mpe_id']) - design_matrix = self.get_object( - DesignMatrix, args['design_id'], matrix=None, - image_file=design_matrix_png, drift_model=drift_model, - hrf_models=hrf_models) - data = self.get_object(Data, args['data_id']) - error_model = self.get_object(ErrorModel, args['error_id']) + # Find list of HRF basis + query_hrf_bases = """ + prefix nidm_DesignMatrix: + prefix nidm_hasHRFBasis: - # Find list of model parameter estimate maps - query_pe_maps = """ -prefix nidm_ParameterEstimateMap: -prefix nidm_inCoordinateSpace: + SELECT DISTINCT * WHERE { + <""" + args['design_id'] + """> nidm_hasHRFBasis: ?hrf_basis . + } + """ + hrf_models = None + sd_hrf = self.graph.query(query_hrf_bases) + if sd_hrf: + hrf_models = list() + # TODO: we probably can avoid the loop below + for row_hrf in sd_hrf: + args_hrf = row_hrf.asdict() + hrf_models.append( + namespace_manager.valid_qualified_name( + args_hrf['hrf_basis'])) + + if 'png_id' in args: + design_matrix_png = self.get_object(Image, args['png_id']) + else: + design_matrix_png = None -SELECT DISTINCT * WHERE { - ?pe_id a nidm_ParameterEstimateMap: ; - nidm_inCoordinateSpace: ?pe_coordspace_id ; - prov:wasGeneratedBy <""" + str(args['mpe_id']) + """> . -} - """ - param_estimates = list() - sd_pe_maps = self.graph.query(query_pe_maps) - if sd_pe_maps: - for row_pe in sd_pe_maps: - args_pe = row_pe.asdict() - pe_map_coordspace = self.get_object( - CoordinateSpace, args_pe['pe_coordspace_id']) - - param_estimates.append(self.get_object( - ParameterEstimateMap, args_pe['pe_id'], - coord_space=pe_map_coordspace, pe_num=None)) - - rms_coord_space = self.get_object( - CoordinateSpace, args['rms_coordspace_id']) - rms_map = self.get_object( - ResidualMeanSquares, args['rms_id'], - coord_space=rms_coord_space) - - mask_coord_space = self.get_object( - CoordinateSpace, args['mask_coordspace_id']) - mask_map = self.get_object( - MaskMap, args['mask_id'], coord_space=mask_coord_space) - - gm_coord_space = self.get_object( - CoordinateSpace, args['gm_coordspace_id']) - grand_mean_map = self.get_object( - GrandMeanMap, args['gm_id'], coord_space=mask_coord_space, - mask_file=None) - - if 'rpv_coordspace_id' in args: - rpv_coord_space = self.get_object( - CoordinateSpace, args['rpv_coordspace_id']) - rpv_map = self.get_object( - ReselsPerVoxelMap, args['rpv_id'], - coord_space=mask_coord_space) - else: - rpv_map = None + if 'drift_model_id' in args: + drift_model = self.get_object( + DriftModel, args['drift_model_id']) + else: + drift_model = None - machine = self.get_object( - ImagingInstrument, args['machine_id']) + design_matrix = self.get_object( + DesignMatrix, args['design_id'], matrix=None, + image_file=design_matrix_png, drift_model=drift_model, + hrf_models=hrf_models) + data = self.get_object(Data, args['data_id']) + error_model = self.get_object(ErrorModel, args['error_id']) - # Find subject or group(s) - query_subjects = """ -prefix nidm_Data: -prefix obo_studygrouppopulation: + # Find list of model parameter estimate maps + query_pe_maps = """ + prefix nidm_ParameterEstimateMap: + prefix nidm_inCoordinateSpace: -SELECT DISTINCT * WHERE { - <""" + str(args['data_id']) + """> a nidm_Data: ; - prov:wasAttributedTo ?person_or_group_id . + SELECT DISTINCT * WHERE { + ?pe_id a nidm_ParameterEstimateMap: ; + nidm_inCoordinateSpace: ?pe_coordspace_id ; + prov:wasGeneratedBy <""" + str(args['mpe_id']) + """> . + } + """ + param_estimates = list() + sd_pe_maps = self.graph.query(query_pe_maps) + if sd_pe_maps: + for row_pe in sd_pe_maps: + args_pe = row_pe.asdict() + pe_map_coordspace = self.get_object( + CoordinateSpace, args_pe['pe_coordspace_id']) + + param_estimates.append(self.get_object( + ParameterEstimateMap, args_pe['pe_id'], + coord_space=pe_map_coordspace, pe_num=None)) + + rms_coord_space = self.get_object( + CoordinateSpace, args['rms_coordspace_id']) + rms_map = self.get_object( + ResidualMeanSquares, args['rms_id'], + coord_space=rms_coord_space) + + mask_coord_space = self.get_object( + CoordinateSpace, args['mask_coordspace_id']) + mask_map = self.get_object( + MaskMap, args['mask_id'], coord_space=mask_coord_space) + + gm_coord_space = self.get_object( + CoordinateSpace, args['gm_coordspace_id']) + grand_mean_map = self.get_object( + GrandMeanMap, args['gm_id'], coord_space=mask_coord_space, + mask_file=None) + + if 'rpv_coordspace_id' in args: + rpv_coord_space = self.get_object( + CoordinateSpace, args['rpv_coordspace_id']) + rpv_map = self.get_object( + ReselsPerVoxelMap, args['rpv_id'], + coord_space=mask_coord_space) + else: + rpv_map = None - {?person_or_group_id a prov:Person .} UNION - {?person_or_group_id a obo_studygrouppopulation: .} . + machine = self.get_object( + ImagingInstrument, args['machine_id']) -} - """ + # Find subject or group(s) + query_subjects = """ + prefix nidm_Data: + prefix obo_studygrouppopulation: - sd_subjects = self.graph.query(query_subjects) - subjects = None - - if sd_subjects: - subjects = list() - for row_sub in sd_subjects: - args_sub = row_sub.asdict() - group = self.get_object( - Group, args_sub['person_or_group_id'], - err_if_none=False) - - if group is None: - # Try loading as a single subject - subject = self.get_object( - Person, args_sub['person_or_group_id']) - subjects.append(subject) - else: - subjects.append(group) + SELECT DISTINCT * WHERE { + <""" + str(args['data_id']) + """> a nidm_Data: ; + prov:wasAttributedTo ?person_or_group_id . - model_fittings.append(ModelFitting( - activity, design_matrix, data, error_model, - param_estimates, rms_map, mask_map, grand_mean_map, - machine, subjects, rpv_map)) + {?person_or_group_id a prov:Person .} UNION + {?person_or_group_id a obo_studygrouppopulation: .} . - con_num = row_num + 1 - else: - raise Exception('No model fitting found') + } + """ + + sd_subjects = self.graph.query(query_subjects) + subjects = None + + if sd_subjects: + subjects = list() + for row_sub in sd_subjects: + args_sub = row_sub.asdict() + group = self.get_object( + Group, args_sub['person_or_group_id'], + err_if_none=False) + + if group is None: + # Try loading as a single subject + subject = self.get_object( + Person, args_sub['person_or_group_id']) + subjects.append(subject) + else: + subjects.append(group) + + model_fittings.append(ModelFitting( + activity, design_matrix, data, error_model, + param_estimates, rms_map, mask_map, grand_mean_map, + machine, subjects, rpv_map)) + + con_num = row_num + 1 + else: + raise Exception('No model fitting found') + elif self.json_file is not None: + model_fittings = ModelFitting.load(self.json, self.software.id) return model_fittings diff --git a/nidmresults/objects/generic.py b/nidmresults/objects/generic.py index 1c5584d..34f786e 100644 --- a/nidmresults/objects/generic.py +++ b/nidmresults/objects/generic.py @@ -68,6 +68,12 @@ def add_attributes(self, attributes): else: self.attributes = attributes + @classmethod + def load(klass, loaded_from, *args, **kwargs): + if type(loaded_from) is dict: + obj = klass.load_from_json(loaded_from, *args, **kwargs) + return obj + class NIDMResultsBundle(NIDMObject): """ @@ -424,6 +430,14 @@ def __init__(self, software_type, version, label=None, feat_version=None, self.prov_type = PROV['Agent'] self.feat_version = feat_version + @classmethod + def load_from_json(klass, json_dict): + soft_type = json_dict['NeuroimagingAnalysisSoftware_type'] + version = json_dict['NeuroimagingAnalysisSoftware_type'] + label = json_dict.get('NeuroimagingAnalysisSoftware_label', None) + soft = NeuroimagingSoftware(soft_type, version, label) + return soft + @classmethod def get_query(klass, oid=None): if oid is None: diff --git a/nidmresults/objects/modelfitting.py b/nidmresults/objects/modelfitting.py index fa049c0..4532ada 100644 --- a/nidmresults/objects/modelfitting.py +++ b/nidmresults/objects/modelfitting.py @@ -21,7 +21,7 @@ from prov.identifier import QualifiedName -class ModelFitting(object): +class ModelFitting(NIDMObject): """ Object representing a Model fitting step: including a @@ -112,6 +112,18 @@ def get_query(klass): """ return query + @classmethod + def load_from_json(klass, json_dict, software_id): + # soft_type = json_dict['NeuroimagingAnalysisSoftware_type'] + # version = json_dict['NeuroimagingAnalysisSoftware_type'] + # label = json_dict.get('NeuroimagingAnalysisSoftware_label', None) + # return NeuroimagingSoftware(soft_type, version, label) + + # TODO: currently assuming list of 1 ==> should be extended + model_fittings = list() + activity = ModelParametersEstimation.load(json_dict, software_id) + print(activity) + class ImagingInstrument(NIDMObject): """ @@ -630,6 +642,11 @@ def get_query(klass, oid=None): """ return query + @classmethod + def load_from_json(self, json_dict, software_id): + est_method = json_dict['ModelParameterEstimation_withEstimationMethod'] + return ModelParametersEstimation(est_method, software_id) + def export(self, nidm_version, export_dir): """ Create prov entities and activities. From cc5cc0968f2bd364f9900ab76e57a08efbbbc012 Mon Sep 17 00:00:00 2001 From: cmaumet Date: Fri, 20 Jul 2018 16:19:30 +0200 Subject: [PATCH 04/19] load DesignMatrix, Data and ErroeModel in json --- nidmresults/objects/modelfitting.py | 96 ++++++++++++++++++++++++++++- 1 file changed, 94 insertions(+), 2 deletions(-) diff --git a/nidmresults/objects/modelfitting.py b/nidmresults/objects/modelfitting.py index 4532ada..b681b17 100644 --- a/nidmresults/objects/modelfitting.py +++ b/nidmresults/objects/modelfitting.py @@ -121,8 +121,33 @@ def load_from_json(klass, json_dict, software_id): # TODO: currently assuming list of 1 ==> should be extended model_fittings = list() + activity = ModelParametersEstimation.load(json_dict, software_id) - print(activity) + design = DesignMatrix.load(json_dict) + data = Data.load(json_dict) + error = ErrorModel.load(json_dict) + param_estimates = ParameterEstimateMap.load(json_dict) + rms_map = ResidualMeanSquares.load(json_dict) + mask_map = MaskMap.load(json_dict) + grand_mean_map = GrandMeanMap.load(json_dict) + machine = Machine.load(json_dict) + subjects = Subject.load(json_dict) + + # self.design_matrix = design + # self.data = data + # self.error_model = error_model + # self.param_estimates = param_estimates + # self.rms_map = rms_map + # self.rpv_map = rpv_map + # self.mask_map = mask_map + # self.grand_mean_map = grand_mean_map + # self.machine = machine + # self.subjects = subjects + + return ModelFitting( + activity, design, data, error, + param_estimates, rms_map, mask_map, grand_mean_map, + machine, subjects, rpv_map=None) class ImagingInstrument(NIDMObject): @@ -305,7 +330,10 @@ def __init__(self, matrix, image_file, regressors=None, self.image = image_file else: self.image = Image(image_file, img_filename) - if not type(regressors) is list: + + # Note: changed to fit regressors passed as loaded json when creating + # NIDM pack from JSON --> check if this cause issue in the tests TODO + if not type(regressors) is not list: regressors = json.loads(regressors) self.regressors = regressors @@ -331,6 +359,28 @@ def __init__(self, matrix, image_file, regressors=None, else: self.label = "Design Matrix" + @classmethod + def load_from_json(klass, json_dict): + if 'DesignMatrix_atLocation' in json_dict: + mat_csv = json_dict['DesignMatrix_atLocation'] + # Note: this could be removed and the csv passed directly + matrix = genfromtxt(mat_csv, delimiter=',') + else: + matrix = json_dict['DesignMatrix_value'] + + # TODO: deal with optional png of design matric + image_file = None + + regressors = json_dict['DesignMatrix_regressorNames'] + # TODO deal with optional arguments + + design = DesignMatrix( + matrix, image_file, regressors=None, + design_type=None, hrf_models=None, drift_model=None, + suffix='', csv_file=None, filename=None, label=None, + oid=None) + return design + @classmethod def get_query(klass, oid=None): if oid is None: @@ -483,6 +533,15 @@ def __init__(self, grand_mean_scaling, target=None, mri_protocol=None, self.label = label self.group_or_sub = group_or_sub + @classmethod + def load_from_json(klass, json_dict): + grand_mean_scaling = json_dict['Data_grandMeanScaling'] + target = json_dict['Data_targetIntensity'] + # TODO deal with optional arguments + data = Data(grand_mean_scaling, mri_protocol=None, + label=None, group_or_sub=None, oid=None) + return data + @classmethod def get_query(klass, oid=None): if oid is None: @@ -550,6 +609,29 @@ def __init__(self, error_distribution, variance_homo, variance_spatial, self.type = NIDM_ERROR_MODEL self.prov_type = PROV['Entity'] + @classmethod + def load_from_json(klass, json_dict): + DEP = { + 'nidm_ConstantParameter': SPATIALLY_GLOBAL, + 'nidm_IndependentParameter': SPATIALLY_LOCAL, + 'nidm_RegularizedParameter': SPATIALLY_REGUL, + } + DIST = { + 'obo_NormalDistribution': STATO_NORMAL_DISTRIBUTION, + } + + error_distribution = DIST[json_dict['ErrorModel_hasErrorDistribution']] + variance_homo = json_dict['ErrorModel_errorVarianceHomogeneous'] + variance_spatial = DEP[ + json_dict['ErrorModel_varianceMapWiseDependence']] + dep = json_dict['ErrorModel_hasErrorDependence'] + dep_spatial = DEP[json_dict['ErrorModel_dependenceMapWiseDependence']] + + error = ErrorModel( + error_distribution, variance_homo, variance_spatial, + dep, dep_spatial, oid=None) + return error + @classmethod def get_query(klass, oid=None): if oid is None: @@ -700,6 +782,16 @@ def __init__(self, coord_space, pe_file=None, pe_num=None, filename=None, self.derfrom = None self.isderfrommap = isderfrommap + @classmethod + def load_from_json(klass, json_dict): + params = json_dict['ParameterEstimateMaps'] + + + version = json_dict['NeuroimagingAnalysisSoftware_type'] + label = json_dict.get('NeuroimagingAnalysisSoftware_label', None) + soft = NeuroimagingSoftware(soft_type, version, label) + return soft + @classmethod def get_query(klass, oid=None): if oid is None: From 9578fb4d250d8c9345f1d0e52140a9cd7c64a60f Mon Sep 17 00:00:00 2001 From: cmaumet Date: Fri, 20 Jul 2018 17:09:09 +0200 Subject: [PATCH 05/19] load ParameterEstimateMaps from json --- nidmresults/graph.py | 5 ++++- nidmresults/objects/generic.py | 26 ++++++++++++++++++++++++++ nidmresults/objects/modelfitting.py | 25 +++++++++++++++++-------- 3 files changed, 47 insertions(+), 9 deletions(-) diff --git a/nidmresults/graph.py b/nidmresults/graph.py index 20138a0..d44ce67 100644 --- a/nidmresults/graph.py +++ b/nidmresults/graph.py @@ -55,6 +55,8 @@ def __init__(self, nidm_zip=None, rdf_file=None, json_file=None, with open(json_file) as json_data: self.json = json.load(json_data) + self.json_path = os.path.dirname(json_file) + # TODO: add validation here of the JSON file according to JSON API self.objects = dict() @@ -698,7 +700,8 @@ def load_modelfitting(self): else: raise Exception('No model fitting found') elif self.json_file is not None: - model_fittings = ModelFitting.load(self.json, self.software.id) + model_fittings = ModelFitting.load(self.json, self.json_path, + self.software.id) return model_fittings diff --git a/nidmresults/objects/generic.py b/nidmresults/objects/generic.py index 34f786e..e2a5d44 100644 --- a/nidmresults/objects/generic.py +++ b/nidmresults/objects/generic.py @@ -198,6 +198,32 @@ def is_talairach(self): else: return False + @classmethod + def load_from_json(klass, json_dict, nifti_file): + COORD_SYS = { + 'MNICoordinateSystem': NIDM_MNI_COORDINATE_SYSTEM, + 'IcbmMni152LinearCoordinateSystem': NIDM_ICBM_MNI152_LINEAR_COORDINATE_SYSTEM, + 'IcbmMni152NonLinear2009aAsymmetricCoordinateSystem': NIDM_ICBM_MNI152_NON_LINEAR2009A_ASYMMETRIC_COORDINATE_SYSTEM, + 'IcbmMni152NonLinear2009aSymmetricCoordinateSystem': NIDM_ICBM_MNI152_NON_LINEAR2009A_SYMMETRIC_COORDINATE_SYSTEM, + 'IcbmMni152NonLinear2009bAsymmetricCoordinateSystem': NIDM_ICBM_MNI152_NON_LINEAR2009B_ASYMMETRIC_COORDINATE_SYSTEM, + 'IcbmMni152NonLinear2009bSymmetricCoordinateSystem': NIDM_ICBM_MNI152_NON_LINEAR2009B_SYMMETRIC_COORDINATE_SYSTEM, + 'IcbmMni152NonLinear2009cAsymmetricCoordinateSystem': NIDM_ICBM_MNI152_NON_LINEAR2009C_ASYMMETRIC_COORDINATE_SYSTEM, + 'IcbmMni152NonLinear2009cSymmetricCoordinateSystem': NIDM_ICBM_MNI152_NON_LINEAR2009C_SYMMETRIC_COORDINATE_SYSTEM, + 'IcbmMni152NonLinear6thGenerationCoordinateSystem': NIDM_ICBM_MNI152_NON_LINEAR6TH_GENERATION_COORDINATE_SYSTEM, + 'Icbm452AirCoordinateSystem': NIDM_ICBM452_AIR_COORDINATE_SYSTEM, + 'Icbm452Warp5CoordinateSystem': NIDM_ICBM452_WARP5_COORDINATE_SYSTEM, + 'Ixi549CoordinateSystem': NIDM_IXI549_COORDINATE_SYSTEM, + 'Mni305CoordinateSystem': NIDM_MNI305_COORDINATE_SYSTEM, + 'TalairachCoordinateSystem': NIDM_TALAIRACH_COORDINATE_SYSTEM, + 'SubjectCoordinateSystem': NIDM_SUBJECT_COORDINATE_SYSTEM, + 'CustomCoordinateSystem': NIDM_CUSTOM_COORDINATE_SYSTEM, + } + coordsys = COORD_SYS[ + json_dict['CoordinateSpace_inWorldCoordinateSystem']] + coord_space = CoordinateSpace(coordsys, nifti_file) + + return coord_space + @classmethod def get_query(klass, oid=None): if oid is None: diff --git a/nidmresults/objects/modelfitting.py b/nidmresults/objects/modelfitting.py index b681b17..418737c 100644 --- a/nidmresults/objects/modelfitting.py +++ b/nidmresults/objects/modelfitting.py @@ -113,7 +113,7 @@ def get_query(klass): return query @classmethod - def load_from_json(klass, json_dict, software_id): + def load_from_json(klass, json_dict, base_dir, software_id): # soft_type = json_dict['NeuroimagingAnalysisSoftware_type'] # version = json_dict['NeuroimagingAnalysisSoftware_type'] # label = json_dict.get('NeuroimagingAnalysisSoftware_label', None) @@ -126,7 +126,7 @@ def load_from_json(klass, json_dict, software_id): design = DesignMatrix.load(json_dict) data = Data.load(json_dict) error = ErrorModel.load(json_dict) - param_estimates = ParameterEstimateMap.load(json_dict) + param_estimates = ParameterEstimateMap.load(json_dict, base_dir) rms_map = ResidualMeanSquares.load(json_dict) mask_map = MaskMap.load(json_dict) grand_mean_map = GrandMeanMap.load(json_dict) @@ -746,16 +746,20 @@ class ParameterEstimateMap(NIDMObject): Object representing an ParameterEstimateMap entity. """ - def __init__(self, coord_space, pe_file=None, pe_num=None, filename=None, + def __init__(self, coord_space=None, pe_file=None, pe_num=None, filename=None, sha=None, label=None, suffix='', model_param_estimation=None, oid=None, fmt=None, derfrom_id=None, derfrom_filename=None, derfrom_fmt=None, derfrom_sha=None, isderfrommap=False): super(ParameterEstimateMap, self).__init__(oid=oid) # Column index in the corresponding design matrix self.num = pe_num + self.coord_space = coord_space + # Parameter Estimate Map is going to be copied over to export_dir if not filename: + if suffix is None and pe_num is not None: + suffix = str(pe_num) filename = 'ParameterEstimate' + suffix + '.nii.gz' self.file = NIDMFile(self.id, pe_file, filename=filename, sha=sha, @@ -783,14 +787,19 @@ def __init__(self, coord_space, pe_file=None, pe_num=None, filename=None, self.isderfrommap = isderfrommap @classmethod - def load_from_json(klass, json_dict): + def load_from_json(klass, json_dict, base_dir): + pe_list = list() params = json_dict['ParameterEstimateMaps'] + for idx, pe_file in enumerate(params): + # FIXME: deal with varying coordsys across maps + coordspace = CoordinateSpace.load_from_json(json_dict, + os.path.join(base_dir, pe_file)) + + pe = ParameterEstimateMap(coordspace, pe_file, idx+1) + pe_list.append(pe) - version = json_dict['NeuroimagingAnalysisSoftware_type'] - label = json_dict.get('NeuroimagingAnalysisSoftware_label', None) - soft = NeuroimagingSoftware(soft_type, version, label) - return soft + return pe_list @classmethod def get_query(klass, oid=None): From 2fa47ba2659634c92f128e3cb1949ef5b6a9ead4 Mon Sep 17 00:00:00 2001 From: cmaumet Date: Fri, 20 Jul 2018 17:11:33 +0200 Subject: [PATCH 06/19] load ResidualMeanSquareMap from json --- nidmresults/objects/modelfitting.py | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/nidmresults/objects/modelfitting.py b/nidmresults/objects/modelfitting.py index 418737c..33e9322 100644 --- a/nidmresults/objects/modelfitting.py +++ b/nidmresults/objects/modelfitting.py @@ -127,7 +127,7 @@ def load_from_json(klass, json_dict, base_dir, software_id): data = Data.load(json_dict) error = ErrorModel.load(json_dict) param_estimates = ParameterEstimateMap.load(json_dict, base_dir) - rms_map = ResidualMeanSquares.load(json_dict) + rms_map = ResidualMeanSquares.load(json_dict, base_dir) mask_map = MaskMap.load(json_dict) grand_mean_map = GrandMeanMap.load(json_dict) machine = Machine.load(json_dict) @@ -889,6 +889,18 @@ def __init__(self, residual_file, coord_space, self.derfrom = None self.isderfrommap = isderfrommap + @classmethod + def load_from_json(klass, json_dict, base_dir): + rms_file = json_dict['ResidualMeanSquaresMap_atLocation'] + + # FIXME: deal with varying coordsys across maps + coordspace = CoordinateSpace.load_from_json(json_dict, + os.path.join(base_dir, rms_file)) + + rms = ResidualMeanSquares(rms_file, coordspace) + + return rms + @classmethod def get_query(klass, oid=None): if oid is None: From aa30d16217b70f55f503b015d44fa620239e2a41 Mon Sep 17 00:00:00 2001 From: cmaumet Date: Fri, 20 Jul 2018 17:14:34 +0200 Subject: [PATCH 07/19] load MaskMap from json --- nidmresults/objects/modelfitting.py | 15 ++++++++++++--- 1 file changed, 12 insertions(+), 3 deletions(-) diff --git a/nidmresults/objects/modelfitting.py b/nidmresults/objects/modelfitting.py index 33e9322..df6c5e3 100644 --- a/nidmresults/objects/modelfitting.py +++ b/nidmresults/objects/modelfitting.py @@ -128,7 +128,7 @@ def load_from_json(klass, json_dict, base_dir, software_id): error = ErrorModel.load(json_dict) param_estimates = ParameterEstimateMap.load(json_dict, base_dir) rms_map = ResidualMeanSquares.load(json_dict, base_dir) - mask_map = MaskMap.load(json_dict) + mask_map = MaskMap.load(json_dict, base_dir) grand_mean_map = GrandMeanMap.load(json_dict) machine = Machine.load(json_dict) subjects = Subject.load(json_dict) @@ -892,11 +892,9 @@ def __init__(self, residual_file, coord_space, @classmethod def load_from_json(klass, json_dict, base_dir): rms_file = json_dict['ResidualMeanSquaresMap_atLocation'] - # FIXME: deal with varying coordsys across maps coordspace = CoordinateSpace.load_from_json(json_dict, os.path.join(base_dir, rms_file)) - rms = ResidualMeanSquares(rms_file, coordspace) return rms @@ -1091,6 +1089,17 @@ def get_query(klass, oid=None): """ return query + @classmethod + def load_from_json(klass, json_dict, base_dir): + mask_file = json_dict['MaskMap_atLocation'] + # FIXME: deal with varying coordsys across maps + coordspace = CoordinateSpace.load_from_json(json_dict, + os.path.join(base_dir, mask_file)) + mask = MaskMap(mask_file, coordspace, False) + + return mask + + def export(self, nidm_version, export_dir): """ Create prov entities and activities. From 99b05febe611e9d74b7d4cbb436c49e2ea0886f3 Mon Sep 17 00:00:00 2001 From: cmaumet Date: Fri, 20 Jul 2018 17:23:11 +0200 Subject: [PATCH 08/19] load GrandMeanMap from json --- nidmresults/objects/modelfitting.py | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/nidmresults/objects/modelfitting.py b/nidmresults/objects/modelfitting.py index df6c5e3..d4163af 100644 --- a/nidmresults/objects/modelfitting.py +++ b/nidmresults/objects/modelfitting.py @@ -129,7 +129,7 @@ def load_from_json(klass, json_dict, base_dir, software_id): param_estimates = ParameterEstimateMap.load(json_dict, base_dir) rms_map = ResidualMeanSquares.load(json_dict, base_dir) mask_map = MaskMap.load(json_dict, base_dir) - grand_mean_map = GrandMeanMap.load(json_dict) + grand_mean_map = GrandMeanMap.load(json_dict, base_dir) machine = Machine.load(json_dict) subjects = Subject.load(json_dict) @@ -1164,6 +1164,16 @@ def get_query(klass, oid=None): """ return query + @classmethod + def load_from_json(klass, json_dict, base_dir): + gm_file = json_dict['GrandMeanMap_atLocation'] + # FIXME: deal with varying coordsys across maps + coordspace = CoordinateSpace.load_from_json(json_dict, + os.path.join(base_dir, gm_file)) + mask = GrandMeanMap(gm_file, gm_file, coordspace) + + return mask + def export(self, nidm_version, export_dir): """ Create prov entities and activities. From 3df1c457f836fa8ae4533d52c1cb5306f28692da Mon Sep 17 00:00:00 2001 From: cmaumet Date: Fri, 20 Jul 2018 17:36:15 +0200 Subject: [PATCH 09/19] load ImagingInstrument from json --- nidmresults/objects/modelfitting.py | 27 ++++++++++++++++++++++----- 1 file changed, 22 insertions(+), 5 deletions(-) diff --git a/nidmresults/objects/modelfitting.py b/nidmresults/objects/modelfitting.py index d4163af..4673dd3 100644 --- a/nidmresults/objects/modelfitting.py +++ b/nidmresults/objects/modelfitting.py @@ -130,7 +130,7 @@ def load_from_json(klass, json_dict, base_dir, software_id): rms_map = ResidualMeanSquares.load(json_dict, base_dir) mask_map = MaskMap.load(json_dict, base_dir) grand_mean_map = GrandMeanMap.load(json_dict, base_dir) - machine = Machine.load(json_dict) + machine = ImagingInstrument.load(json_dict) subjects = Subject.load(json_dict) # self.design_matrix = design @@ -158,14 +158,18 @@ class ImagingInstrument(NIDMObject): def __init__(self, machine_type, label=None, oid=None): super(ImagingInstrument, self).__init__(oid=oid) + machine_label = dict() + machine_label[NIF_MRI] = 'MRI Scanner' + machine_label[NIF_EEG] = 'EEG Machine' + machine_label[NIF_MEG] = 'MEG Machine' + machine_label[NIF_PET] = 'PET Scanner' + machine_label[NIF_SPECT] = 'SPECT Machine' + if not isinstance(machine_type, QualifiedName): machine_type = machine_type.lower() machine_term = dict( mri=NIF_MRI, eeg=NIF_EEG, meg=NIF_MEG, pet=NIF_PET, spect=NIF_SPECT) - machine_label = dict( - mri='MRI Scanner', eeg='EEG Machine', meg='MEG Machine', - pet='PET Scanner', spect='SPECT Machine') if not machine_type.startswith('http:'): self.type = machine_term[machine_type] @@ -175,10 +179,23 @@ def __init__(self, machine_type, label=None, oid=None): self.prov_type = PROV['Agent'] if label is None: - self.label = machine_label[machine_type] + self.label = machine_label[self.type] else: self.label = label + @classmethod + def load_from_json(klass, json_dict): + MACHINES = { + 'nlx_ElectroencephalographyMachine': NIF_EEG, + 'nlx_MagnetoencephalographyMachine': NIF_MEG, + 'nlx_PositronEmissionTomographyScanner': NIF_PET, + 'nlx_SinglePhotonEmissionComputedTomographyScanner': NIF_SPECT, + 'nlx_MagneticResonanceImagingScanner': NIF_MRI + } + machine_type = MACHINES[json_dict['ImagingInstrument_type']] + instrument = ImagingInstrument(machine_type) + return instrument + @classmethod def get_query(klass, oid=None): if oid is None: From a1a9ef149b68a92243a3a29e62c722b3e5f29443 Mon Sep 17 00:00:00 2001 From: cmaumet Date: Fri, 20 Jul 2018 17:41:45 +0200 Subject: [PATCH 10/19] load minimal model fitting from json --- nidmresults/objects/modelfitting.py | 45 +++++++++++++++++------------ 1 file changed, 26 insertions(+), 19 deletions(-) diff --git a/nidmresults/objects/modelfitting.py b/nidmresults/objects/modelfitting.py index 4673dd3..a351c48 100644 --- a/nidmresults/objects/modelfitting.py +++ b/nidmresults/objects/modelfitting.py @@ -43,6 +43,9 @@ def __init__(self, activity, design_matrix, data, error_model, self.machine = machine self.subjects = subjects + # Useful for printing + self.label = 'Model fitting' + @classmethod def get_query(klass): query = """ @@ -114,11 +117,6 @@ def get_query(klass): @classmethod def load_from_json(klass, json_dict, base_dir, software_id): - # soft_type = json_dict['NeuroimagingAnalysisSoftware_type'] - # version = json_dict['NeuroimagingAnalysisSoftware_type'] - # label = json_dict.get('NeuroimagingAnalysisSoftware_label', None) - # return NeuroimagingSoftware(soft_type, version, label) - # TODO: currently assuming list of 1 ==> should be extended model_fittings = list() @@ -131,24 +129,17 @@ def load_from_json(klass, json_dict, base_dir, software_id): mask_map = MaskMap.load(json_dict, base_dir) grand_mean_map = GrandMeanMap.load(json_dict, base_dir) machine = ImagingInstrument.load(json_dict) - subjects = Subject.load(json_dict) - - # self.design_matrix = design - # self.data = data - # self.error_model = error_model - # self.param_estimates = param_estimates - # self.rms_map = rms_map - # self.rpv_map = rpv_map - # self.mask_map = mask_map - # self.grand_mean_map = grand_mean_map - # self.machine = machine - # self.subjects = subjects - - return ModelFitting( + subjects = Group.load(json_dict) + + mf = ModelFitting( activity, design, data, error, param_estimates, rms_map, mask_map, grand_mean_map, machine, subjects, rpv_map=None) + print(mf) + + return mf + class ImagingInstrument(NIDMObject): """ @@ -280,6 +271,22 @@ def get_query(klass, oid=None): """ return query + @classmethod + def load_from_json(klass, json_dict): + groups = json_dict.get('groups', None) + grps = list() + + if groups is not None: + for group in groups: + group_name = group['StudyGroupPopulation_groupName'] + num_subjects = group['StudyGroupPopulation_numberOfSubjects'] + grp = Group(num_subjects, group_name) + grps.append(grp) + else: + grps = Person() + + return grps + def export(self, nidm_version, export_dir): """ Create prov entities and activities. From d1f9da2788c7bf0bfd12e2772f1be223bc688ab8 Mon Sep 17 00:00:00 2001 From: cmaumet Date: Tue, 11 Sep 2018 15:00:04 -0400 Subject: [PATCH 11/19] trying to load from JSON --- nidmresults/graph.py | 318 ++++++++++++++++---------------- nidmresults/objects/contrast.py | 25 ++- 2 files changed, 185 insertions(+), 158 deletions(-) diff --git a/nidmresults/graph.py b/nidmresults/graph.py index d44ce67..acc7744 100644 --- a/nidmresults/graph.py +++ b/nidmresults/graph.py @@ -706,181 +706,185 @@ def load_modelfitting(self): return model_fittings def load_contrasts(self, workaround=False): - if workaround: - warnings.warn('Using workaround: links between contrast weights' + - 'and contrast estimations are not assessed') - con_est_att = "." - else: - con_est_att = """; - prov:used ?conw_id ; - prov:used ?design_id .""" - - query = """ -prefix nidm_DesignMatrix: -prefix nidm_ModelParameterEstimation: -prefix nidm_contrastName: -prefix obo_contrastweightmatrix: -prefix nidm_ContrastEstimation: -prefix nidm_ContrastMap: -prefix nidm_ContrastStandardErrorMap: -prefix nidm_ContrastExplainedMeanSquareMap: -prefix nidm_StatisticMap: -prefix nidm_Inference: -prefix nidm_ConjunctionInference: -prefix spm_PartialConjunctionInference: -prefix nidm_contrastName: - -SELECT DISTINCT * WHERE { - - ?design_id a nidm_DesignMatrix: . - - ?conw_id a obo_contrastweightmatrix: . - - ?conest_id a nidm_ContrastEstimation: """ + con_est_att + """ - - ?mpe_id a nidm_ModelParameterEstimation: ; - prov:used ?design_id . + if self.nidm_zip is not None: + if workaround: + warnings.warn('Using workaround: links between contrast weights' + + 'and contrast estimations are not assessed') + con_est_att = "." + else: + con_est_att = """; + prov:used ?conw_id ; + prov:used ?design_id .""" - { - ?conm_id a nidm_ContrastMap: ; - nidm_inCoordinateSpace: ?conm_coordspace_id ; - prov:wasGeneratedBy ?conest_id . + query = """ + prefix nidm_DesignMatrix: + prefix nidm_ModelParameterEstimation: + prefix nidm_contrastName: + prefix obo_contrastweightmatrix: + prefix nidm_ContrastEstimation: + prefix nidm_ContrastMap: + prefix nidm_ContrastStandardErrorMap: + prefix nidm_ContrastExplainedMeanSquareMap: + prefix nidm_StatisticMap: + prefix nidm_Inference: + prefix nidm_ConjunctionInference: + prefix spm_PartialConjunctionInference: + prefix nidm_contrastName: - ?constdm_id a nidm_ContrastStandardErrorMap: . - } UNION - { - ?constdm_id a nidm_ContrastExplainedMeanSquareMap: . - } . + SELECT DISTINCT * WHERE { - ?constdm_id nidm_inCoordinateSpace: ?constdm_coordspace_id ; - prov:wasGeneratedBy ?conest_id . + ?design_id a nidm_DesignMatrix: . - ?statm_id a nidm_StatisticMap: ; - prov:wasGeneratedBy ?conest_id ; - nidm_contrastName: ?contrast_name ; - nidm_inCoordinateSpace: ?statm_coordspace_id . + ?conw_id a obo_contrastweightmatrix: . - { - ?inf_id a nidm_Inference: . - } UNION { - ?inf_id a nidm_ConjunctionInference: . - } UNION { - ?inf_id a spm_PartialConjunctionInference: . - } . + ?conest_id a nidm_ContrastEstimation: """ + con_est_att + """ - ?inf_id prov:used ?statm_id . + ?mpe_id a nidm_ModelParameterEstimation: ; + prov:used ?design_id . + + { + ?conm_id a nidm_ContrastMap: ; + nidm_inCoordinateSpace: ?conm_coordspace_id ; + prov:wasGeneratedBy ?conest_id . + + ?constdm_id a nidm_ContrastStandardErrorMap: . + } UNION + { + ?constdm_id a nidm_ContrastExplainedMeanSquareMap: . + } . + + ?constdm_id nidm_inCoordinateSpace: ?constdm_coordspace_id ; + prov:wasGeneratedBy ?conest_id . + + ?statm_id a nidm_StatisticMap: ; + prov:wasGeneratedBy ?conest_id ; + nidm_contrastName: ?contrast_name ; + nidm_inCoordinateSpace: ?statm_coordspace_id . + + { + ?inf_id a nidm_Inference: . + } UNION { + ?inf_id a nidm_ConjunctionInference: . + } UNION { + ?inf_id a spm_PartialConjunctionInference: . + } . + + ?inf_id prov:used ?statm_id . + + OPTIONAL { + ?otherstatm_id a nidm_StatisticMap: ; + prov:wasGeneratedBy ?conest_id ; + nidm_inCoordinateSpace: ?otherstatm_coordspace_id . + FILTER (?otherstatm_id != ?statm_id) + } . - OPTIONAL { - ?otherstatm_id a nidm_StatisticMap: ; - prov:wasGeneratedBy ?conest_id ; - nidm_inCoordinateSpace: ?otherstatm_coordspace_id . - FILTER (?otherstatm_id != ?statm_id) - } . + } + """ -} - """ + sd = self.graph.query(query) - sd = self.graph.query(query) + contrasts = dict() + if sd: + con_num = 0 + for row in sd: + args = row.asdict() + contrast_num = str(con_num) + contrast_name = args['contrast_name'] - contrasts = dict() - if sd: - con_num = 0 - for row in sd: - args = row.asdict() - contrast_num = str(con_num) - contrast_name = args['contrast_name'] - - weights = self.get_object( - ContrastWeights, args['conw_id'], - contrast_num=contrast_num) - estimation = self.get_object( - ContrastEstimation, args['conest_id'], - contrast_num=contrast_num) - - contraststd_map_coordspace = self.get_object( - CoordinateSpace, args['constdm_coordspace_id']) - - if 'conm_id' in args: - # T-contrast - contrast_map_coordspace = self.get_object( - CoordinateSpace, args['conm_coordspace_id']) - contrast_map = self.get_object( - ContrastMap, args['conm_id'], - coord_space=contrast_map_coordspace, + weights = self.get_object( + ContrastWeights, args['conw_id'], + contrast_num=contrast_num) + estimation = self.get_object( + ContrastEstimation, args['conest_id'], contrast_num=contrast_num) - stderr_or_expl_mean_sq_map = self.get_object( - ContrastStdErrMap, args['constdm_id'], - coord_space=contraststd_map_coordspace, - contrast_num=contrast_num, is_variance=False, - var_coord_space=None, filepath=None) - else: - # F-contrast - contrast_map = None - stderr_or_expl_mean_sq_map = self.get_object( - ContrastExplainedMeanSquareMap, args['constdm_id'], - coord_space=contraststd_map_coordspace, - contrast_num=contrast_num, stat_file=None, - sigma_sq_file=None) - - stat_map_coordspace = self.get_object( - CoordinateSpace, args['statm_coordspace_id']) - stat_map = self.get_object( - StatisticMap, args['statm_id'], - coord_space=stat_map_coordspace) - - zstat_exist = False - if 'otherstatm_id' in args: - zstat_exist = True - - otherstat_map_coordspace = self.get_object( - CoordinateSpace, args['otherstatm_coordspace_id']) - otherstat_map = self.get_object( - StatisticMap, args['otherstatm_id'], + contraststd_map_coordspace = self.get_object( + CoordinateSpace, args['constdm_coordspace_id']) + + if 'conm_id' in args: + # T-contrast + contrast_map_coordspace = self.get_object( + CoordinateSpace, args['conm_coordspace_id']) + contrast_map = self.get_object( + ContrastMap, args['conm_id'], + coord_space=contrast_map_coordspace, + contrast_num=contrast_num) + + stderr_or_expl_mean_sq_map = self.get_object( + ContrastStdErrMap, args['constdm_id'], + coord_space=contraststd_map_coordspace, + contrast_num=contrast_num, is_variance=False, + var_coord_space=None, filepath=None) + else: + # F-contrast + contrast_map = None + stderr_or_expl_mean_sq_map = self.get_object( + ContrastExplainedMeanSquareMap, args['constdm_id'], + coord_space=contraststd_map_coordspace, + contrast_num=contrast_num, stat_file=None, + sigma_sq_file=None) + + stat_map_coordspace = self.get_object( + CoordinateSpace, args['statm_coordspace_id']) + stat_map = self.get_object( + StatisticMap, args['statm_id'], coord_space=stat_map_coordspace) - if zstat_exist: - con = Contrast( - contrast_num, args['contrast_name'], weights, - estimation, contrast_map, - stderr_or_expl_mean_sq_map, stat_map=otherstat_map, - z_stat_map=stat_map) - else: - con = Contrast( - contrast_num, args['contrast_name'], - weights, estimation, contrast_map, - stderr_or_expl_mean_sq_map, stat_map=stat_map) + zstat_exist = False + if 'otherstatm_id' in args: + zstat_exist = True + + otherstat_map_coordspace = self.get_object( + CoordinateSpace, args['otherstatm_coordspace_id']) + otherstat_map = self.get_object( + StatisticMap, args['otherstatm_id'], + coord_space=stat_map_coordspace) + + if zstat_exist: + con = Contrast( + contrast_num, args['contrast_name'], weights, + estimation, contrast_map, + stderr_or_expl_mean_sq_map, stat_map=otherstat_map, + z_stat_map=stat_map) + else: + con = Contrast( + contrast_num, args['contrast_name'], + weights, estimation, contrast_map, + stderr_or_expl_mean_sq_map, stat_map=stat_map) - # Find list of model parameter estimate maps - query_pe_maps = """ -prefix nidm_ParameterEstimateMap: + # Find list of model parameter estimate maps + query_pe_maps = """ + prefix nidm_ParameterEstimateMap: -SELECT DISTINCT * WHERE { - <""" + str(args['conest_id']) + """> prov:used ?pe_id . - ?pe_id a nidm_ParameterEstimateMap: . -} - """ - pe_ids = () - sd_pe_maps = self.graph.query(query_pe_maps) - if sd_pe_maps: - for row_pe in sd_pe_maps: - args_pe = row_pe.asdict() - pe_ids = pe_ids + (NIIRI.qname(str(args_pe['pe_id'])),) - - mpe_id = NIIRI.qname(str(args['mpe_id'])) - if not (mpe_id, pe_ids) in contrasts: - contrasts[(mpe_id, pe_ids)] = [con] - else: - contrasts[(mpe_id, pe_ids)].append(con) + SELECT DISTINCT * WHERE { + <""" + str(args['conest_id']) + """> prov:used ?pe_id . + ?pe_id a nidm_ParameterEstimateMap: . + } + """ + pe_ids = () + sd_pe_maps = self.graph.query(query_pe_maps) + if sd_pe_maps: + for row_pe in sd_pe_maps: + args_pe = row_pe.asdict() + pe_ids = pe_ids + (NIIRI.qname(str(args_pe['pe_id'])),) - con_num = con_num + 1 + mpe_id = NIIRI.qname(str(args['mpe_id'])) + if not (mpe_id, pe_ids) in contrasts: + contrasts[(mpe_id, pe_ids)] = [con] + else: + contrasts[(mpe_id, pe_ids)].append(con) + + con_num = con_num + 1 - if not contrasts: - raise Exception('No contrast found') + if not contrasts: + raise Exception('No contrast found') + elif self.json_file is not None: + contrasts = Contrast.load(self.json, self.json_path, + self.software.id) return contrasts diff --git a/nidmresults/objects/contrast.py b/nidmresults/objects/contrast.py index 23ce2d9..256dd97 100644 --- a/nidmresults/objects/contrast.py +++ b/nidmresults/objects/contrast.py @@ -17,7 +17,7 @@ from prov.identifier import QualifiedName -class Contrast(object): +class Contrast(NIDMObject): """ Object representing a Contrast Estimation step: including a @@ -39,6 +39,29 @@ def __init__(self, contrast_num, contrast_name, weights, estimation, self.stat_map = stat_map self.z_stat_map = z_stat_map + @classmethod + def load_from_json(klass, json_dict, base_dir, software_id): + contrasts = list() + + activity = ModelParametersEstimation.load(json_dict, software_id) + design = DesignMatrix.load(json_dict) + data = Data.load(json_dict) + error = ErrorModel.load(json_dict) + param_estimates = ParameterEstimateMap.load(json_dict, base_dir) + rms_map = ResidualMeanSquares.load(json_dict, base_dir) + mask_map = MaskMap.load(json_dict, base_dir) + grand_mean_map = GrandMeanMap.load(json_dict, base_dir) + machine = ImagingInstrument.load(json_dict) + subjects = Group.load(json_dict) + + con = Contrast(contrast_num, contrast_name, weights, estimation, + contrast_map, stderr_or_expl_mean_sq_map, stat_map, + z_stat_map) + + contrasts.append(con) + + return mf + class ContrastWeights(NIDMObject): From 9e5e1cc317f8a1321416e9af991616fda32fe5ec Mon Sep 17 00:00:00 2001 From: cmaumet Date: Tue, 11 Sep 2018 15:24:51 -0400 Subject: [PATCH 12/19] Add minimal reading of contrasts --- nidmresults/objects/contrast.py | 71 ++++++++++++++++++++++++++------- 1 file changed, 57 insertions(+), 14 deletions(-) diff --git a/nidmresults/objects/contrast.py b/nidmresults/objects/contrast.py index 256dd97..7190d88 100644 --- a/nidmresults/objects/contrast.py +++ b/nidmresults/objects/contrast.py @@ -43,24 +43,20 @@ def __init__(self, contrast_num, contrast_name, weights, estimation, def load_from_json(klass, json_dict, base_dir, software_id): contrasts = list() - activity = ModelParametersEstimation.load(json_dict, software_id) - design = DesignMatrix.load(json_dict) - data = Data.load(json_dict) - error = ErrorModel.load(json_dict) - param_estimates = ParameterEstimateMap.load(json_dict, base_dir) - rms_map = ResidualMeanSquares.load(json_dict, base_dir) - mask_map = MaskMap.load(json_dict, base_dir) - grand_mean_map = GrandMeanMap.load(json_dict, base_dir) - machine = ImagingInstrument.load(json_dict) - subjects = Group.load(json_dict) - + weights = ContrastWeights.load(json_dict) + estimation = ContrastEstimation.load(json_dict) + contrast_map = ContrastMap.load(json_dict, base_dir) + stderr_or_expl_mean_sq_map = ContrastStdErrMap.load(json_dict, base_dir) + stat_map = StatisticMap.load(json_dict, base_dir) + + contrast_num = '1' # TODO deal with more than 1 contrast + contrast_name = json_dict['Contrasts']['StatisticMap_contrastName'] con = Contrast(contrast_num, contrast_name, weights, estimation, - contrast_map, stderr_or_expl_mean_sq_map, stat_map, - z_stat_map) + contrast_map, stderr_or_expl_mean_sq_map, stat_map) contrasts.append(con) - return mf + return contrasts class ContrastWeights(NIDMObject): @@ -107,6 +103,13 @@ def get_query(klass, oid=None): """ return query + @classmethod + def load_from_json(self, json_dict): + conname = json_dict['Contrasts']['StatisticMap_contrastName'] + weights = json_dict['Contrasts']['ContrastWeightMatrix_value'] + stat_type = json_dict['Contrasts']['StatisticMap_statisticType'] + return ContrastWeights(None, conname, weights, stat_type) + def export(self, nidm_version, export_dir): """ Create prov graph. @@ -201,6 +204,17 @@ def get_query(klass, oid=None): """ return query + @classmethod + def load_from_json(self, json_dict, base_dir): + contrast_file = json_dict['Contrasts']['ContrastMap_atLocation'] + contrast_name = json_dict['Contrasts']['StatisticMap_contrastName'] + contrast_num = '1' # TODO deal with more than 1 contrast + # FIXME: deal with varying coordsys across maps + coordspace = CoordinateSpace.load_from_json(json_dict, + os.path.join(base_dir, contrast_file)) + return ContrastMap(contrast_file, contrast_num, contrast_name, + coordspace) + def export(self, nidm_version, export_dir): """ Create prov graph. @@ -397,6 +411,17 @@ def get_query(klass, oid=None): """ return query + @classmethod + def load_from_json(self, json_dict, base_dir): + filepath = json_dict['Contrasts']['ContrastStandardErrorMap_atLocation'] + contrast_num = '1' # TODO deal with more than 1 contrast + # FIXME: deal with varying coordsys across maps + coordspace = CoordinateSpace.load_from_json(json_dict, + os.path.join(base_dir, filepath)) + is_variance = False + return ContrastStdErrMap(contrast_num, filepath, is_variance, + coordspace, None) + def export(self, nidm_version, export_dir): """ Create prov graph. @@ -543,6 +568,19 @@ def get_query(klass, oid=None): """ return query + @classmethod + def load_from_json(self, json_dict, base_dir): + location = json_dict['Contrasts']['StatisticMap_atLocation'] + stat_type = json_dict['Contrasts']['StatisticMap_statisticType'] + contrast_name = json_dict['Contrasts']['StatisticMap_contrastName'] + dof = json_dict['Contrasts']['StatisticMap_errorDegreesOfFreedom'] + # FIXME: deal with varying coordsys across maps + coordspace = CoordinateSpace.load_from_json(json_dict, + os.path.join(base_dir, location)) + is_variance = False + return StatisticMap(location, stat_type, contrast_name, dof, + coordspace) + def export(self, nidm_version, export_dir): """ Create prov graph. @@ -613,6 +651,11 @@ def get_query(klass, oid=None): """ return query + @classmethod + def load_from_json(self, json_dict): + conname = json_dict['Contrasts']['StatisticMap_contrastName'] + return ContrastEstimation(None, conname) + def export(self, nidm_version, export_dir): """ Create prov graph. From 023e77b072a2997318b7b875e095615c95b259bc Mon Sep 17 00:00:00 2001 From: cmaumet Date: Tue, 11 Sep 2018 17:46:08 -0400 Subject: [PATCH 13/19] inferences read from JSON -- wip --- nidmresults/graph.py | 453 ++++++++++++++++--------------- nidmresults/objects/inference.py | 119 +++++++- 2 files changed, 347 insertions(+), 225 deletions(-) diff --git a/nidmresults/graph.py b/nidmresults/graph.py index acc7744..551843d 100644 --- a/nidmresults/graph.py +++ b/nidmresults/graph.py @@ -889,234 +889,239 @@ def load_contrasts(self, workaround=False): return contrasts def load_inferences(self): - query = """ -prefix nidm_ContrastEstimation: - -prefix nidm_Inference: -prefix nidm_HeightThreshold: -prefix nidm_ExtentThreshold: -prefix nidm_PeakDefinitionCriteria: -prefix nidm_ClusterDefinitionCriteria: -prefix nidm_DisplayMaskMap: -prefix nidm_ExcursionSetMap: -prefix nidm_inCoordinateSpace: -prefix nidm_SearchSpaceMaskMap: -prefix nidm_ConjunctionInference: -prefix spm_PartialConjunctionInference: -prefix nidm_hasClusterLabelsMap: -prefix nidm_hasMaximumIntensityProjection: - -SELECT DISTINCT * WHERE { - ?con_est_id a nidm_ContrastEstimation: . - - { ?inference_id a nidm_Inference: . } - UNION { ?inference_id a nidm_ConjunctionInference: . } - UNION { ?inference_id a spm_PartialConjunctionInference: . } . - - ?inference_id prov:used/prov:wasGeneratedBy ?con_est_id ; - prov:used ?height_thresh_id ; - prov:used ?extent_thresh_id ; - prov:used ?peak_criteria_id ; - prov:used ?cluster_criteria_id . - - ?height_thresh_id a nidm_HeightThreshold: . - - ?extent_thresh_id a nidm_ExtentThreshold: . - - ?peak_criteria_id a nidm_PeakDefinitionCriteria: . - - ?cluster_criteria_id a nidm_ClusterDefinitionCriteria: . - - OPTIONAL { - ?inference_id prov:used ?display_mask_id . - ?display_mask_id a nidm_DisplayMaskMap: ; - nidm_inCoordinateSpace: ?disp_coord_space_id . - } . - - ?exc_set_id a nidm_ExcursionSetMap: ; - nidm_inCoordinateSpace: ?excset_coord_space_id ; - prov:wasGeneratedBy ?inference_id . - - OPTIONAL {?exc_set_id dc:description ?excset_visu_id} - OPTIONAL { - ?exc_set_id nidm_hasClusterLabelsMap: ?cluster_map_id . - ?cluster_map_id nidm_inCoordinateSpace: ?cluster_map_coord_space_id . } - OPTIONAL {?exc_set_id nidm_hasMaximumIntensityProjection: ?mip_id} - - ?search_space_id a nidm_SearchSpaceMaskMap: ; - nidm_inCoordinateSpace: ?search_space_coord_space_id ; - prov:wasGeneratedBy ?inference_id . -} - """ - sd = self.graph.query(query) + if self.nidm_zip is not None: + query = """ + prefix nidm_ContrastEstimation: - inferences = dict() - if sd: - for row in sd: - args = row.asdict() - inference = self.get_object( - InferenceActivity, args['inference_id'], err_if_none=False) + prefix nidm_Inference: + prefix nidm_HeightThreshold: + prefix nidm_ExtentThreshold: + prefix nidm_PeakDefinitionCriteria: + prefix nidm_ClusterDefinitionCriteria: + prefix nidm_DisplayMaskMap: + prefix nidm_ExcursionSetMap: + prefix nidm_inCoordinateSpace: + prefix nidm_SearchSpaceMaskMap: + prefix nidm_ConjunctionInference: + prefix spm_PartialConjunctionInference: + prefix nidm_hasClusterLabelsMap: + prefix nidm_hasMaximumIntensityProjection: - # Find list of equivalent height thresholds - query_equiv_threshs = """ -prefix nidm_equivalentThreshold: + SELECT DISTINCT * WHERE { + ?con_est_id a nidm_ContrastEstimation: . -SELECT DISTINCT * WHERE { - <""" + str(args['height_thresh_id']) + """> nidm_equivalentThreshold: ?equiv_h_thresh_id . -} - """ - equiv_h_threshs = list() - sd_equiv_h_threshs = self.graph.query(query_equiv_threshs) - if sd_equiv_h_threshs: - for row_equiv_h in sd_equiv_h_threshs: - args_hequiv = row_equiv_h.asdict() - - equiv_h_threshs.append(self.get_object( - HeightThreshold, - args_hequiv['equiv_h_thresh_id'])) - - height_thresh = self.get_object( - HeightThreshold, args['height_thresh_id'], - equiv_thresh=equiv_h_threshs) - - # Find list of equivalent extent thresholds - query_equiv_threshs = """ -prefix nidm_equivalentThreshold: - -SELECT DISTINCT * WHERE { - <""" + str(args['extent_thresh_id']) + """> nidm_equivalentThreshold: ?equiv_e_thresh_id . -} - """ - equiv_e_threshs = list() - sd_equiv_e_threshs = self.graph.query(query_equiv_threshs) - if sd_equiv_e_threshs: - for row_equiv_e in sd_equiv_e_threshs: - args_eequiv = row_equiv_e.asdict() - - equiv_e_threshs.append(self.get_object( - ExtentThreshold, args_eequiv['equiv_e_thresh_id'])) - - extent_thresh = self.get_object( - ExtentThreshold, args['extent_thresh_id'], - equiv_thresh=equiv_e_threshs) - peak_criteria = self.get_object( - PeakCriteria, args['peak_criteria_id'], contrast_num=None) - cluster_criteria = self.get_object( - ClusterCriteria, args['cluster_criteria_id'], - contrast_num=None) - - if 'display_mask_id' in args: - disp_coordspace = self.get_object( - CoordinateSpace, args['disp_coord_space_id']) - # TODO we need to deal with more than 1 DisplayMaskMap - disp_mask = [self.get_object( - DisplayMaskMap, args['display_mask_id'], - contrast_num=None, coord_space=disp_coordspace, - mask_num=None)] - else: - disp_mask = None - - if 'excset_visu_id' in args: - excset_visu = self.get_object( - Image, args['excset_visu_id']) - else: - excset_visu = None - - if 'cluster_map_id' in args: - clustermap_coordspace = self.get_object( - CoordinateSpace, args['cluster_map_coord_space_id']) - cluster_map = self.get_object( - ClusterLabelsMap, args['cluster_map_id'], - coord_space=clustermap_coordspace) - else: - cluster_map = None - - if 'mip_id' in args: - mip = self.get_object(Image, args['mip_id']) - else: - mip = None - - excset_coordspace = self.get_object( - CoordinateSpace, args['excset_coord_space_id']) - excursion_set = self.get_object( - ExcursionSet, args['exc_set_id'], - coord_space=excset_coordspace, visu=excset_visu, - clust_map=cluster_map, mip=mip) - - searchspace_coordspace = self.get_object( - CoordinateSpace, args['search_space_coord_space_id']) - search_space = self.get_object( - SearchSpace, args['search_space_id'], - coord_space=searchspace_coordspace) - - # TODO - software_id = self.software.id - - # Find list of clusters - query_clusters = """ -prefix nidm_SupraThresholdCluster: -prefix nidm_ClusterCenterOfGravity: -prefix nidm_clusterLabelId: - -SELECT DISTINCT * WHERE { - ?cluster_id a nidm_SupraThresholdCluster: ; - nidm_clusterLabelId: ?cluster_num ; - prov:wasDerivedFrom <""" + str(args['exc_set_id']) + """> . - - OPTIONAL { - ?cog_id a nidm_ClusterCenterOfGravity: ; - prov:wasDerivedFrom ?cluster_id . + { ?inference_id a nidm_Inference: . } + UNION { ?inference_id a nidm_ConjunctionInference: . } + UNION { ?inference_id a spm_PartialConjunctionInference: . } . + + ?inference_id prov:used/prov:wasGeneratedBy ?con_est_id ; + prov:used ?height_thresh_id ; + prov:used ?extent_thresh_id ; + prov:used ?peak_criteria_id ; + prov:used ?cluster_criteria_id . + + ?height_thresh_id a nidm_HeightThreshold: . + + ?extent_thresh_id a nidm_ExtentThreshold: . + + ?peak_criteria_id a nidm_PeakDefinitionCriteria: . + + ?cluster_criteria_id a nidm_ClusterDefinitionCriteria: . + + OPTIONAL { + ?inference_id prov:used ?display_mask_id . + ?display_mask_id a nidm_DisplayMaskMap: ; + nidm_inCoordinateSpace: ?disp_coord_space_id . + } . + + ?exc_set_id a nidm_ExcursionSetMap: ; + nidm_inCoordinateSpace: ?excset_coord_space_id ; + prov:wasGeneratedBy ?inference_id . + + OPTIONAL {?exc_set_id dc:description ?excset_visu_id} + OPTIONAL { + ?exc_set_id nidm_hasClusterLabelsMap: ?cluster_map_id . + ?cluster_map_id nidm_inCoordinateSpace: ?cluster_map_coord_space_id . } + OPTIONAL {?exc_set_id nidm_hasMaximumIntensityProjection: ?mip_id} + + ?search_space_id a nidm_SearchSpaceMaskMap: ; + nidm_inCoordinateSpace: ?search_space_coord_space_id ; + prov:wasGeneratedBy ?inference_id . } -} - """ - clusters = list() - sd_clusters = self.graph.query(query_clusters) - if sd_clusters: - for row_cluster in sd_clusters: - args_cl = row_cluster.asdict() - - if 'cog_id' in args_cl: - cog = self.get_object( - CenterOfGravity, args_cl['cog_id'], - cluster_num=args_cl['cluster_num'].toPython()) - else: - cog = None - - # Find list of peaks - query_peaks = """ -prefix nidm_Peak: - -SELECT DISTINCT * WHERE { - ?peak_id a nidm_Peak: ; - prov:wasDerivedFrom <""" + str(args_cl['cluster_id']) + """> . -} - """ - peaks = list() - sd_peaks = self.graph.query(query_peaks) - if sd_peaks: - for row_peak in sd_peaks: - args_peak = row_peak.asdict() - - peaks.append(self.get_object( - Peak, args_peak['peak_id'])) - - clusters.append(self.get_object( - Cluster, args_cl['cluster_id'], peaks=peaks, - cog=cog)) - - # Dictionary of (key, value) pairs where key is the identifier - # of a ContrastEstimation object and value is an object of type - # Inference describing the inference step in NIDM-Results (main - # activity: Inference) - # TODO: if key exist we need to append! - inferences[NIIRI.qname(args['con_est_id'])] = [ - Inference( - inference, height_thresh, extent_thresh, - peak_criteria, cluster_criteria, disp_mask, - excursion_set, clusters, search_space, software_id)] + """ + sd = self.graph.query(query) + + inferences = dict() + if sd: + for row in sd: + args = row.asdict() + inference = self.get_object( + InferenceActivity, args['inference_id'], err_if_none=False) + + # Find list of equivalent height thresholds + query_equiv_threshs = """ + prefix nidm_equivalentThreshold: + + SELECT DISTINCT * WHERE { + <""" + str(args['height_thresh_id']) + """> nidm_equivalentThreshold: ?equiv_h_thresh_id . + } + """ + equiv_h_threshs = list() + sd_equiv_h_threshs = self.graph.query(query_equiv_threshs) + if sd_equiv_h_threshs: + for row_equiv_h in sd_equiv_h_threshs: + args_hequiv = row_equiv_h.asdict() + + equiv_h_threshs.append(self.get_object( + HeightThreshold, + args_hequiv['equiv_h_thresh_id'])) + + height_thresh = self.get_object( + HeightThreshold, args['height_thresh_id'], + equiv_thresh=equiv_h_threshs) + + # Find list of equivalent extent thresholds + query_equiv_threshs = """ + prefix nidm_equivalentThreshold: + + SELECT DISTINCT * WHERE { + <""" + str(args['extent_thresh_id']) + """> nidm_equivalentThreshold: ?equiv_e_thresh_id . + } + """ + equiv_e_threshs = list() + sd_equiv_e_threshs = self.graph.query(query_equiv_threshs) + if sd_equiv_e_threshs: + for row_equiv_e in sd_equiv_e_threshs: + args_eequiv = row_equiv_e.asdict() + + equiv_e_threshs.append(self.get_object( + ExtentThreshold, args_eequiv['equiv_e_thresh_id'])) + + extent_thresh = self.get_object( + ExtentThreshold, args['extent_thresh_id'], + equiv_thresh=equiv_e_threshs) + peak_criteria = self.get_object( + PeakCriteria, args['peak_criteria_id'], contrast_num=None) + cluster_criteria = self.get_object( + ClusterCriteria, args['cluster_criteria_id'], + contrast_num=None) + + if 'display_mask_id' in args: + disp_coordspace = self.get_object( + CoordinateSpace, args['disp_coord_space_id']) + # TODO we need to deal with more than 1 DisplayMaskMap + disp_mask = [self.get_object( + DisplayMaskMap, args['display_mask_id'], + contrast_num=None, coord_space=disp_coordspace, + mask_num=None)] + else: + disp_mask = None + + if 'excset_visu_id' in args: + excset_visu = self.get_object( + Image, args['excset_visu_id']) + else: + excset_visu = None + + if 'cluster_map_id' in args: + clustermap_coordspace = self.get_object( + CoordinateSpace, args['cluster_map_coord_space_id']) + cluster_map = self.get_object( + ClusterLabelsMap, args['cluster_map_id'], + coord_space=clustermap_coordspace) + else: + cluster_map = None + + if 'mip_id' in args: + mip = self.get_object(Image, args['mip_id']) + else: + mip = None + + excset_coordspace = self.get_object( + CoordinateSpace, args['excset_coord_space_id']) + excursion_set = self.get_object( + ExcursionSet, args['exc_set_id'], + coord_space=excset_coordspace, visu=excset_visu, + clust_map=cluster_map, mip=mip) + + searchspace_coordspace = self.get_object( + CoordinateSpace, args['search_space_coord_space_id']) + search_space = self.get_object( + SearchSpace, args['search_space_id'], + coord_space=searchspace_coordspace) + + # TODO + software_id = self.software.id + + # Find list of clusters + query_clusters = """ + prefix nidm_SupraThresholdCluster: + prefix nidm_ClusterCenterOfGravity: + prefix nidm_clusterLabelId: + + SELECT DISTINCT * WHERE { + ?cluster_id a nidm_SupraThresholdCluster: ; + nidm_clusterLabelId: ?cluster_num ; + prov:wasDerivedFrom <""" + str(args['exc_set_id']) + """> . + + OPTIONAL { + ?cog_id a nidm_ClusterCenterOfGravity: ; + prov:wasDerivedFrom ?cluster_id . + } + } + """ + clusters = list() + sd_clusters = self.graph.query(query_clusters) + if sd_clusters: + for row_cluster in sd_clusters: + args_cl = row_cluster.asdict() + + if 'cog_id' in args_cl: + cog = self.get_object( + CenterOfGravity, args_cl['cog_id'], + cluster_num=args_cl['cluster_num'].toPython()) + else: + cog = None + + # Find list of peaks + query_peaks = """ + prefix nidm_Peak: + + SELECT DISTINCT * WHERE { + ?peak_id a nidm_Peak: ; + prov:wasDerivedFrom <""" + str(args_cl['cluster_id']) + """> . + } + """ + peaks = list() + sd_peaks = self.graph.query(query_peaks) + if sd_peaks: + for row_peak in sd_peaks: + args_peak = row_peak.asdict() + + peaks.append(self.get_object( + Peak, args_peak['peak_id'])) + + clusters.append(self.get_object( + Cluster, args_cl['cluster_id'], peaks=peaks, + cog=cog)) + + # Dictionary of (key, value) pairs where key is the identifier + # of a ContrastEstimation object and value is an object of type + # Inference describing the inference step in NIDM-Results (main + # activity: Inference) + # TODO: if key exist we need to append! + inferences[NIIRI.qname(args['con_est_id'])] = [ + Inference( + inference, height_thresh, extent_thresh, + peak_criteria, cluster_criteria, disp_mask, + excursion_set, clusters, search_space, software_id)] + + elif self.json_file is not None: + inferences = Inference.load(self.json, self.json_path, + self.software.id) return inferences diff --git a/nidmresults/objects/inference.py b/nidmresults/objects/inference.py index f720323..8cdcf84 100644 --- a/nidmresults/objects/inference.py +++ b/nidmresults/objects/inference.py @@ -17,7 +17,7 @@ from prov.model import Identifier -class Inference(object): +class Inference(NIDMObject): """ Object representing an Inference step: including an Inference activity, its @@ -40,6 +40,29 @@ def __init__( self.disp_mask = disp_mask self.search_space = search_space + @classmethod + def load_from_json(klass, json_dict, base_dir, software_id): + inferences = list() + + inference = InferenceActivity.load(json_dict) + height_thresh = HeightThreshold.load(json_dict) + extent_thresh = ExtentThreshold.load(json_dict) + peak_criteria = PeakCriteria.load(json_dict) + cluster_criteria = ClusterCriteria.load(json_dict) + # disp_mask = DisplayMaskMap.load(json_dict, base_dir) + disp_mask = None # TODO + excursion_set = ExcursionSet.load(json_dict, base_dir) + clusters = Cluster.load(json_dict) + search_space = SearchSpaceMaskMap.load(json_dict, base_dir) + + inf = Inference(inference, height_thresh, extent_thresh, + peak_criteria, cluster_criteria, disp_mask, excursion_set, + clusters, search_space, software_id) + + inferences.append(inf) + + return inferences + class InferenceActivity(NIDMObject): @@ -101,6 +124,11 @@ def get_query(klass, oid=None): """ return query + @classmethod + def load_from_json(klass, json_dict): + inf_act = InferenceActivity() + return inf_act + def export(self, nidm_version, export_dir): """ Create prov entities and activities. @@ -186,6 +214,16 @@ def get_query(klass, oid=None): """ return query + @classmethod + def load_from_json(klass, json_dict, base_dir): + + location = json_dict['Inferences']["ExcursionSetMap_atLocation"] + coordspace = CoordinateSpace.load_from_json(json_dict, + os.path.join(base_dir, location)) + + exc = ExcursionSet(location, coordspace) + return exc + def export(self, nidm_version, export_dir): """ Create prov entities and activities. @@ -360,6 +398,16 @@ def get_query(klass, oid=None): """ return query + @classmethod + def load_from_json(klass, json_dict): + + threshold_type = json_dict['Inferences']["HeightThreshold_type"] + threshold_value = json_dict['Inferences']["HeightThreshold_value"] + + ht = HeightThreshold(threshold_type=threshold_type, + value=threshold_value) + return ht + def export(self, version, export_dir): """ Create prov entities and activities. @@ -485,6 +533,15 @@ def get_query(klass, oid=None): """ return query + @classmethod + def load_from_json(klass, json_dict): + threshold_type = json_dict['Inferences']["ExtentThreshold_type"] + threshold_value = json_dict['Inferences']["ExtentThreshold_clusterSizeInVoxels"] + + et = ExtentThreshold(threshold_type=threshold_type, + value=threshold_value) + return et + def export(self, version, export_dir): """ Create prov entities and activities. @@ -597,6 +654,25 @@ def get_query(klass, oid=None): """ return query + @classmethod + def load_from_json(klass, json_dict): + + clusters = json_dict['Inferences']['Clusters'] + clusts = list() + + clid = 1 + if clusts is not None: + for cl in clusts: + size = cl['SupraThresholdCluster_clusterSizeInVoxels'] + pFWER = cl['SupraThresholdCluster_pValueFWER'] + + peaks = load_from_json(clust['peaks']) + clust = Cluster(clid, size, pFWER, peaks) + clusts.append(clust) + + return clusters + + def export(self, nidm_version, export_dir): """ Create prov entities and activities. @@ -755,6 +831,14 @@ def get_query(klass, oid=None): """ return query + @classmethod + def load_from_json(klass, json_dict): + peak_dist = json_dict["PeakDefinitionCriteria_minDistanceBetweenPeaks"] + num_peak = json_dict["PeakDefinitionCriteria_maxNumberOfPeaksPerCluster"] + + pc = PeakCriteria(None, peak_dist, num_peak) + return pc + def export(self, nidm_version, export_dir): """ Create prov entities and activities. @@ -809,6 +893,14 @@ def get_query(klass, oid=None): """ return query + @classmethod + def load_from_json(klass, json_dict): + connectivity = json_dict["ClusterDefinitionCriteria_hasConnectivityCriterion"] + + cc = ClusterCriteria(None, connectivity) + + return cc + def export(self, nidm_version, export_dir): """ Create prov entities and activities. @@ -1192,6 +1284,31 @@ def get_query(klass, oid=None): """ return query + @classmethod + def load_from_json(klass, json_dict): + pks = list() + + pid = 1 + for pk in pks: + + # "": 17.5207633972168, + # "Coordinate_coordinateVector": [-60,-25,11], + # "": 4.440892098500626e-16, + # "": 0, + # "": 1.191565917138381e-11 + + equiv_z = pk['Peak_equivalentZStatistic'] + p_unc = pk.get('Peak_pValueUncorrected', None) + p_fwer = pk.get('Peak_pValueFWER', None) + p_fdr = pk.get('Peak_qValueFDR', None) + value = pk.get('Peak_value', None) + + peaks = load_from_json(clust['peaks']) + clust = Peak(equiv_z, p_unc, p_fwer, p_fdr=p_fdr, value=value) + clusts.append(clust) + + return clusters + def __str__(self): return '%s \tz=%.2f \tp=%.2e (unc.) \t%s' % ( self.label, self.equiv_z, self.p_unc, str(self.coordinate)) From fed22ef04608a9ffc2edcd4009aaf0512af2f6fd Mon Sep 17 00:00:00 2001 From: cmaumet Date: Wed, 12 Sep 2018 10:17:19 -0400 Subject: [PATCH 14/19] Reading JSON not crashing --- nidmresults/graph.py | 100 +++++++++++++++++-------------- nidmresults/objects/generic.py | 23 +++++++ nidmresults/objects/inference.py | 35 ++++++++++- 3 files changed, 110 insertions(+), 48 deletions(-) diff --git a/nidmresults/graph.py b/nidmresults/graph.py index 551843d..617238c 100644 --- a/nidmresults/graph.py +++ b/nidmresults/graph.py @@ -22,6 +22,7 @@ import zipfile import csv import warnings +import datetime class NIDMResults(): @@ -492,63 +493,69 @@ def load_software(self): return software def load_bundle_export(self): - # query = """ - # prefix nidm_softwareVersion: - # prefix nidm_NIDMResultsExport: + if self.nidm_zip is not None: + # query = """ + # prefix nidm_softwareVersion: + # prefix nidm_NIDMResultsExport: - # SELECT DISTINCT * WHERE - # { - # ?bundle_id a prov:Bundle . + # SELECT DISTINCT * WHERE + # { + # ?bundle_id a prov:Bundle . - # ?exporter_id a prov:SoftwareAgent . + # ?exporter_id a prov:SoftwareAgent . - # ?export_id a nidm_NIDMResultsExport: ; - # prov:wasAssociatedWith ?exporter_id . + # ?export_id a nidm_NIDMResultsExport: ; + # prov:wasAssociatedWith ?exporter_id . - # } - # """ + # } + # """ - query = """ -prefix nidm_softwareVersion: -prefix nidm_NIDMResultsExport: + query = """ + prefix nidm_softwareVersion: + prefix nidm_NIDMResultsExport: -SELECT DISTINCT * WHERE - { - ?bundle_id a prov:Bundle ; - prov:qualifiedGeneration ?blank_node . + SELECT DISTINCT * WHERE + { + ?bundle_id a prov:Bundle ; + prov:qualifiedGeneration ?blank_node . - ?blank_node a prov:Generation ; - prov:activity ?export_id ; - prov:atTime ?export_time . + ?blank_node a prov:Generation ; + prov:activity ?export_id ; + prov:atTime ?export_time . - ?exporter_id a prov:SoftwareAgent . + ?exporter_id a prov:SoftwareAgent . - ?export_id a nidm_NIDMResultsExport: ; - prov:wasAssociatedWith ?exporter_id . + ?export_id a nidm_NIDMResultsExport: ; + prov:wasAssociatedWith ?exporter_id . - } - """ + } + """ - sd = self.graph.query(query) + sd = self.graph.query(query) - # if len(sd) > 1: - # raise Exception('More than one result found for query:' + query) + # if len(sd) > 1: + # raise Exception('More than one result found for query:' + query) - exporter = None - export = None - bundle = None - if sd: - for row in sd: - args = row.asdict() - exporter = self.get_object( - ExporterSoftware, args['exporter_id']) - export = self.get_object(NIDMResultsExport, args['export_id']) - bundle = self.get_object(NIDMResultsBundle, args['bundle_id']) - export_time = args['export_time'].toPython() - else: - raise Exception('No results found for query:' + query) + exporter = None + export = None + bundle = None + if sd: + for row in sd: + args = row.asdict() + exporter = self.get_object( + ExporterSoftware, args['exporter_id']) + export = self.get_object(NIDMResultsExport, args['export_id']) + bundle = self.get_object(NIDMResultsBundle, args['bundle_id']) + export_time = args['export_time'].toPython() + else: + raise Exception('No results found for query:' + query) + elif self.json_file is not None: + bundle = NIDMResultsBundle.load(self.json) + exporter = ExporterSoftware.load(self.json) + export = NIDMResultsExport.load(self.json) + export_time = str(datetime.datetime.now().time()) return (bundle, exporter, export, export_time) @@ -700,8 +707,9 @@ def load_modelfitting(self): else: raise Exception('No model fitting found') elif self.json_file is not None: - model_fittings = ModelFitting.load(self.json, self.json_path, - self.software.id) + model_fittings = list() + model_fittings.append(ModelFitting.load(self.json, self.json_path, + self.software.id)) return model_fittings diff --git a/nidmresults/objects/generic.py b/nidmresults/objects/generic.py index e2a5d44..1e25d8d 100644 --- a/nidmresults/objects/generic.py +++ b/nidmresults/objects/generic.py @@ -109,6 +109,13 @@ def get_query(klass, oid=None): """ return query + @classmethod + def load_from_json(klass, json_dict): + nidm_version = json_dict["NIDMResults_version"] + bd = NIDMResultsBundle(nidm_version) + + return bd + def export(self, nidm_version, export_dir): """ Create prov entity. @@ -556,6 +563,14 @@ def get_query(klass, oid=None): """ return query + @classmethod + def load_from_json(klass, json_dict): + software_type = json_dict["NIDMResultsExporter_type"] + version = json_dict["NIDMResultsExporter_softwareVersion"] + exp = ExporterSoftware(software_type, version) + + return exp + def export(self, nidm_version, export_dir): """ Create prov entities and activities. @@ -599,6 +614,14 @@ def get_query(klass, oid=None): """ return query + @classmethod + def load_from_json(klass, json_dict): + software_type = json_dict["NIDMResultsExporter_type"] + version = json_dict["NIDMResultsExporter_softwareVersion"] + exp = ExporterSoftware(software_type, version) + + return exp + def export(self, nidm_version, export_dir): """ Create prov entities and activities. diff --git a/nidmresults/objects/inference.py b/nidmresults/objects/inference.py index 8cdcf84..0612ed1 100644 --- a/nidmresults/objects/inference.py +++ b/nidmresults/objects/inference.py @@ -50,10 +50,10 @@ def load_from_json(klass, json_dict, base_dir, software_id): peak_criteria = PeakCriteria.load(json_dict) cluster_criteria = ClusterCriteria.load(json_dict) # disp_mask = DisplayMaskMap.load(json_dict, base_dir) - disp_mask = None # TODO + disp_mask = None # TODO excursion_set = ExcursionSet.load(json_dict, base_dir) clusters = Cluster.load(json_dict) - search_space = SearchSpaceMaskMap.load(json_dict, base_dir) + search_space = SearchSpace.load(json_dict, base_dir) inf = Inference(inference, height_thresh, extent_thresh, peak_criteria, cluster_criteria, disp_mask, excursion_set, @@ -1091,6 +1091,37 @@ def get_query(klass, oid=None): """ return query + @classmethod + def load_from_json(klass, json_dict, base_dir): + search_space_file = json_dict["Inferences"]["SearchSpaceMaskMap_atLocation"] + vol_in_voxels = json_dict["Inferences"]["SearchSpaceMaskMap_searchVolumeInVoxels"] + vol_in_units = json_dict["Inferences"]["SearchSpaceMaskMap_searchVolumeInUnits"] + vol_in_resels = json_dict["Inferences"]["SearchSpaceMaskMap_searchVolumeInResels"] + resel_size_in_voxels = json_dict["Inferences"]["SearchSpaceMaskMap_reselSizeInVoxels"] + unused = json_dict["Inferences"]["SearchSpaceMaskMap_searchVolumeReselsGeometry"] + noise_fwhm_in_voxels = json_dict["Inferences"]["SearchSpaceMaskMap_noiseFWHMInVoxels"] + noise_fwhm_in_units = json_dict["Inferences"]["SearchSpaceMaskMap_noiseFWHMInUnits"] + random_field_stationarity = json_dict["Inferences"]["SearchSpaceMaskMap_randomFieldStationarity"] + + # "SearchSpaceMaskMap_expectedNumberOfVoxelsPerCluster": 4.028346559086133, + # "SearchSpaceMaskMap_expectedNumberOfClusters": 0.05129329438754778, + # "SearchSpaceMaskMap_heightCriticalThresholdFWE05": 4.852417456895391, + # "SearchSpaceMaskMap_heightCriticalThresholdFDR05": 5.763953685760498, + # "SearchSpaceMaskMap_smallestSignificantClusterSizeInVoxelsFWE05": 12, + # "SearchSpaceMaskMap_smallestSignificantClusterSizeInVoxelsFDR05": 29, + + coordspace = CoordinateSpace.load_from_json( + json_dict, + os.path.join(base_dir, search_space_file)) + + sm = SearchSpace( + search_space_file, vol_in_voxels, vol_in_units, + vol_in_resels, resel_size_in_voxels, + random_field_stationarity, noise_fwhm_in_voxels, + noise_fwhm_in_units, coordspace) + + return sm + # Generate prov for search space entity generated by the inference activity def export(self, version, export_dir): """ From 49538b865d16cc0e073e3ab4b9b40c01c42561bc Mon Sep 17 00:00:00 2001 From: cmaumet Date: Wed, 12 Sep 2018 14:12:35 -0400 Subject: [PATCH 15/19] wip: writing modelfitting and contrast --- nidmresults/exporter.py | 4 +++- nidmresults/graph.py | 16 ++++++++++++---- nidmresults/objects/generic.py | 1 + nidmresults/objects/modelfitting.py | 11 ++++------- 4 files changed, 20 insertions(+), 12 deletions(-) diff --git a/nidmresults/exporter.py b/nidmresults/exporter.py index 3d54af1..27a437b 100644 --- a/nidmresults/exporter.py +++ b/nidmresults/exporter.py @@ -75,7 +75,6 @@ def __init__(self, version, out_dir, zipped=True): # A temp directory that will contain the exported data self.export_dir = tempfile.mkdtemp(prefix="nidm-", dir=out_path) - self.prepend_path = '' def parse(self): @@ -292,12 +291,14 @@ def export(self): # Add contrast estimation steps analysis_masks = dict() + for (model_fitting_id, pe_ids), contrasts in list( self.contrasts.items()): for contrast in contrasts: model_fitting = self._get_model_fitting(model_fitting_id) # for contrast in contrasts: # contrast.estimation.used(model_fitting.rms_map) + self.bundle.used(contrast.estimation.id, model_fitting.rms_map.id) # contrast.estimation.used(model_fitting.mask_map) @@ -718,6 +719,7 @@ def save_prov_to_files(self, showattributes=False): zf.close() # Need to move up before deleting the folder os.chdir("..") + print("DELETING IT!!!") shutil.rmtree(os.path.join("..", self.export_dir)) # ttl_fid = open(ttl_file, 'w'); diff --git a/nidmresults/graph.py b/nidmresults/graph.py index 617238c..6c7c772 100644 --- a/nidmresults/graph.py +++ b/nidmresults/graph.py @@ -34,10 +34,12 @@ def __init__(self, nidm_zip=None, rdf_file=None, json_file=None, workaround=False, to_replace=dict()): self.zip_path = None self.nidm_zip = None + self.prepend_path = None if nidm_zip is not None: self.study_name = os.path.basename(nidm_zip).replace(".nidm.zip", "") self.zip_path = nidm_zip + self.prepend_path = zip_path # Load the turtle file with zipfile.ZipFile(self.zip_path, 'r') as z: @@ -57,6 +59,7 @@ def __init__(self, nidm_zip=None, rdf_file=None, json_file=None, self.json = json.load(json_data) self.json_path = os.path.dirname(json_file) + self.prepend_path = self.json_path # TODO: add validation here of the JSON file according to JSON API @@ -891,8 +894,13 @@ def load_contrasts(self, workaround=False): if not contrasts: raise Exception('No contrast found') elif self.json_file is not None: - contrasts = Contrast.load(self.json, self.json_path, - self.software.id) + contrasts = dict() + con = Contrast.load(self.json, self.json_path, self.software.id) + # TODO: get actual ids -- if more than 1 + mpe_id = self.model_fittings[0].activity.id + pe_ids = (self.model_fittings[0].param_estimates[0].id,) + + contrasts[(mpe_id, pe_ids)] = con return contrasts @@ -1128,7 +1136,7 @@ def load_inferences(self): excursion_set, clusters, search_space, software_id)] elif self.json_file is not None: - inferences = Inference.load(self.json, self.json_path, + inferences = Inference.load(self.json, self.json_path, self.software.id) return inferences @@ -1147,7 +1155,7 @@ def serialize(self, destination, fmt="nidm", overwrite=False, # exporter.exporter = ExporterSoftware( # 'nidmresults', nidmresults.__version__) exporter.software = self.software - exporter.prepend_path = self.zip_path + exporter.prepend_path = self.prepend_path exporter.exporter = self.exporter exporter.bundle_ent = self.bundle exporter.export_act = self.export_act diff --git a/nidmresults/objects/generic.py b/nidmresults/objects/generic.py index 1e25d8d..85315b8 100644 --- a/nidmresults/objects/generic.py +++ b/nidmresults/objects/generic.py @@ -347,6 +347,7 @@ def export(self, nidm_version, export_dir, prepend_path): if self.temporary: os.remove(self.path) + else: new_file = self.path diff --git a/nidmresults/objects/modelfitting.py b/nidmresults/objects/modelfitting.py index a351c48..0e6af3f 100644 --- a/nidmresults/objects/modelfitting.py +++ b/nidmresults/objects/modelfitting.py @@ -136,8 +136,6 @@ def load_from_json(klass, json_dict, base_dir, software_id): param_estimates, rms_map, mask_map, grand_mean_map, machine, subjects, rpv_map=None) - print(mf) - return mf @@ -283,7 +281,7 @@ def load_from_json(klass, json_dict): grp = Group(num_subjects, group_name) grps.append(grp) else: - grps = Person() + grps.append(Person()) return grps @@ -1190,12 +1188,11 @@ def get_query(klass, oid=None): @classmethod def load_from_json(klass, json_dict, base_dir): - gm_file = json_dict['GrandMeanMap_atLocation'] + gm_file = os.path.join(base_dir, json_dict['GrandMeanMap_atLocation']) # FIXME: deal with varying coordsys across maps - coordspace = CoordinateSpace.load_from_json(json_dict, - os.path.join(base_dir, gm_file)) + coordspace = CoordinateSpace.load_from_json(json_dict, gm_file) mask = GrandMeanMap(gm_file, gm_file, coordspace) - + return mask def export(self, nidm_version, export_dir): From 9f3a969b7138d756a19fa9427b64c351166a7210 Mon Sep 17 00:00:00 2001 From: cmaumet Date: Thu, 13 Sep 2018 14:21:57 -0400 Subject: [PATCH 16/19] Read/write json without error --- nidmresults/exporter.py | 1 - nidmresults/graph.py | 13 +++++++++++-- nidmresults/objects/inference.py | 21 +++++++++++---------- 3 files changed, 22 insertions(+), 13 deletions(-) diff --git a/nidmresults/exporter.py b/nidmresults/exporter.py index 27a437b..d02872f 100644 --- a/nidmresults/exporter.py +++ b/nidmresults/exporter.py @@ -719,7 +719,6 @@ def save_prov_to_files(self, showattributes=False): zf.close() # Need to move up before deleting the folder os.chdir("..") - print("DELETING IT!!!") shutil.rmtree(os.path.join("..", self.export_dir)) # ttl_fid = open(ttl_file, 'w'); diff --git a/nidmresults/graph.py b/nidmresults/graph.py index 6c7c772..5e8fcad 100644 --- a/nidmresults/graph.py +++ b/nidmresults/graph.py @@ -1136,8 +1136,17 @@ def load_inferences(self): excursion_set, clusters, search_space, software_id)] elif self.json_file is not None: - inferences = Inference.load(self.json, self.json_path, - self.software.id) + inferences = dict() + + # TODO: get actual ids -- if more than 1 + mpe_id = self.model_fittings[0].activity.id + pe_ids = (self.model_fittings[0].param_estimates[0].id,) + + con_id = self.contrasts[(mpe_id, pe_ids)][0].estimation.id + + inferences[con_id] = Inference.load( + self.json, self.json_path, + self.software.id) return inferences diff --git a/nidmresults/objects/inference.py b/nidmresults/objects/inference.py index 0612ed1..87ac571 100644 --- a/nidmresults/objects/inference.py +++ b/nidmresults/objects/inference.py @@ -661,16 +661,16 @@ def load_from_json(klass, json_dict): clusts = list() clid = 1 - if clusts is not None: - for cl in clusts: + if clusters is not None: + for cl in clusters: size = cl['SupraThresholdCluster_clusterSizeInVoxels'] pFWER = cl['SupraThresholdCluster_pValueFWER'] - peaks = load_from_json(clust['peaks']) + peaks = Peak.load_from_json(cl['Peaks']) clust = Cluster(clid, size, pFWER, peaks) clusts.append(clust) - return clusters + return clusts def export(self, nidm_version, export_dir): @@ -1099,8 +1099,8 @@ def load_from_json(klass, json_dict, base_dir): vol_in_resels = json_dict["Inferences"]["SearchSpaceMaskMap_searchVolumeInResels"] resel_size_in_voxels = json_dict["Inferences"]["SearchSpaceMaskMap_reselSizeInVoxels"] unused = json_dict["Inferences"]["SearchSpaceMaskMap_searchVolumeReselsGeometry"] - noise_fwhm_in_voxels = json_dict["Inferences"]["SearchSpaceMaskMap_noiseFWHMInVoxels"] - noise_fwhm_in_units = json_dict["Inferences"]["SearchSpaceMaskMap_noiseFWHMInUnits"] + noise_fwhm_in_voxels = json.dumps(json_dict["Inferences"]["SearchSpaceMaskMap_noiseFWHMInVoxels"]) + noise_fwhm_in_units = json.dumps(json_dict["Inferences"]["SearchSpaceMaskMap_noiseFWHMInUnits"]) random_field_stationarity = json_dict["Inferences"]["SearchSpaceMaskMap_randomFieldStationarity"] # "SearchSpaceMaskMap_expectedNumberOfVoxelsPerCluster": 4.028346559086133, @@ -1181,7 +1181,8 @@ def export(self, version, export_dir): self.noise_roughness),) # Create "Search Space Mask map" entity - self.add_attributes(atts) + for att in atts: + self.add_attributes((att,)) class Coordinate(NIDMObject): @@ -1335,10 +1336,10 @@ def load_from_json(klass, json_dict): value = pk.get('Peak_value', None) peaks = load_from_json(clust['peaks']) - clust = Peak(equiv_z, p_unc, p_fwer, p_fdr=p_fdr, value=value) - clusts.append(clust) + peak = Peak(equiv_z, p_unc, p_fwer, p_fdr=p_fdr, value=value) + pks.append(peak) - return clusters + return pks def __str__(self): return '%s \tz=%.2f \tp=%.2e (unc.) \t%s' % ( From 8188fc546c92240050d88f23b29ca196962309ab Mon Sep 17 00:00:00 2001 From: cmaumet Date: Thu, 13 Sep 2018 14:51:53 -0400 Subject: [PATCH 17/19] empty json if pack --- nidmresults/graph.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/nidmresults/graph.py b/nidmresults/graph.py index 5e8fcad..58474fa 100644 --- a/nidmresults/graph.py +++ b/nidmresults/graph.py @@ -38,8 +38,9 @@ def __init__(self, nidm_zip=None, rdf_file=None, json_file=None, if nidm_zip is not None: self.study_name = os.path.basename(nidm_zip).replace(".nidm.zip", "") + self.nidm_zip = nidm_zip self.zip_path = nidm_zip - self.prepend_path = zip_path + self.prepend_path = self.zip_path # Load the turtle file with zipfile.ZipFile(self.zip_path, 'r') as z: @@ -51,6 +52,7 @@ def __init__(self, nidm_zip=None, rdf_file=None, json_file=None, # Parse turtle into RDF graph self.graph = self.parse(rdf_data) + self.json_file = None else: self.study_name = os.path.basename(json_file).replace(".json", "") From 25be8e4901dae8b1fd021b9c209c54182f341fdb Mon Sep 17 00:00:00 2001 From: cmaumet Date: Thu, 13 Sep 2018 14:58:39 -0400 Subject: [PATCH 18/19] add missing var --- nidmresults/graph.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/nidmresults/graph.py b/nidmresults/graph.py index 551843d..f38567e 100644 --- a/nidmresults/graph.py +++ b/nidmresults/graph.py @@ -35,7 +35,9 @@ def __init__(self, nidm_zip=None, rdf_file=None, json_file=None, self.nidm_zip = None if nidm_zip is not None: + self.json_file = None self.study_name = os.path.basename(nidm_zip).replace(".nidm.zip", "") + self.nidm_zip = nidm_zip self.zip_path = nidm_zip # Load the turtle file From 8d3918847842b81731db5e873eb1da68b9ea12db Mon Sep 17 00:00:00 2001 From: cmaumet Date: Thu, 13 Sep 2018 15:19:11 -0400 Subject: [PATCH 19/19] remove unused var --- nidmresults/graph.py | 12 +++++------- 1 file changed, 5 insertions(+), 7 deletions(-) diff --git a/nidmresults/graph.py b/nidmresults/graph.py index 58474fa..888c77f 100644 --- a/nidmresults/graph.py +++ b/nidmresults/graph.py @@ -33,12 +33,10 @@ class NIDMResults(): def __init__(self, nidm_zip=None, rdf_file=None, json_file=None, workaround=False, to_replace=dict()): self.zip_path = None - self.nidm_zip = None self.prepend_path = None if nidm_zip is not None: self.study_name = os.path.basename(nidm_zip).replace(".nidm.zip", "") - self.nidm_zip = nidm_zip self.zip_path = nidm_zip self.prepend_path = self.zip_path @@ -468,7 +466,7 @@ def get_object(self, klass, oid=None, err_if_none=True, **kwargs): return(to_return) def load_software(self): - if self.nidm_zip is not None: + if self.zip_path is not None: query = """ prefix nidm_ModelParameterEstimation: @@ -498,7 +496,7 @@ def load_software(self): return software def load_bundle_export(self): - if self.nidm_zip is not None: + if self.zip_path is not None: # query = """ # prefix nidm_softwareVersion: @@ -566,7 +564,7 @@ def load_bundle_export(self): def load_modelfitting(self): - if self.nidm_zip is not None: + if self.zip_path is not None: sd = self.graph.query(ModelFitting.get_query()) model_fittings = list() @@ -719,7 +717,7 @@ def load_modelfitting(self): return model_fittings def load_contrasts(self, workaround=False): - if self.nidm_zip is not None: + if self.zip_path is not None: if workaround: warnings.warn('Using workaround: links between contrast weights' + 'and contrast estimations are not assessed') @@ -907,7 +905,7 @@ def load_contrasts(self, workaround=False): return contrasts def load_inferences(self): - if self.nidm_zip is not None: + if self.zip_path is not None: query = """ prefix nidm_ContrastEstimation: