From dea1d2e93291590a8e6eeaf6838b6564a8eda574 Mon Sep 17 00:00:00 2001 From: ehennestad Date: Sun, 3 Nov 2024 20:43:24 +0100 Subject: [PATCH 01/29] Add function for running nwbinspector on tutorial files during testing #484 --- +tests/+unit/TutorialTest.m | 111 +++++++++++++++++++++++++++++------- +tests/requirements.txt | 3 +- nwbtest.m | 1 + 3 files changed, 93 insertions(+), 22 deletions(-) diff --git a/+tests/+unit/TutorialTest.m b/+tests/+unit/TutorialTest.m index 2d542129..90dce0c9 100644 --- a/+tests/+unit/TutorialTest.m +++ b/+tests/+unit/TutorialTest.m @@ -61,12 +61,7 @@ function setupClass(testCase) setenv('PYTHONPATH', fileparts(mfilename('fullpath'))); nwbClearGenerated() - end - end - - methods (TestClassTeardown) - function tearDownClass(testCase) %#ok - %generateCore() + testCase.addTeardown(@generateCore) end end @@ -79,30 +74,27 @@ function setupMethod(testCase) methods (Test) function testTutorial(testCase, tutorialFile) %#ok + % Intentionally capturing output, in order for tests to cover + % code which overloads display methods for nwb types/objects. C = evalc( 'run(tutorialFile)' ); %#ok - testCase.testReadTutorialNwbFileWithPynwb() + + testCase.readTutorialNwbFileWithPynwb() + %testCase.inspectTutorialFileWithNwbInspector() end end methods - function testReadTutorialNwbFileWithPynwb(testCase) + function readTutorialNwbFileWithPynwb(testCase) % Retrieve all files generated by tutorial - nwbListing = dir('*.nwb'); - - for i = 1:numel(nwbListing) - nwbFilename = nwbListing(i).name; - if any(strcmp(nwbFilename, tests.unit.TutorialTest.SkippedFiles)) - continue - end - + nwbFileNameList = testCase.listNwbFiles(); + for nwbFilename = nwbFileNameList try try - io = py.pynwb.NWBHDF5IO(nwbListing(i).name); + io = py.pynwb.NWBHDF5IO(nwbFilename); nwbObject = io.read(); testCase.verifyNotEmpty(nwbObject, 'The NWB file should not be empty.'); io.close() - catch ME if strcmp(ME.identifier, 'MATLAB:undefinedVarOrClass') && ... contains(ME.message, 'py.pynwb.NWBHDF5IO') @@ -110,8 +102,8 @@ function testReadTutorialNwbFileWithPynwb(testCase) pythonExecutable = tests.util.getPythonPath(); cmd = sprintf('"%s" -B -m read_nwbfile_with_pynwb %s',... pythonExecutable, nwbFilename); - status = system(cmd); + if status ~= 0 error('Failed to read NWB file "%s" using pynwb', nwbFilename) end @@ -119,20 +111,97 @@ function testReadTutorialNwbFileWithPynwb(testCase) rethrow(ME) end end - catch ME error(ME.message) %testCase.verifyFail(sprintf('Failed to read file %s with error: %s', nwbListing(i).name, ME.message)); end end end + + function inspectTutorialFileWithNwbInspector(testCase) + % Retrieve all files generated by tutorial + nwbFileNameList = testCase.listNwbFiles(); + for nwbFilename = nwbFileNameList + results = py.list(py.nwbinspector.inspect_nwbfile(nwbfile_path=nwbFilename)); + + if isempty(cell(results)) + return + end + + results = testCase.convertNwbInspectorResultsToStruct(results); + T = struct2table(results); disp(T) + + for j = 1:numel(results) + testCase.verifyLessThanOrEqual(results(j).importance, 0, ... + sprintf('Message: %s\nLocation: %s\n File: %s\n', ... + string(results(j).message), results(j).location, results(j).filepath)) + end + end + end + end + + methods (Access = private) + function nwbFileNames = listNwbFiles(testCase) + nwbListing = dir('*.nwb'); + nwbFileNames = string( {nwbListing.name} ); + nwbFileNames = setdiff(nwbFileNames, testCase.SkippedFiles); + assert(isrow(nwbFileNames), 'Expected output to be a row vector') + if ~isscalar(nwbFileNames) + if iscolumn(nwbFileNames) + nwbFileNames = transpose(nwbFileNames); + end + end + end + end + + methods (Static) + function resultsOut = convertNwbInspectorResultsToStruct(resultsIn) + CHECK_IGNORE = "check_image_series_external_file_valid"; + + C = cell(resultsIn); + + resultsOut = struct(... + 'importance', {}, ... + 'severity', {}, ... + 'location', {}, ... + 'filepath', {}, ... + 'check_name', {}, ... + 'ignore', {}); + + for i = 1:numel(C) + resultsOut(i).importance = double( py.getattr(C{i}.importance, 'value') ); + resultsOut(i).severity = double( py.getattr(C{i}.severity, 'value') ); + + try + resultsOut(i).location = string(C{i}.location); + catch + resultsOut(i).location = "N/A"; + end + + resultsOut(i).message = string(C{i}.message); + resultsOut(i).filepath = string(C{i}.file_path); + resultsOut(i).check_name = string(C{i}.check_function_name); + resultsOut(i).ignore = any(strcmp(CHECK_IGNORE, resultsOut(i).check_name)); + + % Special case to ignore + if resultsOut(i).location == "/acquisition/ExternalVideos" && ... + resultsOut(i).check_name == "check_timestamps_match_first_dimension" + resultsOut(i).ignore = true; + end + end + resultsOut([resultsOut.ignore]) = []; + end end end function tutorialNames = listTutorialFiles() % listTutorialFiles - List names of all tutorial files (exclude skipped files) rootPath = getMatNwbRootDirectory(); - L = dir(fullfile(rootPath, 'tutorials')); + L = cat(1, ... + dir(fullfile(rootPath, 'tutorials', '*.mlx')), ... + dir(fullfile(rootPath, 'tutorials', '*.m')) ... + ); + L( [L.isdir] ) = []; % Ignore folders tutorialNames = setdiff({L.name}, tests.unit.TutorialTest.SkippedTutorials); end diff --git a/+tests/requirements.txt b/+tests/requirements.txt index da0e7629..cc9afd91 100644 --- a/+tests/requirements.txt +++ b/+tests/requirements.txt @@ -1,2 +1,3 @@ pynwb -hdf5plugin \ No newline at end of file +hdf5plugin +nwbinspector diff --git a/nwbtest.m b/nwbtest.m index 18a5f86a..a1f2ed15 100644 --- a/nwbtest.m +++ b/nwbtest.m @@ -46,6 +46,7 @@ ws = pwd; nwbClearGenerated(); % Clear default files if any. + cleanupObj = onCleanup(@() generateCore); cleaner = onCleanup(@generateCore); % Regenerate core when finished pvcell = struct2pvcell(parser.Unmatched); From a0af8720c72e433bc231e2a60f5401c0bda2af66 Mon Sep 17 00:00:00 2001 From: ehennestad Date: Sun, 3 Nov 2024 21:14:36 +0100 Subject: [PATCH 02/29] Fix intro tutorial --- +tests/+unit/TutorialTest.m | 8 +- tutorials/html/intro.html | 239 ++++++++++++++++++------------------ tutorials/intro.mlx | Bin 220876 -> 221084 bytes 3 files changed, 128 insertions(+), 119 deletions(-) diff --git a/+tests/+unit/TutorialTest.m b/+tests/+unit/TutorialTest.m index 90dce0c9..8360dfdc 100644 --- a/+tests/+unit/TutorialTest.m +++ b/+tests/+unit/TutorialTest.m @@ -14,6 +14,10 @@ MatNwbDirectory end + properties (Constant) + NwbInspectorSeverityLevel = 1 + end + properties (TestParameter) % TutorialFile - A cell array where each cell is the name of a % tutorial file. testTutorial will run on each file individually @@ -79,7 +83,7 @@ function testTutorial(testCase, tutorialFile) %#ok C = evalc( 'run(tutorialFile)' ); %#ok testCase.readTutorialNwbFileWithPynwb() - %testCase.inspectTutorialFileWithNwbInspector() + testCase.inspectTutorialFileWithNwbInspector() end end @@ -132,7 +136,7 @@ function inspectTutorialFileWithNwbInspector(testCase) T = struct2table(results); disp(T) for j = 1:numel(results) - testCase.verifyLessThanOrEqual(results(j).importance, 0, ... + testCase.verifyLessThan(results(j).importance, testCase.NwbInspectorSeverityLevel, ... sprintf('Message: %s\nLocation: %s\n File: %s\n', ... string(results(j).message), results(j).location, results(j).filepath)) end diff --git a/tutorials/html/intro.html b/tutorials/html/intro.html index 9fb4c9db..6f261e98 100644 --- a/tutorials/html/intro.html +++ b/tutorials/html/intro.html @@ -1,28 +1,22 @@ -Introduction to MatNWB

Behavior Data

This tutorial will guide you in writing behavioral data to NWB.

Creating an NWB File

Create an NWBFile object with the required fields (session_description, identifier, and session_start_time) and additional metadata.
nwb = NwbFile( ...
'session_description', 'mouse in open exploration',...
'identifier', 'Mouse5_Day3', ...
'session_start_time', datetime(2018, 4, 25, 2, 30, 3), ...
'general_experimenter', 'My Name', ... % optional
'general_session_id', 'session_1234', ... % optional
'general_institution', 'University of My Institution', ... % optional
'general_related_publications', 'DOI:10.1016/j.neuron.2016.12.011'); % optional
nwb

SpatialSeries: Storing continuous spatial data

SpatialSeries is a subclass of TimeSeries that represents data in space, such as the spatial direction e.g., of gaze or travel or position of an animal over time.
Create data that corresponds to x, y position over time.
position_data = [linspace(0, 10, 50); linspace(0, 8, 50)];
In SpatialSeries data, the first dimension is always time (in seconds), the second dimension represents the x, y position. SpatialSeries data should be stored as one continuous stream as it is acquired, not by trials as is often reshaped fro analysis. Data can be trial-aligned on-the-fly using the trials table. See the trials tutorial for further information.
For position data reference_frame indicates the zero-position, e.g. the 0,0 point might be the bottom-left corner of an enclosure, as viewed fromvteh tracking camera.
timestamps = linspace(0, 50)/ 200;
position_spatial_series = types.core.SpatialSeries( ...
'description', 'Postion (x, y) in an open field.', ...
'data', position_data, ...
'timestamps', timestamps, ...
'reference_frame', '(0,0) is the bottom left corner.' ...
)

Position: Storing position measured over time

To help data analysis and visualiztion tools know that this SpatialSeries obejct represents the position of the subject, store the SpatialSeries object inside a Position object, which can hold one or more SpatialSeries objects.
position = types.core.Position();
position.spatialseries.set('SpatialSeries', position_spatial_series);

Create a Behavior Processing Module

Create a processing module called "behavior" for storing behavioral data in the NWBFile, then add the Position object to the processing module.
behavior_processing_module = types.core.ProcessingModule('description', 'stores behavioral data.');
behavior_processing_module.nwbdatainterface.set("Position", position);
nwb.processing.set("behavior", behavior_processing_module);

CompassDirection: Storing view angle measured over time

Analogous to how position can be stored, we can create a SpatialSeries object for representing the view angle of the subject.
For direction data reference from indicates the zero direction, for instance in this case "straight ahead" is 0 radians.
view_angle_data = linspace(0, 4, 50);
direction_spatial_series = types.core.SpatialSeries( ...
'description', 'View angle of the subject measured in radians.', ...
'data', view_angle_data, ...
'timestamps', timestamps, ...
'reference_frame', 'straight ahead', ...
'data_unit', 'radians' ...
);
direction = types.core.CompassDirection();
direction.spatialseries.set('spatial_series', direction_spatial_series);
We can add a CompassDirection object to the behavior processing module the same way we have added the position data.
%behavior_processing_module = types.core.ProcessingModule("stores behavioral data."); % if you have not already created it
behavior_processing_module.nwbdatainterface.set('CompassDirection', direction);
%nwb.processing.set('behavior', behavior_processing_module); % if you have not already added it

BehaviorTimeSeries: Storing continuous behavior data

BehavioralTimeSeries is an interface for storing continuous behavior data, such as the speed of a subject.
speed_data = linspace(0, 0.4, 50);
 
speed_time_series = types.core.TimeSeries( ...
'data', speed_data, ...
'starting_time_rate', 10.0, ... % Hz
'description', 'he speed of the subject measured over time.', ...
'data_unit', 'm/s' ...
);
 
behavioral_time_series = types.core.BehavioralTimeSeries();
behavioral_time_series.timeseries.set('speed', speed_time_series);
 
%behavior_processing_module = types.core.ProcessingModule("stores behavioral data."); % if you have not already created it
behavior_processing_module.nwbdatainterface.set('BehavioralTimeSeries', behavioral_time_series);
%nwb.processing.set('behavior', behavior_processing_module); % if you have not already added it

BehavioralEvents: Storing behavioral events

BehavioralEvents is an interface for storing behavioral events. We can use it for storing the timing and amount of rewards (e.g. water amount) or lever press times.
reward_amount = [1.0, 1.5, 1.0, 1.5];
event_timestamps = [1.0, 2.0, 5.0, 6.0];
 
time_series = types.core.TimeSeries( ...
'data', reward_amount, ...
'timestamps', event_timestamps, ...
'description', 'The water amount the subject received as a reward.', ...
'data_unit', 'ml' ...
);
 
behavioral_events = types.core.BehavioralEvents();
behavioral_events.timeseries.set('lever_presses', time_series);
 
%behavior_processing_module = types.core.ProcessingModule("stores behavioral data."); % if you have not already created it
behavior_processing_module.nwbdatainterface.set('BehavioralEvents', behavioral_events);
%nwb.processing.set('behavior', behavior_processing_module); % if you have not already added it
Storing only the timestsamps of the events is possible with the ndx-events NWB extension. You can also add labels associated with the events with this extension. You can find information about installation and example usage here.

BehavioralEpochs: Storing intervals of behavior data

BehavioralEpochs is for storing intervals of behavior data. BehavioralEpochs uses IntervalSeries to represent the time intervals. Create an IntervalSeries object that represents the time intervals when hte animal was running. IntervalSeries uses 1 to indicate the beginning of an interval and -1 to indicate the end.
run_intervals = types.core.IntervalSeries( ...
'description', 'Intervals when the animal was running.', ...
'data', [1, -1, 1, -1, 1, -1], ...
'timestamps', [0.5, 1.5, 3.5, 4.0, 7.0, 7.3] ...
);
 
behavioral_epochs = types.core.BehavioralEpochs();
behavioral_epochs.intervalseries.set('running', run_intervals);
You can add more than one IntervalSeries to a BehavioralEpochs object.
sleep_intervals = types.core.IntervalSeries( ...
'description', 'Intervals when the animal was sleeping', ...
'data', [1, -1, 1, -1], ...
'timestamps', [15.0, 30.0, 60.0, 95.0] ...
);
behavioral_epochs.intervalseries.set('sleeping', sleep_intervals);
 
% behavior_processing_module = types.core.ProcessingModule("stores behavioral data.");
% behavior_processing_module.nwbdatainterface.set('BehavioralEvents', behavioral_events);
% nwb.processing.set('behavior', behavior_processing_module);

Another approach: TimeIntervals

Using TimeIntervals to represent time intervals is often preferred over BehavioralEpochs and IntervalSeries. TimeIntervals is a subclass of DynamicTable, which offers flexibility for tabular data by allowing the addition of optional columns which are not defined in the standard.
sleep_intervals = types.core.TimeIntervals( ...
'description', 'Intervals when the animal was sleeping.', ...
'colnames', {'start_time', 'stop_time', 'stage'} ...
);
 
sleep_intervals.addRow('start_time', 0.3, 'stop_time', 0.35, 'stage', 1);
sleep_intervals.addRow('start_time', 0.7, 'stop_time', 0.9, 'stage', 2);
sleep_intervals.addRow('start_time', 1.3, 'stop_time', 3.0, 'stage', 3);
 
nwb.intervals.set('sleep_intervals', sleep_intervals);

EyeTracking: Storing continuous eye-tracking data of gaze direction

EyeTracking is for storing eye-tracking data which represents direction of gaze as measured by an eye tracking algorithm. An EyeTracking object holds one or more SpatialSeries objects that represent the gaze direction over time extracted from a video.
eye_position_data = [linspace(-20, 30, 50); linspace(30, -20, 50)];
 
right_eye_position = types.core.SpatialSeries( ...
'description', 'The position of the right eye measured in degrees.', ...
'data', eye_position_data, ...
'starting_time_rate', 50.0, ... % Hz
'reference_frame', '(0,0) is middle', ...
'data_unit', 'degrees' ...
);
 
left_eye_position = types.core.SpatialSeries( ...
'description', 'The position of the right eye measured in degrees.', ...
'data', eye_position_data, ...
'starting_time_rate', 50.0, ... % Hz
'reference_frame', '(0,0) is middle', ...
'data_unit', 'degrees' ...
);
 
eye_tracking = types.core.EyeTracking();
eye_tracking.spatialseries.set('right_eye_position', right_eye_position);
eye_tracking.spatialseries.set('left_eye_position', left_eye_position);
 
% behavior_processing_module = types.core.ProcessingModule("stores behavioral data.");
behavior_processing_module.nwbdatainterface.set('EyeTracking', eye_tracking);
% nwb.processing.set('behavior', behavior_processing_module);

PupilTracking: Storing continuous eye-tracking data of pupil size

PupilTracking is for storing eye-tracking data which represents pupil size. PupilTracking hold one or more TimeSeries obejcts taht canrepresent different features such as the dilaltion of the pupil measured over time by a pupil tracking algorithm.
pupil_diameter = types.core.TimeSeries( ...
'description', 'Pupil diameter extracted from the video of the right eye.', ...
'data', linspace(0.001, 0.002, 50), ...
'starting_time_rate', 20.0, ... % Hz
'data_unit', 'meters' ...
);
 
pupil_tracking = types.core.PupilTracking();
pupil_tracking.timeseries.set('pupil_diameter', pupil_diameter);
 
% behavior_processing_module = types.core.ProcessingModule("stores behavioral data.");
behavior_processing_module.nwbdatainterface.set('PupilTracking', pupil_tracking);
% nwb.processing.set('behavior', behavior_processing_module);

Writing the behavior data to an NWB file

All of the above commands build an NWBFile object in-memory. To write this file, use nwbExport.
nwbExport(nwb, 'test_behavior.nwb');
+.S3 { border-left: 0.994318px solid rgb(217, 217, 217); border-right: 0.994318px solid rgb(217, 217, 217); border-top: 0.994318px solid rgb(217, 217, 217); border-bottom: 0px none rgb(33, 33, 33); border-radius: 4px 4px 0px 0px; padding: 6px 45px 0px 13px; line-height: 18.004px; min-height: 0px; white-space: nowrap; color: rgb(33, 33, 33); font-family: Menlo, Monaco, Consolas, "Courier New", monospace; font-size: 14px; } +.S4 { border-left: 0.994318px solid rgb(217, 217, 217); border-right: 0.994318px solid rgb(217, 217, 217); border-top: 0px none rgb(33, 33, 33); border-bottom: 0px none rgb(33, 33, 33); border-radius: 0px; padding: 0px 45px 0px 13px; line-height: 18.004px; min-height: 0px; white-space: nowrap; color: rgb(33, 33, 33); font-family: Menlo, Monaco, Consolas, "Courier New", monospace; font-size: 14px; } +.S5 { border-left: 0.994318px solid rgb(217, 217, 217); border-right: 0.994318px solid rgb(217, 217, 217); border-top: 0px none rgb(33, 33, 33); border-bottom: 0.994318px solid rgb(217, 217, 217); border-radius: 0px; padding: 0px 45px 4px 13px; line-height: 18.004px; min-height: 0px; white-space: nowrap; color: rgb(33, 33, 33); font-family: Menlo, Monaco, Consolas, "Courier New", monospace; font-size: 14px; } +.S6 { color: rgb(33, 33, 33); padding: 10px 0px 6px 17px; background: rgb(255, 255, 255) none repeat scroll 0% 0% / auto padding-box border-box; font-family: Menlo, Monaco, Consolas, "Courier New", monospace; font-size: 14px; overflow-x: hidden; line-height: 17.234px; } +/* Styling that is common to warnings and errors is in diagnosticOutput.css */.embeddedOutputsErrorElement { min-height: 18px; max-height: 550px;} +.embeddedOutputsErrorElement .diagnosticMessage-errorType { overflow: auto;} +.embeddedOutputsErrorElement.inlineElement {} +.embeddedOutputsErrorElement.rightPaneElement {} +/* Styling that is common to warnings and errors is in diagnosticOutput.css */.embeddedOutputsWarningElement { min-height: 18px; max-height: 550px;} +.embeddedOutputsWarningElement .diagnosticMessage-warningType { overflow: auto;} +.embeddedOutputsWarningElement.inlineElement {} +.embeddedOutputsWarningElement.rightPaneElement {} +/* Copyright 2015-2023 The MathWorks, Inc. *//* In this file, styles are not scoped to rtcContainer since they could be in the Dojo Tooltip */.diagnosticMessage-wrapper { font-family: Menlo, Monaco, Consolas, "Courier New", monospace; font-size: 12px;} +.diagnosticMessage-wrapper.diagnosticMessage-warningType { /*This fallback value will be used for appdesigner warnings*/ color: var(--rtc-warning-output-color, var(--mw-color-matlabWarning));} +.diagnosticMessage-wrapper.diagnosticMessage-warningType a { /*This fallback value will be used for appdesigner warnings*/ color: var(--rtc-warning-output-color, var(--mw-color-matlabWarning)); text-decoration: underline;} +.rtcThemeDefaultOverride .diagnosticMessage-wrapper.diagnosticMessage-warningType,.rtcThemeDefaultOverride .diagnosticMessage-wrapper.diagnosticMessage-warningType a { color: var(--mw-color-matlabWarning) !important;} +.diagnosticMessage-wrapper.diagnosticMessage-errorType { /*This fallback value will be used in appdesigner error tooltip text*/ color: var(--rtc-error-output-color, var(--mw-color-matlabErrors));} +.diagnosticMessage-wrapper.diagnosticMessage-errorType a { /*This fallback value will be used in appdesigner error tooltip text*/ color: var(--rtc-error-output-color, var(--mw-color-matlabErrors)); text-decoration: underline;} +.rtcThemeDefaultOverride .diagnosticMessage-wrapper.diagnosticMessage-errorType,.rtcThemeDefaultOverride .diagnosticMessage-wrapper.diagnosticMessage-errorType a { color: var(--mw-color-matlabErrors) !important;} +.diagnosticMessage-wrapper .diagnosticMessage-messagePart,.diagnosticMessage-wrapper .diagnosticMessage-causePart { white-space: pre-wrap;} +.diagnosticMessage-wrapper .diagnosticMessage-stackPart { white-space: pre;} +.embeddedOutputsTextElement,.embeddedOutputsVariableStringElement { white-space: pre; word-wrap: initial; min-height: 18px; max-height: 550px;} +.embeddedOutputsTextElement .textElement,.embeddedOutputsVariableStringElement .textElement { overflow: auto;} +.textElement,.rtcDataTipElement .textElement { padding-top: 2px;} +.embeddedOutputsTextElement.inlineElement,.embeddedOutputsVariableStringElement.inlineElement {} +.inlineElement .textElement {} +.embeddedOutputsTextElement.rightPaneElement,.embeddedOutputsVariableStringElement.rightPaneElement { min-height: 16px;} +.rightPaneElement .textElement { padding-top: 2px; padding-left: 9px;} +.S7 { border-left: 0.994318px solid rgb(217, 217, 217); border-right: 0.994318px solid rgb(217, 217, 217); border-top: 0.994318px solid rgb(217, 217, 217); border-bottom: 0.994318px solid rgb(217, 217, 217); border-radius: 4px; padding: 6px 45px 4px 13px; line-height: 18.004px; min-height: 0px; white-space: nowrap; color: rgb(33, 33, 33); font-family: Menlo, Monaco, Consolas, "Courier New", monospace; font-size: 14px; } +.S8 { margin: 10px 10px 9px 4px; padding: 0px; line-height: 21px; min-height: 0px; white-space: pre-wrap; color: rgb(33, 33, 33); font-family: Helvetica, Arial, sans-serif; font-style: normal; font-size: 14px; font-weight: 400; text-align: left; } +.S9 { border-left: 0.994318px solid rgb(217, 217, 217); border-right: 0.994318px solid rgb(217, 217, 217); border-top: 0px none rgb(33, 33, 33); border-bottom: 0.994318px solid rgb(217, 217, 217); border-radius: 0px 0px 4px 4px; padding: 0px 45px 4px 13px; line-height: 18.004px; min-height: 0px; white-space: nowrap; color: rgb(33, 33, 33); font-family: Menlo, Monaco, Consolas, "Courier New", monospace; font-size: 14px; } +.S10 { margin: 15px 10px 5px 4px; padding: 0px; line-height: 18px; min-height: 0px; white-space: pre-wrap; color: rgb(33, 33, 33); font-family: Helvetica, Arial, sans-serif; font-style: normal; font-size: 17px; font-weight: 700; text-align: left; }

Behavior Data

This tutorial will guide you in writing behavioral data to NWB.

Creating an NWB File

Create an NWBFile object with the required fields (session_description, identifier, and session_start_time) and additional metadata.
nwb = NwbFile( ...
'session_description', 'mouse in open exploration',...
'identifier', 'Mouse5_Day3', ...
'session_start_time', datetime(2018, 4, 25, 2, 30, 3, 'TimeZone', 'local'), ...
'general_experimenter', 'My Name', ... % optional
'general_session_id', 'session_1234', ... % optional
'general_institution', 'University of My Institution', ... % optional
'general_related_publications', 'DOI:10.1016/j.neuron.2016.12.011'); % optional
nwb
nwb =
NwbFile with properties: + + nwb_version: '2.7.0' + file_create_date: [] + identifier: 'Mouse5_Day3' + session_description: 'mouse in open exploration' + session_start_time: {[2018-04-25T02:30:03.000000+02:00]} + timestamps_reference_time: [] + acquisition: [0×1 types.untyped.Set] + analysis: [0×1 types.untyped.Set] + general: [0×1 types.untyped.Set] + general_data_collection: '' + general_devices: [0×1 types.untyped.Set] + general_experiment_description: '' + general_experimenter: 'My Name' + general_extracellular_ephys: [0×1 types.untyped.Set] + general_extracellular_ephys_electrodes: [] + general_institution: 'University of My Institution' + general_intracellular_ephys: [0×1 types.untyped.Set] + general_intracellular_ephys_experimental_conditions: [] + general_intracellular_ephys_filtering: '' + general_intracellular_ephys_intracellular_recordings: [] + general_intracellular_ephys_repetitions: [] + general_intracellular_ephys_sequential_recordings: [] + general_intracellular_ephys_simultaneous_recordings: [] + general_intracellular_ephys_sweep_table: [] + general_keywords: '' + general_lab: '' + general_notes: '' + general_optogenetics: [0×1 types.untyped.Set] + general_optophysiology: [0×1 types.untyped.Set] + general_pharmacology: '' + general_protocol: '' + general_related_publications: 'DOI:10.1016/j.neuron.2016.12.011' + general_session_id: 'session_1234' + general_slices: '' + general_source_script: '' + general_source_script_file_name: '' + general_stimulus: '' + general_subject: [] + general_surgery: '' + general_virus: '' + intervals: [0×1 types.untyped.Set] + intervals_epochs: [] + intervals_invalid_times: [] + intervals_trials: [] + processing: [0×1 types.untyped.Set] + scratch: [0×1 types.untyped.Set] + stimulus_presentation: [0×1 types.untyped.Set] + stimulus_templates: [0×1 types.untyped.Set] + units: [] + +Warning: The following required properties are missing for instance for type "NwbFile": + timestamps_reference_time

SpatialSeries: Storing continuous spatial data

SpatialSeries is a subclass of TimeSeries that represents data in space, such as the spatial direction e.g., of gaze or travel or position of an animal over time.
Create data that corresponds to x, y position over time.
position_data = [linspace(0, 10, 50); linspace(0, 8, 50)];
In SpatialSeries data, the first dimension is always time (in seconds), the second dimension represents the x, y position. SpatialSeries data should be stored as one continuous stream as it is acquired, not by trials as is often reshaped fro analysis. Data can be trial-aligned on-the-fly using the trials table. See the trials tutorial for further information.
For position data reference_frame indicates the zero-position, e.g. the 0,0 point might be the bottom-left corner of an enclosure, as viewed fromvteh tracking camera.
timestamps = linspace(0, 50, 50)/ 200;
position_spatial_series = types.core.SpatialSeries( ...
'description', 'Postion (x, y) in an open field.', ...
'data', position_data, ...
'timestamps', timestamps, ...
'reference_frame', '(0,0) is the bottom left corner.' ...
)
position_spatial_series =
SpatialSeries with properties: + + reference_frame: '(0,0) is the bottom left corner.' + starting_time_unit: 'seconds' + timestamps_interval: 1 + timestamps_unit: 'seconds' + data: [2×50 double] + comments: 'no comments' + control: [] + control_description: '' + data_continuity: '' + data_conversion: 1 + data_offset: 0 + data_resolution: -1 + data_unit: 'meters' + description: 'Postion (x, y) in an open field.' + starting_time: [] + starting_time_rate: [] + timestamps: [0 0.0051 0.0102 0.0153 0.0204 0.0255 0.0306 0.0357 0.0408 0.0459 0.0510 0.0561 0.0612 0.0663 0.0714 0.0765 0.0816 0.0867 0.0918 0.0969 0.1020 0.1071 0.1122 0.1173 0.1224 0.1276 0.1327 0.1378 0.1429 0.1480 0.1531 … ] (1×50 double) +

Position: Storing position measured over time

To help data analysis and visualiztion tools know that this SpatialSeries obejct represents the position of the subject, store the SpatialSeries object inside a Position object, which can hold one or more SpatialSeries objects.
position = types.core.Position();
position.spatialseries.set('SpatialSeries', position_spatial_series);

Create a Behavior Processing Module

Create a processing module called "behavior" for storing behavioral data in the NWBFile, then add the Position object to the processing module.
behavior_processing_module = types.core.ProcessingModule('description', 'stores behavioral data.');
behavior_processing_module.nwbdatainterface.set("Position", position);
nwb.processing.set("behavior", behavior_processing_module);

CompassDirection: Storing view angle measured over time

Analogous to how position can be stored, we can create a SpatialSeries object for representing the view angle of the subject.
For direction data reference from indicates the zero direction, for instance in this case "straight ahead" is 0 radians.
view_angle_data = linspace(0, 4, 50);
direction_spatial_series = types.core.SpatialSeries( ...
'description', 'View angle of the subject measured in radians.', ...
'data', view_angle_data, ...
'timestamps', timestamps, ...
'reference_frame', 'straight ahead', ...
'data_unit', 'radians' ...
);
direction = types.core.CompassDirection();
direction.spatialseries.set('spatial_series', direction_spatial_series);
We can add a CompassDirection object to the behavior processing module the same way we have added the position data.
%behavior_processing_module = types.core.ProcessingModule("stores behavioral data."); % if you have not already created it
behavior_processing_module.nwbdatainterface.set('CompassDirection', direction);
%nwb.processing.set('behavior', behavior_processing_module); % if you have not already added it

BehaviorTimeSeries: Storing continuous behavior data

BehavioralTimeSeries is an interface for storing continuous behavior data, such as the speed of a subject.
speed_data = linspace(0, 0.4, 50);
 
speed_time_series = types.core.TimeSeries( ...
'data', speed_data, ...
'starting_time', 1.0, ... % NB: Important to set starting_time when using starting_time_rate
'starting_time_rate', 10.0, ... % Hz
'description', 'he speed of the subject measured over time.', ...
'data_unit', 'm/s' ...
);
 
behavioral_time_series = types.core.BehavioralTimeSeries();
behavioral_time_series.timeseries.set('speed', speed_time_series);
 
%behavior_processing_module = types.core.ProcessingModule("stores behavioral data."); % if you have not already created it
behavior_processing_module.nwbdatainterface.set('BehavioralTimeSeries', behavioral_time_series);
%nwb.processing.set('behavior', behavior_processing_module); % if you have not already added it

BehavioralEvents: Storing behavioral events

BehavioralEvents is an interface for storing behavioral events. We can use it for storing the timing and amount of rewards (e.g. water amount) or lever press times.
reward_amount = [1.0, 1.5, 1.0, 1.5];
event_timestamps = [1.0, 2.0, 5.0, 6.0];
 
time_series = types.core.TimeSeries( ...
'data', reward_amount, ...
'timestamps', event_timestamps, ...
'description', 'The water amount the subject received as a reward.', ...
'data_unit', 'ml' ...
);
 
behavioral_events = types.core.BehavioralEvents();
behavioral_events.timeseries.set('lever_presses', time_series);
 
%behavior_processing_module = types.core.ProcessingModule("stores behavioral data."); % if you have not already created it
behavior_processing_module.nwbdatainterface.set('BehavioralEvents', behavioral_events);
%nwb.processing.set('behavior', behavior_processing_module); % if you have not already added it
Storing only the timestsamps of the events is possible with the ndx-events NWB extension. You can also add labels associated with the events with this extension. You can find information about installation and example usage here.

BehavioralEpochs: Storing intervals of behavior data

BehavioralEpochs is for storing intervals of behavior data. BehavioralEpochs uses IntervalSeries to represent the time intervals. Create an IntervalSeries object that represents the time intervals when hte animal was running. IntervalSeries uses 1 to indicate the beginning of an interval and -1 to indicate the end.
run_intervals = types.core.IntervalSeries( ...
'description', 'Intervals when the animal was running.', ...
'data', [1, -1, 1, -1, 1, -1], ...
'timestamps', [0.5, 1.5, 3.5, 4.0, 7.0, 7.3] ...
);
 
behavioral_epochs = types.core.BehavioralEpochs();
behavioral_epochs.intervalseries.set('running', run_intervals);
You can add more than one IntervalSeries to a BehavioralEpochs object.
sleep_intervals = types.core.IntervalSeries( ...
'description', 'Intervals when the animal was sleeping', ...
'data', [1, -1, 1, -1], ...
'timestamps', [15.0, 30.0, 60.0, 95.0] ...
);
behavioral_epochs.intervalseries.set('sleeping', sleep_intervals);
 
% behavior_processing_module = types.core.ProcessingModule("stores behavioral data.");
% behavior_processing_module.nwbdatainterface.set('BehavioralEvents', behavioral_events);
% nwb.processing.set('behavior', behavior_processing_module);

Another approach: TimeIntervals

Using TimeIntervals to represent time intervals is often preferred over BehavioralEpochs and IntervalSeries. TimeIntervals is a subclass of DynamicTable, which offers flexibility for tabular data by allowing the addition of optional columns which are not defined in the standard.
sleep_intervals = types.core.TimeIntervals( ...
'description', 'Intervals when the animal was sleeping.', ...
'colnames', {'start_time', 'stop_time', 'stage'} ...
);
 
sleep_intervals.addRow('start_time', 0.3, 'stop_time', 0.35, 'stage', 1);
sleep_intervals.addRow('start_time', 0.7, 'stop_time', 0.9, 'stage', 2);
sleep_intervals.addRow('start_time', 1.3, 'stop_time', 3.0, 'stage', 3);
 
nwb.intervals.set('sleep_intervals', sleep_intervals);

EyeTracking: Storing continuous eye-tracking data of gaze direction

EyeTracking is for storing eye-tracking data which represents direction of gaze as measured by an eye tracking algorithm. An EyeTracking object holds one or more SpatialSeries objects that represent the gaze direction over time extracted from a video.
eye_position_data = [linspace(-20, 30, 50); linspace(30, -20, 50)];
 
right_eye_position = types.core.SpatialSeries( ...
'description', 'The position of the right eye measured in degrees.', ...
'data', eye_position_data, ...
'starting_time', 1.0, ... % NB: Important to set starting_time when using starting_time_rate
'starting_time_rate', 50.0, ... % Hz
'reference_frame', '(0,0) is middle', ...
'data_unit', 'degrees' ...
);
 
left_eye_position = types.core.SpatialSeries( ...
'description', 'The position of the right eye measured in degrees.', ...
'data', eye_position_data, ...
'starting_time', 1.0, ... % NB: Important to set starting_time when using starting_time_rate
'starting_time_rate', 50.0, ... % Hz
'reference_frame', '(0,0) is middle', ...
'data_unit', 'degrees' ...
);
 
eye_tracking = types.core.EyeTracking();
eye_tracking.spatialseries.set('right_eye_position', right_eye_position);
eye_tracking.spatialseries.set('left_eye_position', left_eye_position);
 
% behavior_processing_module = types.core.ProcessingModule("stores behavioral data.");
behavior_processing_module.nwbdatainterface.set('EyeTracking', eye_tracking);
% nwb.processing.set('behavior', behavior_processing_module);

PupilTracking: Storing continuous eye-tracking data of pupil size

PupilTracking is for storing eye-tracking data which represents pupil size. PupilTracking hold one or more TimeSeries obejcts taht canrepresent different features such as the dilaltion of the pupil measured over time by a pupil tracking algorithm.
pupil_diameter = types.core.TimeSeries( ...
'description', 'Pupil diameter extracted from the video of the right eye.', ...
'data', linspace(0.001, 0.002, 50), ...
'starting_time', 1.0, ... % NB: Important to set starting_time when using starting_time_rate
'starting_time_rate', 20.0, ... % Hz
'data_unit', 'meters' ...
);
 
pupil_tracking = types.core.PupilTracking();
pupil_tracking.timeseries.set('pupil_diameter', pupil_diameter);
 
% behavior_processing_module = types.core.ProcessingModule("stores behavioral data.");
behavior_processing_module.nwbdatainterface.set('PupilTracking', pupil_tracking);
% nwb.processing.set('behavior', behavior_processing_module);

Writing the behavior data to an NWB file

All of the above commands build an NWBFile object in-memory. To write this file, use nwbExport.
% Save to tutorials/tutorial_nwb_files folder
nwbFilePath = misc.getTutorialNwbFilePath('behavior_tutorial.nwb');
nwbExport(nwb, nwbFilePath);
fprintf('Exported NWB file to "%s"\n', 'behavior_tutorial.nwb')
Exported NWB file to "behavior_tutorial.nwb"

\ No newline at end of file From e1b8d57a228fa506172b36a037b2c98b7423b6b7 Mon Sep 17 00:00:00 2001 From: ehennestad Date: Tue, 5 Nov 2024 22:19:26 +0100 Subject: [PATCH 05/29] Update behavior tutorial. - Add note about why dimension of SpatialSeries data is transposed wrt the type specification - Fix typos --- tutorials/behavior.mlx | Bin 8264 -> 8412 bytes tutorials/html/behavior.html | 54 +++++++++++++++++++---------------- 2 files changed, 29 insertions(+), 25 deletions(-) diff --git a/tutorials/behavior.mlx b/tutorials/behavior.mlx index b5f8fe01118d076df6dbcdd04f9f74da75628d07..5894a6f889e9e29c76255b28d3131203f5519ebc 100644 GIT binary patch delta 4209 zcmV-%5RUK2K-@vF#R3XqLS=)j4gdg9T9eNL8-HzY+c*;bo?pQ;P}srRvYj;D-E`AK z+O*gL+g`BE-r}xT7_>y&+)AQPq~f@T`|me1)T=Egb{yI5<_grYC2B|xXP$ZCP(S_k zCh*y{h_qDUS)=dt8cc+)@?h+#=11`e`k{l{;j(?~0aT(E{ z4`%KQHaorM{#nCFsK8x=|JvzhdLHjbqy*j&X-hJS1pNdpsR!(>3v$b%slbK zGxmNq#8quE$8ioA{A*Tlzu9KZK*d_1{38@FaWnNn^zF^;(VUt1#5;WAWN^Xf$MAEB ze$EQG3sV&MwRP0%|IlVn+wABB{(raGaS#3ju`9UwPZeU@rmtM?HxCOm$08J&zvuLO{b$`joKVD(3LW6lGpB##^nd#O=Hc@q zO<2=UpI+1qK_8|tKJI@2P78g?J|I#8Ai1CiGK^u-nYJwm7i$5h$sE?%m*Hndw(A#V zS!T_a2D?R^!tN2KRBVP|tYB%JG5m~&j#OZ%3~;LpPj|mFKu&kRb5$g|rP({+Q_6a{{luSVkY9(!pCn@r3pUqE*LHz$^af!pqmUa zW>YA=kG-@W0nQKD3-g<*5KJSu2sdpuFAVU$W4asO(>?|?lr#1bxPMQKZ2_Y8;r~ew zu%meK2fFqN0p*C@u<(lUDB|r zNfKBKH9)v1Fo*U5`>XIoL1*xONv0h3vziHzt2XM6HR40?&VVYmt&P{I$E>@^)^0Y~ zcYOL@q0fFPr%OCHFMn3FTMs{yF57H20iT4kcjs5XoWHbUi);mVKt*6yQSpAhcynUe zncW7&0O?m+cn-Uoh)6(q1Gs}e4o(uEBJFjJKTl{80lSpYEa?xg0c01~G(jHfNdw<} z58?%@>5oU;3p3ZSQm_snk>JCXTm=Jw8Q=|RNE2KeO|)6441XKWVUbX%r~%qUBT)SH zgikTz0I=ua_sq4_4h4qH1-2un@u?2?dH zqfG{vuJ_tK7=NA&4GZLWVu;;vd8mw0K?i4rY6E_4q6N^cuXK#61*>>1#SE7&QbEFS z?tUi!5ylnO5a*!ML97u>HCVx-y*sf+u*;5mz2^s+y-ZvUhzd~H;Th5>2NlXG=|Lf8 zT8Nb{m8P|8Au=B#Qt%{hq8QmJhC{f%@VjKL3%B#pLw}?EN1?^S(qjKGyWSxoT(&N( zur5xMo<6KURJcrLl}8DgTnB=qOnA8uRBKX4v5D}fR-01Dpji7_>KJh13#m0gz5?z4 z9IBc1-V7##9tB`0X`KbvqUcJKD-u()wgvQhJ&Lc8JAfXT30wl?8Yi8q&^Gf;X>2f6 z^r#%C(8H8Zi8J4aM>TxlI!{+v$APP*Qj-v+af2iMb;EKT9{U|G?Aj=DBG>W z=k=p|nj2x(oQmb+KzU4vUQ2HLxI zu+G7!Qe%whK%6GvK4mDEL}WGs1hXROikm9$BeEp~LYae=#-W8JD?3@W$C!CCb-zJo zK7Wulu?V+SIFPZx3=X)3~DiUd}{Nj z2ieFUB5eHoDoMz7890*8xFU#L81uGaWSQZM{SD$!8f=yfCBbI1-J+=ZrmJazsTRG#gAuav!XgH&2WgXrtNK-DoZ@L{Y9mBc4<^ z9?i6q=IxHhEhDyTAx-_hNg4~}dx4c(#F;RCi+jW`GjVyCebXv>Ybhw@uvjKxrGF@A z6|)yNRt36IXo>xQQ|Md8hytpX`S53gWA4Ac z#;k4*#s#hmy7ld%u$|h&An@x-UTloMOWbu>;zd#!ohdb@eJ~l=vQF?Z0~G@@0YF4z z#v`n$q|LV(h<%jYJ;Y?JkNf-#<4IdPP(v7GM=(f+bjCgskoKLE1XAJKr+?=MG{r$_ zr_DAz!jBXDc;@t=;laHSzBGtcQO)8Q+AHI%tBEM_vV_CniV1mb_o{i)^DRj1R|2tJ z@>LjYMQj0o6De7QURvS-EwM;rw{IVOEeFcLYZ>r0V(vz>p)zqPP(@E}Qd4D(bY%59}_79j8MWMb{6pH$MD5x?*D_7E@WKaZ=McMVBA$f7nZe?jYD_ynRY?qM%*Nzsg;eTq*?y`1j?_2>t zVV9)`ZxgO>AI+92tuM!c{@=R=Qg!76eIcgx>N<5{mE=}kU#;YQ zidv5;q4$hFe}pTUdatLdg<&jAnu51mLYezVZ?}@&GBRUZgnwqYpi|(Up8GJM5Dd&K z5a;|Pa^S9W6glA0=Xlz2acjlMq4cGh({bbm(S9{i(iQAidr%eB&4?Wofr{6-1#-2q~0 zV%wfmTgjm{9DmfZ1D2gjn9?44mcxHUmZIW}^qFp=)X zJmi6NujqKr7IQDsS!e2L;jLbNK1GePmdA zCY!m+kAqMrWO-z>lAajJFh7=u1y3H2JXRboBku3m|9>jU%C_!TDb@l;z}zT{{hwy( z^gq^fP76QySTz5e1yT#cb%4vis#&W{qURhh(19x_88vYCfYh3lzF(sBV}a1ox`g^m zLpnw`G9z@1lS79pr;|I4Iq@prvHpF-*K=`|oXo1M-w<=r$&PN3GNebby;UVfUwa<3 zFy(Ckr++}^KH1M@##-Pgo|DfC>{6bC%4PKf2udN6$&c1?e+ZkHys_Y$O&iV~#Tj)(@*^L(*# zgFBgfzRQh}{D0=}iTOC=4Ao0~FJL5Ql8X(iYO9xZ2-C?+7uTS)Bo9Dn=~|81u8nhD z1x_V0luob44A%pBWtZ_(`kVV@Q+$kR`K6vCKy{vpD^7T;gK3?DE<&9M(-wOA_a{tPcBf@#ZMuM9d;i_0??a8F1CNXwaR+fu?$$d}mR`~A^?vYVX z15biw!h*OpO7a$7bwMM|o?mbz?fEuvC6{b`l|9c(!<}BQk8yXecSLUA-VOUvRbh9j z5D=$du+)~X)wa5%XmsAY)Oe`|u)Ici=6}0%;cGIkrKx2j{dNrO@5#$uOXr=^*SjXK zTpiW?%GGnu&L^-}cZWv47t9Tcy3_F1U{GB`OO3@2yg)RFV(Rp^PIIb0# z;qPmDJ=Qi*oDsPji8rEK!+A6I0gFrVSX}nfv-H~lrZGUaqxawg%P_--Yjw%Z1V=P6 zklJ;|!dxZo-{*I8H5jDSP|KS3G`i(Y3%<2kfh613A3h%*Vw5(^Pv|RQM9$at11qB{Olkr^I3U5QOoKi2hpoLE&VPd%MofaWJ z-G*8rf8WFgXqKkeR6`K02!bAj#7;(hRJL+b+IQEDf>u)KK!XXMvV_rZR;#eLaB<=k zp3woE_7~wXnaG`26EzG&GUOu{_hF&@2nekMH$VwLny zm-XxuvjrP}0}5h7WrM5^002;0la3xy0+bPx;T|I#8lhN<9RUCU83F(R8vp(Jfq8yVU8yl0)A2$Mz9g`s&8b!-EU^7x8=l+BYV5K0(ESO8ghm+zj=_``M>t_$qhk5ow%=nd55qrfA$J>S)MYicJJ&9|x zaYvUkshNq5iX`{hLi#?N#?ljPsbVHWwttAE0fyN)84xt`U`Wg;_U^YA&Xz-)368U| z@#tPgf>ZZA#O~M|>5C?%LH&uOH|&h5@n_*05R92IGZQQl|BhuOJT{TS_cZHh)Cd)h zJfYo4&JBF;G7w;BBs7V-K+I!lBW-wOMn(qWn67irllTNkH4uiQWVaP5=(nil<$tF? zeSG=q`Q`JE;bP1#*t^9T8R{^{agG@LYuAjt-DT}S#aiIhqZKf5J@;YO?al1boIiZx zXMEyp^olP};pYndoN2fVEDHSEIT;Lp=(5LMc5(*)yX`Qk6`RptmUWD5lIi9Nq-GbP9NX$pbRzGZ=8zg z*H9A7m?cva27SLR-M@)aBz%y7H=4&|U%Dh$jU-?F?d|z+;0y=DC;iV(DB?(k4#?<< zGdyty!(sdQX^|$(>8DT68n(Uy{(rsJ{ebFEQ@ zN>4S)HJaD%Eay}<;@8Z$0e|G2M?wpvP96jhhG=&|67Gy~P57E=6o9OYOf%t3oh}M# z%D)00N6bY0O8EF~t~9|X-UZC&p$y_KtlneF|q)&qF0HO}z|JeYrqj>QLy7uYm_6V|-gn!Z;Fr*|ke&MrzSK1Jtb-~qT-1F^D3c6jRFEwa81C9*A@((+BXt@(14|)2RjR!f#4r3s$>qF_RE^qu-k;-y;VL z{v*I?e?U{Nn~TZIo+X{5&T;OFoWvDbPvmG}I_(l8MSs6hc3a1H2;lEjfJ1A@3X<=$ zA}IS=dGUVgaV#&rf_h@FptZgao~4x*CR-G5>BI$9%a!1s5y+CN9tGbaR^RpvZwIn; z%ts08t5Upw%})hC%KI86Ggh{R?6%dSPBp$F@1v^|>9oOi*7xCn)Ulbwy5IzaM+VhO zMb^q~34h2yTPLSrTV~qSW$Sy{i{YtY&b5A(mO=993WIBiYNkMYzwy;`2&vQ*Q!h}b z8H7(6$|aeTP1V4y2)g2?#`}PL34u@+V5RwDVam!uR^z!~UPdb?gOr=qxDVdNTP98htkG;87PPAqQPgX&h(|R6LOboGeRF_t z!-(x%NYi`_k;d%#YEA7TUZzan;ws*YOkM70Wvh%~{6V4XrzQ-0Q=0HNfGVZfQB3oWt#Zwh_u1W`cMvK;lu32z2rV_K83FBE)9qM1Gu6z)Ypn~QJ;?m?KRNKl{70Em4rg_8-F^gp)--8 zSKK4O_!#ilR$qKSxdctbHH;mG7;8S=zHXi1xT}Ib%xf_FDxp&P23Q zSF8i!rZ)o@bat-XS;2pja$n)*3z*eENIGMAyeCIWv^I65MR&MfvisbPba1Iaps>$9 z0&f$k@1D+va!kPjD$m5Q5O)oL~$6R z?_}kktxgS5+-Vs;DeX-qJ&^3IN{JKx_g34B&}$&njKfjRRVjKYbFsdH{9BPeF zYAWX+hh5+k!r8)~PwOJhj{_V2!pA9oJjVF;d;4+vsWu)Yt6v)r(tq7*JZM2D6G$N@ zO!&aSxG1v5%ivi60y4aq0b-TN@m`hnQCDe6{abByCIjkd-`@G*&S0ewnJ`q13Crn`Ewrz6oP|! z3F@4mlMLKV4oU_*`hOD7doHf67&(@{G)p=x*&^C6XG*$){bG;Nftmb*2f@az+p>g{ z+n}+Z5vP9MK;AUw?w4J&kA8_(zeYL*t9lMeF7Ec`bC5ZAXXgw{DVcRN=@5RU3in@nHl3`%FO(sDJN=IW0Kzg2@O_92n`p zT82E3?j@af*@5>WolRIzcPOg~*s^95UtA|1g^Co0k7J+Pas(du0hm^cw1}S`tww>R z$7Y$U{5S}8LY7CiEa{1f4DF5{DnA=AAuKiTH`ppE#;&neDX{{sLGDzu{!hDf8XI?6 z&I>>IRJ8w_Wq(bT;W{AFU(}*gCNXeMD|Fz>Sw;zOMw=fgUuRWib%!rS@5$Yo@crHc zGU{pMNpMWK`)z~$ca_&q&`7iAD{-VfbOw>+ii@vv=lMQxXD}FI+C3PYP}FyDzkX7; zi@UVz57{p`YAe@zS6#9*I)q(nyt3i1JV$sCyL93Auv`2lbjTdH z7dHyb@b~pK+TAWtoDsPliMOKLz~M6X0e6z(8Mx#fOS+$a8vz?56gvhFK8Oqpd^lI{ zxtW0_22#7uRG7=8{k!~bZU&>28d_P~9yJlSvgyFLE-R3Po0B=jCuRbHEj0n`O|=w< z*m|T}|01YfL%rUp`sur5`}uonpZyO|O9KRxPahhyvJsyM1-~w87ao&V8Y+LnFcgOG zd5Wg@UE48)G~2k)aHWZfGaBxi?f{Ex*R<8a+gm0~R1z%xTjvjNMQTLDn)zdK3~n8u3Y4t5Ip+T{{SBMWF=^ zCV0vdPQO{*goA~v6X)=P4&b!E3J=Lh?tL>-ecvbjbl~FtoW4HXKMh|cjXB;*0WK#9 z?VP+!Z||xVE>o7Kgv|-hDn1u!CbGpkr@!WZICUJXXw&>B%?Rg& z&8n1%St{7

Behavior Data

This tutorial will guide you in writing behavioral data to NWB.

Creating an NWB File

Create an NWBFile object with the required fields (session_description, identifier, and session_start_time) and additional metadata.
nwb = NwbFile( ...
'session_description', 'mouse in open exploration',...
'identifier', 'Mouse5_Day3', ...
'session_start_time', datetime(2018, 4, 25, 2, 30, 3, 'TimeZone', 'local'), ...
'general_experimenter', 'My Name', ... % optional
'general_session_id', 'session_1234', ... % optional
'general_institution', 'University of My Institution', ... % optional
'general_related_publications', 'DOI:10.1016/j.neuron.2016.12.011'); % optional
nwb
nwb =
NwbFile with properties: +.S9 { border-left: 1px solid rgb(217, 217, 217); border-right: 1px solid rgb(217, 217, 217); border-top: 0px none rgb(33, 33, 33); border-bottom: 1px solid rgb(217, 217, 217); border-radius: 0px 0px 4px 4px; padding: 0px 45px 4px 13px; line-height: 18.004px; min-height: 0px; white-space: nowrap; color: rgb(33, 33, 33); font-family: Menlo, Monaco, Consolas, "Courier New", monospace; font-size: 14px; } +.S10 { margin: 15px 10px 5px 4px; padding: 0px; line-height: 18px; min-height: 0px; white-space: pre-wrap; color: rgb(33, 33, 33); font-family: Helvetica, Arial, sans-serif; font-style: normal; font-size: 17px; font-weight: 700; text-align: left; }

Behavior Data

This tutorial will guide you in writing behavioral data to NWB.

Creating an NWB File

Create an NWBFile object with the required fields (session_description, identifier, and session_start_time) and additional metadata.
nwb = NwbFile( ...
'session_description', 'mouse in open exploration',...
'identifier', 'Mouse5_Day3', ...
'session_start_time', datetime(2018, 4, 25, 2, 30, 3, 'TimeZone', 'local'), ...
'general_experimenter', 'My Name', ... % optional
'general_session_id', 'session_1234', ... % optional
'general_institution', 'University of My Institution', ... % optional
'general_related_publications', 'DOI:10.1016/j.neuron.2016.12.011'); % optional
nwb
nwb =
NwbFile with properties: nwb_version: '2.7.0' file_create_date: [] @@ -87,7 +87,7 @@ units: [] Warning: The following required properties are missing for instance for type "NwbFile": - timestamps_reference_time

SpatialSeries: Storing continuous spatial data

SpatialSeries is a subclass of TimeSeries that represents data in space, such as the spatial direction e.g., of gaze or travel or position of an animal over time.
Create data that corresponds to x, y position over time.
position_data = [linspace(0, 10, 50); linspace(0, 8, 50)];
In SpatialSeries data, the first dimension is always time (in seconds), the second dimension represents the x, y position. SpatialSeries data should be stored as one continuous stream as it is acquired, not by trials as is often reshaped fro analysis. Data can be trial-aligned on-the-fly using the trials table. See the trials tutorial for further information.
For position data reference_frame indicates the zero-position, e.g. the 0,0 point might be the bottom-left corner of an enclosure, as viewed fromvteh tracking camera.
timestamps = linspace(0, 50, 50)/ 200;
position_spatial_series = types.core.SpatialSeries( ...
'description', 'Postion (x, y) in an open field.', ...
'data', position_data, ...
'timestamps', timestamps, ...
'reference_frame', '(0,0) is the bottom left corner.' ...
)
position_spatial_series =
SpatialSeries with properties: + timestamps_reference_time

SpatialSeries: Storing continuous spatial data

SpatialSeries is a subclass of TimeSeries that represents data in space, such as the spatial direction e.g., of gaze or travel or position of an animal over time.
Create data that corresponds to x, y position over time.
position_data = [linspace(0, 10, 50); linspace(0, 8, 50)]; % 2 x nT array
In SpatialSeries data, the first dimension is always time (in seconds), the second dimension represents the x, y position. However, as described in the dimensionMapNoDataPipes tutorial, when a MATLAB array is exported to HDF5, the array is transposed. Therefore, in order to correctly export the data, in MATLAB the last dimension of an array should be time. SpatialSeries data should be stored as one continuous stream as it is acquired, not by trials as is often reshaped for analysis. Data can be trial-aligned on-the-fly using the trials table. See the trials tutorial for further information.
For position data reference_frame indicates the zero-position, e.g. the 0,0 point might be the bottom-left corner of an enclosure, as viewed from the tracking camera.
timestamps = linspace(0, 50, 50)/ 200;
position_spatial_series = types.core.SpatialSeries( ...
'description', 'Postion (x, y) in an open field.', ...
'data', position_data, ...
'timestamps', timestamps, ...
'reference_frame', '(0,0) is the bottom left corner.' ...
)
position_spatial_series =
SpatialSeries with properties: reference_frame: '(0,0) is the bottom left corner.' starting_time_unit: 'seconds' @@ -106,15 +106,15 @@ starting_time: [] starting_time_rate: [] timestamps: [0 0.0051 0.0102 0.0153 0.0204 0.0255 0.0306 0.0357 0.0408 0.0459 0.0510 0.0561 0.0612 0.0663 0.0714 0.0765 0.0816 0.0867 0.0918 0.0969 0.1020 0.1071 0.1122 0.1173 0.1224 0.1276 0.1327 0.1378 0.1429 0.1480 0.1531 … ] (1×50 double) -

Position: Storing position measured over time

To help data analysis and visualiztion tools know that this SpatialSeries obejct represents the position of the subject, store the SpatialSeries object inside a Position object, which can hold one or more SpatialSeries objects.
position = types.core.Position();
position.spatialseries.set('SpatialSeries', position_spatial_series);

Create a Behavior Processing Module

Create a processing module called "behavior" for storing behavioral data in the NWBFile, then add the Position object to the processing module.
behavior_processing_module = types.core.ProcessingModule('description', 'stores behavioral data.');
behavior_processing_module.nwbdatainterface.set("Position", position);
nwb.processing.set("behavior", behavior_processing_module);

CompassDirection: Storing view angle measured over time

Analogous to how position can be stored, we can create a SpatialSeries object for representing the view angle of the subject.
For direction data reference from indicates the zero direction, for instance in this case "straight ahead" is 0 radians.
view_angle_data = linspace(0, 4, 50);
direction_spatial_series = types.core.SpatialSeries( ...
'description', 'View angle of the subject measured in radians.', ...
'data', view_angle_data, ...
'timestamps', timestamps, ...
'reference_frame', 'straight ahead', ...
'data_unit', 'radians' ...
);
direction = types.core.CompassDirection();
direction.spatialseries.set('spatial_series', direction_spatial_series);
We can add a CompassDirection object to the behavior processing module the same way we have added the position data.
%behavior_processing_module = types.core.ProcessingModule("stores behavioral data."); % if you have not already created it
behavior_processing_module.nwbdatainterface.set('CompassDirection', direction);
%nwb.processing.set('behavior', behavior_processing_module); % if you have not already added it

BehaviorTimeSeries: Storing continuous behavior data

BehavioralTimeSeries is an interface for storing continuous behavior data, such as the speed of a subject.
speed_data = linspace(0, 0.4, 50);
 
speed_time_series = types.core.TimeSeries( ...
'data', speed_data, ...
'starting_time', 1.0, ... % NB: Important to set starting_time when using starting_time_rate
'starting_time_rate', 10.0, ... % Hz
'description', 'he speed of the subject measured over time.', ...
'data_unit', 'm/s' ...
);
 
behavioral_time_series = types.core.BehavioralTimeSeries();
behavioral_time_series.timeseries.set('speed', speed_time_series);
 
%behavior_processing_module = types.core.ProcessingModule("stores behavioral data."); % if you have not already created it
behavior_processing_module.nwbdatainterface.set('BehavioralTimeSeries', behavioral_time_series);
%nwb.processing.set('behavior', behavior_processing_module); % if you have not already added it

BehavioralEvents: Storing behavioral events

BehavioralEvents is an interface for storing behavioral events. We can use it for storing the timing and amount of rewards (e.g. water amount) or lever press times.
reward_amount = [1.0, 1.5, 1.0, 1.5];
event_timestamps = [1.0, 2.0, 5.0, 6.0];
 
time_series = types.core.TimeSeries( ...
'data', reward_amount, ...
'timestamps', event_timestamps, ...
'description', 'The water amount the subject received as a reward.', ...
'data_unit', 'ml' ...
);
 
behavioral_events = types.core.BehavioralEvents();
behavioral_events.timeseries.set('lever_presses', time_series);
 
%behavior_processing_module = types.core.ProcessingModule("stores behavioral data."); % if you have not already created it
behavior_processing_module.nwbdatainterface.set('BehavioralEvents', behavioral_events);
%nwb.processing.set('behavior', behavior_processing_module); % if you have not already added it
Storing only the timestsamps of the events is possible with the ndx-events NWB extension. You can also add labels associated with the events with this extension. You can find information about installation and example usage here.

BehavioralEpochs: Storing intervals of behavior data

BehavioralEpochs is for storing intervals of behavior data. BehavioralEpochs uses IntervalSeries to represent the time intervals. Create an IntervalSeries object that represents the time intervals when hte animal was running. IntervalSeries uses 1 to indicate the beginning of an interval and -1 to indicate the end.
run_intervals = types.core.IntervalSeries( ...
'description', 'Intervals when the animal was running.', ...
'data', [1, -1, 1, -1, 1, -1], ...
'timestamps', [0.5, 1.5, 3.5, 4.0, 7.0, 7.3] ...
);
 
behavioral_epochs = types.core.BehavioralEpochs();
behavioral_epochs.intervalseries.set('running', run_intervals);
You can add more than one IntervalSeries to a BehavioralEpochs object.
sleep_intervals = types.core.IntervalSeries( ...
'description', 'Intervals when the animal was sleeping', ...
'data', [1, -1, 1, -1], ...
'timestamps', [15.0, 30.0, 60.0, 95.0] ...
);
behavioral_epochs.intervalseries.set('sleeping', sleep_intervals);
 
% behavior_processing_module = types.core.ProcessingModule("stores behavioral data.");
% behavior_processing_module.nwbdatainterface.set('BehavioralEvents', behavioral_events);
% nwb.processing.set('behavior', behavior_processing_module);

Another approach: TimeIntervals

Using TimeIntervals to represent time intervals is often preferred over BehavioralEpochs and IntervalSeries. TimeIntervals is a subclass of DynamicTable, which offers flexibility for tabular data by allowing the addition of optional columns which are not defined in the standard.
sleep_intervals = types.core.TimeIntervals( ...
'description', 'Intervals when the animal was sleeping.', ...
'colnames', {'start_time', 'stop_time', 'stage'} ...
);
 
sleep_intervals.addRow('start_time', 0.3, 'stop_time', 0.35, 'stage', 1);
sleep_intervals.addRow('start_time', 0.7, 'stop_time', 0.9, 'stage', 2);
sleep_intervals.addRow('start_time', 1.3, 'stop_time', 3.0, 'stage', 3);
 
nwb.intervals.set('sleep_intervals', sleep_intervals);

EyeTracking: Storing continuous eye-tracking data of gaze direction

EyeTracking is for storing eye-tracking data which represents direction of gaze as measured by an eye tracking algorithm. An EyeTracking object holds one or more SpatialSeries objects that represent the gaze direction over time extracted from a video.
eye_position_data = [linspace(-20, 30, 50); linspace(30, -20, 50)];
 
right_eye_position = types.core.SpatialSeries( ...
'description', 'The position of the right eye measured in degrees.', ...
'data', eye_position_data, ...
'starting_time', 1.0, ... % NB: Important to set starting_time when using starting_time_rate
'starting_time_rate', 50.0, ... % Hz
'reference_frame', '(0,0) is middle', ...
'data_unit', 'degrees' ...
);
 
left_eye_position = types.core.SpatialSeries( ...
'description', 'The position of the right eye measured in degrees.', ...
'data', eye_position_data, ...
'starting_time', 1.0, ... % NB: Important to set starting_time when using starting_time_rate
'starting_time_rate', 50.0, ... % Hz
'reference_frame', '(0,0) is middle', ...
'data_unit', 'degrees' ...
);
 
eye_tracking = types.core.EyeTracking();
eye_tracking.spatialseries.set('right_eye_position', right_eye_position);
eye_tracking.spatialseries.set('left_eye_position', left_eye_position);
 
% behavior_processing_module = types.core.ProcessingModule("stores behavioral data.");
behavior_processing_module.nwbdatainterface.set('EyeTracking', eye_tracking);
% nwb.processing.set('behavior', behavior_processing_module);

PupilTracking: Storing continuous eye-tracking data of pupil size

PupilTracking is for storing eye-tracking data which represents pupil size. PupilTracking hold one or more TimeSeries obejcts taht canrepresent different features such as the dilaltion of the pupil measured over time by a pupil tracking algorithm.
pupil_diameter = types.core.TimeSeries( ...
'description', 'Pupil diameter extracted from the video of the right eye.', ...
'data', linspace(0.001, 0.002, 50), ...
'starting_time', 1.0, ... % NB: Important to set starting_time when using starting_time_rate
'starting_time_rate', 20.0, ... % Hz
'data_unit', 'meters' ...
);
 
pupil_tracking = types.core.PupilTracking();
pupil_tracking.timeseries.set('pupil_diameter', pupil_diameter);
 
% behavior_processing_module = types.core.ProcessingModule("stores behavioral data.");
behavior_processing_module.nwbdatainterface.set('PupilTracking', pupil_tracking);
% nwb.processing.set('behavior', behavior_processing_module);

Writing the behavior data to an NWB file

All of the above commands build an NWBFile object in-memory. To write this file, use nwbExport.
% Save to tutorials/tutorial_nwb_files folder
nwbFilePath = misc.getTutorialNwbFilePath('behavior_tutorial.nwb');
nwbExport(nwb, nwbFilePath);
fprintf('Exported NWB file to "%s"\n', 'behavior_tutorial.nwb')
Exported NWB file to "behavior_tutorial.nwb"
+

Position: Storing position measured over time

To help data analysis and visualization tools know that this SpatialSeries object represents the position of the subject, store the SpatialSeries object inside a Position object, which can hold one or more SpatialSeries objects.
position = types.core.Position();
position.spatialseries.set('SpatialSeries', position_spatial_series);

Create a Behavior Processing Module

Create a processing module called "behavior" for storing behavioral data in the NWBFile, then add the Position object to the processing module.
behavior_processing_module = types.core.ProcessingModule('description', 'stores behavioral data.');
behavior_processing_module.nwbdatainterface.set("Position", position);
nwb.processing.set("behavior", behavior_processing_module);

CompassDirection: Storing view angle measured over time

Analogous to how position can be stored, we can create a SpatialSeries object for representing the view angle of the subject.
For direction data reference_frame indicates the zero direction, for instance in this case "straight ahead" is 0 radians.
view_angle_data = linspace(0, 4, 50);
direction_spatial_series = types.core.SpatialSeries( ...
'description', 'View angle of the subject measured in radians.', ...
'data', view_angle_data, ...
'timestamps', timestamps, ...
'reference_frame', 'straight ahead', ...
'data_unit', 'radians' ...
);
direction = types.core.CompassDirection();
direction.spatialseries.set('spatial_series', direction_spatial_series);
We can add a CompassDirection object to the behavior processing module the same way we have added the position data.
%behavior_processing_module = types.core.ProcessingModule("stores behavioral data."); % if you have not already created it
behavior_processing_module.nwbdatainterface.set('CompassDirection', direction);
%nwb.processing.set('behavior', behavior_processing_module); % if you have not already added it

BehaviorTimeSeries: Storing continuous behavior data

BehavioralTimeSeries is an interface for storing continuous behavior data, such as the speed of a subject.
speed_data = linspace(0, 0.4, 50);
 
speed_time_series = types.core.TimeSeries( ...
'data', speed_data, ...
'starting_time', 1.0, ... % NB: Important to set starting_time when using starting_time_rate
'starting_time_rate', 10.0, ... % Hz
'description', 'he speed of the subject measured over time.', ...
'data_unit', 'm/s' ...
);
 
behavioral_time_series = types.core.BehavioralTimeSeries();
behavioral_time_series.timeseries.set('speed', speed_time_series);
 
%behavior_processing_module = types.core.ProcessingModule("stores behavioral data."); % if you have not already created it
behavior_processing_module.nwbdatainterface.set('BehavioralTimeSeries', behavioral_time_series);
%nwb.processing.set('behavior', behavior_processing_module); % if you have not already added it

BehavioralEvents: Storing behavioral events

BehavioralEvents is an interface for storing behavioral events. We can use it for storing the timing and amount of rewards (e.g. water amount) or lever press times.
reward_amount = [1.0, 1.5, 1.0, 1.5];
event_timestamps = [1.0, 2.0, 5.0, 6.0];
 
time_series = types.core.TimeSeries( ...
'data', reward_amount, ...
'timestamps', event_timestamps, ...
'description', 'The water amount the subject received as a reward.', ...
'data_unit', 'ml' ...
);
 
behavioral_events = types.core.BehavioralEvents();
behavioral_events.timeseries.set('lever_presses', time_series);
 
%behavior_processing_module = types.core.ProcessingModule("stores behavioral data."); % if you have not already created it
behavior_processing_module.nwbdatainterface.set('BehavioralEvents', behavioral_events);
%nwb.processing.set('behavior', behavior_processing_module); % if you have not already added it
Storing only the timestamps of the events is possible with the ndx-events NWB extension. You can also add labels associated with the events with this extension. You can find information about installation and example usage here.

BehavioralEpochs: Storing intervals of behavior data

BehavioralEpochs is for storing intervals of behavior data. BehavioralEpochs uses IntervalSeries to represent the time intervals. Create an IntervalSeries object that represents the time intervals when the animal was running. IntervalSeries uses 1 to indicate the beginning of an interval and -1 to indicate the end.
run_intervals = types.core.IntervalSeries( ...
'description', 'Intervals when the animal was running.', ...
'data', [1, -1, 1, -1, 1, -1], ...
'timestamps', [0.5, 1.5, 3.5, 4.0, 7.0, 7.3] ...
);
 
behavioral_epochs = types.core.BehavioralEpochs();
behavioral_epochs.intervalseries.set('running', run_intervals);
You can add more than one IntervalSeries to a BehavioralEpochs object.
sleep_intervals = types.core.IntervalSeries( ...
'description', 'Intervals when the animal was sleeping', ...
'data', [1, -1, 1, -1], ...
'timestamps', [15.0, 30.0, 60.0, 95.0] ...
);
behavioral_epochs.intervalseries.set('sleeping', sleep_intervals);
 
% behavior_processing_module = types.core.ProcessingModule("stores behavioral data.");
% behavior_processing_module.nwbdatainterface.set('BehavioralEvents', behavioral_events);
% nwb.processing.set('behavior', behavior_processing_module);

Another approach: TimeIntervals

Using TimeIntervals to represent time intervals is often preferred over BehavioralEpochs and IntervalSeries. TimeIntervals is a subclass of DynamicTable, which offers flexibility for tabular data by allowing the addition of optional columns which are not defined in the standard DynamicTable class.
sleep_intervals = types.core.TimeIntervals( ...
'description', 'Intervals when the animal was sleeping.', ...
'colnames', {'start_time', 'stop_time', 'stage'} ...
);
 
sleep_intervals.addRow('start_time', 0.3, 'stop_time', 0.35, 'stage', 1);
sleep_intervals.addRow('start_time', 0.7, 'stop_time', 0.9, 'stage', 2);
sleep_intervals.addRow('start_time', 1.3, 'stop_time', 3.0, 'stage', 3);
 
nwb.intervals.set('sleep_intervals', sleep_intervals);

EyeTracking: Storing continuous eye-tracking data of gaze direction

EyeTracking is for storing eye-tracking data which represents direction of gaze as measured by an eye tracking algorithm. An EyeTracking object holds one or more SpatialSeries objects that represent the gaze direction over time extracted from a video.
eye_position_data = [linspace(-20, 30, 50); linspace(30, -20, 50)];
 
right_eye_position = types.core.SpatialSeries( ...
'description', 'The position of the right eye measured in degrees.', ...
'data', eye_position_data, ...
'starting_time', 1.0, ... % NB: Important to set starting_time when using starting_time_rate
'starting_time_rate', 50.0, ... % Hz
'reference_frame', '(0,0) is middle', ...
'data_unit', 'degrees' ...
);
 
left_eye_position = types.core.SpatialSeries( ...
'description', 'The position of the right eye measured in degrees.', ...
'data', eye_position_data, ...
'starting_time', 1.0, ... % NB: Important to set starting_time when using starting_time_rate
'starting_time_rate', 50.0, ... % Hz
'reference_frame', '(0,0) is middle', ...
'data_unit', 'degrees' ...
);
 
eye_tracking = types.core.EyeTracking();
eye_tracking.spatialseries.set('right_eye_position', right_eye_position);
eye_tracking.spatialseries.set('left_eye_position', left_eye_position);
 
% behavior_processing_module = types.core.ProcessingModule("stores behavioral data.");
behavior_processing_module.nwbdatainterface.set('EyeTracking', eye_tracking);
% nwb.processing.set('behavior', behavior_processing_module);

PupilTracking: Storing continuous eye-tracking data of pupil size

PupilTracking is for storing eye-tracking data which represents pupil size. PupilTracking holds one or more TimeSeries objects that can represent different features such as the dilation of the pupil measured over time by a pupil tracking algorithm.
pupil_diameter = types.core.TimeSeries( ...
'description', 'Pupil diameter extracted from the video of the right eye.', ...
'data', linspace(0.001, 0.002, 50), ...
'starting_time', 1.0, ... % NB: Important to set starting_time when using starting_time_rate
'starting_time_rate', 20.0, ... % Hz
'data_unit', 'meters' ...
);
 
pupil_tracking = types.core.PupilTracking();
pupil_tracking.timeseries.set('pupil_diameter', pupil_diameter);
 
% behavior_processing_module = types.core.ProcessingModule("stores behavioral data.");
behavior_processing_module.nwbdatainterface.set('PupilTracking', pupil_tracking);
% nwb.processing.set('behavior', behavior_processing_module);

Writing the behavior data to an NWB file

All of the above commands build an NWBFile object in-memory. To write this file, use nwbExport.
% Save to tutorials/tutorial_nwb_files folder
nwbFilePath = misc.getTutorialNwbFilePath('behavior_tutorial.nwb');
nwbExport(nwb, nwbFilePath);
fprintf('Exported NWB file to "%s"\n', 'behavior_tutorial.nwb')
Exported NWB file to "behavior_tutorial.nwb"

\ No newline at end of file +--> +
\ No newline at end of file diff --git a/tutorials/icephys.mlx b/tutorials/icephys.mlx index 9d5db56c6cbcb8836da051a2ee74e92fe35020be..6be91a05578f2d07d52c22b019571c9ce76091d1 100644 GIT binary patch delta 9690 zcmZ8{1yCJb()PvS;vu+8aCdjVxC9IC?r_nILvWX%!8N#Ra0u@10fGeg03Yve{kwm5 zs-BstneJ2P>FGJAtGj=fFl41O;HrRNVBZ4(0K_+t1Mc1dQ3Rm?fSm+fa!BVL1EZnp zPA?RBasvX+XJn01-=25{tcndY<+eXU9_sQ?|U z)R%R?6z{QIJ)K0>iyGOeEEtV6f(C9}VWfc}`xRoFpDZViI#Y!?a3g%JUp zMeIbg3VH8#B1mXdx%g`52Tt3@M@tq#zlsKuZw2ctZ9B)xo&^GHpYoEX3IDce>oxxt z?VCkOU;zN+HyazenA^FsF#mI|O8TzU$BH6)_8duj$7RbyEo$!xUB(kKh>+fIThR!S zOD?o~@?j5@?#rXc8XCU1awW1nmRL;W;cQZ6FOd^0f&HMBpP;E^$Q@U&CtnFem#6v8 z)msuY(iTx?Y4`nsN{?sdopJ(qNu%py4%-jglb%8Og6o7Tc4A%#C-!WQrwTdf;0l{V z0-lLN9Ga0%N=cFKojkV@y&K<67LEZVL#5+PrEO`}@4WOPzo}d2EM<8)y&~3c|6x$6 zC`n5Dajf7(>+`j^_>I!tYw@`lA*MD8>{K8Cp>oyL{Cwm-oco)YfX==xirk|j6;&)E z6Ahve5z*-EX4{}0pN)rf9PNgJ%C#3F4yaXgQFbFn9&L;AO9d$>4f?GzXb_s>MFGKe zLc3;8>Kg5Do7S>fyNtfsvLI;$niite&x#^-_Z&Iwv(qGt__es6(7MU$9x$8E`DOnr zFZBKyl8Hp9#n4RKCEsEWTCTDB*{A*zE2AkP+E`GXFh2~-!R|UGZP|M!d%C(n2ISJd zHO0veReMm~BV)O0KAyJixN<~YC6TP#D3RV}s(jwc;127o3;7W5?K4d3CJH27pvXhp zy$*hK1TAkugGdWT*VO13$IkzuV1|?sT`@`UL$qN5&e-$OUWwfIAuhz7@wzoUg zKX%yVh19*|mTX;^Y{MQ8dpKm9an}#K(?tEwk@ocP_6+XH`>?X#!008(=a4%Hcp^LK zSQ8#8qo9jS{Gg|FLgh*wEH;!1W?J}JVUqB2W!Q!0=Q2t0y>F{?*OBPE*(3?6n!u1 zGX<}8SOzT>)yoLO48u4dFX@U^Q3jy-&1f1ua|L*^3$l4isyg-4=l9ZmXt<6)+kIS! zgLHSJ?L@SV-3V^~`X@OcxcIp$>ha|a=Y*Ekz^;fJB z>N}|G^mBbAg9Y|HM6p(5?P~`Y8frDvtHPuqYvy~DPv&Ek)Jis_p1P+J^mGgz+OMjx zK?6^K!iU)>dY33BK}#?zf7ylTw|vMSiG%Z<^U`(i zdaZi!fuKo^uvqR?K*y`E$ADi`0%@l8Lb=RVH<2mRVW<1l%BF3z=g9`kl2F}uNc_ND z0h_-1BtrW^8m^=6fyZQ;ynIJ_Hp^kYpTE0%>*j>p1>JsMtq6{&ZX6&fu2+<`ZqTa< zZ36Aqx8_f!ek=S!^*(m#Ip2ei-8XEUJ11s zQR1H!n@#j>{rnqz{GD1e3!uJALbh!BIgQ8}5 zp@`=nKk7RRqIe#}B<=cpc)uqizAG@%Re!Z?uZk+Dc<%IlJz>z;wTcTwbif08?vJAVZ)fdNTv~w6#f2{x~Z=`eGy#8)^|RU=+RB^eVt9k z_#VBdH^~%_*8^5T0Jd(9y;{?nU?|~x2F&17a7$`Kr)Nz`+mH8OoLETG*-saK`-_nW zY_lExQO~`DQEvgk!Aw}fVV9C_Ze1tl!%0!>gTy1DNcDY+&D>5Kxgc=tuJzs3v2EPn zcKJSeq7pF#xS!6GSi$fPAcRR2dSE4n$`TRK(e7nP&BR|OX{UaL*POO1(=*wYJr&Vb zwref~=rX%W)hSlF!F_*^qAj|1Qd-#MFsylaA={t#VJX5X(3UD{6nivApf3`}KNy&$nd z+uA(x1%nS4FBO(GHM*fu(aL`_9ODMkS$2my9O zBMq-4PWVKTMfCndLg>RDNR&hv<&`%jP;uI-pkLZo4IS{ur2Vkd(0|U>`_i0YD**~9 zikGw&dFCBRF~2kBC`7GtFUea=L|^gDWJR5~-@AWNNr3n)O|Oz}8g@M&v}fMA>%3cd z`TY%XPROvMSWnN)(gTO?z$^X^^R|LuD%*5dp`jO#`l`RlxfLJ{_hi9SZ}OB%h@#8p zbYs-2vLiS%E(1+!GbpXDB8KL~o+l3f0k+%$hcmRF_pvc(OwI3={7N$BQk7nxOI5A_ zSDEkRgct*GlO+3!LI=vOE zfCGwlu08(NJjzu#QY@T`sPLL$G||qav!sl~ZU|ie6WJ)X{Vm`Q_Go*XmWRVkVsc2L zXU9V;Ph_~Yw?6<@4Z0!?$vd{;97CRJ8h}Z5J_Yd>Z;!44-VyC49{Ta_!4qR2d4&wg zJ@jkpbdJg+Sm$ zGCzXI<;zS$=2G$8YM$;%P03dp;E>2< zSVq%$av?{EsYq$`IU!U!1U;C`-^R}4$A%yv`58Ut9rP(nk^F+doq8idjSt&>jh6Xw zU%u?y&!bLb*AFO1*4W_RjA;#!{j_IU)47L8?Ya)s+J1HRT8+!Um=U0oCN&wuJbj_0 z(i?EgvtR*+yZqod?3+@-Jg&+1`e;%zB2+?!g2F0y=h(3H+O_9ReEG*GZP$JH>?|Hq zy!dLO+M*V@ujk7z1RfL}iAMr8;hEJ94iCks)4x(riJNui zyZo^8k!GbhXLMLat)uLF^g*qIScXqcBL~#dOkdqjd~;Q~?ZSeRt5JzM)it+UEbmW{ z*C!NMP~dyCWh!Yjz2h~un}f>vJvs&XgE!XqeZ#^B!oWe#wdpJR$M0&c#c=SIJ+yHk z(YIQVrI3D(@*cNht^F$Elk{5Z?4Zt;!{_>_*OP4#e0J06SVDMyvG>-)O{)SA?^t`L zly6ZsX7A$kOS9dXlR;u`UJ5X2?y#~kX3-xVbXh4?EbCChZnsFMQoiME6I#_mcFI2* z$9L>1t{V=1I(KU|_Hu6AR^~$2>yZ_a^w2ot+Gy?4N6TijvR6K&xn&;erRtS`BC+nX z4Va#MU--xB65l-REK(w+b%6nn(R>u;9w=SHDZj&F#U480AkSW>V5c3O{=;W&;G7a& z(0{VCHoLavdSN7qhP@FThnk%jLP8?!L!wTB&cYgp)D@@S+FV7-*q6yLzG}?=3JbGy zexNziScoelt(uphW~)3^R^wWaqxMs`TD*MFRaLwd(U;h)Ci9y#%wDu57Kypr0+oQr zZi*yAmF}9#1ve9;1hgpqvb_&KyO4}Y?#9*8Bf2WrNZwn?`%+*k;D5I%4-Q$})&Yjb1D1q(+5lg$H7x;vh}s9xqW{^33)H*~_% zpO4+DQNbf|daWb-f-|n?GvQ)kg2}X5W>D%ul%ox%pXN|AMY5Ay%k6w8Yit~xjskLp zn(7eSp$~JuuVg6npdgnd1RIhzB!`AfR#(t9aPe4N1T>Dr3-~AVLk?&rs56{~`WA#z z32B6IYJ1E-&oG08WG%JF5Ou4^wxL%SgtOqHZ%ZejtEcUQc49oR{sZDq ziCj_0#x=^P!{h03kiBlTjWn^zWo|74gg$!?uHSXu(~Q1fwebiA786&ej<@rTLvJb~ zrPY3pNcFk>Va!j+kLCId4?o8sJNQx#2ZMhLCHMksQtPX`R5$&@a?<%xQJVj~(m@cc zv4TH5k?f_8qq4j#Xq%IzIIpy&o#Ju1N+`%c-wDs^Tuqj$4^lCv$5aF)&{ts9Z`Vav z!hBA(`D*5#V~4eC&I}E2zC|9%5eIkWv$fh8S*76$Ly$zCy`zZ4_VZ{-XYOa7dI6oP zKpo!KlAJb~z(*FT{@k=I)ZS77(a};#-?$qv+10r%{}`vc&7MV%xp38%M6B%l^HkxA zDi_n&tFu?kUC8gIjd<+Whr3t)J^uLNA4L?)1YTTQ$f+j1!vPz+I!6&ByJ$9Cf{D_9 z0R94FweK#a>Rd+%GXgT5Wa}vRqT<1kETw14=8QZ>SQYcIq71oVS-=5oBm|5Mg1}R6 z@v7%DC3%zN0P=!BSuHvRj@tOFZjLYUX6TB<-AF?T7LeQ>1rJcDZ!6V~C!huebxq zz?@j!sns3by1`cFh0O_s~A_7oF$mN9k%gU8{i&5H73N43-j{u`F# zypLTLEUVLtImf|Seya@3?)OEnX~&Z&W*O#NO=0_kAd$*xyy&e+FmHlnGF5$x%C zT7$%BScqePt#TvoK2Qh8RlfvqM?4HuJI3Q8PT&C8bO zen&GMd@rA~*!no`&n2JvurtUnD}N7xB+M^5kZqe+gH-g4I3 zJQr8lrlRDS|2i51KLe!V!?s~QHzQsEILd$~+{rZF1y9|b=hLk-L{F|Xso@r<*xB%X zewSTKB4yq+@7_#2ds>NEOTPhfr=e`1A8{@A(EO00e3){V%X_-sK7+dYxLGI&M=c;o z4pE9X&B`Lfv4_>8sjQvi_&s zZ7l0S|CF>z*HNiT@Z(i~&gQtwokLNx9bJl9Zq}lvST+h{{zIwR_=mVh-{zvAM9@+4 z7nyaYUi(Uf)1_kgj%2_~eibP)YuIOKdB}TfWs=9lm5Gz&cx7)G@>YgP6789v0i)^` zU`NzLt|AMoXwkGCF_w;8g;f@U~Wiz;nM;&Ja2d6{fG%xbv0@5-la<&P7CqkboRFTpj8Y5`*A%GoxR})4@Jz@h^u)rt;Xe^01toZ9D8Ni33AAw??G&xU;R>uMn=L5S zt@APzP*4_a%StT}zy=>q&EiRjKtYl`3*;KY(5>jV$v^?Ez8;glFzQ>#3N&kf>W_mM}I1N3x@7J9S%nO;+r``RiS1a?#JzUfoNUm8+F+ z;a?RQe(K;J_nK?tuS;^RoEM#s>AwOiQ#1B+Kf;5iC9|M+X@xA+!k`NW=po7U4wn+WeWFH(#Hb|0tM_}Db&D{+lYdb`=#fQ9T`CHJ8Su`y}GDs^#ZcnA{N5@ja>Hu z{Fl1yK0b~V98)nOiQW&g$ApB<9~X*UHqC0P#Vm4vh<+!&=P_liizSHn9`1-eVs_u^ z$~f2QQyQ4oVRFf%&Xx)Sc|kfwCBuySHBcmATD?A12(>i$tzONPfmnTfsUt7r4oc%^ zDvJ{PjY2xr3S=wo@gQ9uBQm@dB%~*I4_a1ep1q} zL()ZFdfTcp5WGWdI{w4 z+mQr2>(c0I+%PZ}0a0Yy8uS_iC-`&KB965MOb!U8x|n|z?+k1L;y0>gRY|gHq%(n9 z7K8U##`M{{Jrkt?Ni=Z3)xIm|X?&*BWqN7K{=o4%-!5;i$g)fOGd@I&h-D%TpEU|IEX;!m1dg0h*{3e=T%Ep$2%|fs_+H^l1 zRy@6V14`b12l9c2T~nq=Qr2N6`Lk*Q&x29zC{OC`QK3L_(KnM;o}>-NLcsZ`{=jo$ zfo&0GIaN{A3LA+zwli~^Wf?Hc@U|l!>SE+O-$?KX=2wbN1K>#tjvsFXiz;&Hq@nR2 z|9u%)1~PDa?Ivf?Yr2xZR6NJ#4nz&yiP0>CKnwXAs|YVhX{V6k zNn$}g%`2K*sptcpl$%=25;AYcBFfsxluJSH`zJ z%y6ZRsezM^O1OF$YxkVJD@SZQG`Ia@SxRrd}(b)(+knW(FGs;j4www ztBowy27zn0%&cerG4aVcR|Z7i&Y&FeZchZ^h{h#BRw58XBWO+vxfA}+pJG@MT#s&1 ztIMU@%33_JB7}(ge5E_b~~>M+r(#k z{F>*xMU>ZRE$v7dRvkO|*f>;dw-pz4yDT z+X3MhY-^a&PNHcVcp-Amrq1~@@hN3bv!Y1zkQ>7tV}>=7r*Vohi5=4un)??3C$W1S zzWQiVYOFpuxW1wyI28_(NW;x0BChpgX2!uvs-Zu}Qxu_BhG^P?jVWRM35r%0*O&3* zrmm~}G*YdI160od-iQ0ASD1fN#*qJ3Z`ej zU&H}M7>56-#uqxi8&j2QAp}##N>C*=}N38Y);!gb~n+4Seq=thT!z3S+ z=8vw33=GX&=mWy^Hu-8E*kjVw$UM->w&-x%lB}x*&a+1w&jf9b1*2Pgd>LTZ-!&?3 zyN4fb5<|?e0!bF<7WaRB0ZJcWuF(ZvoM*4KTrtPLBk1E?9>ZmXp7gu~8FGZvkxx?I zL?)&L3>+&dXi6TWF+|Kv3XZF}v}Zs>OVAQ*2)7Y&1XWphdQutO`3_WuvLBePxY*{w01YFw`4H;ctHJv8N@xk3x_VbK~5Y}j2U zG+kM#J#JM76-F{>Xl6I>=AZf$AgRzVbU3*`^d!i%vqtfY;SoPMp1*GiCnfNmOgEDv zJAyc!iDvUlMSPjEV#tsj-oO$|{6Ix!<%YJ0CT{s58%{8iY{^{o>BJup$b~HWqBbXw zavzAC{7dbUma=*TJ|=_)PhPx_%fJje(!A=U0&PG@2qFCf+2TF*b;IAMmmJd4_l4ajL00%RRW`8~1aa<1*){X(v#&R*gq$_XnhJ zUEQ^6@;jI*=blXHEO4ZZVoPeE0nCIyib!MBlehM&AQRu zWrMhm)Kq?02UsR;j~;0RHQQAT8+SEnOTH$A!Ac}`CaNUzc(bKFx0yu~${ma!ckkk} z|4}Qbn1SAe;Dk&W@s32)X<&Nu@Dqd|F+!y z#J8hc$lU)sgX}#KC6f8`=&1--D6z6r+17C#g0O#NcDLP#s!^6g|-R{i(x>BjJ9U= z#P&6uBSFkl%hk<4N@m;BQ#If!uE>5g^w$B@K*2Pak3Jf3&gB(DNkBjGx{xvx3f~g2 z{B6o`Vm_}v4YKeJkH!WrHKJ1-k)LMndSk zoe333d9?*4az)0fIABhBe5tq0d7QvVu-u#jUuSrb5t{l&FXnhfc8C$?Q*)s5rnIod zhb?Y9-qv+~VGFh`v^9vgumtkY0NVwa-!(74sGIeMh8U67jQbw(bxMWw>*<|jZqWI@ z#kBQsA;a9|8z{Ep;wS|XD)eE1mg-Os6#vu#7-9Dl|EI^vsHpd=&j(PT0dj@?u zj)VV{cHXW{^8i+TXvv(Uli`py@SUbQ<^4elPNOU2@WsJDdm%4;bBq2VnfO#{@IchBMJZ zy#e|SFmHf;16(r`Ej${e-2)D?Ce*?lB5Z{2*Y>@|4WCQ}tz%WomZ>4ekw9YXg z6+~INj|>m~{7D#YdqZvI8QHo$*n&n;A!`~;d95$Qm<(7sT?HNMmM~w)<{Ui3fv?6@ zxej3}ld70mW3Zw{qnZ-in9pY&2&>p!a*iW;s=}aidyE_CEL&L>8gNMLY*vS zgdZS}VI>-P8#|Src=j#v#cb=SAJ3pcNc-ov)NnH!lI$#9{lx)&?(5!XIkA?>2931n^rv zIlcE?yoxNQyv(P0B@WSvLIE~WgLA1d_A5V64{AY4nfKGLEvmROG zzl<355nhBy3=Q~xzzqi=hrr~rxi+cCT%!}e^Dn|+!;Z?(jJWJ6QQOJH&`~}lZzdVN zM*NkkN8AyS=jbFg?U`w9!1(luD_V}Ylq|(Lwv>y279^d=FMu<`yUNu-;bcmAK?C`dhq-$viyze z#v7>9FeiXSX^fLV^tYfRm<$i#nHy9?Y4X7)n7`3A{1ytMyIi#g@bT&Pd3ojew@d!= zO*F~YA1lRBoh&0*>178_zfi|T4m#FLu`fy!3!8tCcjE^Jnmy^Qih6O6!TO*dDgw`kH@*UfPL4%V@X8v+Ha;>^&LWcOp~AY zD#`FCe;4jWVt%7ym_ZqBPzmy%mYsSn;P;05o6f+(&w<#+udpfa>KaPp=LlFfIxx~GzpEwGWZ=Z)vQUCLz&=+C+pC`X-$B;u4da8i#FOyz z!`uvCd`)v>ecr9;8DGGWJCS@+;cWi6XdT`n<0RYuR5V`{nC_jdPyR7QJ_cI+aj%kk zS3bzg80tI$BbjgRf@t(3C^u|PJsW&|_!WdQzn1MV$8KK8OmZKw_518bb5`l|ItRii zpsy1KKY34oWJb_-TSlxURy&}A!`k_7bCy&E1O*KM{5J;uPX|%j#T1Z76Y#(NFK^@l zZ{*(&`?t&g*A4T(1O2DT<4ug@Ukd+Eu`A7Q8puTY&(iY0T@wGu{d@HOw2^5b#lM99 zH!S|I(2q3W43O+!i&Pr(43Getk}OSr2KY~}4mlvr?w=?2>1h!&Kyu>$u2ye!3ICc; zdw<&C43Lo+@Sp$kZ3@JHiC_$-q0a)Di2;B4^IszPL;r{n{^Q=LjQ?wlzx?JeHH6`R QtG&4qDLniL@a^UQ0W);cnE(I) delta 11685 zcmZv?18`>D)-C+Rw(WG1j&0jX$F^-hv2CMc+vwP~ZKFHJ?e{yk{&UXv-(5BKu3CGn zx#pZT_o`ZRkL>jC*QMX_6lK7`(E$Je^k0yyV%I<$vV#BsBVYgkS{l_k1;<(vJuogObv{r~C@{zx+Lk&=Ypw0w55KZF5wp@={P3B;Jjy( zWrIT>pqQDpKS)P7Xy1sn106W8DJ%nFiolPvvk%HGcWJgVGmoK8dC+ak*||~lNN-Rx z!~uV+_Z=qL-^lbrfhp2Am^uHNsa};om!da9D08?EQL0t9vo89jXs=l|YM{ZbfhFQW zjAupamAaaz3|FBUhJ)N|7#@@_P>QcU38RK=N9lzOxO-I$r|}$L5&%TMz5!l_gR>!r z!)FsX{CfCYov6RE@vqeUD!tjG3x(oT{A3?vu?h}Ha@z|v=OXPcMfniO?gIJm#OkKa zF_QjGEDSgR0QWc5`c9@c&J6Vb+^dszK_%7Jv%L*Kd$F8IMpg3E?Yz?h} zSuJQD-ZHFhuw+RdYuULsPk&}Ks~&Ad_-8w`zRNd~7lWtFHmFgQ+FzWE@li|%CC*xe zWE0LRHE7Z6)q@au>8RpKfQoX-f4`X`GnQ9b4U}mT!l1#k)xI^flra;uDG*I9J z4;q$LmXx<8VUb=okolbrDG7EXYc5K)@$?+q<4Ki>YtZrhj4WQYhzVColoj92_LcT7%9i z?UrzTmv6P0sGu99$)rOI% zcucuvPlRa79-dO$>XqE-8Pd4Jv^xj9wWhvvL{RWxUjafNbbKcJ`W9Y1oRP%2O%<%i zE_cU4kLz~$?xl6@O(ImIg&R0EylRocnHjPrZOXHoJ*nxGR#rjPG8xdZOAXrw9&@xo zv`e@=p{@=k7|pE$I>D@&>f?f$sEwM~py+B-fZ?jGT`6uTJZZt1YF^?5f6fT(Telf- zcW++L7(RaAcx`sMeEqK=QHx3zD=`27gQ%oUFg&2Q{RU^tmq1?NN>m716`8<1@A}$g ze?(Fa?YjJ&S!pL8I3(R1duhWVN^x7y5a3Pd`orl>>YA%aRH?yaA|l7++Jw==PnBqc2q47VhL=FiKHQj_$)7`K=yo@3I{Rd3F8JU zP8OoMX?Z`j1Gj*#pRWxob_7eiOz!g5!$7pPswyHs;86%jbH;@@Jp#0I1_9)!3olef z+owgWZ%D(4ED$UA$EwYbX-lJDXB{MPytoa{J8(bt z$lFAT9#y2d5t^-*O)y0Ji4!ZguOkmc*kXQK-SOvn9!g7D&)Ux){m;x%^Rl6JHuzUd zWY{yMv<~f05Rr;Agl~P8G2TuXTO0zv89GL)lNt}AGW&Q=K!X0E_O%f;0;K z>Z^p3>vZ4b<~eIZnj!@5(&h*79kaW_l!2b#eL-SA@RT;=ezmdo<3@f@QR-K08V7{jUY~LPJV8C-KP5>=+)!^$3O}c za?znCx^|w+=D64y`TLK?TE&<`K1kBlo|Dmf6`b)%#EtHx>5Cw>l7SfzBpjF#_$54? znnnBX9jA}n5>PqY0c%4NbwK0{!c>=h6<-$XfXwv{8^cR}2tc#M?4tTln-Uv?ggqYQn%vwHdQ#lH4qa#D*>tCSZs;vnhGyf z$kP{yRpN1?(Z48L2FlV`JQ|p$ZkIFNe&P0AL6r8a$>pQfDC|F9U&aj2ST(uN9y2mw7Fmx^M_)8p!JaX4x^m@^wT^xMIw zkcA(1#=eAYlL2I9d4t-1x6Rv*vNz9RfH(7*5!N>rCU!KFs2h%gp#+LHQtyGTNMfdZPsVdM>P}6k zcR<~KZP8IWT8F;1<;4tm2IH9Y;Dp>G^e=~$SitRDu`qBX(g9}AW+7xMcD|G%XQE9B z1n`+prTZ;&X16u-{y;=uVi4L0#Pu`|x?s8E)0Yhhv*HS+(+WpD*d%5F=om9Uks1A9 zAdw(PJ!)M#`;~{l&T71*NWlqOa|G`4&>?IZB+tN}_T}oleWB-W5AOBevw}q>=WD4b zKL=oXZ;SG?3IgJ@O5%Oquu95&>i-IL|GMW>3!9-=T&TYBZEb7>Ll-Uyf!r^@LY-)E zLzY=e@=Ny0ApNL+G|6>%+6+p8YtI1{^Qf(KhB7=vM#o<6c$RqD3TwsssLa(gbELU- zyDLyK9^{=3w-Zp+uK3V1NMN}}^l=iGAiIw65~4^$tOL>}6PHnL2daX*UO+wk?4O17 zpQXxp+~K1K<0?=Yx@kKAtkD|;8w;nHey|>ce7B+|u+B|YHz@Cp&o>SG%HerNh3JLt z9h@ac5{A?o&Y0Vcm7)U;cB9UV=Yjo=Boqpcj-{ql3WL!eM9}?l_c62q?CJE=Poprr zvbT={tO6H?)F7Kx-AWpzxfp^$9-u1i?1=CxP@aQIRORveZ(O4@c;K_xQf_M8vGHNJUgMuZot! z>UEd=(Kq8?z6_~Q`R83e`dlHRX&V#w#$&jXxEEM|-hvvSUK~Q9(WdURFUDS$2;LwBFT$jX<@%2k2MN=*<^g^R3F0Dfc=-Dts-`xJE0KYCee{uXw}(^ z1GL8GHRNw7F7ciLbSZWs8Xzg;LyBAd?=mKb+GRE#^I;F8i%+980Vb9 zVN$@CLDjXcacQVrC9NS=mV7frOeN;KD~6$(%A^1dtMgmYKbW4+EA{*@1GjXF797jo zFsEY|ld?#@p?3B2d8;_*a$9H0OYLG`8c^5G=@;L=H%of@{|aPh>RhW}r-t_~_s4~x zwW+{}h!SH3AkzYNKt}}HGhYNnynU|c$9@Bo;LrJV`VO_LernOq%bkt7nkZQrDPo$< zDlMS#-SF%nm^v^MyP<iB;jiI+CZdlc8gnL2e0-{1CNmUr@WZP8sCcll% z1DHqMio?Q8{2EKhyN^iU0lu-I+yvkGAAUyAz!25o6TC&0m+z=;#cncczZXGQrZNF@ zUrMlxx~@_jKfcsi`PK0|F>upbKZtQ_y9EmVkcgwqB?<^Os3t>NA%X!(IqbQ|O{tKqb!pgQBY#tUUAy19Oesj6nf~+Y}y2+f?gBT6X8h#%Cwo=ERJQk z#CP{B0KP@SfG}3?ss-tZxls9GDy(gXqm%o%>VrX-vqdBECgmhVCBzH-%P0lN9D*Q97Rw=9WK7$3bShjZZp~#s$_*dxCTvBVLBDiN3y)SbVOy9 zV-2?4sal;x^`M-pV3&6X5@)icat^?CFcvI<6HYf4yXHYLCj^Jiu3|%rnF<3kn{?^Q zN0^aIT1~%AwAp1pVe8;?*(2Mp(U6w*!>zOA8)`#B+%`_{n|*m!kiKzk;K2yEL-=cL z3DH5Pg`Mgax>~P?c!m>Dv-VK@S3E-}jRGc@A^P(iI)60wt*Adru<&dWLNqR-{IY6N zic9^zGhPF{U7*1yhu=)AT70H1XkPWJEoHU5ccig7vnUoK=U9iL$EozUt%$8b9JT3H zIhhi0H@l}t<{pF47BRXn@W#g=!lve|9HC90-V$Qe&YklWD@R7)9|s1Gwd^MkM8ZFV z6Y*iFAxZGaQVyaSZcRMx#`mQu%<3t&ClG2=2=W9WEF)@|220ob3J@K+8WpAUxEEHDi9gV913A%EOnc~oRqlNr@om0 z92sTd)dm&Rhv2qqc9&uGDAG2?=vs}*ow@?&7=>?D#mLHm_oqJ#c>GZ3B$Y-|MZto0 zFvt%a76Jp{y^#ok;Pn~;A=X=x2T3PnACzPSu%*sIyaMC`V2I=|@XZnwMFkF--CR^O zcJ|?ZX4XjgD*$}Rb3QCoS6%-5^hQl-4AzcJ1IH^XJ`J(o+La-ROP);#VJGd-*vSE5 zRMWiZlLUc4`zO*C<*67;NlKSkci7HAmJ>U4 z;UhM6;ruQE`(f^6VC|4$NxLf9iZQ*jK17~_=uiLF{iJ!CxIY44pF}7Kj|Wpf?(%?N zwPH2A5nVPkiPZ^MU_D0EX3^1S1i2=N>AW`iY6B!d%R!{w)*)IlPd)O^=&l5qHkc@K z9^snXh{ETX}?Ui zki++!xnOc{i}TwkA_%588(xD?i8ZNO(0wG`d^~D@`>ga0 zUtH@Uc)IXvMnD@mDl1$Q1aL&-`@@Ejt8fMyxZPH^{mo6mP5>9f$a^dtpv34vDDIie zmrMleiV{aqp{=Xz9lC=Kn%Jox#6`JPSGytA5K(HsZd$&!cpXhgrZe8_<(#9Bir|zi zEt$|>QHBnADy^(5(&=a)brrY;wIAsEpyBfr!$5#Av*+gX3sxv3DOy%NjE3c0>U;bl zP^GNBH_285tV@EHD=0uwD%y9Y?|a6+o}tfhOG<}UmC26@>EzSEKU6di)r9UFD_iuq zD24kuD(=j}m8i^ER_dyWFNfNmZ{bV)ZFF08)n9VsNu@09X!{;q!(%W;bK;62g*ol! z!}15zOk)qbqKkC7P)i)RB_lma-^}FAfI~8f&bU3dD(!RcB`%YRFS>-eO?CkX`6V zpFgYPz_ABJK`0_!sE{nAY)jpxM@v1xNZM7W@ipd;(#O#ZJ*?qR*^139!Ufazfd#Z( z1hS8=C(dDb(`F&IqDpXoDDgb=n=D z%d(c7+2+2*$d-)vFt><5+DSda0B_$Y8p0&D+ILA0Ovh95sEc~bc6(_=h}tqF1uL~n zP8oxRLtDDwfidmeJz!)+j2V!q-!2EMmGEj?>Dxf(d`KJJ60e5Ea4>MF|CG`+EE8jk zhUJpNe|X4l%Tmta^5>Ta2hoBYJFO8ov&jZ9F!mKxCBcS|&A;qAC77{c0J91-_2rpt z9?^q(9M#eE1jcfVsJeq&`;9Vf?fkUw#Mo$-s2-k0%PW*Cc3~ch^K`q|r#z!wT@5Ht zuqy*9@FGrb3uHB*<$fci{*2&+7*_J4EP{uSj8dgi6&~IxW&DNArkrsr70nK>c^N$( z94WYm5<+%+nNi&T6+r|74U{RO4uQ31lbpIF4*5!du~=%^1shISNnFeFM#~7eeot7O zkkHy$YiDX;PkEyZ@-!baJE~^ zRcE{s0Lpv^R$_$B6t6wpTo0AjV(+Erc#b+rm>2Q${eXg|h}F8B9LR)iSi4AtYHdBN zU!!}EGHvysOB41&Svg*;s!*|&dZI=Vl;F8?q|yRD?q?PF!=b}03rozDS|w_NsMHc|%4 z*-GCu{vr|mS76UW2S=C+cYpj6*$b!En07|BcE3-yyHa2V%oO{$}1Df>1Qfd&0Gn!x^U{8vA->%>n}Vp|+^=KZ;)aqH$7Cl-VY1khLqR_4Fcv=M7QWh2!BCnPUT*f7 z)-URdf}ffL{p@1zGT`lv4yM~iP%5RN$?e>(|=AV7LV zczQ(`vsd*@m;k}KS7sPchR@9u@l=~fTjJ;mk;uZXZ2becH#6YdLo@CJT;coU}-(k37f*u3Jqjc0Z(O_CxNdE}~<7Y;n$H#`(8HUVQ5^uEr zb9*-|^tR0Q<+}^{H z;xiUQHsAsrvaIbysU&S5<$S#mvV5t33r5c@wm6Xq&gk13%8uQhdyO3|og88W!}t49 z&aUWMEJcsK{HeSHEvqy+*Bp>DPrFrSb`BR+!oc(_9b7%2xbisbvT(dz@Ac4ebW}CV z%G}9Z;`sGA+E4HohJe=y9rAfaV(=7V*EwN(5TF22+U`mJwK}vMArDsS(x_+gb9F-P zCnyrp-AF`Y9>4qpZ_+nUkG9K|_{zgL+^f%$g#8equ!msytcF7wTzx2k%oaluICj-7 z1prkZ%=4Jt08;uzCA|ovd7tEgmhTEz05%zye82e86yjht6?Fp;>nMhZQK1_iB9)?}t;-pZ zxP`$aea}jR!l_jDf(^k%A;}No&6p6v3}A9Z-A%nFQC<9d4x5K6kjBW(JEt&6{)U*M z#Lo!zfy||uo^C3XT~y|>9gG{9e?QzU$}-}T|6WjNeb>1DbvyM(Bw_m%rSe9`1QLLr zG$76i)K7h{y+o(tTX@sl)v3=TSFHcLn8)zR~H9Y7a+Zdt<93oh0_K* z=I2gn@3VyI%qptXT5Jx@c~L}XR?gtrpxx8{odq#!>Z&5SOrRQTF7F8cY|m;g*tQ@n zIhD4fYTQVP0~Y{s`iKWAUtGXX)8jB#tRC;V_HnYOULxHdD)wSV@9M+r<6&-XJMI8Q zTQ=i>@x-`)Ab8|p4jAb8n(73r`m`uR6?Uzj?ZmF9C{VC%p~yhw>4HJd`>_k|)?mO- zgm5^oL_B&BMouUZ&0RT{ugo4PGINwIrWwWv+fUptG%Wt*!5YsW2ie>bU6eftj-JS7Y+H7@ghh09(WcQ^KtZh`cFC=Zg?D- z4i_HUefJ13--NKfQ{5YGoR8a=%UthS2APtiT}K`mD(_8x{2j^60qgnQVr+9Q`&Tw( zZsXHO^jAEaurOF8gvr@GSDz`f24$4K-xI;wAyf(XnA;nL=c(q$LeJdWdIluDv(dn` z1H0qyjEs}qtd7fj|eY9(OgR4}-TmqZ{ zvVku4>a*LxzU^(rGTn=zcm{En%t9p+4XAd%G)TM+G~#8dc-jG@PlzY39S zXP_{yV1TeXsD~)c87CBkF_g5~wd_d|nA~Wg8(0vrt3k1N=?l&u>7Be|?H`plunQyV zjHJ|>`5Hkmo?C)~8n#4(W{odltCtcB zkC<5LvHTXLzWg9`e-hfZjpYt4HEg9N@|`g`0U!olmf0G+)u1rry4>co%CcHSYeMuI z5zH%YnVs&C>YRLc@G{e;*uLX=BXI;?f)^IAMpG%NbBBUo!K44Qb-Vh{6w>UZ<3tdUF$T*jfdW_n9b9+ak z13<1A%*AObyB=)-m?z+)Ip+Zp4 zlMLKrZVmMoqNQ#_8mMB@VAxqBsN2yWM@+dx8kefrSAT z74x5LBMo@~eW2Q(gi48^*IqC#zFnb#2@@uW)wOtwy4X00AkquQc<$&UCu(DmO#>F} z8a7`;FeAdaeb(PnrLbrrAu5sLlkWW6y2WBI%HK`bG%P_)x(LCO)WIJB&uwu`7bG}@ zsAzuGB1Yjm+iZ&;-)=RLa=Uv=fOJp(;;G#60J`Ww$F^#@Y$6U-vL#k5I{3a=kMvx&K++5xmyQ)>9BTN5TQ zkBNyCVHYCFZxGEFA<9Uq(c!;$#C`9RF66*KN9`a3R2Al6>WErY4FtrDfz+G>^14Bd z%k9ga?bXj(OcoIV4*95H5vjN!9L=D1xY95jGwuq53-B>66^8Y9x| z26r{}Bn!X&=o}a0aCUA!a(qh;xBg_2JuET`>di|dc!zpDW|yrwH+WxC&vU8@m7BoO z-?JwsR8g~1MD>2k(iAXI11%=JG8?d^t%xbwMC^RGXIk>3ItVW~_GQ~qM4X@4f6|vm z(hAcwj;bav1$N@n=UB9}HZAHjxW&n|5R9lwebC)<=G1c#>ISRhkoJJK4j5@_f&2!U zumb=zL174p5|gyoNDl4|{Wr0)mg|7)Dgq!U)VMe{RLu!P-9l{>K)xhCELWwl=pMr{ zvdHKJhDUfSwD_=v1K)?keCA48&@#(|qg~f;K=%r)80TUO{2ThBi|;|zDz1uN{sQ1% z)*27O6PX8)%a_FHZLJ&0B~qHvJrT+gAR@qUVC|^n zWlq}f`|?P0Uzwpsprxd3+7I4`sBQyl^sO>FIJLuJOFSNp;M;GJ%F7}TSc3J1scuTo z^qD>-7%ON$sB20GPB2$kH?Q(X=l3isdWeV1x^EQ*Vw~wX35|Y|%PaEO)K6j`I zoUABzcOh^xh|xJVD~}KGDfECkPAQRfnpir?Ki;h=g1_oGoXs7>LJdHEdL@hxbdn+< z-<(jgsh;7908NalHiCj8#maBh#9lZwT#*eyt$%d{M*9r*Lma&!bzitRGafiAYTHaf zcB5E!zTnJD;LwjO`fEosG{!`D-2R|e}^Wi0lwbF?He3(q|trPeRl+nh1Ju! z5Xx$*xjRA~$8byZ@P#~+&B5c`>Dv?(+h`UebO6nco{$O~q>cAWEjV+j}ied~<&(o{#;kzywNR^eHY582CbD-!<8w1gJis ztiPM%0Y3KpE!j2v_iD8LnG?a$92OK+Bhc;+fl8ZCSH9g#jPiMt9C(QdKYdDZPO5t4 z+ub0G(u>?^)XHg>H?sw%1^TF@$K55*e+$#MJfYX;{8LzaLS_2~gs zAd1sV5expSG44a#-tCJC6p|J0&m%v29cxU{rQ-tkJj#l1K8Ue)gdVZm@ux^^lSvDZ z;<=x$*q#icdImpPxjKf+Ya*2uo}<1C`#O7Uq;Fq*kVSCmZ5imFl7hb|*n98!A_fyH zI~7^Qu*a1A31d8#3S!~&eI(nS_PiR=0UkoRCO`-caC}($M|c7$c3QGMH-sVI#ie6P zlo{R+Ni-SU9V=RUtAacA1iXpl(V5K?VJ)ejkU~zll8r;*{h~T0A+9l7<3aOK97^GO zpTO(cxByd7>&e5#-S4Xc+#Sx*cmKNTLUFY$!L@{L-wQtmUiXN3Nt4>~tt~W(o2koLu*4itx0p&wl>dg z|CecZw1*{5RS|Em7{L}xm7j8C+o?o%>kOH~MIkfiD0r$OmP@q(%g5_K6>2gO)>J1v z9+54-JJj_d7F*QOkm7E;wl!I?>YTH=-6#*rff-lyh%Z>M)LHPNkhkRH0Wrj@>}up$ zxHp5@7S#>uoS|;|X?8$s0)|Sk{D0A@*oe6#asOPrdc)nX0*v#w=XMec7?J5@dHG`} zVJjZJvSrpZYE7f$?l)0MW!RLm^YLtkNMOLm_DPjK>O;Tt#i)%&#fvNU-4Rr9b(@<+ z+S{76(i>(HKf4+om3=T{0cW8dvy?PEOX-K8w8PrIAwc!Su$-I3ySTuTZw@(?+hMyo zzGda|VfYCM*kLlK6LF+eRQTXkL_+KfphoT-)D`Tkl#0{+$eFIo{X%?md-n?hQAxFK zE1e0`8i1S;*c6+oH4>V-X=U>?tKB&)S@!91n&+OH{U%v$7_=ks3ACn^EWW`n2Mv5( ziSaSUCtmCBXivJ4!>oDFqhg6(cV;7w*Lmk|pDZp(V-e{>vX5F@qWouq~$Equ6%z zckC{PmAX~2ul_b8ag-!ZMs}ozoQfF{$hg=D0oJ)i{Vbj%0Ig_ts zt$I})-K|wzY^N|ab-*z&>7dZC$Xk6a-1jFm|6rIGKlQ1d8#sg~;;XjJt!kcB-AK?C z38Dq}iYe+Nu%MsJf%yg{!@-sGi=;_vFKr}r_fCCYkc<$>_3mE zpf74cOvGVK#2|kG`WIk-0sa>dtxUv_;vV4toM6Me(;@!?>Mx-G0;bhF9afGOcGcSE zG8^l!Rtq_8eh`)%IA=9Lj1IZ04zrDCEWAK0EF#;ntYM`W1mAzs#x2#wrk@=}0jxsGBGtQ}ydVpWA)qlQHQovfJxGW%h}PF!}F?iAI7MVWN>$OeDNv{!3{eGZ)$SAN*rxwSX-KL z^~(?H7BcZ&dSQZ_a-H@@>I(QrSS*#SC9lHNE*;i&$_h!>bY|R9E)TfoW@Qd17KJR( z?5ybmHD{A)qDToR!`GnjV|Rh~%U^>}JoV6LX-~l*0dv_}Lsy!WN6mk=3|Zg)T~`0Z zY4tV`n*KM=`8386SO#DpVjv?j1h9CmlP;gqfGYd1x*4WskC z8);S(%O5LBzJUMs*^$rty6>;g>}iCfu#CW19<;ws2YdSsNfDuv#&V*Gn41UH&I@sd z?4eK!jpW&rp02K5`F6ItlX|uw@G53WyIKptrz5Q%Rg`sXtP5OnkUyfGMc|M);Qvr` zRWIJKaagBWH52hv?qH918h(LLwqay}4qlzA_0Fxy-s<@c7z970fj$9&e;2;;EuME|c2*+1le`4OGQISxzw zKbugR&NwVCC?Zvw$2ctEzeY0u8PUj3YZ-?nA^i7l^fzX_|5~qP5;kph5Eeg8X95<9 z;D3LD|3m%Pa*tDKd=s$r1pltyzb`5O%hsRie<*lqiX*THX?6c&@Bb^CzecG3i##{; WFEalqEK(ZeBrG!I>MZPE#{U7}sOS~| From 9973d6b8f9ff6761675f13ea2051f38a03917478 Mon Sep 17 00:00:00 2001 From: ehennestad Date: Tue, 5 Nov 2024 23:08:06 +0100 Subject: [PATCH 08/29] Update run_tests.yml --- .github/workflows/run_tests.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/run_tests.yml b/.github/workflows/run_tests.yml index 0d6230c5..a1eebfd7 100644 --- a/.github/workflows/run_tests.yml +++ b/.github/workflows/run_tests.yml @@ -29,6 +29,7 @@ jobs: python -m pip install -U pip pip install -r +tests/requirements.txt echo "HDF5_PLUGIN_PATH=$(python -c "import hdf5plugin; print(hdf5plugin.PLUGINS_PATH)")" >> "$GITHUB_ENV" + echo "NWBINSPECTOR_PATH=$( python -m pip show nwbinspector | grep ^Location: | awk '{print $2}' )" >> "$GITHUB_ENV" - name: Install MATLAB uses: matlab-actions/setup-matlab@v2 with: From 1f090df41dc07d9b767c386efa60efa53fd43a0b Mon Sep 17 00:00:00 2001 From: ehennestad Date: Tue, 5 Nov 2024 23:17:00 +0100 Subject: [PATCH 09/29] Update TutorialTest.m --- +tests/+unit/TutorialTest.m | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/+tests/+unit/TutorialTest.m b/+tests/+unit/TutorialTest.m index 82e42aad..6bf2f43c 100644 --- a/+tests/+unit/TutorialTest.m +++ b/+tests/+unit/TutorialTest.m @@ -34,6 +34,9 @@ % SkippedFiles - Name of exported nwb files to skip reading with pynwb SkippedFiles = {'testFileWithDataPipes.nwb'} % does not produce a valid nwb file + + % PythonDependencies - Package dependencies for running pynwb tutorials + PythonDependencies = {'nwbinspector'} end methods (TestClassSetup) @@ -61,6 +64,13 @@ function setupClass(testCase) % % insert(py.sys.path,int32(0),pynwbPath); % % end + + + pythonEnv = pyenv(); + disp(pythonEnv.Executable) + + fprintf('PYTHONPATH: %s\n', getenv('PYTHONPATH') ) + % % Alternative: Use python script for reading file with pynwb setenv('PYTHONPATH', fileparts(mfilename('fullpath'))); From 5b30a12784e2432d91d22fa579af5a252e856f66 Mon Sep 17 00:00:00 2001 From: ehennestad Date: Tue, 5 Nov 2024 23:25:35 +0100 Subject: [PATCH 10/29] Update TutorialTest.m --- +tests/+unit/TutorialTest.m | 26 ++++++++++++++++++++++---- 1 file changed, 22 insertions(+), 4 deletions(-) diff --git a/+tests/+unit/TutorialTest.m b/+tests/+unit/TutorialTest.m index 6bf2f43c..30a99db6 100644 --- a/+tests/+unit/TutorialTest.m +++ b/+tests/+unit/TutorialTest.m @@ -64,10 +64,7 @@ function setupClass(testCase) % % insert(py.sys.path,int32(0),pynwbPath); % % end - - - pythonEnv = pyenv(); - disp(pythonEnv.Executable) + installNWBInspector() fprintf('PYTHONPATH: %s\n', getenv('PYTHONPATH') ) @@ -226,3 +223,24 @@ function inspectTutorialFileWithNwbInspector(testCase) function folderPath = getMatNwbRootDirectory() folderPath = fileparts(fileparts(fileparts(mfilename('fullpath')))); end + +function installNWBInspector() + pythonInfo = pyenv; + pythonExecutable = pythonInfo.Executable; + systemCommand = sprintf("%s -m pip install %s", pythonExecutable, 'nwbinspector'); + [status, ~] = system(systemCommand); + systemCommand = sprintf("%s -m pip show nwbinspector | grep ^Location: | awk '{print $2}'", pythonExecutable); + [~, nwbInspectorPath] = system(systemCommand); + checkAndUpdatePythonPath(nwbInspectorPath, 'nwbinspector') +end + +function checkAndUpdatePythonPath(installLocation, packageName) + pyPath = py.sys.path(); + pyPath = string(pyPath); + pyPath(pyPath=="") = []; + + if ~any( contains(pyPath, installLocation) ) + fprintf("Adding %s location to pythonpath\n", packageName) + py.sys.path().append(installLocation) + end +end From 04315f5e9b8425a3b1a711498b2797109ff6ffc1 Mon Sep 17 00:00:00 2001 From: ehennestad Date: Tue, 5 Nov 2024 23:27:46 +0100 Subject: [PATCH 11/29] Update TutorialTest.m --- +tests/+unit/TutorialTest.m | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/+tests/+unit/TutorialTest.m b/+tests/+unit/TutorialTest.m index 30a99db6..2b6101f0 100644 --- a/+tests/+unit/TutorialTest.m +++ b/+tests/+unit/TutorialTest.m @@ -64,12 +64,11 @@ function setupClass(testCase) % % insert(py.sys.path,int32(0),pynwbPath); % % end - installNWBInspector() - - fprintf('PYTHONPATH: %s\n', getenv('PYTHONPATH') ) + %installNWBInspector() % % Alternative: Use python script for reading file with pynwb - setenv('PYTHONPATH', fileparts(mfilename('fullpath'))); + pythonPath = getenv('PYTHONPATH'); + setenv('PYTHONPATH', [fileparts(mfilename('fullpath')), ':', pythonPath]); nwbClearGenerated() testCase.addTeardown(@generateCore) From 62a240e346540eda788baec9d09a1ee3d1ffd2c8 Mon Sep 17 00:00:00 2001 From: ehennestad Date: Tue, 5 Nov 2024 23:55:04 +0100 Subject: [PATCH 12/29] Try o set up python path --- +tests/+system/PyNWBIOTest.m | 2 +- +tests/+unit/TutorialTest.m | 10 ++++++---- +tests/+util/addFolderToPythonPath.m | 16 ++++++++++++++++ 3 files changed, 23 insertions(+), 5 deletions(-) create mode 100644 +tests/+util/addFolderToPythonPath.m diff --git a/+tests/+system/PyNWBIOTest.m b/+tests/+system/PyNWBIOTest.m index d08c2053..c81ab493 100644 --- a/+tests/+system/PyNWBIOTest.m +++ b/+tests/+system/PyNWBIOTest.m @@ -34,7 +34,7 @@ function testInFromPyNWB(testCase) methods function [status, cmdout] = runPyTest(testCase, testName) - setenv('PYTHONPATH', fileparts(mfilename('fullpath'))); + tests.util.addFolderToPythonPath( fileparts(mfilename('fullpath')) ) envPath = fullfile('+tests', 'env.mat'); if 2 == exist(envPath, 'file') diff --git a/+tests/+unit/TutorialTest.m b/+tests/+unit/TutorialTest.m index 2b6101f0..392e09be 100644 --- a/+tests/+unit/TutorialTest.m +++ b/+tests/+unit/TutorialTest.m @@ -64,12 +64,12 @@ function setupClass(testCase) % % insert(py.sys.path,int32(0),pynwbPath); % % end - %installNWBInspector() + installNWBInspector() % % Alternative: Use python script for reading file with pynwb - pythonPath = getenv('PYTHONPATH'); - setenv('PYTHONPATH', [fileparts(mfilename('fullpath')), ':', pythonPath]); - + tests.util.addFolderToPythonPath( fileparts(mfilename('fullpath')) ) + disp(getenv('PYTHONPATH')) + nwbClearGenerated() testCase.addTeardown(@generateCore) end @@ -242,4 +242,6 @@ function checkAndUpdatePythonPath(installLocation, packageName) fprintf("Adding %s location to pythonpath\n", packageName) py.sys.path().append(installLocation) end + disp('py path') + disp( py.sys.path() ) end diff --git a/+tests/+util/addFolderToPythonPath.m b/+tests/+util/addFolderToPythonPath.m new file mode 100644 index 00000000..2e93013a --- /dev/null +++ b/+tests/+util/addFolderToPythonPath.m @@ -0,0 +1,16 @@ +function addFolderToPythonPath(folderPath) + pythonPath = getenv('PYTHONPATH'); + if isempty(pythonPath) + updatedPythonPath = folderPath; + else + if ~contains(pythonPath, folderPath) + updatedPythonPath = strjoin({pythonPath, folderPath}, pathsep); + else + return + end + end + setenv('PYTHONPATH', updatedPythonPath); + disp('updated python path:') + disp(updatedPythonPath) +end + From 4f3e11f1151082b3fda98b711f7f9624f4ed0a8a Mon Sep 17 00:00:00 2001 From: ehennestad Date: Wed, 6 Nov 2024 00:16:01 +0100 Subject: [PATCH 13/29] ... --- +tests/+unit/TutorialTest.m | 5 ++--- .github/workflows/run_tests.yml | 1 + 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/+tests/+unit/TutorialTest.m b/+tests/+unit/TutorialTest.m index 392e09be..2f70e667 100644 --- a/+tests/+unit/TutorialTest.m +++ b/+tests/+unit/TutorialTest.m @@ -68,8 +68,7 @@ function setupClass(testCase) % % Alternative: Use python script for reading file with pynwb tests.util.addFolderToPythonPath( fileparts(mfilename('fullpath')) ) - disp(getenv('PYTHONPATH')) - + nwbClearGenerated() testCase.addTeardown(@generateCore) end @@ -229,7 +228,7 @@ function installNWBInspector() systemCommand = sprintf("%s -m pip install %s", pythonExecutable, 'nwbinspector'); [status, ~] = system(systemCommand); systemCommand = sprintf("%s -m pip show nwbinspector | grep ^Location: | awk '{print $2}'", pythonExecutable); - [~, nwbInspectorPath] = system(systemCommand); + [~, nwbInspectorPath] = system(strtrim(systemCommand)); checkAndUpdatePythonPath(nwbInspectorPath, 'nwbinspector') end diff --git a/.github/workflows/run_tests.yml b/.github/workflows/run_tests.yml index a1eebfd7..4ad95079 100644 --- a/.github/workflows/run_tests.yml +++ b/.github/workflows/run_tests.yml @@ -30,6 +30,7 @@ jobs: pip install -r +tests/requirements.txt echo "HDF5_PLUGIN_PATH=$(python -c "import hdf5plugin; print(hdf5plugin.PLUGINS_PATH)")" >> "$GITHUB_ENV" echo "NWBINSPECTOR_PATH=$( python -m pip show nwbinspector | grep ^Location: | awk '{print $2}' )" >> "$GITHUB_ENV" + echo $( python -m pip show nwbinspector | grep ^Location: | awk '{print $2}' ) - name: Install MATLAB uses: matlab-actions/setup-matlab@v2 with: From 392df70134fe203356db81522651071ebcb6a6aa Mon Sep 17 00:00:00 2001 From: ehennestad Date: Wed, 6 Nov 2024 00:35:23 +0100 Subject: [PATCH 14/29] ... --- +tests/+unit/TutorialTest.m | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/+tests/+unit/TutorialTest.m b/+tests/+unit/TutorialTest.m index 2f70e667..d2ec5821 100644 --- a/+tests/+unit/TutorialTest.m +++ b/+tests/+unit/TutorialTest.m @@ -228,8 +228,8 @@ function installNWBInspector() systemCommand = sprintf("%s -m pip install %s", pythonExecutable, 'nwbinspector'); [status, ~] = system(systemCommand); systemCommand = sprintf("%s -m pip show nwbinspector | grep ^Location: | awk '{print $2}'", pythonExecutable); - [~, nwbInspectorPath] = system(strtrim(systemCommand)); - checkAndUpdatePythonPath(nwbInspectorPath, 'nwbinspector') + [~, nwbInspectorPath] = system(systemCommand); + checkAndUpdatePythonPath(strtrim(nwbInspectorPath), 'nwbinspector') end function checkAndUpdatePythonPath(installLocation, packageName) From e8c7c510d6dab79744e5a35fb90d9afda86a3a79 Mon Sep 17 00:00:00 2001 From: ehennestad Date: Wed, 6 Nov 2024 00:36:10 +0100 Subject: [PATCH 15/29] .. --- +tests/+unit/TutorialTest.m | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/+tests/+unit/TutorialTest.m b/+tests/+unit/TutorialTest.m index d2ec5821..3b4d63ff 100644 --- a/+tests/+unit/TutorialTest.m +++ b/+tests/+unit/TutorialTest.m @@ -64,11 +64,12 @@ function setupClass(testCase) % % insert(py.sys.path,int32(0),pynwbPath); % % end - installNWBInspector() % % Alternative: Use python script for reading file with pynwb tests.util.addFolderToPythonPath( fileparts(mfilename('fullpath')) ) + installNWBInspector() + nwbClearGenerated() testCase.addTeardown(@generateCore) end From 007bfcc8eae0a5cccddcf876edf105e4e93986df Mon Sep 17 00:00:00 2001 From: ehennestad Date: Wed, 6 Nov 2024 02:29:29 +0100 Subject: [PATCH 16/29] ... --- +tests/+unit/TutorialTest.m | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/+tests/+unit/TutorialTest.m b/+tests/+unit/TutorialTest.m index 3b4d63ff..ec9a5e72 100644 --- a/+tests/+unit/TutorialTest.m +++ b/+tests/+unit/TutorialTest.m @@ -132,7 +132,12 @@ function inspectTutorialFileWithNwbInspector(testCase) % Retrieve all files generated by tutorial nwbFileNameList = testCase.listNwbFiles(); for nwbFilename = nwbFileNameList - results = py.list(py.nwbinspector.inspect_nwbfile(nwbfile_path=nwbFilename)); + try + results = py.list(py.nwbinspector.inspect_nwbfile(nwbfile_path=nwbFilename)); + catch + [s,m] = system(sprintf('nwbinspector %s', nwbFilename)) + disp(m) + end if isempty(cell(results)) return From d9a050fbcae074f8aafc8d57c326e5879d3a1322 Mon Sep 17 00:00:00 2001 From: ehennestad Date: Wed, 6 Nov 2024 09:53:38 +0100 Subject: [PATCH 17/29] test --- +tests/+unit/TutorialTest.m | 4 ++++ .github/workflows/run_tests.yml | 3 ++- 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/+tests/+unit/TutorialTest.m b/+tests/+unit/TutorialTest.m index ec9a5e72..34f583a1 100644 --- a/+tests/+unit/TutorialTest.m +++ b/+tests/+unit/TutorialTest.m @@ -70,6 +70,10 @@ function setupClass(testCase) installNWBInspector() + disp(pyenv) + + py.nwbinspector.inspect_nwbfile('test.nwb') + nwbClearGenerated() testCase.addTeardown(@generateCore) end diff --git a/.github/workflows/run_tests.yml b/.github/workflows/run_tests.yml index 4ad95079..167ee994 100644 --- a/.github/workflows/run_tests.yml +++ b/.github/workflows/run_tests.yml @@ -12,6 +12,7 @@ on: push: branches: - master + - 484-use-nwb-inspector-for-tutorial-files jobs: run_tests: @@ -38,7 +39,7 @@ jobs: - name: Run tests uses: matlab-actions/run-command@v2 with: - command: results = assertSuccess(nwbtest); assert(~isempty(results), 'No tests ran'); + command: results = assertSuccess(nwbtest('Name', 'tests.unit.Tutorial*')); assert(~isempty(results), 'No tests ran'); - name: Upload JUnit results if: always() uses: actions/upload-artifact@v4 From ef82026e8398a8eaff66dca6f8150a9f23197c98 Mon Sep 17 00:00:00 2001 From: ehennestad Date: Wed, 6 Nov 2024 10:06:47 +0100 Subject: [PATCH 18/29] ... --- +tests/+unit/TutorialTest.m | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/+tests/+unit/TutorialTest.m b/+tests/+unit/TutorialTest.m index 34f583a1..212f326c 100644 --- a/+tests/+unit/TutorialTest.m +++ b/+tests/+unit/TutorialTest.m @@ -72,6 +72,22 @@ function setupClass(testCase) disp(pyenv) + generator = py.pkgutil.iter_modules(); + + py.dir(generator) + methodNext = py.getattr(generator, '__next__'); + finished = false; + moduleNames = string.empty; + while ~finished + try + module = methodNext(); + moduleNames(end+1) = string(module.name); + catch; + finished = true; + end + end + moduleNames' + py.nwbinspector.inspect_nwbfile('test.nwb') nwbClearGenerated() From d96560755551a8f5413dccae428b33162a8c03bc Mon Sep 17 00:00:00 2001 From: ehennestad Date: Wed, 6 Nov 2024 13:13:21 +0100 Subject: [PATCH 19/29] Update TutorialTest.m --- +tests/+unit/TutorialTest.m | 134 ++++++++++++++++++++++++------------ 1 file changed, 91 insertions(+), 43 deletions(-) diff --git a/+tests/+unit/TutorialTest.m b/+tests/+unit/TutorialTest.m index 212f326c..18aea587 100644 --- a/+tests/+unit/TutorialTest.m +++ b/+tests/+unit/TutorialTest.m @@ -51,7 +51,8 @@ function setupClass(testCase) testCase.applyFixture(matlab.unittest.fixtures.PathFixture(rootPath)); testCase.applyFixture(matlab.unittest.fixtures.PathFixture(tutorialsFolder)); - % Note: The following seems to not be working on the azure pipeline + % Note: The following seems to not be working on the azure + % pipeline / github runner. % Keep for reference % % % Make sure pynwb is installed in MATLAB's Python Environment @@ -64,32 +65,12 @@ function setupClass(testCase) % % insert(py.sys.path,int32(0),pynwbPath); % % end + % This is also not working on github runner + % installNWBInspector() % % Alternative: Use python script for reading file with pynwb tests.util.addFolderToPythonPath( fileparts(mfilename('fullpath')) ) - installNWBInspector() - - disp(pyenv) - - generator = py.pkgutil.iter_modules(); - - py.dir(generator) - methodNext = py.getattr(generator, '__next__'); - finished = false; - moduleNames = string.empty; - while ~finished - try - module = methodNext(); - moduleNames(end+1) = string(module.name); - catch; - finished = true; - end - end - moduleNames' - - py.nwbinspector.inspect_nwbfile('test.nwb') - nwbClearGenerated() testCase.addTeardown(@generateCore) end @@ -154,16 +135,18 @@ function inspectTutorialFileWithNwbInspector(testCase) for nwbFilename = nwbFileNameList try results = py.list(py.nwbinspector.inspect_nwbfile(nwbfile_path=nwbFilename)); + results = testCase.convertNwbInspectorResultsToStruct(results); catch - [s,m] = system(sprintf('nwbinspector %s', nwbFilename)) - disp(m) + [~, m] = system(sprintf('nwbinspector %s --levels importance', nwbFilename)); + results = testCase.parseInspectorTextOutput(m); end if isempty(cell(results)) return end + + results = testCase.filterResults(results); - results = testCase.convertNwbInspectorResultsToStruct(results); T = struct2table(results); disp(T) for j = 1:numel(results) @@ -191,21 +174,10 @@ function inspectTutorialFileWithNwbInspector(testCase) methods (Static) function resultsOut = convertNwbInspectorResultsToStruct(resultsIn) - CHECK_IGNORE = [... - "check_image_series_external_file_valid", ... - "check_regular_timestamps" - ]; + resultsOut = tests.unit.TutorialTest.getEmptyNwbInspectorResultStruct(); + C = cell(resultsIn); - - resultsOut = struct(... - 'importance', {}, ... - 'severity', {}, ... - 'location', {}, ... - 'filepath', {}, ... - 'check_name', {}, ... - 'ignore', {}); - for i = 1:numel(C) resultsOut(i).importance = double( py.getattr(C{i}.importance, 'value') ); resultsOut(i).severity = double( py.getattr(C{i}.severity, 'value') ); @@ -219,14 +191,74 @@ function inspectTutorialFileWithNwbInspector(testCase) resultsOut(i).message = string(C{i}.message); resultsOut(i).filepath = string(C{i}.file_path); resultsOut(i).check_name = string(C{i}.check_function_name); - resultsOut(i).ignore = any(strcmp(CHECK_IGNORE, resultsOut(i).check_name)); + end + end + + function resultsOut = parseInspectorTextOutput(systemCommandOutput) + resultsOut = tests.unit.TutorialTest.getEmptyNwbInspectorResultStruct(); + + importanceLevels = containers.Map(... + ["BEST_PRACTICE_SUGGESTION", ... + "BEST_PRACTICE_VIOLATION", ... + "CRITICAL", ... + "PYNWB_VALIDATION", ... + "ERROR"], 0:4 ); + + lines = splitlines(systemCommandOutput); + count = 0; + for i = 1:numel(lines) + % Example line: + % '.0 Importance.BEST_PRACTICE_VIOLATION: behavior_tutorial.nwb - check_regular_timestamps - 'SpatialSeries' object at location '/processing/behavior/Position/SpatialSeries' + % ^2 ^1 ^2 ^ ^ ^ 3 + % [-----------importance------------] [------filepath------] [------check_name------] [-----------------location----------------] + % Splitting and components is exemplified above. + + if ~isempty(regexp( lines{i}, '^\.\d{1}', 'once' ) ) + count = count+1; + % Split line into separate components + splitLine = strsplit(lines{i}, ':'); + splitLine = [... + strsplit(splitLine{1}, ' '), ... + strsplit(splitLine{2}, '-') ... + ]; + + resultsOut(count).importance = importanceLevels( extractAfter(splitLine{2}, 'Importance.') ); + resultsOut(count).filepath = string(strtrim( splitLine{3} )); + resultsOut(count).check_name = string(strtrim(splitLine{4} )); + + locationInfo = strsplit(splitLine{end}, 'at location'); + resultsOut(count).location = string(strtrim(eval(locationInfo{2}))); + resultsOut(count).message = string(strtrim(lines{i+1})); + end + end + end + function emptyResults = getEmptyNwbInspectorResultStruct() + emptyResults = struct(... + 'importance', {}, ... + 'severity', {}, ... + 'location', {}, ... + 'filepath', {}, ... + 'check_name', {}, ... + 'ignore', {}); + end + + function resultsOut = filterResults(resultsIn) + CHECK_IGNORE = [... + "check_image_series_external_file_valid", ... + "check_regular_timestamps" + ]; + + for i = 1:numel(resultsIn) + resultsIn(i).ignore = any(strcmp(CHECK_IGNORE, resultsIn(i).check_name)); + % Special case to ignore - if resultsOut(i).location == "/acquisition/ExternalVideos" && ... - resultsOut(i).check_name == "check_timestamps_match_first_dimension" - resultsOut(i).ignore = true; + if resultsIn(i).location == "/acquisition/ExternalVideos" && ... + resultsIn(i).check_name == "check_timestamps_match_first_dimension" + resultsIn(i).ignore = true; end end + resultsOut = resultsIn; resultsOut([resultsOut.ignore]) = []; end end @@ -270,3 +302,19 @@ function checkAndUpdatePythonPath(installLocation, packageName) disp('py path') disp( py.sys.path() ) end + +function listPythonModules() + generator = py.pkgutil.iter_modules(); + methodNext = py.getattr(generator, '__next__'); + finished = false; + moduleNames = string.empty; + while ~finished + try + module = methodNext(); + moduleNames(end+1) = string(module.name); + catch; + finished = true; + end + end + moduleNames' +end \ No newline at end of file From 0c8c551c304e01942beccac7c9aa6f071c42e502 Mon Sep 17 00:00:00 2001 From: ehennestad Date: Wed, 6 Nov 2024 13:18:26 +0100 Subject: [PATCH 20/29] Fix TutorialTest --- +tests/+unit/TutorialTest.m | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/+tests/+unit/TutorialTest.m b/+tests/+unit/TutorialTest.m index 18aea587..6e2776a0 100644 --- a/+tests/+unit/TutorialTest.m +++ b/+tests/+unit/TutorialTest.m @@ -141,7 +141,7 @@ function inspectTutorialFileWithNwbInspector(testCase) results = testCase.parseInspectorTextOutput(m); end - if isempty(cell(results)) + if isempty(results) return end @@ -225,9 +225,12 @@ function inspectTutorialFileWithNwbInspector(testCase) resultsOut(count).importance = importanceLevels( extractAfter(splitLine{2}, 'Importance.') ); resultsOut(count).filepath = string(strtrim( splitLine{3} )); resultsOut(count).check_name = string(strtrim(splitLine{4} )); - - locationInfo = strsplit(splitLine{end}, 'at location'); - resultsOut(count).location = string(strtrim(eval(locationInfo{2}))); + try + locationInfo = strsplit(splitLine{end}, 'at location'); + resultsOut(count).location = string(strtrim(eval(locationInfo{2}))); + catch + resultsOut(count).location = 'N/A'; + end resultsOut(count).message = string(strtrim(lines{i+1})); end end From 9d530c4680c2fdd9232d137053217aad8787dfbc Mon Sep 17 00:00:00 2001 From: ehennestad Date: Wed, 6 Nov 2024 15:37:20 +0100 Subject: [PATCH 21/29] Tests are working on GItHub Actions --- +tests/+unit/TutorialTest.m | 44 +++++++++++++++++++++------------ .github/workflows/run_tests.yml | 4 +-- 2 files changed, 29 insertions(+), 19 deletions(-) diff --git a/+tests/+unit/TutorialTest.m b/+tests/+unit/TutorialTest.m index 6e2776a0..dba26544 100644 --- a/+tests/+unit/TutorialTest.m +++ b/+tests/+unit/TutorialTest.m @@ -3,12 +3,18 @@ % % This test will test most tutorial files (while skipping tutorials with % dependencies) If the tutorial creates an nwb file, the test will also try -% to open this with pynwb. -% -% Note: -% - Requires MATLAB XXXX to run py.* commands. -% - pynwb must be installed in the python environment returned by -% pyenv() +% to open this with pynwb and run nwbinspector on the file. + +% Notes: +% - Requires MATLAB 2019b or later to run py.* commands. +% +% - pynwb must be installed in the python environment returned by pyenv() +% +% - Running NWBInspector as a Python package within MATLAB on GitHub runners +% currently encounters compatibility issues between the HDF5 library and +% h5py. As a workaround in this test, the CLI interface is used to run +% NWBInspector and the results are manually parsed. This approach is not +% ideal, and hopefully can be improved upon. properties MatNwbDirectory @@ -38,6 +44,10 @@ % PythonDependencies - Package dependencies for running pynwb tutorials PythonDependencies = {'nwbinspector'} end + + properties (Access = private) + NWBInspectorMode = "python" + end methods (TestClassSetup) function setupClass(testCase) @@ -65,12 +75,18 @@ function setupClass(testCase) % % insert(py.sys.path,int32(0),pynwbPath); % % end - % This is also not working on github runner - % installNWBInspector() - % % Alternative: Use python script for reading file with pynwb tests.util.addFolderToPythonPath( fileparts(mfilename('fullpath')) ) + % This is also not working on github runner: + % installNWBInspector() + + try + py.nwbinspector.is_module_installed('nwbinspector') + catch + testCase.NWBInspectorMode = "CLI"; + end + nwbClearGenerated() testCase.addTeardown(@generateCore) end @@ -124,7 +140,6 @@ function readTutorialNwbFileWithPynwb(testCase) end catch ME error(ME.message) - %testCase.verifyFail(sprintf('Failed to read file %s with error: %s', nwbListing(i).name, ME.message)); end end end @@ -133,10 +148,10 @@ function inspectTutorialFileWithNwbInspector(testCase) % Retrieve all files generated by tutorial nwbFileNameList = testCase.listNwbFiles(); for nwbFilename = nwbFileNameList - try + if testCase.NWBInspectorMode == "python" results = py.list(py.nwbinspector.inspect_nwbfile(nwbfile_path=nwbFilename)); results = testCase.convertNwbInspectorResultsToStruct(results); - catch + elseif testCase.NWBInspectorMode == "CLI" [~, m] = system(sprintf('nwbinspector %s --levels importance', nwbFilename)); results = testCase.parseInspectorTextOutput(m); end @@ -146,8 +161,7 @@ function inspectTutorialFileWithNwbInspector(testCase) end results = testCase.filterResults(results); - - T = struct2table(results); disp(T) + % T = struct2table(results); disp(T) for j = 1:numel(results) testCase.verifyLessThan(results(j).importance, testCase.NwbInspectorSeverityLevel, ... @@ -302,8 +316,6 @@ function checkAndUpdatePythonPath(installLocation, packageName) fprintf("Adding %s location to pythonpath\n", packageName) py.sys.path().append(installLocation) end - disp('py path') - disp( py.sys.path() ) end function listPythonModules() diff --git a/.github/workflows/run_tests.yml b/.github/workflows/run_tests.yml index 167ee994..4f7d5aab 100644 --- a/.github/workflows/run_tests.yml +++ b/.github/workflows/run_tests.yml @@ -12,7 +12,6 @@ on: push: branches: - master - - 484-use-nwb-inspector-for-tutorial-files jobs: run_tests: @@ -30,7 +29,6 @@ jobs: python -m pip install -U pip pip install -r +tests/requirements.txt echo "HDF5_PLUGIN_PATH=$(python -c "import hdf5plugin; print(hdf5plugin.PLUGINS_PATH)")" >> "$GITHUB_ENV" - echo "NWBINSPECTOR_PATH=$( python -m pip show nwbinspector | grep ^Location: | awk '{print $2}' )" >> "$GITHUB_ENV" echo $( python -m pip show nwbinspector | grep ^Location: | awk '{print $2}' ) - name: Install MATLAB uses: matlab-actions/setup-matlab@v2 @@ -39,7 +37,7 @@ jobs: - name: Run tests uses: matlab-actions/run-command@v2 with: - command: results = assertSuccess(nwbtest('Name', 'tests.unit.Tutorial*')); assert(~isempty(results), 'No tests ran'); + command: results = assertSuccess(nwbtest()); assert(~isempty(results), 'No tests ran'); - name: Upload JUnit results if: always() uses: actions/upload-artifact@v4 From e7145bb5a04c78896799a4178871ed0b7be69ccf Mon Sep 17 00:00:00 2001 From: ehennestad Date: Wed, 6 Nov 2024 16:52:19 +0100 Subject: [PATCH 22/29] Update TutorialTest.m --- +tests/+unit/TutorialTest.m | 48 ++++--------------------------------- 1 file changed, 5 insertions(+), 43 deletions(-) diff --git a/+tests/+unit/TutorialTest.m b/+tests/+unit/TutorialTest.m index dba26544..3e10c3e4 100644 --- a/+tests/+unit/TutorialTest.m +++ b/+tests/+unit/TutorialTest.m @@ -78,9 +78,8 @@ function setupClass(testCase) % % Alternative: Use python script for reading file with pynwb tests.util.addFolderToPythonPath( fileparts(mfilename('fullpath')) ) - % This is also not working on github runner: - % installNWBInspector() - + % Todo: More explicitly check if this is run on a github runner + % or not? try py.nwbinspector.is_module_installed('nwbinspector') catch @@ -117,12 +116,12 @@ function readTutorialNwbFileWithPynwb(testCase) nwbFileNameList = testCase.listNwbFiles(); for nwbFilename = nwbFileNameList try - try + if testCase.NWBInspectorMode == "python" io = py.pynwb.NWBHDF5IO(nwbFilename); nwbObject = io.read(); testCase.verifyNotEmpty(nwbObject, 'The NWB file should not be empty.'); io.close() - catch ME + elseif testCase.NWBInspectorMode == "CLI" if strcmp(ME.identifier, 'MATLAB:undefinedVarOrClass') && ... contains(ME.message, 'py.pynwb.NWBHDF5IO') @@ -151,7 +150,7 @@ function inspectTutorialFileWithNwbInspector(testCase) if testCase.NWBInspectorMode == "python" results = py.list(py.nwbinspector.inspect_nwbfile(nwbfile_path=nwbFilename)); results = testCase.convertNwbInspectorResultsToStruct(results); - elseif testCase.NWBInspectorMode == "CLI" + elseif testCase.NWBInspectorMode == "CLI" [~, m] = system(sprintf('nwbinspector %s --levels importance', nwbFilename)); results = testCase.parseInspectorTextOutput(m); end @@ -296,40 +295,3 @@ function inspectTutorialFileWithNwbInspector(testCase) function folderPath = getMatNwbRootDirectory() folderPath = fileparts(fileparts(fileparts(mfilename('fullpath')))); end - -function installNWBInspector() - pythonInfo = pyenv; - pythonExecutable = pythonInfo.Executable; - systemCommand = sprintf("%s -m pip install %s", pythonExecutable, 'nwbinspector'); - [status, ~] = system(systemCommand); - systemCommand = sprintf("%s -m pip show nwbinspector | grep ^Location: | awk '{print $2}'", pythonExecutable); - [~, nwbInspectorPath] = system(systemCommand); - checkAndUpdatePythonPath(strtrim(nwbInspectorPath), 'nwbinspector') -end - -function checkAndUpdatePythonPath(installLocation, packageName) - pyPath = py.sys.path(); - pyPath = string(pyPath); - pyPath(pyPath=="") = []; - - if ~any( contains(pyPath, installLocation) ) - fprintf("Adding %s location to pythonpath\n", packageName) - py.sys.path().append(installLocation) - end -end - -function listPythonModules() - generator = py.pkgutil.iter_modules(); - methodNext = py.getattr(generator, '__next__'); - finished = false; - moduleNames = string.empty; - while ~finished - try - module = methodNext(); - moduleNames(end+1) = string(module.name); - catch; - finished = true; - end - end - moduleNames' -end \ No newline at end of file From 67c4fb73c254a04b92f1280be7d7996b6512385b Mon Sep 17 00:00:00 2001 From: ehennestad Date: Wed, 6 Nov 2024 17:17:41 +0100 Subject: [PATCH 23/29] Fix --- +tests/+unit/TutorialTest.m | 22 +++++++++++----------- .github/workflows/run_tests.yml | 2 +- 2 files changed, 12 insertions(+), 12 deletions(-) diff --git a/+tests/+unit/TutorialTest.m b/+tests/+unit/TutorialTest.m index 3e10c3e4..1b801e93 100644 --- a/+tests/+unit/TutorialTest.m +++ b/+tests/+unit/TutorialTest.m @@ -122,20 +122,20 @@ function readTutorialNwbFileWithPynwb(testCase) testCase.verifyNotEmpty(nwbObject, 'The NWB file should not be empty.'); io.close() elseif testCase.NWBInspectorMode == "CLI" - if strcmp(ME.identifier, 'MATLAB:undefinedVarOrClass') && ... - contains(ME.message, 'py.pynwb.NWBHDF5IO') + % if strcmp(ME.identifier, 'MATLAB:undefinedVarOrClass') && ... + % contains(ME.message, 'py.pynwb.NWBHDF5IO') - pythonExecutable = tests.util.getPythonPath(); - cmd = sprintf('"%s" -B -m read_nwbfile_with_pynwb %s',... - pythonExecutable, nwbFilename); - status = system(cmd); + pythonExecutable = tests.util.getPythonPath(); + cmd = sprintf('"%s" -B -m read_nwbfile_with_pynwb %s',... + pythonExecutable, nwbFilename); + status = system(cmd); - if status ~= 0 - error('Failed to read NWB file "%s" using pynwb', nwbFilename) - end - else - rethrow(ME) + if status ~= 0 + error('Failed to read NWB file "%s" using pynwb', nwbFilename) end + % else + % rethrow(ME) + % end end catch ME error(ME.message) diff --git a/.github/workflows/run_tests.yml b/.github/workflows/run_tests.yml index 4f7d5aab..5a84fa11 100644 --- a/.github/workflows/run_tests.yml +++ b/.github/workflows/run_tests.yml @@ -37,7 +37,7 @@ jobs: - name: Run tests uses: matlab-actions/run-command@v2 with: - command: results = assertSuccess(nwbtest()); assert(~isempty(results), 'No tests ran'); + command: results = assertSuccess(nwbtest); assert(~isempty(results), 'No tests ran'); - name: Upload JUnit results if: always() uses: actions/upload-artifact@v4 From 2d2ee09c4deefb5b267513499862c5aeab032fa0 Mon Sep 17 00:00:00 2001 From: ehennestad Date: Wed, 6 Nov 2024 20:44:09 +0100 Subject: [PATCH 24/29] Update untypedSetTest.m Suppress output in test --- +tests/+unit/untypedSetTest.m | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/+tests/+unit/untypedSetTest.m b/+tests/+unit/untypedSetTest.m index 4d18716f..29aad172 100644 --- a/+tests/+unit/untypedSetTest.m +++ b/+tests/+unit/untypedSetTest.m @@ -32,13 +32,15 @@ function testCreateSetFromNvPairsPlusFunctionHandle(testCase) end function testDisplayEmptyObject(testCase) - emptyUntypedSet = types.untyped.Set(); - disp(emptyUntypedSet) + emptyUntypedSet = types.untyped.Set(); %#ok + C = evalc( 'disp(emptyUntypedSet)' ); + testCase.verifyClass(C, 'char') end function testDisplayScalarObject(testCase) - scalarSet = types.untyped.Set('a',1) - disp(scalarSet) + scalarSet = types.untyped.Set('a', 1); %#ok + C = evalc( 'disp(scalarSet)' ); + testCase.verifyClass(C, 'char') end function testGetSetSize(testCase) From 47b0498856583be30a484e7f816c079c986c3a0b Mon Sep 17 00:00:00 2001 From: ehennestad Date: Wed, 6 Nov 2024 20:44:44 +0100 Subject: [PATCH 25/29] Add fixture for clearing generated types when running tests --- +tests/+fixtures/ResetGeneratedTypesFixture.m | 18 ++++++++++++++++++ +tests/+sanity/GenerationTest.m | 10 ++++++++-- +tests/+unit/TutorialTest.m | 17 ++++++++--------- +tests/+util/addFolderToPythonPath.m | 3 --- +tests/+util/getProjectDirectory.m | 3 +++ 5 files changed, 37 insertions(+), 14 deletions(-) create mode 100644 +tests/+fixtures/ResetGeneratedTypesFixture.m create mode 100644 +tests/+util/getProjectDirectory.m diff --git a/+tests/+fixtures/ResetGeneratedTypesFixture.m b/+tests/+fixtures/ResetGeneratedTypesFixture.m new file mode 100644 index 00000000..d4914364 --- /dev/null +++ b/+tests/+fixtures/ResetGeneratedTypesFixture.m @@ -0,0 +1,18 @@ +classdef ResetGeneratedTypesFixture < matlab.unittest.fixtures.Fixture + % ResetGeneratedTypesFixture - Fixture for reseting generated NWB classes. + % + % ResetGeneratedTypesFixture clears all the generated types from the + % matnwb folder. When the fixture is set up, NWB types class files are + % deleted. When the fixture is torn down, generateCore is called to + % regenerate the NWB types classes for the latest NWB version + + methods + function setup(fixture) + fixture.addTeardown( @generateCore ) + nwbClearGenerated() + + % Todo: Should get all generated namespaces and regenerate all + % when tearing down. + end + end +end diff --git a/+tests/+sanity/GenerationTest.m b/+tests/+sanity/GenerationTest.m index 1b30f54c..d09e9b64 100644 --- a/+tests/+sanity/GenerationTest.m +++ b/+tests/+sanity/GenerationTest.m @@ -5,8 +5,14 @@ methods (TestClassSetup) function setupClass(testCase) - rootPath = fullfile(fileparts(mfilename('fullpath')), '..', '..'); - testCase.applyFixture(matlab.unittest.fixtures.PathFixture(rootPath)); + + import matlab.unittest.fixtures.PathFixture + import tests.fixtures.ResetGeneratedTypesFixture + + rootPath = tests.util.getProjectDirectory(); + testCase.applyFixture( PathFixture(rootPath) ); + + testCase.applyFixture( ResetGeneratedTypesFixture ); end end diff --git a/+tests/+unit/TutorialTest.m b/+tests/+unit/TutorialTest.m index 1b801e93..d3f1213f 100644 --- a/+tests/+unit/TutorialTest.m +++ b/+tests/+unit/TutorialTest.m @@ -51,8 +51,12 @@ methods (TestClassSetup) function setupClass(testCase) + + import tests.fixtures.ResetGeneratedTypesFixture + % Get the root path of the matnwb repository - rootPath = getMatNwbRootDirectory(); + rootPath = tests.util.getProjectDirectory(); + tutorialsFolder = fullfile(rootPath, 'tutorials'); testCase.MatNwbDirectory = rootPath; @@ -81,13 +85,12 @@ function setupClass(testCase) % Todo: More explicitly check if this is run on a github runner % or not? try - py.nwbinspector.is_module_installed('nwbinspector') + py.nwbinspector.is_module_installed('nwbinspector'); catch testCase.NWBInspectorMode = "CLI"; end - nwbClearGenerated() - testCase.addTeardown(@generateCore) + testCase.applyFixture( ResetGeneratedTypesFixture ); end end @@ -282,7 +285,7 @@ function inspectTutorialFileWithNwbInspector(testCase) function tutorialNames = listTutorialFiles() % listTutorialFiles - List names of all tutorial files (exclude skipped files) - rootPath = getMatNwbRootDirectory(); + rootPath = tests.util.getProjectDirectory(); L = cat(1, ... dir(fullfile(rootPath, 'tutorials', '*.mlx')), ... dir(fullfile(rootPath, 'tutorials', '*.m')) ... @@ -291,7 +294,3 @@ function inspectTutorialFileWithNwbInspector(testCase) L( [L.isdir] ) = []; % Ignore folders tutorialNames = setdiff({L.name}, tests.unit.TutorialTest.SkippedTutorials); end - -function folderPath = getMatNwbRootDirectory() - folderPath = fileparts(fileparts(fileparts(mfilename('fullpath')))); -end diff --git a/+tests/+util/addFolderToPythonPath.m b/+tests/+util/addFolderToPythonPath.m index 2e93013a..ac21c350 100644 --- a/+tests/+util/addFolderToPythonPath.m +++ b/+tests/+util/addFolderToPythonPath.m @@ -10,7 +10,4 @@ function addFolderToPythonPath(folderPath) end end setenv('PYTHONPATH', updatedPythonPath); - disp('updated python path:') - disp(updatedPythonPath) end - diff --git a/+tests/+util/getProjectDirectory.m b/+tests/+util/getProjectDirectory.m new file mode 100644 index 00000000..8cdad9fa --- /dev/null +++ b/+tests/+util/getProjectDirectory.m @@ -0,0 +1,3 @@ +function projectDirectory = getProjectDirectory() + projectDirectory = fullfile(fileparts(mfilename('fullpath')), '..', '..'); +end From 9bef2213173ca9061fd5b319c747d3745efae367 Mon Sep 17 00:00:00 2001 From: ehennestad Date: Wed, 6 Nov 2024 20:55:58 +0100 Subject: [PATCH 26/29] Fix typo --- +tests/+fixtures/ResetGeneratedTypesFixture.m | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/+tests/+fixtures/ResetGeneratedTypesFixture.m b/+tests/+fixtures/ResetGeneratedTypesFixture.m index d4914364..d6fb8ac6 100644 --- a/+tests/+fixtures/ResetGeneratedTypesFixture.m +++ b/+tests/+fixtures/ResetGeneratedTypesFixture.m @@ -1,5 +1,5 @@ classdef ResetGeneratedTypesFixture < matlab.unittest.fixtures.Fixture - % ResetGeneratedTypesFixture - Fixture for reseting generated NWB classes. + % ResetGeneratedTypesFixture - Fixture for resetting generated NWB classes. % % ResetGeneratedTypesFixture clears all the generated types from the % matnwb folder. When the fixture is set up, NWB types class files are From 8206b59f53aaccf07ce645d30497119f49df2875 Mon Sep 17 00:00:00 2001 From: ehennestad Date: Thu, 7 Nov 2024 10:19:02 +0100 Subject: [PATCH 27/29] Improve Fixture description and improve function names --- +tests/+fixtures/ResetGeneratedTypesFixture.m | 9 +++++---- +tests/+unit/TutorialTest.m | 8 ++++---- 2 files changed, 9 insertions(+), 8 deletions(-) diff --git a/+tests/+fixtures/ResetGeneratedTypesFixture.m b/+tests/+fixtures/ResetGeneratedTypesFixture.m index d6fb8ac6..1df3a5b0 100644 --- a/+tests/+fixtures/ResetGeneratedTypesFixture.m +++ b/+tests/+fixtures/ResetGeneratedTypesFixture.m @@ -1,10 +1,11 @@ classdef ResetGeneratedTypesFixture < matlab.unittest.fixtures.Fixture % ResetGeneratedTypesFixture - Fixture for resetting generated NWB classes. % - % ResetGeneratedTypesFixture clears all the generated types from the - % matnwb folder. When the fixture is set up, NWB types class files are - % deleted. When the fixture is torn down, generateCore is called to - % regenerate the NWB types classes for the latest NWB version + % ResetGeneratedTypesFixture clears all the generated classes for NWB + % types from the matnwb folder. When the fixture is set up, all generated + % class files for NWB types are deleted. When the fixture is torn down, + % generateCore is called to regenerate the classes for NWB types of the + % latest NWB version methods function setup(fixture) diff --git a/+tests/+unit/TutorialTest.m b/+tests/+unit/TutorialTest.m index d3f1213f..252b1b33 100644 --- a/+tests/+unit/TutorialTest.m +++ b/+tests/+unit/TutorialTest.m @@ -155,14 +155,14 @@ function inspectTutorialFileWithNwbInspector(testCase) results = testCase.convertNwbInspectorResultsToStruct(results); elseif testCase.NWBInspectorMode == "CLI" [~, m] = system(sprintf('nwbinspector %s --levels importance', nwbFilename)); - results = testCase.parseInspectorTextOutput(m); + results = testCase.parseNWBInspectorTextOutput(m); end if isempty(results) return end - results = testCase.filterResults(results); + results = testCase.filterNWBInspectorResults(results); % T = struct2table(results); disp(T) for j = 1:numel(results) @@ -210,7 +210,7 @@ function inspectTutorialFileWithNwbInspector(testCase) end end - function resultsOut = parseInspectorTextOutput(systemCommandOutput) + function resultsOut = parseNWBInspectorTextOutput(systemCommandOutput) resultsOut = tests.unit.TutorialTest.getEmptyNwbInspectorResultStruct(); importanceLevels = containers.Map(... @@ -262,7 +262,7 @@ function inspectTutorialFileWithNwbInspector(testCase) 'ignore', {}); end - function resultsOut = filterResults(resultsIn) + function resultsOut = filterNWBInspectorResults(resultsIn) CHECK_IGNORE = [... "check_image_series_external_file_valid", ... "check_regular_timestamps" From b228a21e1e644c525e5ecd4f45acf1b28fcee1c4 Mon Sep 17 00:00:00 2001 From: ehennestad Date: Fri, 8 Nov 2024 14:34:13 +0100 Subject: [PATCH 28/29] Use OutOfProcess execution mode for python in matlan for github action - Simplifies the reading of nwb files using pynwb in TutorialTest --- +tests/+fixtures/ResetGeneratedTypesFixture.m | 3 -- +tests/+unit/TutorialTest.m | 50 +++---------------- .github/workflows/run_tests.yml | 6 ++- 3 files changed, 13 insertions(+), 46 deletions(-) diff --git a/+tests/+fixtures/ResetGeneratedTypesFixture.m b/+tests/+fixtures/ResetGeneratedTypesFixture.m index 1df3a5b0..312881c7 100644 --- a/+tests/+fixtures/ResetGeneratedTypesFixture.m +++ b/+tests/+fixtures/ResetGeneratedTypesFixture.m @@ -11,9 +11,6 @@ function setup(fixture) fixture.addTeardown( @generateCore ) nwbClearGenerated() - - % Todo: Should get all generated namespaces and regenerate all - % when tearing down. end end end diff --git a/+tests/+unit/TutorialTest.m b/+tests/+unit/TutorialTest.m index 252b1b33..e238e381 100644 --- a/+tests/+unit/TutorialTest.m +++ b/+tests/+unit/TutorialTest.m @@ -56,7 +56,6 @@ function setupClass(testCase) % Get the root path of the matnwb repository rootPath = tests.util.getProjectDirectory(); - tutorialsFolder = fullfile(rootPath, 'tutorials'); testCase.MatNwbDirectory = rootPath; @@ -65,26 +64,10 @@ function setupClass(testCase) testCase.applyFixture(matlab.unittest.fixtures.PathFixture(rootPath)); testCase.applyFixture(matlab.unittest.fixtures.PathFixture(tutorialsFolder)); - % Note: The following seems to not be working on the azure - % pipeline / github runner. - % Keep for reference - - % % % Make sure pynwb is installed in MATLAB's Python Environment - % % args = py.list({py.sys.executable, "-m", "pip", "install", "pynwb"}); - % % py.subprocess.check_call(args); - % % - % % % Add pynwb to MATLAB's python environment path - % % pynwbPath = getenv('PYNWB_PATH'); - % % if count(py.sys.path, pynwbPath) == 0 - % % insert(py.sys.path,int32(0),pynwbPath); - % % end - - % % Alternative: Use python script for reading file with pynwb - tests.util.addFolderToPythonPath( fileparts(mfilename('fullpath')) ) - - % Todo: More explicitly check if this is run on a github runner - % or not? - try + % Check if it is possible to call py.nwbinspector.* functions. + % When running these tests on Github Actions, calling + % py.nwbinspector does not work, whereas the CLI can be used instead. + try py.nwbinspector.is_module_installed('nwbinspector'); catch testCase.NWBInspectorMode = "CLI"; @@ -119,27 +102,10 @@ function readTutorialNwbFileWithPynwb(testCase) nwbFileNameList = testCase.listNwbFiles(); for nwbFilename = nwbFileNameList try - if testCase.NWBInspectorMode == "python" - io = py.pynwb.NWBHDF5IO(nwbFilename); - nwbObject = io.read(); - testCase.verifyNotEmpty(nwbObject, 'The NWB file should not be empty.'); - io.close() - elseif testCase.NWBInspectorMode == "CLI" - % if strcmp(ME.identifier, 'MATLAB:undefinedVarOrClass') && ... - % contains(ME.message, 'py.pynwb.NWBHDF5IO') - - pythonExecutable = tests.util.getPythonPath(); - cmd = sprintf('"%s" -B -m read_nwbfile_with_pynwb %s',... - pythonExecutable, nwbFilename); - status = system(cmd); - - if status ~= 0 - error('Failed to read NWB file "%s" using pynwb', nwbFilename) - end - % else - % rethrow(ME) - % end - end + io = py.pynwb.NWBHDF5IO(nwbFilename); + nwbObject = io.read(); + testCase.verifyNotEmpty(nwbObject, 'The NWB file should not be empty.'); + io.close() catch ME error(ME.message) end diff --git a/.github/workflows/run_tests.yml b/.github/workflows/run_tests.yml index 5a84fa11..f9649886 100644 --- a/.github/workflows/run_tests.yml +++ b/.github/workflows/run_tests.yml @@ -37,7 +37,11 @@ jobs: - name: Run tests uses: matlab-actions/run-command@v2 with: - command: results = assertSuccess(nwbtest); assert(~isempty(results), 'No tests ran'); + command: | + # Using pynwb requires pyenv "ExecutionMode" set to "OutOfProcess" + pyenv("ExecutionMode", "OutOfProcess"); + results = assertSuccess(nwbtest); + assert(~isempty(results), 'No tests ran'); - name: Upload JUnit results if: always() uses: actions/upload-artifact@v4 From fdd9b75b92d38ae57562b794ab23298da815c4e3 Mon Sep 17 00:00:00 2001 From: ehennestad Date: Fri, 8 Nov 2024 14:51:48 +0100 Subject: [PATCH 29/29] Update run_tests.yml --- .github/workflows/run_tests.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/workflows/run_tests.yml b/.github/workflows/run_tests.yml index f9649886..ad7aee46 100644 --- a/.github/workflows/run_tests.yml +++ b/.github/workflows/run_tests.yml @@ -38,7 +38,6 @@ jobs: uses: matlab-actions/run-command@v2 with: command: | - # Using pynwb requires pyenv "ExecutionMode" set to "OutOfProcess" pyenv("ExecutionMode", "OutOfProcess"); results = assertSuccess(nwbtest); assert(~isempty(results), 'No tests ran');