diff --git a/.github/workflows/test_and_deploy.yml b/.github/workflows/test_and_deploy.yml index 59060356b..9cf92c374 100644 --- a/.github/workflows/test_and_deploy.yml +++ b/.github/workflows/test_and_deploy.yml @@ -41,7 +41,7 @@ jobs: steps: # these libraries enable testing on Qt on linux - - uses: pyvista/setup-headless-display-action@v2 + - uses: pyvista/setup-headless-display-action@v3 with: qt: true - name: Cache Test Data diff --git a/docs/source/_static/napari_plugin_with_poses_as_points.png b/docs/source/_static/napari_plugin_with_poses_as_points.png new file mode 100644 index 000000000..5604e85c0 Binary files /dev/null and b/docs/source/_static/napari_plugin_with_poses_as_points.png differ diff --git a/docs/source/community/index.md b/docs/source/community/index.md index 269520ff8..437e7a2bc 100644 --- a/docs/source/community/index.md +++ b/docs/source/community/index.md @@ -5,6 +5,7 @@ develop a new feature, or improve the documentation. To help you get started, we have prepared a statement on the project's [mission and scope](target-mission), a [roadmap](target-roadmaps) outlining our current priorities, and a detailed [contributing guide](target-contributing). +(target-get-in-touch)= ```{include} ../snippets/get-in-touch.md ``` diff --git a/docs/source/user_guide/gui.md b/docs/source/user_guide/gui.md new file mode 100644 index 000000000..b252882d9 --- /dev/null +++ b/docs/source/user_guide/gui.md @@ -0,0 +1,194 @@ +(target-gui)= +# Graphical User Interface + +The `movement` graphical user interface (GUI), powered by our custom plugin for +[napari](napari:), makes it easy to view and explore `movement` +motion tracks. Currently, you can use it to +visualise 2D [poses datasets](target-poses-and-bboxes-dataset) +as points overlaid on video frames. + +:::{warning} +The GUI is still in early stages of development but we are working on ironing +out the [kinks](target-plugin-known-issues). +Please [get in touch](target-get-in-touch) +if you find any bugs or have suggestions for improvements! +::: + +The `napari` plugin is shipped with the `movement` package starting from +version `0.1.0`. To use it, you need to +[install the package](target-installation) via a method that +includes the `napari` dependency. + + +## Launch the GUI + +To launch the `movement` GUI, type the following command in your terminal: + +```sh +movement launch +``` + +This is equivalent to running `napari -w movement` and will open the `napari` +window with the `movement` widget docked on the +right-hand side, as in the [screenshot](target-widget-screenshot) below. + +In `napari`, data is typically loaded into [layers](napari:guides/layers.html), +which can be reordered and toggled for visibility in the layers list panel. +For example, keypoint data can be added as a +[points layer](napari:howtos/layers/points.html), +while image stacks (including videos) can be added as +[image layers](napari:howtos/layers/image.html). +Below, we'll explain how to do this. + +## Load a background layer + +Though this is not strictly necessary, it is usually informative to +view the keypoints overlaid on a background that provides +some spatial context. You can either [load the video](target-load-video) +corresponding to the poses dataset, or a [single image](target-load-frame), +e.g., a still frame derived from that video. +You can do this by dragging and dropping the corresponding file onto the +`napari` window or by using the `File > Open File(s)` menu option. +Please read the following sections for detailed information +and some important considerations. + +(target-load-video)= +### Load a video + +When trying to load a video file into `napari`, you will be prompted +via a pop-up dialog to select the reader. +Choose the `video` reader—corresponding to the +[`napari-video`](https://github.com/janclemenslab/napari-video) +plugin—and click `OK`. You can optionally select to remember this reader +for all files with the same extension. + +`napari-video` will load the video as an image stack with a slider +at the bottom that you can use to navigate through frames. +You may also use the left and right arrow keys to navigate +frame-by-frame. + +Clicking on the play button will start the video playback at a default +rate of 10 frames per second. You can adjust that by right-clicking on the +play button or by opening the `napari > Preferences` menu +(`File > Preferences` on Windows) and changing +the `Playback frames per second` setting. + +(target-video-playback-limitations)= +:::{admonition} Video playback limitations +:class: warning + +- The video playback may freeze or stutter if you click on the slider to jump + to a specific frame. We recommended pausing the playback before such jumps. +- `napari-video` may struggle to play videos at a high frame rate, depending + on your hardware, the video resolution and codec. If you experience + performance issues, such as the video freezing or skipping frames, + try reducing the playback frames per second or fall back to + using a [single image](target-load-frame) as a background. +::: + + +(target-load-frame)= +### Load an image + +This usually means using a still frame extracted from the video, but in theory +you could use any image that's in the same coordinate system as the +tracking data. For example, you could use a schematic diagram of the arena, +as long as it has the same width and height as the video and is +properly aligned with the tracking data. + +::: {dropdown} Extracting a still frame from a video +:color: info +:icon: info + +You can use the command line tool [`ffmpeg`](https://www.ffmpeg.org/) +to extract a still frame from a video. + +To extract the first frame of a video: + +```sh +ffmpeg -i video.mp4 -frames:v 1 first-frame.png +``` + +To extract a frame at a specific time stamp (e.g. at 2 seconds): + +```sh +ffmpeg -i video.mp4 -ss 00:00:02 -frames:v 1 frame-2sec.png +``` +::: + +Dragging and dropping the image file onto the `napari` window +(or opening it via the `File` menu) will load the image +as a single 2D frame without a slider. + +## Load the poses dataset + +Now you are ready to load some pose tracks over your chosen background layer. + +On the right-hand side of the window you should see +an expanded `Load poses` menu. To load pose data in napari: +1. Select the `source software` from the dropdown menu. +2. Set the `fps` (frames per second) of the video the pose data refers to. Note this will only affect the units of the time variable shown when hovering over a keypoint. If the `fps` is not known, you can set it to 1, which will effectively make the time variable equal to the frame number. +3. Select the file containing the predicted poses. The path can be directly pasted or you can use the file browser button. +4. Click `Load`. + +The data should be loaded into the viewer as a +[points layer](napari:howtos/layers/points.html). +By default, it is added at the top of the layer list. + +::: {note} +See [supported formats](target-supported-formats) for more information on +the expected software and file formats. +::: + + +You will see a view similar to the one below: + +(target-widget-screenshot)= + +![napari widget with poses dataset loaded](../_static/napari_plugin_with_poses_as_points.png) + +The keypoints are represented as points, colour-coded by +keypoint ID for single-individual datasets, or by individual ID for +multi-individual datasets. These IDs can be also displayed as text +next to the points by enabling the `display text` option from the +layer controls panel. + +Hovering with your mouse over a point +(with the points layer selected) will +bring up a tooltip containing the names of the individual and keypoint, +the point-wise confidence score (provided by the source software), +and the time in seconds (calculated based on the frame number and +the `fps` value you provided). + +Using the slider at the bottom of the window, you can move through +the frames of the dataset, and the points and video will update +in sync. + +::: {admonition} Stay tuned +Though the display style of the points layer is currently fixed, we are +working on adding more customisation options in future releases, such as +enabling you to change the point size, colour, or shape. + +We are also working on enabling the visualisation of +[bounding boxes datasets](target-poses-and-bboxes-dataset) in the plugin. +::: + + +(target-plugin-known-issues)= +## Known Issues + +1. The aforementioned + [video playback limitations](target-video-playback-limitations), + which are inherited from the `napari-video` plugin. + +2. Sometimes `napari` may show the following error message: + + ```console + wrapped C/C++ object of type QtDimSliderWidget has been deleted + ``` + + This tends to happen when you delete a points layer containing the + pose data, load a new dataset, and then press on the play button (see + [GitHub issue](https://github.com/neuroinformatics-unit/movement/issues/433) + for more details). If you encounter this error, close the + `napari` window and relaunch the GUI. diff --git a/docs/source/user_guide/index.md b/docs/source/user_guide/index.md index e8caa8cf6..05af4a084 100644 --- a/docs/source/user_guide/index.md +++ b/docs/source/user_guide/index.md @@ -8,7 +8,7 @@ Before you dive deeper, we highly recommend reading about the structure and usage of [movement datasets](movement_dataset.md), which are a central concept in the package. -::::{grid} 1 2 2 3 +::::{grid} 1 1 2 2 :gutter: 3 :::{grid-item-card} {fas}`wrench;sd-text-primary` Installation @@ -32,6 +32,13 @@ Load and save tracking data. Learn about our data structures. ::: +:::{grid-item-card} {fas}`line-chart;sd-text-primary` Graphical User Interface +:link: gui +:link-type: doc + +Use our `napari` plugin to view and explore your data interactively. +::: + :::: @@ -42,4 +49,5 @@ Learn about our data structures. installation input_output movement_dataset +gui ``` diff --git a/movement/napari/_loader_widgets.py b/movement/napari/_loader_widgets.py index 6e3dc8d36..0950caf24 100644 --- a/movement/napari/_loader_widgets.py +++ b/movement/napari/_loader_widgets.py @@ -8,12 +8,12 @@ from napari.viewer import Viewer from qtpy.QtWidgets import ( QComboBox, + QDoubleSpinBox, QFileDialog, QFormLayout, QHBoxLayout, QLineEdit, QPushButton, - QSpinBox, QWidget, ) @@ -56,11 +56,20 @@ def _create_source_software_widget(self): def _create_fps_widget(self): """Create a spinbox for selecting the frames per second (fps).""" - self.fps_spinbox = QSpinBox() + self.fps_spinbox = QDoubleSpinBox() self.fps_spinbox.setObjectName("fps_spinbox") - self.fps_spinbox.setMinimum(1) - self.fps_spinbox.setMaximum(1000) - self.fps_spinbox.setValue(30) + self.fps_spinbox.setMinimum(0.1) + self.fps_spinbox.setMaximum(1000.0) + self.fps_spinbox.setValue(1.0) + self.fps_spinbox.setDecimals(2) + # How much we increment/decrement when the user clicks the arrows + self.fps_spinbox.setSingleStep(1) + # Add a tooltip + self.fps_spinbox.setToolTip( + "Set the frames per second of the tracking data.\n" + "This just affects the displayed time when hovering over a point\n" + "(it doesn't set the playback speed)." + ) self.layout().addRow("fps:", self.fps_spinbox) def _create_file_path_widget(self): @@ -124,9 +133,6 @@ def _on_load_clicked(self): self.file_name = Path(file_path).name self._add_points_layer() - self._set_playback_fps(fps) - logger.debug(f"Set napari playback speed to {fps} fps.") - def _add_points_layer(self): """Add the predicted poses to the viewer as a Points layer.""" # Style properties for the napari Points layer @@ -144,12 +150,6 @@ def _add_points_layer(self): self.viewer.add_points(self.data[:, 1:], **points_style.as_kwargs()) logger.info("Added poses dataset as a napari Points layer.") - @staticmethod - def _set_playback_fps(fps: int): - """Set the playback speed for the napari viewer.""" - settings = get_settings() - settings.application.playback_fps = fps - @staticmethod def _enable_layer_tooltips(): """Toggle on tooltip visibility for napari layers. diff --git a/pyproject.toml b/pyproject.toml index 577a80a37..14bdd7895 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -48,7 +48,9 @@ entry-points."napari.manifest".movement = "movement.napari:napari.yaml" [project.optional-dependencies] napari = [ "napari[all]>=0.5.0", - "brainglobe-utils[qt]>=0.6" # needed for collapsible widgets + "brainglobe-utils[qt]>=0.6", # needed for collapsible widgets + "napari-video", + "pyvideoreader>=0.5.3", # since switching to depend on openCV-headless ] dev = [ "pytest", diff --git a/tests/test_unit/test_napari_plugin/test_poses_loader_widget.py b/tests/test_unit/test_napari_plugin/test_poses_loader_widget.py index da0c7c8ea..772b107e7 100644 --- a/tests/test_unit/test_napari_plugin/test_poses_loader_widget.py +++ b/tests/test_unit/test_napari_plugin/test_poses_loader_widget.py @@ -8,7 +8,7 @@ import pytest from napari.settings import get_settings from pytest import DATA_PATHS -from qtpy.QtWidgets import QComboBox, QLineEdit, QPushButton, QSpinBox +from qtpy.QtWidgets import QComboBox, QDoubleSpinBox, QLineEdit, QPushButton from movement.napari._loader_widgets import PosesLoader @@ -25,7 +25,7 @@ def test_poses_loader_widget_instantiation(make_napari_viewer_proxy): # Check that the expected widgets are present in the layout expected_widgets = [ (QComboBox, "source_software_combo"), - (QSpinBox, "fps_spinbox"), + (QDoubleSpinBox, "fps_spinbox"), (QLineEdit, "file_path_edit"), (QPushButton, "load_button"), (QPushButton, "browse_button"), @@ -160,7 +160,6 @@ def test_on_load_clicked_with_valid_file_path( "Converted poses dataset to a napari Tracks array.", "Tracks array shape: (2170, 4)", "Added poses dataset as a napari Points layer.", - "Set napari playback speed to 60 fps.", } log_messages = {record.getMessage() for record in caplog.records} assert expected_log_messages <= log_messages @@ -168,6 +167,3 @@ def test_on_load_clicked_with_valid_file_path( # Check that a Points layer was added to the viewer points_layer = poses_loader_widget.viewer.layers[0] assert points_layer.name == f"poses: {file_path.name}" - - # Check that the playback fps was set correctly - assert get_settings().application.playback_fps == 60