serve aux files from static, updated docs

.gitignore

@@ -11,6 +11,7 @@ __pycache__
 # Data
 data
 *.csv
+.zarrcade
 
 # Zarrcade databases
 *.db

README.md

@@ -19,27 +19,36 @@ Zarrcade is a web application for easily browsing, searching, and visualizing co
 
 ## Getting Started
 
+
 ### 1. Install miniforge
 
 [Install miniforge](https://docs.conda.io/en/latest/miniforge.html) if you don't already have it.
 
+
 ### 2. Initialize the conda environment
 
 ```bash
 conda env create -f environment.yml
 conda activate zarrcade
 ```
 
+
+### (Optional) Try an example
+
+See the [Example](#example) section below to try out the example before working with your own data.
+
+
 ### 3. Create OME-Zarr images
 
-If necessary, convert your image(s) to OME-Zarr format, e.g. using bioformats2raw:
+If your images are not already in OME-Zarr format, you will need to convert them, e.g. using bioformats2raw:
 
 ```bash
 bioformats2raw -w 128 -h 128 -z 64 --compression zlib /path/to/input /path/to/zarr
 ```
 
 If you have many images to convert, we recommend using the [nf-omezarr Nextflow pipeline](https://github.com/JaneliaSciComp/nf-omezarr) to efficiently run bioformats2raw on a collection of images. This pipeline also lets you scale the conversion processes to your available compute resources (cluster, cloud, etc).
 
+
 ### 4. Import images and metadata into Zarrcade
 
 You can import images into Zarrcade using the provided command line script:
@@ -48,9 +57,7 @@ You can import images into Zarrcade using the provided command line script:
 bin/import.py -d /root/data/dir -c mycollection
 ```
 
-This will automatically create a local Sqlite database containing a Zarrcade **collection** named "mycollection" and populate it with information about the images in the specified directory. 
-
-By default, this will also create MIPs and thumbnails for each image in a folder named `.zarrcade` within the root data directory. You can change this location by setting the `--aux-path` parameter. You can disable the creation of MIPs and thumbnails by setting the `--no-aux` flag. The brightness of the MIPs can be adjusted using the `--p-lower` and `--p-upper` parameters.
+This will automatically create a local Sqlite database containing a Zarrcade **collection** named "mycollection" and populate it with information about the images in the specified directory. By default, this will also create MIPs and thumbnails for each image in `./static/.zarrcade`. 
 
 To add extra metadata about the images, you can provide a CSV file with the `-i` flag:
 
@@ -66,11 +73,8 @@ relative/path/to/ome1.zarr,JKF6363,Blu
 relative/path/to/ome2.zarr,JDH3562,Blu
 ```
 
-To try an example, use the following command:
+Read more about the import options in the [Data Import](./docs/DataImport.md) section of the documentation.
 
-```bash
-bin/import.py -d s3://janelia-data-examples/fly-efish -c flyefish -m docs/flyefish-example.csv
-```
 
 ### 5. Run the Zarrcade web application
 
@@ -83,6 +87,24 @@ uvicorn zarrcade.serve:app --host 0.0.0.0 --reload
 Your images and annotations will be indexed and browseable at [http://0.0.0.0:8000](http://0.0.0.0:8000). Read the documentation below for more details on how to configure the web UI and deploy the service in production.
 
 
+## Example
+
+To try an example, follow steps 1 and 2 above and use the following command to import the example data:
+
+```bash
+bin/import.py -d s3://janelia-data-examples/fly-efish -c flyefish -m docs/flyefish-example.csv
+```
+
+Copy the example settings.yaml file to your working directory and start the server:
+
+```bash
+cp docs/settings.yaml.example settings.yaml
+uvicorn zarrcade.serve:app --host 0.0.0.0 --reload
+```
+
+The example should be visible at [http://0.0.0.0:8000](http://0.0.0.0:8000).
+
+
 ## Documentation
 
 * [Overview](./docs/Overview.md) - learn about the data model and overall architecture

bin/import.py

@@ -61,8 +61,8 @@ def slugify(value, allow_unicode=False):
         help='Path to the CSV file containing additional metadata')
     parser.add_argument('-x', '--no-aux', action=argparse.BooleanOptionalAction, default=False,
         help="Don't create auxiliary images or thumbnails.")
-    parser.add_argument('-a', '--aux-path', type=str, default=".zarrcade",
-        help='Path to the folder containing auxiliary images (non-absolute paths are relative to data_url).')
+    parser.add_argument('-a', '--aux-path', type=str, default="static/.zarrcade",
+        help='Local path to the folder for auxiliary images.')
     parser.add_argument('--aux-image-name', type=str, default='zmax.png',
         help='Filename of the main auxiliary image.')
     parser.add_argument('--thumbnail-name', type=str, default='zmax_sm.jpg',
@@ -161,35 +161,33 @@ def get_aux_path(zarr_path, filename):
             logger.info(f"Processing {image.path}")
             metadata = image.image_metadata
             updated_obj = {}
-            aux_abspath = None
+            aux_path = None
 
             if args.aux_image_name and (not metadata or not metadata.aux_image_path):
                 aux_path = get_aux_path(image.path, args.aux_image_name)
-                aux_abspath = fs.get_absolute_path(aux_path)
 
                 updated_obj['aux_image_path'] = aux_path
 
                 if fs.exists(aux_path):
-                    logger.trace(f"Found auxiliary file: {aux_abspath}")
+                    logger.trace(f"Found auxiliary file: {aux_path}")
                 else:
-                    logger.trace(f"Creating auxiliary file: {aux_abspath}")
-                    zarr_path = fs.get_absolute_path(image.image_path)
-                    create_parent_dirs(aux_abspath)
-                    make_mip_from_zarr(zarr_path, aux_abspath, p_lower=args.p_lower, p_upper=args.p_upper)
-                    logger.info(f"Wrote {aux_abspath}")
+                    logger.trace(f"Creating auxiliary file: {aux_path}")
+                    create_parent_dirs(aux_path)
+                    store = fs.get_store(image.image_path)
+                    make_mip_from_zarr(store, aux_path, p_lower=args.p_lower, p_upper=args.p_upper)
+                    logger.info(f"Wrote {aux_path}")
 
             if args.thumbnail_name and (not metadata or not metadata.thumbnail_path):
                 tb_path = get_aux_path(image.path, args.thumbnail_name)
-                tb_abspath = fs.get_absolute_path(tb_path)  
                 updated_obj['thumbnail_path'] = tb_path
 
                 if fs.exists(tb_path):
-                    logger.trace(f"Found thumbnail: {tb_abspath}")
-                elif aux_abspath:
-                    logger.trace(f"Creating thumbnail: {tb_abspath}")
-                    create_parent_dirs(tb_abspath)
-                    make_thumbnail(aux_abspath, tb_abspath)
-                    logger.info(f"Wrote {tb_abspath}")
+                    logger.trace(f"Found thumbnail: {tb_path}")
+                elif aux_path:
+                    logger.trace(f"Creating thumbnail: {tb_path}")
+                    create_parent_dirs(tb_path)
+                    make_thumbnail(aux_path, tb_path)
+                    logger.info(f"Wrote {tb_path}")
                 else:
                     logger.trace(f"Cannot make thumbnail for {path} without aux image")
 

docs/DataImport.md

@@ -0,0 +1,32 @@
+# Data Import
+
+You can import images into Zarrcade using the provided command line script:
+
+```bash
+bin/import.py -d /root/data/dir -c mycollection
+```
+
+This will automatically create a local Sqlite database containing a Zarrcade **collection** named "mycollection" and populate it with information about the images in the specified directory. You can also add a label to the collection by setting the `--collection-label` parameter. This label will be displayed in the web UI when choosing the collection to view.
+
+
+## Annotations
+
+You can add additional annotations to the images by providing a CSV file with the `-m` flag. The CSV file's first column must be a relative path to the OME-Zarr image within the root data directory. The remaining columns can be any annotations that will be searched and displayed within the gallery.
+
+You can modify the service configuration to control how the annotations are displayed and searched in the gallery. See the [Configuration](./Configuration.md) section for more details.
+
+
+## Auxiliary Images
+
+By default, the import script will also create MIPs and thumbnails for each image in `./static/.zarrcade`. You can disable this by setting the `--no-aux` flag. You can change the output file location by setting the `--aux-path` parameter. Zarrcade will proxy the files automatically from within the `static` folder, but this may not be suitable for a production deployment. Instead, you can store the auxiliary images in the same directory as the OME-Zarr files. You will need to upload the files to your S3 bucket or other storage. Then, set `aux_image_mode: relative` in your `settings.yaml` to let Zarrcade know that your auxiliary files are stored relative to your data. 
+
+
+## Auxiliary Image Options
+
+Currently, Zarrcade supports creation of Maximum Intensity Projection (MIP) and thumbnails of the MIPs. You can control the brightness of the MIPs using the `--p-lower` and `--p-upper` parameters.
+
+
+## Custom Auxiliary Image Generation
+
+You can create your own auxiliary images and thumbnails by setting the `--aux-path` parameter to the location where Zarrcade should find the files. Organize the folder structure to match the OME-Zarr files, and set the `aux_image_name` and `thumbnail_name` parameters to the names of the files containing the auxiliary images and thumbnails, respectively. During the import process, Zarrcade will detect that the files exist and will not attempt to generate them.
+

docs/settings.yaml.example

@@ -0,0 +1,24 @@
+log_level: INFO
+
+base_url: http://localhost:8000
+
+database:
+  url: sqlite:///database.db
+  debug_sql: False
+
+filters:
+  - column_name: "Driver line"
+    data_type: string
+    filter_type: dropdown
+  - column_name: "Probes"
+    data_type: csv
+    filter_type: dropdown
+  - column_name: "Region"
+  - column_name: "Zoom"
+
+title_column_name: "Image Name"
+
+hide_columns:
+  - "Probes"
+  - "Image Name"
+

templates/collection.html

@@ -27,7 +27,7 @@
         {% if not dbimage.image_metadata.thumbnail_path %}
             <img src="{{  url_for('static', path='zarr.jpg') }}" alt="Default thumbnail" class="thumbnail" />
         {% else %}
-            <img src="{{ get_relative_path_url(dbimage, dbimage.image_metadata.thumbnail_path) }}" alt="Image thumbnail" class="thumbnail" />
+            <img src="{{ get_aux_path_url(dbimage, dbimage.image_metadata.thumbnail_path, request) }}" alt="Image thumbnail" class="thumbnail" />
         {% endif %}
             <div class="overlay">
 

templates/details.html

@@ -36,7 +36,7 @@ <h2>{{ get_title(dbimage) | safe }}</h2>
 
 <div class="content">
     {% if dbimage.image_metadata and dbimage.image_metadata.aux_image_path %}
-    <img src="{{ get_relative_path_url(dbimage, dbimage.image_metadata.aux_image_path) }}" alt="Image thumbnail" class="image" />
+    <img src="{{ get_aux_path_url(dbimage, dbimage.image_metadata.aux_image_path, request) }}" alt="Image thumbnail" class="image" />
     {% else %}
     <img src="{{  url_for('static', path='zarr.jpg') }}" alt="Default thumbnail" class="image" />
     {% endif %}
@@ -54,7 +54,7 @@ <h2>{{ get_title(dbimage) | safe }}</h2>
         <tr><td>Compression:</td><td><span class="data">{{ image.compression }}</span></td></tr>
         {% if dbimage.image_metadata %}
             {% for attr in column_map.keys() %}
-                {% if attr not in settings.details.hide_columns %}
+                {% if attr not in settings.hide_columns %}
                 <tr><td>{{ column_map[attr] }}:</td><td>{{ getattr(dbimage.image_metadata, attr) | safe }}</td></tr>
                 {% endif %}
             {% endfor %}

templates/form.html

@@ -13,7 +13,7 @@
         {% if filter.filter_type == FilterType.dropdown %}
             <select name="{{ filter.db_name }}" onchange="submitForm()">
                 <option value="">Select...</option>
-                {% for key, value in filter_values | dictsort %}
+                {% for key, value in filter._values | dictsort %}
                 <option value="{{ key }}" {% if filter_params.get(filter.db_name) == key %}selected{% endif %}>
                     {{ key }} <!--({{ value }})-->
                 </option>

zarrcade/database.py

@@ -198,7 +198,7 @@ def add_metadata_column(self, db_name, original_name):
         """
         if db_name in self.column_map:
             # Column already exists
-            logger.info(f"Column {db_name} already exists")
+            logger.debug(f"Column {db_name} already exists")
             return
 
         with self.engine.connect() as connection:
@@ -339,7 +339,7 @@ def persist_image(self, collection: str, image: Image, metadata_id: int) -> bool
                     db_image.image_metadata_id = metadata_id
                     db_image.set_image(image)
                     session.commit()
-                    logger.debug(f"Updated image {image_path}")
+                    logger.info(f"Updated image {image_path}")
                     return False
                 else:
                     # Insert new record
@@ -353,7 +353,7 @@ def persist_image(self, collection: str, image: Image, metadata_id: int) -> bool
                     new_image.set_image(image)
                     session.add(new_image)
                     session.commit()
-                    logger.debug(f"Inserted image {image_path}")
+                    logger.info(f"Inserted image {image_path}")
                     return True
 
             except SQLAlchemyError as e:
@@ -400,7 +400,6 @@ def persist_images(
                             image=image,
                             metadata_id=metadata_id
                 )
-                logger.info(f"Persisted {image.get_path()}")
                 count += 1
             else:
                 logger.debug(f"Skipping image missing metadata: {image.get_path()}")

zarrcade/filestore.py

@@ -68,6 +68,12 @@ def get_absolute_path(self, relative_path):
         """
         return os.path.join(self.fsroot, relative_path)
 
+
+    def get_url(self, relative_path):
+        """ Returns the full URL to the given path.
+        """
+        return os.path.join(self.data_url, relative_path)
+
 
     def exists(self, relative_path):
         """ Returns true if a file or folder exists at the given relative path.

zarrcade/serve.py

@@ -17,7 +17,7 @@
 from zarrcade.filestore import get_filestore
 from zarrcade.database import Database, DBImage
 from zarrcade.viewers import Viewer, Neuroglancer
-from zarrcade.settings import get_settings, DataType, FilterType
+from zarrcade.settings import get_settings, DataType, FilterType, AuxImageMode
 
 # Create the API
 app = FastAPI(
@@ -57,7 +57,7 @@ async def startup_event():
     app.base_url = str(app.settings.base_url)
     logger.info(f"User-specified base URL is {app.base_url}")
 
-    db_url = str(app.settings.database.db_url)
+    db_url = str(app.settings.database.url)
     logger.info(f"User-specified database URL is {db_url}")
     app.db = Database(db_url)
 
@@ -142,6 +142,17 @@ def get_relative_path_url(dbimage: DBImage, relative_path: str):
     return get_proxy_url(dbimage.collection, relative_path)
 
 
+def get_aux_path_url(dbimage: DBImage, relative_path: str, request: Request):
+    """ Return a web-accessible URL to the given relative path.
+    """
+    if app.settings.aux_image_mode == AuxImageMode.relative:
+        return get_relative_path_url(dbimage, relative_path)
+    elif app.settings.aux_image_mode == AuxImageMode.local:
+        return request.url_for('static', path=relative_path)
+    else:
+        raise ValueError(f"Unknown aux image mode: {app.settings.aux_image_mode}")
+
+
 def get_viewer_url(dbimage: DBImage, viewer: Viewer):
     """ Returns a web-accessible URL that opens the given image 
         in the specified viewer.
@@ -262,6 +273,7 @@ async def collection(request: Request, collection: str = '', search_string: str
             "dbimages": result['images'],
             "get_viewer_url": get_viewer_url,
             "get_relative_path_url": get_relative_path_url,
+            "get_aux_path_url": get_aux_path_url,
             "get_data_url": get_data_url,
             "get_title": get_title,
             "get_query_string": partial(get_query_string, request.query_params),
@@ -289,6 +301,7 @@ async def details(request: Request, collection: str, image_id: str):
             "column_map": app.db.column_map,
             "get_viewer_url": get_viewer_url,
             "get_relative_path_url": get_relative_path_url,
+            "get_aux_path_url": get_aux_path_url,
             "get_title": get_title,
             "get_data_url": get_data_url,
             "getattr": getattr