diff --git a/.circleci/config.yml b/.circleci/config.yml index d3ba4a00c..2eb8b7119 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -1,3 +1,4 @@ +--- version: 2.1 executors: toxandnode: @@ -26,7 +27,7 @@ commands: # consume the exit code # command: PYTEST_ADDOPTS=--forked tox -e << parameters.env >> | cat; test ${PIPESTATUS[0]} -eq 0 # command: PYTEST_ADDOPTS="--reruns=3 --numprocesses=0" tox -e << parameters.env >> | cat; test ${PIPESTATUS[0]} -eq 0 - command: PYTEST_NUMPROCESSES=3 PYTEST_ADDOPTS="--reruns=3" tox -e << parameters.env >> | cat; test ${PIPESTATUS[0]} -eq 0 + command: COVERAGE_CORE=sysmon PYTEST_NUMPROCESSES=3 PYTEST_ADDOPTS="--reruns=3" tox -e << parameters.env >> | cat; test ${PIPESTATUS[0]} -eq 0 switchpython: description: "Upgrade python" parameters: @@ -82,7 +83,6 @@ commands: curl -k --retry 60 -f --retry-all-errors --retry-delay 1 -s -o /dev/null $KEYCLOAK_URL echo 'Updating keycloak token lifespan...' python -W ignore ./.circleci/dcm4chee/update_access_token_lifespan.py - echo 'Creating keycloak access token...' # Now create the token export DICOMWEB_TEST_TOKEN=$(python -W ignore ./.circleci/dcm4chee/create_keycloak_token.py) @@ -132,13 +132,12 @@ commands: - run: name: Install Codecov client command: | - curl -Os https://uploader.codecov.io/latest/linux/codecov - chmod +x codecov + curl -Os https://uploader.codecov.io/latest/linux/codecov + chmod +x codecov - run: name: Upload coverage command: | - ./codecov --disable search pycov gcov --file build/test/coverage/py_coverage.xml,build/test/coverage/cobertura-coverage.xml - + ./codecov --disable search pycov gcov --file build/test/coverage/py_coverage.xml,build/test/coverage/cobertura-coverage.xml jobs: testdocker: machine: @@ -151,8 +150,8 @@ jobs: - run: name: Publish the images to Docker Hub command: | - echo "$DOCKERHUB_PASS" | docker login -u "$DOCKERHUB_USERNAME" --password-stdin - docker push girder/tox-and-node:latest + echo "$DOCKERHUB_PASS" | docker login -u "$DOCKERHUB_USERNAME" --password-stdin + docker push girder/tox-and-node:latest py38: machine: image: ubuntu-2204:current @@ -308,7 +307,6 @@ jobs: command: | touch package.json gh-pages --dotfiles --message "Update documentation" --dist build/docs --no-history - workflows: version: 2 ci: diff --git a/.circleci/dcm4chee/auth-docker-compose.yml b/.circleci/dcm4chee/auth-docker-compose.yml index d70529341..f569e6635 100644 --- a/.circleci/dcm4chee/auth-docker-compose.yml +++ b/.circleci/dcm4chee/auth-docker-compose.yml @@ -1,3 +1,4 @@ +--- volumes: db_data: {} arc_data: {} @@ -5,7 +6,6 @@ volumes: ldap_config: {} mysql: {} keycloak: {} - services: ldap: image: dcm4che/slapd-dcm4chee:2.6.5-31.2 diff --git a/.circleci/dcm4chee/docker-compose.yml b/.circleci/dcm4chee/docker-compose.yml index 4a225fad1..b81489ac9 100644 --- a/.circleci/dcm4chee/docker-compose.yml +++ b/.circleci/dcm4chee/docker-compose.yml @@ -1,11 +1,10 @@ +--- version: "3" - volumes: db_data: {} arc_data: {} ldap_data: {} ldap_config: {} - services: ldap: image: dcm4che/slapd-dcm4chee:2.6.5-31.2 diff --git a/.git-blame-ignore-revs b/.git-blame-ignore-revs index 14822961b..10d74ecab 100644 --- a/.git-blame-ignore-revs +++ b/.git-blame-ignore-revs @@ -13,3 +13,5 @@ 23ab7f011abaa71d0a4904a9da599174c38f64c3 # PR 1257: Use ruff for linting dcda95e659a4eaa73cae85ab02f1b89059e63c32 +# PR 1563: Use yamlfix for linting +f517ecb2b7d8c454d1374156452be42b87ca3fd1 diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml index b1ec39094..4944c5508 100644 --- a/.github/workflows/docker.yml +++ b/.github/workflows/docker.yml @@ -1,3 +1,4 @@ +--- name: Docker Package on: push: @@ -5,11 +6,9 @@ on: branches: - master pull_request: - env: REGISTRY: ghcr.io IMAGE_NAME: ${{ github.repository }} - jobs: build-and-publish-base: runs-on: ubuntu-latest @@ -40,14 +39,16 @@ jobs: push: ${{ github.actor != 'dependabot[bot]' }} tags: ${{ steps.meta.outputs.tags }} labels: ${{ steps.meta.outputs.labels }} - build-and-publish-targets: runs-on: ubuntu-latest if: github.event_name == 'push' || github.event_name == 'workflow_dispatch' || ( github.event_name == 'pull_request' && github.event.pull_request.head.repo.full_name == github.repository ) strategy: fail-fast: false matrix: - target: ["geo", "jupyter", "jupyter-geo"] + target: + - "geo" + - "jupyter" + - "jupyter-geo" steps: - uses: actions/checkout@v4 with: diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 14f8826a5..a2ff44bef 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -1,95 +1,107 @@ +--- # Initially run # pre-commit install # See https://pre-commit.com for more information # See https://pre-commit.com/hooks.html for more hooks repos: -- repo: https://github.com/pre-commit/pre-commit-hooks - rev: v4.5.0 - hooks: - - id: check-added-large-files - - id: check-ast - - id: check-builtin-literals - - id: check-case-conflict - - id: check-docstring-first - - id: check-executables-have-shebangs - - id: check-json - - id: check-merge-conflict - - id: check-shebang-scripts-are-executable - # - id: check-symlinks - # - id: check-toml - # - id: check-xml - - id: check-yaml - - id: debug-statements - - id: destroyed-symlinks - - id: detect-private-key - - id: double-quote-string-fixer - - id: end-of-file-fixer - - id: fix-byte-order-marker - - id: forbid-new-submodules - - id: mixed-line-ending - - id: no-commit-to-branch - - id: trailing-whitespace -- repo: https://github.com/pre-commit/pygrep-hooks - rev: v1.10.0 - hooks: - - id: python-no-eval - - id: python-no-log-warn - - id: rst-backticks - - id: rst-directive-colons - - id: rst-inline-touching-normal - - id: text-unicode-replacement-char -- repo: https://github.com/Lucas-C/pre-commit-hooks-markup - rev: v1.0.1 - hooks: - - id: rst-linter - files: README.rst - name: rst-linter of README.rst -- repo: https://github.com/codespell-project/codespell - rev: v2.2.6 - hooks: - - id: codespell - args: - - --ignore-words-list - - "hist,indext,pixelx,thex,subtile,slippy,fram" -- repo: https://github.com/syntaqx/git-hooks - rev: v0.0.18 - hooks: - - id: circleci-config-validate -- repo: https://github.com/ThisIsManta/stylus-supremacy - rev: v2.17.5 - hooks: - - id: stylus-supremacy - args: - - '--options' - - './girder/girder_large_image/web_client/package.json' -- repo: https://github.com/asottile/pyupgrade - rev: v3.15.2 - hooks: - - id: pyupgrade - args: - - --py38-plus - - --keep-percent-format -- repo: https://github.com/astral-sh/ruff-pre-commit - # Ruff version. - rev: v0.3.5 - hooks: - - id: ruff - args: [--fix, --exit-non-zero-on-fix] - types_or: [python, pyi, jupyter] -- repo: https://github.com/pre-commit/mirrors-autopep8 - rev: v2.0.4 - hooks: - - id: autopep8 -- repo: https://github.com/PyCQA/flake8 - rev: 7.0.0 - hooks: - - id: flake8 -- repo: https://github.com/pycqa/isort - rev: 5.13.2 - hooks: - - id: isort - name: isort (python) -- repo: https://github.com/asottile/yesqa - rev: v1.5.0 - hooks: - - id: yesqa + - repo: https://github.com/pre-commit/pre-commit-hooks + rev: v4.6.0 + hooks: + - id: check-added-large-files + - id: check-ast + - id: check-builtin-literals + - id: check-case-conflict + - id: check-docstring-first + - id: check-executables-have-shebangs + - id: check-json + - id: check-merge-conflict + - id: check-shebang-scripts-are-executable + # - id: check-symlinks + # - id: check-toml + # - id: check-xml + - id: check-yaml + - id: debug-statements + - id: destroyed-symlinks + - id: detect-private-key + - id: double-quote-string-fixer + - id: end-of-file-fixer + - id: fix-byte-order-marker + - id: forbid-new-submodules + - id: mixed-line-ending + - id: no-commit-to-branch + - id: trailing-whitespace + - repo: https://github.com/pre-commit/pygrep-hooks + rev: v1.10.0 + hooks: + - id: python-no-eval + - id: python-no-log-warn + - id: rst-backticks + - id: rst-directive-colons + - id: rst-inline-touching-normal + - id: text-unicode-replacement-char + - repo: https://github.com/Lucas-C/pre-commit-hooks-markup + rev: v1.0.1 + hooks: + - id: rst-linter + files: README.rst + name: rst-linter of README.rst + - repo: https://github.com/codespell-project/codespell + rev: v2.3.0 + hooks: + - id: codespell + args: + - --ignore-words-list + - "hist,indext,pixelx,thex,subtile,slippy,fram" + - repo: https://github.com/syntaqx/git-hooks + rev: v0.0.18 + hooks: + - id: circleci-config-validate + - repo: https://github.com/ThisIsManta/stylus-supremacy + rev: v2.17.5 + hooks: + - id: stylus-supremacy + args: + - '--options' + - './girder/girder_large_image/web_client/package.json' + - repo: https://github.com/asottile/pyupgrade + rev: v3.16.0 + hooks: + - id: pyupgrade + args: + - --py38-plus + - --keep-percent-format + - repo: https://github.com/astral-sh/ruff-pre-commit + rev: v0.5.0 + hooks: + - id: ruff + args: + - --fix + - --exit-non-zero-on-fix + types_or: + - python + - pyi + - jupyter + - repo: https://github.com/pre-commit/mirrors-autopep8 + rev: v2.0.4 + hooks: + - id: autopep8 + - repo: https://github.com/PyCQA/flake8 + rev: 7.1.0 + hooks: + - id: flake8 + - repo: https://github.com/pycqa/isort + rev: 5.13.2 + hooks: + - id: isort + name: isort (python) + - repo: https://github.com/asottile/yesqa + rev: v1.5.0 + hooks: + - id: yesqa + - repo: https://github.com/lyz-code/yamlfix + rev: 1.16.0 + hooks: + - id: yamlfix + args: + - -c + - pyproject.toml diff --git a/CHANGELOG.md b/CHANGELOG.md index 6eea4e49f..846a42fcc 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,15 +1,28 @@ # Change Log +## 1.29.2 + +### Improvements +- Show a loading spinner on the image display in geojs in girder ([#1559](../../pull/1559)) +- Better handle images that are composed of a folder and an item ([#1561](../../pull/1561)) +- Allow specifying which sources are checked with canReadList ([#1562](../../pull/1562)) + +### Bug Fixes +- Fix a compositing error in transformed multi source images ([#1560](../../pull/1560)) + ## 1.29.1 ### Improvements - Improved zarr sink metadata handling ([#1508](../../pull/1508)) +- Speed up decoding jp2k tiff with an optional library ([#1555](../../pull/1555)) ### Changes - Work with newer python-mapnik ([#1550](../../pull/1550)) +- Use the new official yaml mime-type of application/yaml ([#1558](../../pull/1558)) ### Bug Fixes - Fix an issue emitting rectangles in geojson ([#1552](../../pull/1552)) +- Fix an issue writing zarr channel metadata ([#1557](../../pull/1557)) ## 1.29.0 diff --git a/codecov.yml b/codecov.yml index 0f3b15b70..3f76626ae 100644 --- a/codecov.yml +++ b/codecov.yml @@ -1,3 +1,4 @@ +--- comment: false coverage: status: diff --git a/girder/girder_large_image/__init__.py b/girder/girder_large_image/__init__.py index 77f25fdb5..455bd70ec 100644 --- a/girder/girder_large_image/__init__.py +++ b/girder/girder_large_image/__init__.py @@ -158,7 +158,7 @@ def _updateJob(event): datetime.timedelta(seconds=30))) -def checkForLargeImageFiles(event): +def checkForLargeImageFiles(event): # noqa file = event.info logger.info('Handling file %s (%s)', file['_id'], file['name']) possible = False @@ -178,10 +178,51 @@ def checkForLargeImageFiles(event): return try: ImageItem().createImageItem(item, file, createJob=False) + return except Exception: - # We couldn't automatically set this as a large image - girder.logger.info( - 'Saved file %s cannot be automatically used as a largeImage' % str(file['_id'])) + pass + # Check for files that are from folder/image style images. This is custom + # per folder/image format + imageFolderRecords = { + 'mrxs': { + 'match': r'^Slidedat.ini$', + 'up': 1, + 'folder': r'^(.*)$', + 'image': '\\1.mrxs', + }, + 'vsi': { + 'match': r'^.*\.ets$', + 'up': 2, + 'folder': r'^_(\.*)_$', + 'image': '\\1.vsi', + }, + } + for check in imageFolderRecords.values(): + if re.match(check['match'], file['name']): + try: + folderId = item['folderId'] + folder = None + for _ in range(check['up']): + folder = Folder().load(folderId, force=True) + if not folder: + break + folderId = folder['parentId'] + if not folder or not re.match(check['folder'], folder['name']): + continue + imageName = re.sub(check['folder'], check['image'], folder['name']) + parentItem = Item().findOne({'folderId': folder['parentId'], 'name': imageName}) + if not parentItem: + continue + files = list(Item().childFiles(item=parentItem, limit=2)) + if len(files) == 1: + parentFile = files[0] + ImageItem().createImageItem(parentItem, parentFile, createJob=False) + return + except Exception: + pass + # We couldn't automatically set this as a large image + girder.logger.info( + 'Saved file %s cannot be automatically used as a largeImage' % str(file['_id'])) def removeThumbnails(event): @@ -251,6 +292,8 @@ def handleFileSave(event): for mimeType, ext, std in [ ('text/yaml', '.yaml', True), ('text/yaml', '.yml', True), + ('application/yaml', '.yaml', True), + ('application/yaml', '.yml', True), ('application/vnd.geo+json', '.geojson', True), ]: if ext not in mimetypes.types_map: @@ -499,14 +542,14 @@ def yamlConfigFileWrite(folder, name, user, yaml_config): item = Item().createItem(name, user, folder, reuseExisting=True) existingFiles = list(Item().childFiles(item)) if (len(existingFiles) == 1 and - existingFiles[0]['mimeType'] == 'text/yaml' and + existingFiles[0]['mimeType'] == 'application/yaml' and existingFiles[0]['name'] == name): upload = Upload().createUploadToFile( existingFiles[0], user, size=len(yaml_config)) else: upload = Upload().createUpload( user, name, 'item', item, size=len(yaml_config), - mimeType='text/yaml', save=True) + mimeType='application/yaml', save=True) newfile = Upload().handleChunk(upload, yaml_config) with _configWriteLock: for entry in list(Item().childFiles(item)): diff --git a/girder/girder_large_image/girder_tilesource.py b/girder/girder_large_image/girder_tilesource.py index 0ca4f3863..a314a7341 100644 --- a/girder/girder_large_image/girder_tilesource.py +++ b/girder/girder_large_image/girder_tilesource.py @@ -5,7 +5,7 @@ from girder.exceptions import FilePathException, ValidationException from girder.models.file import File from girder.models.item import Item -from large_image import tilesource +from large_image import config, tilesource from large_image.constants import SourcePriority from large_image.exceptions import TileSourceAssetstoreError, TileSourceError @@ -164,6 +164,9 @@ def getGirderTileSourceName(item, file=None, *args, **kwargs): # noqa properties = {} if localPath: properties['_geospatial_source'] = tilesource.isGeospatial(localPath) + ignored_names = config.getConfig('all_sources_ignored_names') + ignoreName = (ignored_names and re.search( + ignored_names, baseName, flags=re.IGNORECASE)) sourceList = [] for sourceName in availableSources: if not getattr(availableSources[sourceName], 'girderSource', False): @@ -183,7 +186,7 @@ def getGirderTileSourceName(item, file=None, *args, **kwargs): # noqa if ext in sourceExtensions: priority = min(priority, sourceExtensions[ext]) fallback = False - if priority >= SourcePriority.MANUAL: + if priority >= SourcePriority.MANUAL or (ignoreName and fallback): continue propertiesClash = any( getattr(availableSources[sourceName], k, False) != v diff --git a/girder/girder_large_image/web_client/stylesheets/imageViewerSelectWidget.styl b/girder/girder_large_image/web_client/stylesheets/imageViewerSelectWidget.styl index a7cb189f6..dbd81f260 100644 --- a/girder/girder_large_image/web_client/stylesheets/imageViewerSelectWidget.styl +++ b/girder/girder_large_image/web_client/stylesheets/imageViewerSelectWidget.styl @@ -6,6 +6,11 @@ width auto margin-left 25px + .image-viewer-loading + position relative + width 0 + height 0 + .image-viewer border 1px solid #f0f0f0 width 100% diff --git a/girder/girder_large_image/web_client/templates/imageViewerSelectWidget.pug b/girder/girder_large_image/web_client/templates/imageViewerSelectWidget.pug index 599e1cd36..974adfb33 100644 --- a/girder/girder_large_image/web_client/templates/imageViewerSelectWidget.pug +++ b/girder/girder_large_image/web_client/templates/imageViewerSelectWidget.pug @@ -7,5 +7,7 @@ option(value=viewer.name) #{viewer.label} .image-controls #vue-container +.image-viewer-loading.hidden + span.icon-spin1.animate-spin(title="Loading image") each viewer in viewers .image-viewer.hidden(id=viewer.name) diff --git a/girder/girder_large_image/web_client/views/imageViewerWidget/geojs.js b/girder/girder_large_image/web_client/views/imageViewerWidget/geojs.js index 2131bc138..c009f0ba2 100644 --- a/girder/girder_large_image/web_client/views/imageViewerWidget/geojs.js +++ b/girder/girder_large_image/web_client/views/imageViewerWidget/geojs.js @@ -25,6 +25,7 @@ var GeojsImageViewerWidget = ImageViewerWidget.extend({ root = __webpack_public_path__ || root; // eslint-disable-line } catch (err) { } root = root.replace(/\/$/, ''); + $(this.el).parent().find('.image-viewer-loading').removeClass('hidden'); $.when( ImageViewerWidget.prototype.initialize.call(this, settings).then(() => { if (this.metadata.geospatial) { @@ -81,6 +82,7 @@ var GeojsImageViewerWidget = ImageViewerWidget.extend({ this.viewer = geo.map(params.map); params.layer.autoshareRenderer = false; this._layer = this.viewer.createLayer('osm', params.layer); + this._layer.onIdle(() => $(this.el).parent().find('.image-viewer-loading').addClass('hidden')); if (this.metadata.frames && this.metadata.frames.length > 1) { const maxTextures = Math.max(1, Math.min(16, Math.ceil( this.metadata.frames.length / 1024))); diff --git a/girder/girder_large_image/web_client/views/itemViewCodemirror.js b/girder/girder_large_image/web_client/views/itemViewCodemirror.js index 7786a1727..021db1a4c 100644 --- a/girder/girder_large_image/web_client/views/itemViewCodemirror.js +++ b/girder/girder_large_image/web_client/views/itemViewCodemirror.js @@ -156,6 +156,7 @@ const Formats = { Formats['application/vnd.geo+json'] = Formats['application/json']; Formats['text/x-yaml'] = Formats['text/yaml']; Formats['application/x-yaml'] = Formats['text/yaml']; +Formats['application/yaml'] = Formats['text/yaml']; function lintGirderIni(text, callback) { return restRequest({ diff --git a/large_image/tilesource/__init__.py b/large_image/tilesource/__init__.py index 1d2eec26b..7942aeddf 100644 --- a/large_image/tilesource/__init__.py +++ b/large_image/tilesource/__init__.py @@ -208,6 +208,7 @@ def canRead(*args, **kwargs) -> bool: def canReadList( pathOrUri: Union[str, PosixPath], mimeType: Optional[str] = None, + availableSources: Optional[Dict[str, Type[FileTileSource]]] = None, *args, **kwargs) -> List[Tuple[str, bool]]: """ Check if large_image can read a path or uri via each source. @@ -218,16 +219,18 @@ def canReadList( :param pathOrUri: either a file path or a fixed source via large_image://. :param mimeType: the mimetype of the file, if known. + :param availableSources: an ordered dictionary of sources to try. If None, + use the primary list of sources. :returns: A list of tuples of (source name, canRead). """ - if not len(AvailableTileSources): + if availableSources is None and not len(AvailableTileSources): loadTileSources() sourceList = getSortedSourceList( - AvailableTileSources, pathOrUri, mimeType, *args, **kwargs) + availableSources or AvailableTileSources, pathOrUri, mimeType, *args, **kwargs) result = [] for entry in sorted(sourceList): sourceName = entry[-1] - result.append((sourceName, AvailableTileSources[sourceName].canRead( + result.append((sourceName, (availableSources or AvailableTileSources)[sourceName].canRead( pathOrUri, *args, **kwargs))) return result diff --git a/large_image/tilesource/base.py b/large_image/tilesource/base.py index 2bad507c1..ab0278e78 100644 --- a/large_image/tilesource/base.py +++ b/large_image/tilesource/base.py @@ -1501,6 +1501,7 @@ def _getTileFromEmptyLevel(self, x: int, y: int, z: int, **kwargs) -> PIL.Image. :param z: original level. :returns: tile in PIL format. """ + lastlog = time.time() basez = z scale = 1 dirlist = self._nonemptyLevelsList(kwargs.get('frame')) @@ -1514,11 +1515,16 @@ def _getTileFromEmptyLevel(self, x: int, y: int, z: int, **kwargs) -> PIL.Image. min(self.sizeX, self.tileWidth * scale), min(self.sizeY, self.tileHeight * scale))) maxX = 2.0 ** (z + 1 - self.levels) * self.sizeX / self.tileWidth maxY = 2.0 ** (z + 1 - self.levels) * self.sizeY / self.tileHeight - for newX in range(scale): - for newY in range(scale): + for newY in range(scale): + for newX in range(scale): if ((newX or newY) and ((x * scale + newX) >= maxX or (y * scale + newY) >= maxY)): continue + if time.time() - lastlog > 10: + self.logger.info( + 'Compositing tile from higher resolution tiles x=%d y=%d z=%d', + x * scale + newX, y * scale + newY, z) + lastlog = time.time() subtile = self.getTile( x * scale + newX, y * scale + newY, z, pilImageAllowed=True, numpyAllowed=False, diff --git a/large_image/tilesource/utilities.py b/large_image/tilesource/utilities.py index 618c3b5a8..e7b750442 100644 --- a/large_image/tilesource/utilities.py +++ b/large_image/tilesource/utilities.py @@ -650,7 +650,7 @@ def getPaletteColors(value: Union[str, List[Union[str, float, Tuple[float, ...]] palette = ['#0000', mpl.colors.to_hex(str(value))] else: cmap = mpl.colormaps.get_cmap(str(value)) if hasattr(getattr( - mpl, 'colormaps', None), 'get_cmap') else mpl.cm.get_cmap( # type: ignore + mpl, 'colormaps', None), 'get_cmap') else mpl.cm.get_cmap( str(value)) palette = [mpl.colors.to_hex(cmap(i)) for i in range(cmap.N)] except (ImportError, ValueError, AttributeError): diff --git a/pyproject.toml b/pyproject.toml index f663fd7a5..f95faddae 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -72,3 +72,8 @@ inline-quotes = "single" [tool.ruff.lint.mccabe] max-complexity = 14 + +[tool.yamlfix] +line_length = 200 +preserve_quotes = true +sequence_style = "block_style" diff --git a/requirements-dev.txt b/requirements-dev.txt index 40047d065..aabe62500 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -13,7 +13,7 @@ girder-jobs>=3.0.3 -e sources/pil[all] -e sources/rasterio[all] -e sources/test --e sources/tiff +-e sources/tiff[all] -e sources/tifffile -e sources/vips -e sources/zarr diff --git a/requirements-test-core.txt b/requirements-test-core.txt index fdf662b3c..67ef51093 100644 --- a/requirements-test-core.txt +++ b/requirements-test-core.txt @@ -11,7 +11,7 @@ sources/openslide sources/pil[all] sources/rasterio[all] sources/test -sources/tiff +sources/tiff[all] sources/tifffile sources/vips sources/zarr diff --git a/requirements-test.txt b/requirements-test.txt index f4a77725d..71dedb92f 100644 --- a/requirements-test.txt +++ b/requirements-test.txt @@ -13,7 +13,7 @@ sources/openslide sources/pil[all] sources/rasterio[all] sources/test -sources/tiff +sources/tiff[all] sources/tifffile sources/vips sources/zarr diff --git a/setup.py b/setup.py index 605e8a475..2318a87ac 100644 --- a/setup.py +++ b/setup.py @@ -71,6 +71,7 @@ def prerelease_local_scheme(version): f'large-image-source-multi[all]{limit_version}', f'large-image-source-pil[all]{limit_version}', f'large-image-source-rasterio[all]{limit_version}', + f'large-image-source-tiff[all]{limit_version}', }) # The common packages are ones that will install on Ubuntu, OSX, and Windows # from pypi with all needed dependencies. diff --git a/sources/bioformats/large_image_source_bioformats/__init__.py b/sources/bioformats/large_image_source_bioformats/__init__.py index c04b6ac0f..4496b7282 100644 --- a/sources/bioformats/large_image_source_bioformats/__init__.py +++ b/sources/bioformats/large_image_source_bioformats/__init__.py @@ -183,6 +183,7 @@ class BioformatsFileTileSource(FileTileSource, metaclass=LruCacheMetaclass): extensions = { None: SourcePriority.FALLBACK, 'czi': SourcePriority.PREFERRED, + 'ets': SourcePriority.LOW, # part of vsi 'lif': SourcePriority.MEDIUM, 'vsi': SourcePriority.PREFERRED, } diff --git a/sources/multi/large_image_source_multi/__init__.py b/sources/multi/large_image_source_multi/__init__.py index a5869f8c2..21abd152f 100644 --- a/sources/multi/large_image_source_multi/__init__.py +++ b/sources/multi/large_image_source_multi/__init__.py @@ -964,7 +964,7 @@ def _mergeTiles(self, base, tile, x, y): tileA = tile[:, :, -1].astype(float) / fullAlphaValue(tile.dtype) outA = tileA + baseA * (1 - tileA) base[y:y + tile.shape[0], x:x + tile.shape[1], :-1] = ( - np.where(tileA[..., np.newaxis], tile[:, :, :-1], 0) + + np.where(tileA[..., np.newaxis], tile[:, :, :-1] * tileA[..., np.newaxis], 0) + base[y:y + tile.shape[0], x:x + tile.shape[1], :-1] * baseA[..., np.newaxis] * (1 - tileA[..., np.newaxis]) ) / np.where(outA[..., np.newaxis], outA[..., np.newaxis], 1) diff --git a/sources/openslide/large_image_source_openslide/__init__.py b/sources/openslide/large_image_source_openslide/__init__.py index f6a987133..f6210c18c 100644 --- a/sources/openslide/large_image_source_openslide/__init__.py +++ b/sources/openslide/large_image_source_openslide/__init__.py @@ -48,6 +48,7 @@ class OpenslideFileTileSource(FileTileSource, metaclass=LruCacheMetaclass): None: SourcePriority.MEDIUM, 'bif': SourcePriority.LOW, # Ventana 'dcm': SourcePriority.LOW, # DICOM + 'ini': SourcePriority.LOW, # Part of mrxs 'mrxs': SourcePriority.PREFERRED, # MIRAX 'ndpi': SourcePriority.PREFERRED, # Hamamatsu 'scn': SourcePriority.LOW, # Leica diff --git a/sources/pil/setup.py b/sources/pil/setup.py index 7bc848be7..0e1304298 100644 --- a/sources/pil/setup.py +++ b/sources/pil/setup.py @@ -55,10 +55,10 @@ def prerelease_local_scheme(version): ], extras_require={ 'all': [ - 'rawpy ; python_version < "3.12"', + 'rawpy', 'pillow-heif', 'pillow-jxl-plugin', - 'pillow-jpls ; python_version < "3.12"', + 'pillow-jpls', ], 'girder': f'girder-large-image{limit_version}', }, diff --git a/sources/tiff/large_image_source_tiff/tiff_reader.py b/sources/tiff/large_image_source_tiff/tiff_reader.py index 5e603e7f5..ffe850b83 100644 --- a/sources/tiff/large_image_source_tiff/tiff_reader.py +++ b/sources/tiff/large_image_source_tiff/tiff_reader.py @@ -803,9 +803,18 @@ def getTile(self, x, y, asarray=False): # Write JPEG End Of Image marker imageBuffer.write(b'\xff\xd9') return imageBuffer.getvalue() - # Get the whole frame, which is in a JPEG or JPEG 2000 format, and + # Get the whole frame, which is in a JPEG or JPEG 2000 format + frame = self._getJpegFrame(tileNum, True) + # For JP2K, see if we can convert it faster than PIL + if self._tiffInfo.get('compression') in {33003, 33005}: + try: + import openjpeg + + return openjpeg.decode(frame) + except Exception: + pass # convert it to a PIL image - imageBuffer.write(self._getJpegFrame(tileNum, True)) + imageBuffer.write(frame) image = PIL.Image.open(imageBuffer) # Converting the image mode ensures that it gets loaded once and is in # a form we expect. If this isn't done, then PIL can load the image diff --git a/sources/tiff/setup.py b/sources/tiff/setup.py index 28a9fb2aa..8ff60a6e4 100644 --- a/sources/tiff/setup.py +++ b/sources/tiff/setup.py @@ -56,6 +56,9 @@ def prerelease_local_scheme(version): 'tifftools>=1.2.0', ], extras_require={ + 'all': [ + 'pylibjpeg-openjpeg', + ], 'girder': f'girder-large-image{limit_version}', }, keywords='large_image, tile source', diff --git a/sources/zarr/large_image_source_zarr/__init__.py b/sources/zarr/large_image_source_zarr/__init__.py index f3788f9a1..c48490b7b 100644 --- a/sources/zarr/large_image_source_zarr/__init__.py +++ b/sources/zarr/large_image_source_zarr/__init__.py @@ -678,7 +678,7 @@ def _writeInternalMetadata(self): with self._threadLock and self._processLock: name = str(self._tempdir.name).split('/')[-1] arrays = dict(self._zarr.arrays()) - channel_axis = self._axes.get('s') or self._axes.get('c') + channel_axis = self._axes.get('c', self._axes.get('s')) datasets = [] axes = [] channels = [] @@ -709,7 +709,7 @@ def _writeInternalMetadata(self): elif a == 'z': rdefs['defaultZ'] = 0 axes.append(axis_metadata) - if channel_axis and len(arrays) > 0: + if channel_axis is not None and len(arrays) > 0: base_array = list(arrays.values())[0] base_shape = base_array.shape for c in range(base_shape[channel_axis]): @@ -721,7 +721,13 @@ def _writeInternalMetadata(self): 'inverted': False, 'label': f'Band {c + 1}', } - channel_data = base_array[..., c] + slicing = tuple( + slice(None) + if k != ('c' if 'c' in self._axes else 's') + else c + for k, v in self._axes.items() + ) + channel_data = base_array[slicing] channel_min = np.min(channel_data) channel_max = np.max(channel_data) channel_metadata['window'] = { diff --git a/test.Dockerfile b/test.Dockerfile index 6c9930033..50435dcd8 100644 --- a/test.Dockerfile +++ b/test.Dockerfile @@ -13,8 +13,7 @@ ENV DEBIAN_FRONTEND=noninteractive \ LANG=en_US.UTF-8 \ PYENV_ROOT="/.pyenv" \ PATH="/.pyenv/bin:/.pyenv/shims:$PATH" \ - OLD_PYTHON_VERSIONS="3.6.15" \ - PYTHON_VERSIONS="3.9 3.8 3.7 3.10 3.11 3.12" + PYTHON_VERSIONS="3.11 3.8 3.9 3.10 3.12" # Consumers of this package aren't expecting an existing ubuntu user (there # wasn't one in the ubuntu:22.04 base) @@ -65,9 +64,6 @@ RUN apt-get update && \ rdfind \ # core girder \ gcc \ - libpython3-dev \ - python3-pip \ - python3-venv \ cmake \ iptables \ dnsutils \ @@ -89,17 +85,6 @@ RUN git clone "https://github.com/universal-ctags/ctags.git" "./ctags" && \ cd .. && \ rm -rf ./ctags -# Install an older openssl for Python 3.6 -RUN curl -O https://www.openssl.org/source/old/1.1.1/openssl-1.1.1t.tar.gz && \ - tar -xf openssl-1.1.1t.tar.gz && \ - cd openssl-1.1.1t/ && \ - ./config shared -Wl,-rpath=/opt/openssl11/lib --prefix=/opt/openssl11 && \ - make --silent -j `nproc` && \ - make install --silent -j `nproc` &&\ - cd .. && \ - rm -r /opt/openssl11/share/doc && \ - rm -r openssl-1.1.1* - RUN pyenv update && \ pyenv install --list && \ echo $PYTHON_VERSIONS | xargs -P `nproc` -n 1 pyenv install && \ @@ -107,19 +92,18 @@ RUN pyenv update && \ export CPPFLAGS=-I/opt/openssl11/include && \ export LDFLAGS="-L/opt/openssl11/lib -Wl,-rpath,/opt/openssl11/lib" && \ export CONFIGURE_OPTS="--with-openssl=/opt/openssl11" && \ - echo $OLD_PYTHON_VERSIONS | xargs -P `nproc` -n 1 pyenv install && \ # ensure newest pip and setuptools for all python versions \ - echo $PYTHON_VERSIONS $OLD_PYTHON_VERSIONS | xargs -n 1 bash -c 'pyenv global "${0}" && pip install -U setuptools pip' && \ + echo $PYTHON_VERSIONS | xargs -n 1 bash -c 'pyenv global "${0}" && pip install -U setuptools pip' && \ pyenv global $(pyenv versions --bare) && \ find $PYENV_ROOT/versions -type d '(' -name '__pycache__' -o -name 'test' -o -name 'tests' ')' -exec rm -rfv '{}' + >/dev/null && \ find $PYENV_ROOT/versions -type f '(' -name '*.py[co]' -o -name '*.exe' ')' -exec rm -fv '{}' + >/dev/null && \ - echo $PYTHON_VERSIONS $OLD_PYTHON_VERSIONS | tr " " "\n" > $PYENV_ROOT/version && \ + echo $PYTHON_VERSIONS | tr " " "\n" > $PYENV_ROOT/version && \ find / -xdev -name __pycache__ -type d -exec rm -r {} \+ && \ rm -rf /tmp/* /var/tmp/* && \ # This makes duplicate python library files hardlinks of each other \ - rdfind -minsize 524288 -makehardlinks true -makeresultsfile false /.pyenv + rdfind -minsize 32768 -makehardlinks true -makeresultsfile false /.pyenv -RUN for ver in $PYTHON_VERSIONS $OLD_PYTHON_VERSIONS; do \ +RUN for ver in $PYTHON_VERSIONS; do \ pyenv local $ver && \ python -m pip install --no-cache-dir -U pip && \ python -m pip install --no-cache-dir tox wheel && \ @@ -128,7 +112,7 @@ RUN for ver in $PYTHON_VERSIONS $OLD_PYTHON_VERSIONS; do \ pyenv rehash && \ find / -xdev -name __pycache__ -type d -exec rm -r {} \+ && \ rm -rf /tmp/* /var/tmp/* && \ - rdfind -minsize 524288 -makehardlinks true -makeresultsfile false /.pyenv + rdfind -minsize 32768 -makehardlinks true -makeresultsfile false /.pyenv # Note: to actually run tox on python 3.6, you have to do # pip install 'tox<4.5' 'virtualenv<20.22' diff --git a/test/lisource_compare.py b/test/lisource_compare.py index b2d0deb42..fcf5bc3e5 100755 --- a/test/lisource_compare.py +++ b/test/lisource_compare.py @@ -139,7 +139,11 @@ def source_compare(sourcePath, opts): # noqa else: sys.stdout.write('%s\n' % sourcePath) sys.stdout.flush() - canread = large_image.canReadList(sourcePath) + sublist = { + k: v for k, v in large_image.tilesource.AvailableTileSources.items() + if (getattr(opts, 'skipsource', None) is None or k not in opts.skipsource) and + (getattr(opts, 'usesource', None) is None or k in opts.usesource)} + canread = large_image.canReadList(sourcePath, availableSources=sublist) large_image.cache_util.cachesClear() slen = max([len(source) for source, _ in canread] + [10]) sys.stdout.write('Source' + ' ' * (slen - 6)) diff --git a/test/test_files/.large_image_config.yaml b/test/test_files/.large_image_config.yaml index 24693f87d..9aa030d74 100644 --- a/test/test_files/.large_image_config.yaml +++ b/test/test_files/.large_image_config.yaml @@ -1,3 +1,4 @@ +--- access: # Show to user and higher user: @@ -10,23 +11,18 @@ access: min-width: 200 # Show these columns columns: - - - type: image + - type: image value: thumbnail title: Thumbnail - - - type: image + - type: image value: label title: Label - - - type: image + - type: image value: macro title: Macro - - - type: record + - type: record value: name - - - type: metadata + - type: metadata value: Stain format: text # Only show this record for entries that match a particular value @@ -34,42 +30,32 @@ access: - type: record value: name match: "\\.svs$" - - - type: metadata + - type: metadata value: Classification format: number - - - type: metadata + - type: metadata value: gloms.length - - - type: record + - type: record value: size - - - type: record + - type: record value: controls defaultSort: - - - type: metadata + - type: metadata value: Classification dir: up - - - type: record + - type: record value: name dir: down itemListDialog: # Show these columns columns: - - - type: image + - type: image value: thumbnail title: Thumbnail - - - type: record + - type: record value: name - - - type: metadata + - type: metadata value: Stain format: text - - - type: record + - type: record value: size diff --git a/test/test_files/multi_band.yml b/test/test_files/multi_band.yml index fe249c983..72b55422d 100644 --- a/test/test_files/multi_band.yml +++ b/test/test_files/multi_band.yml @@ -9,9 +9,9 @@ sources: tileHeight: 256 sizeX: 10000 sizeY: 7500 - fractal: True + fractal: true # c,z,t,xy OR frames frames: "4,5,1,1" - monochrome: False + monochrome: false # multiband bands: "red,green,blue,ir1,ir2,grey" diff --git a/test/test_files/multi_test_source.yml b/test/test_files/multi_test_source.yml index 5d207f4ee..98f8ff939 100644 --- a/test/test_files/multi_test_source.yml +++ b/test/test_files/multi_test_source.yml @@ -9,7 +9,7 @@ sources: tileHeight: 256 sizeX: 100000 sizeY: 75000 - fractal: True + fractal: true # c,z,t,xy OR frames frames: "4,10,5,3" - monochrome: False + monochrome: false diff --git a/test/test_files/multi_test_source3.yml b/test/test_files/multi_test_source3.yml index d5df67e87..84cab1774 100644 --- a/test/test_files/multi_test_source3.yml +++ b/test/test_files/multi_test_source3.yml @@ -5,6 +5,6 @@ sources: params: sizeX: 2000 sizeY: 1250 - fractal: True + fractal: true # c,z,t,xy OR frames frames: "1,3,1,1" diff --git a/test/test_files/multi_test_source_axes.yml b/test/test_files/multi_test_source_axes.yml index 496c8f96e..77aff0ea0 100644 --- a/test/test_files/multi_test_source_axes.yml +++ b/test/test_files/multi_test_source_axes.yml @@ -13,6 +13,6 @@ sources: tileHeight: 256 sizeX: 10000 sizeY: 7500 - fractal: True + fractal: true frames: "c=4,i=5,j=3" - monochrome: False + monochrome: false diff --git a/test/test_files/multi_test_source_bands.yml b/test/test_files/multi_test_source_bands.yml index a3f697096..39943728e 100644 --- a/test/test_files/multi_test_source_bands.yml +++ b/test/test_files/multi_test_source_bands.yml @@ -11,7 +11,7 @@ sources: tileHeight: 256 sizeX: 10000 sizeY: 6000 - fractal: True - monochrome: False + fractal: true + monochrome: false # multiband bands: "red=400-12000,green=0-65535,blue=800-4000,alpha=0-65535" diff --git a/test/test_files/multi_test_source_style.yml b/test/test_files/multi_test_source_style.yml index f9e0e5c17..60103a504 100644 --- a/test/test_files/multi_test_source_style.yml +++ b/test/test_files/multi_test_source_style.yml @@ -8,13 +8,12 @@ sources: tileHeight: 256 sizeX: 1000 sizeY: 750 - fractal: True - monochrome: False + fractal: true + monochrome: false bands: "red=0-4000" style: bands: - - - min: 0 - max: 4000 - color: #ffffff - band: 1 + - min: 0 + max: 4000 + color: # ffffff + band: 1 diff --git a/test/test_sink.py b/test/test_sink.py index f6c948d35..1b59f8f31 100644 --- a/test/test_sink.py +++ b/test/test_sink.py @@ -431,6 +431,19 @@ def testMetadata(tmp_path): assert rdefs.get('defaultZ') == 0 +def testChannelNames(tmp_path): + output_file = tmp_path / 'test.db' + sink = large_image_source_zarr.new() + + for c in range(5): + sink.addTile(np.random.random((4, 4, 3)), c=c) + + sink.channelNames = ['a', 'b', 'c', 'd', 'e'] + sink.write(output_file) + written = large_image.open(output_file) + assert len(written.metadata['channels']) == 5 + + def testAddAssociatedImages(tmp_path): output_file = tmp_path / 'test.db' sink = large_image_source_zarr.new() diff --git a/tox.ini b/tox.ini index 59001ad0c..0a0bd8e02 100644 --- a/tox.ini +++ b/tox.ini @@ -10,7 +10,7 @@ skip_missing_interpreters = true toxworkdir = {toxinidir}/build/tox [testenv] -passenv = PYTEST_*,DICOMWEB_TEST_URL,DICOMWEB_TEST_TOKEN +passenv = PYTEST_*,COVERAGE_*,DICOMWEB_TEST_URL,DICOMWEB_TEST_TOKEN extras = memcached redis @@ -172,9 +172,13 @@ deps = flake8-isort flake8-quotes ruff + yamlfix +allowlist_externals = + find commands = ruff check large_image sources utilities girder girder_annotation examples docs test flake8 + find . \( -name '*.yaml' -o -name '*.yml' \) -not -path './build/*' -not -path '*/node_modules/*' -exec yamlfix -c pyproject.toml --check {} \+ [testenv:type] description = Check python types @@ -225,11 +229,15 @@ deps = isort unify ruff + yamlfix +allowlist_externals = + find commands = isort . autopep8 -ria large_image sources utilities girder girder_annotation examples docs test unify --in-place --recursive large_image sources utilities girder girder_annotation examples docs test ruff check large_image sources utilities girder girder_annotation examples docs test --fix + find . \( -name '*.yaml' -o -name '*.yml' \) -not -path './build/*' -not -path '*/node_modules/*' -exec yamlfix -c pyproject.toml {} \+ [testenv:lintclient] description = Lint the girder large_image plugin client @@ -459,3 +467,8 @@ exclude = (?x)( | (^|/)test/ | (^|/)test_.*/ ) + +[yamlfix] +line_length = 200 +preserve_quotes = True +sequence_style = YamlNodeStyle = YamlNodeStyle.BLOCK_STYLE