Skip to content

Commit

Permalink
NDPI: first version of precompressed tile reading
Browse files Browse the repository at this point in the history
  • Loading branch information
@melissalinkert
melissalinkert committed Apr 9, 2024
1 parent c1c59a0 commit 32c9fec
Show file tree
Hide file tree
Showing 3 changed files with 232 additions and 53 deletions.
10 changes: 10 additions & 0 deletions components/formats-bsd/src/loci/formats/services/JPEGTurboService.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -120,6 +120,16 @@ byte[] getTile(byte[] buf, int xCoordinate, int yCoordinate, int width,
*/
byte[] getCompressedTile(int xTile, int yTile) throws IOException;

/**
* Similar to getTile(int, int), but returns the JPEG-compressed bytes.
*
* @param data preallocated array for storing tile
* @param xTile column index of the tile
* @param yTile row index of the tile
* @return JPEG-compressed bytes
*/
byte[] getCompressedTile(byte[] data, int xTile, int yTile) throws IOException;

/**
* Free resources associated with the initialized stream.
*/
Expand Down
11 changes: 10 additions & 1 deletion components/formats-bsd/src/loci/formats/services/JPEGTurboServiceImpl.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -356,12 +356,21 @@ public byte[] getCompressedTile(int tileX, int tileY) throws IOException {
}

byte[] data = new byte[(int) dataLength];
return getCompressedTile(data, tileX, tileY);
}

@Override
public byte[] getCompressedTile(byte[] data, int tileX, int tileY) throws IOException {
if (header == null) {
header = getFixedHeader();
}

int offset = 0;
System.arraycopy(header, 0, data, offset, header.length);
offset += header.length;

start = tileX + (tileY * xTiles * mult);
int mult = tileHeight / mcuHeight; // was restartInterval
int start = tileX + (tileY * xTiles * mult);
for (int row=0; row<tileHeight/mcuHeight; row++) {
int end = start + 1;

Expand Down
264 changes: 212 additions & 52 deletions components/formats-gpl/src/loci/formats/in/NDPIReader.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -35,6 +35,8 @@
import loci.formats.FormatException;
import loci.formats.FormatTools;
import loci.formats.MetadataTools;
import loci.formats.codec.Codec;
import loci.formats.codec.CodecOptions;
import loci.formats.meta.MetadataStore;
import loci.formats.services.JPEGTurboService;
import loci.formats.services.JPEGTurboServiceImpl;
Expand Down Expand Up @@ -119,6 +121,106 @@ public NDPIReader() {
canSeparateSeries = false;
}

// -- ICompressedTileReader API methods --

@Override
public int getTileRows(int no) {
FormatTools.assertId(currentId, true, 1);

int ifdIndex = getIFDIndex(getCoreIndex(), no);
//service.initialize();
return service.getTileRows();
}

@Override
public int getTileColumns(int no) {
FormatTools.assertId(currentId, true, 1);

int ifdIndex = getIFDIndex(getCoreIndex(), no);
//service.initialize();
return service.getTileColumns();
}

@Override
public byte[] openCompressedBytes(int no, int x, int y) throws FormatException, IOException {
FormatTools.assertId(currentId, true, 1);

int ifdIndex = getIFDIndex(getCoreIndex(), no);
IFD ifd = ifds.get(ifdIndex);

if (useTiffParser(ifd)) {
// TODO
return null;
}

if (initializedSeries != getCoreIndex() || initializedPlane != no) {
resetStream(ifd);
initializedSeries = getCoreIndex();
initializedPlane = no;
}
if ((x % service.getTileWidth()) != 0) {
throw new FormatException("Invalid x: " + x + " must be multiple of " +
service.getTileWidth());
}
if ((y % service.getTileHeight()) != 0) {
throw new FormatException("Invalid y: " + y + " must be multiple of " +
service.getTileHeight());
}
int tileRow = y / service.getTileHeight();
int tileColumn = x / service.getTileWidth();
return service.getCompressedTile(tileColumn, tileRow);
}

@Override
public byte[] openCompressedBytes(int no, byte[] buf, int x, int y) throws FormatException, IOException {
FormatTools.assertId(currentId, true, 1);

int ifdIndex = getIFDIndex(getCoreIndex(), no);
IFD ifd = ifds.get(ifdIndex);

if (useTiffParser(ifd)) {
// TODO
return buf;
}

if (initializedSeries != getCoreIndex() || initializedPlane != no) {
resetStream(ifd);
initializedSeries = getCoreIndex();
initializedPlane = no;
}
if ((x % service.getTileWidth()) != 0) {
throw new FormatException("Invalid x: " + x + " must be multiple of " +
service.getTileWidth());
}
if ((y % service.getTileHeight()) != 0) {
throw new FormatException("Invalid y: " + y + " must be multiple of " +
service.getTileHeight());
}
int tileRow = y / service.getTileHeight();
int tileColumn = x / service.getTileWidth();
service.getCompressedTile(buf, tileColumn, tileRow);
return buf;
}

@Override
public Codec getTileCodec(int no) throws FormatException, IOException {
FormatTools.assertId(currentId, true, 1);
int ifdIndex = getIFDIndex(getCoreIndex(), no);
IFD ifd = ifds.get(ifdIndex);
return ifd.getCompression().getCodec();
}

@Override
public CodecOptions getTileCodecOptions(int no, int x, int y) throws FormatException, IOException {
FormatTools.assertId(currentId, true, 1);
int ifdIndex = getIFDIndex(getCoreIndex(), no);
IFD ifd = ifds.get(ifdIndex);
CodecOptions options = ifd.getCompression().getCompressionCodecOptions(ifd);
options.width = getOptimalTileWidth();
options.height = getOptimalTileHeight();
return options;
}

// -- IFormatReader API methods --

/* (non-Javadoc)
Expand Down Expand Up @@ -185,56 +287,7 @@ else if (useTiffParser(ifds.get(ifdIndex))) {

if (initializedSeries != getCoreIndex() || initializedPlane != no) {
IFD ifd = ifds.get(ifdIndex);

long offset = ifd.getStripOffsets()[0];
long byteCount = ifd.getStripByteCounts()[0];
if (in != null) {
in.close();
}
in = new RandomAccessInputStream(currentId);
in.seek(offset);
in.setLength(offset + byteCount);

try {
service.close();
long[] markers = ifd.getIFDLongArray(MARKER_TAG);
long[] markerHighBytes = ifd.getIFDLongArray(MARKER_TAG_HIGH_BYTES);
if (!use64Bit) {
for (int i=0; i<markers.length; i++) {
markers[i] = markers[i] & 0xffffffffL;
}
}
else if (markerHighBytes != null) {
// 64-bit offsets expected
// markers need to be reconstructed from MARKER_TAG (lower 32 bits)
// and MARKER_TAG_HIGH_BYTES (upper 32 bits)
for (int i=0; i<markers.length; i++) {
if (i < markerHighBytes.length) {
markers[i] = markers[i] & 0xffffffffL;
markers[i] += (markerHighBytes[i] << 32);
}
}
}
else {
// 64-bit offsets expected, but upper 32 bits not found
// this can happen in sub-resolution IFDs
// try to correct for offset overflow by adding 4GB to offsets, if appropriate
LOGGER.debug("Optional tag {} missing or unreadable", MARKER_TAG_HIGH_BYTES);
for (int i=1; i<markers.length; i++) {
if (markers[i] < markers[i - 1]) {
markers[i] += (long) Math.pow(2, 32);
}
}
}
if (markers != null) {
service.setRestartMarkers(markers);
}
service.initialize(in, getSizeX(), getSizeY());
}
catch (ServiceException e) {
throw new FormatException(e);
}

resetStream(ifd);
initializedSeries = getCoreIndex();
initializedPlane = no;
}
Expand Down Expand Up @@ -314,14 +367,54 @@ public void close(boolean fileOnly) throws IOException {
@Override
public int getOptimalTileWidth() {
FormatTools.assertId(currentId, true, 1);
return 1024;

int no = 0;
int ifdIndex = getIFDIndex(getCoreIndex(), no);
IFD ifd = ifds.get(ifdIndex);
try {
if (useTiffParser(ifd)) {
// TODO:
return 1024;
}

if (initializedSeries != getCoreIndex() || initializedPlane != no) {
resetStream(ifd);
initializedSeries = getCoreIndex();
initializedPlane = no;
}
}
catch (FormatException|IOException e) {
return 1024;
}

return service.getTileWidth();
}

/* @see loci.formats.IFormatReader#getOptimalTileHeight() */
@Override
public int getOptimalTileHeight() {
FormatTools.assertId(currentId, true, 1);
return 1024;

int no = 0;
int ifdIndex = getIFDIndex(getCoreIndex(), no);
IFD ifd = ifds.get(ifdIndex);
try {
if (useTiffParser(ifd)) {
// TODO:
return 1024;
}

if (initializedSeries != getCoreIndex() || initializedPlane != no) {
resetStream(ifd);
initializedSeries = getCoreIndex();
initializedPlane = no;
}
}
catch (FormatException|IOException e) {
return 1024;
}

return service.getTileHeight();
}

// -- Internal FormatReader API methods --
Expand Down Expand Up @@ -794,4 +887,71 @@ private int getBitsPerSample(int mode) {
return -1;
}

/**
* Get the array of restart markers (offsets) for the given IFD.
*
* Reads both the marker tag and marker high bytes tag (if present)
* to build the array of 64-bit offsets.
*/
private long[] getMarkers(IFD ifd) throws FormatException {
long[] markers = ifd.getIFDLongArray(MARKER_TAG);
long[] markerHighBytes = ifd.getIFDLongArray(MARKER_TAG_HIGH_BYTES);
if (!use64Bit) {
for (int i=0; i<markers.length; i++) {
markers[i] = markers[i] & 0xffffffffL;
}
}
else if (markerHighBytes != null) {
// 64-bit offsets expected
// markers need to be reconstructed from MARKER_TAG (lower 32 bits)
// and MARKER_TAG_HIGH_BYTES (upper 32 bits)
for (int i=0; i<markers.length; i++) {
if (i < markerHighBytes.length) {
markers[i] = markers[i] & 0xffffffffL;
markers[i] += (markerHighBytes[i] << 32);
}
}
}
else {
// 64-bit offsets expected, but upper 32 bits not found
// this can happen in sub-resolution IFDs
// try to correct for offset overflow by adding 4GB to offsets, if appropriate
LOGGER.debug("Optional tag {} missing or unreadable", MARKER_TAG_HIGH_BYTES);
for (int i=1; i<markers.length; i++) {
if (markers[i] < markers[i - 1]) {
markers[i] += (long) Math.pow(2, 32);
}
}
}
return markers;
}

private void resetStream(IFD ifd) throws IOException, FormatException {
// get the size and location of the single tile
long offset = ifd.getStripOffsets()[0];
long byteCount = ifd.getStripByteCounts()[0];

// reopen the stream
if (in != null) {
in.close();
}
in = new RandomAccessInputStream(currentId);
in.seek(offset);
// don't allow seeking past the end of the tile
in.setLength(offset + byteCount);

// initialize stream for the current plane
try {
service.close();
long[] markers = getMarkers(ifd);
if (markers != null) {
service.setRestartMarkers(markers);
}
service.initialize(in, getSizeX(), getSizeY());
}
catch (ServiceException e) {
throw new FormatException(e);
}
}

}

0 comments on commit 32c9fec

Please sign in to comment.