Bintable data better support, tile compressed image (gzip and rice), additional things

.gitignore

@@ -5,6 +5,9 @@ fitswasm/test
 fitswasm/Cargo.lock
 fitswasm/target
 fitswasm/pkg
+samples/*
 fits-rs-test-files.tar*
 .DS_Store
-samples
+samples
+
+publish-test-files.sh

CHANGELOG

@@ -1,12 +1,25 @@
 # CHANGELOG.md
 
-## 0.3.0 (unreleased)
+## 0.3.0
 
 Features:
 
-- BREAKING API change. Provide an iterator over the HDU list
+- Provide an iterator over the HDU list
 - #16: Parsing of comments, history, continued, card as enum.
-- BREAKING API change. `get` method on the Header object now takes a `&str` instead of a `[u8; 8]` slice.
+- #20 Basic support of Bintable
+- #20 Faster parsing using the benefit of the Seek trait to skip data block not wanted
+- Seek to a specific pixel in the ImageData struct
+- Possibility to access the raw bytes of a binary table for cursor typed readers
+- Fits is cloneable as long as the provided reader is cloneable
+- #20 Tiled image convention for storing compressed images in FITS binary tables
+    * Compression supported, GZIP, GZIP2 and RICE on u8, i16, i32 and f32. H_compress and PLI0 are not supported
+    * Dithering techniques for floating point texture not well tested (test samples are welcome)
+    * `NULL_PIXEL_MASK` column and `ZMASKCMP` keyword is not supported
+
+Breaking changes:
+
+- The iterator over HDUs is more rust idiomatic but introduces a different usage of the API
+- `Header.get` method now takes a `&str` instead of a `[u8; 8]` slice.
 
 Bugfixes:
 

Cargo.toml

@@ -16,19 +16,31 @@ exclude = [
 ]
 
 # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
-
 [dependencies]
 byteorder = "1.4.2"
 serde = { version = "1.0.152", features = ["derive"] }
 serde_json = "1.0"
 futures = "0.3.31"
 async-trait = "0.1.66"
 quick-error = "2.0.1"
+flate2 = "1.0.35"
+log = "0.4"
 
 [dev-dependencies]
 test-case = "3.0.0"
 tokio = { version = "1.26.0", features = ["rt", "macros"]}
+image = { version = "0.25.5" }
 
 [profile.release]
 lto = true
 opt-level = 3
+
+[profile.test]
+incremental = true
+opt-level = 0
+debug = 1
+lto = true
+debug-assertions = true
+overflow-checks = true
+rpath = false
+codegen-units = 512

README.md

@@ -1,16 +1,17 @@
-Fits reader written in pure Rust
---------------------------------
+FITS file reader written in pure Rust
+-------------------------------------
 
 [![](https://img.shields.io/crates/v/fitsrs.svg)](https://crates.io/crates/fitsrs)
 [![](https://img.shields.io/crates/d/fitsrs.svg)](https://crates.io/crates/fitsrs)
 [![API Documentation on docs.rs](https://docs.rs/fitsrs/badge.svg)](https://docs.rs/fitsrs/)
 ![testing CI](https://github.com/cds-astro/fitsrs/actions/workflows/rust.yml/badge.svg)
 
-This crate is under development, it was initiated for reading fits HiPS tile, i.e. generated from hipsgen.
+This crate is under development, it was initiated for reading FITS images mapped onto HEALPix cell in the sky (See the [HiPS IVOA](https://www.ivoa.net/documents/HiPS/) standard) for using inside the [Aladin Lite](https://github.com/cds-astro/aladin-lite) web sky atlas.
 
-This fits parser only supports image data (not tables), and does not know anything about WCS parsing.
-For WCS parsing, see [wcsrs](https://github.com/cds-astro/wcs-rs).
-This parser is able to parse extension HDUs. Ascii tables and binary tables are still not properly parsed, only the list bytes of their data block can be retrieved but no interpretation/parsing is done on it.
+Currently, fitsrs supports reading multiple HDU and is mainly dedicated to image extension reading.
+For interpreting WCS keywords, see [wcs-rs](https://github.com/cds-astro/wcs-rs).
+A very new support of binary table extension has been added. This has been done mainly for supporting the tiled image convention for storing compressed images in binary tables. This stores tile images inside variable length arrays of a binary table.
+The ASCII table extension parsing has not been implemented but it is possible to get an iterator over the data bytes as well as its mandatory cards from the header.
 
 Contributing
 ------------
@@ -34,89 +35,41 @@ To Do list
 ----------
 
 * [X] Support single typed data block (i.e. image type data)
-* [X] Single HDU parsing, header and data units
-* [X] Support big fits file parsing that may not fit in memory (iterator usage)
-* [X] Async reading (experimental and not tested)
-* [X] Keep CARD comment
-* [ ] Support compressed fits files (https://fits.gsfc.nasa.gov/registry/tilecompression.html)
-* [ ] Support data table (each column can have a specific types)
-* [X] Support of multiple HDU, fits extensions (in progress, only the header is parsed)
-* [ ] WCS parsing, see [wcsrs](https://github.com/cds-astro/wcs-rs)
+* [X] Single HDU parsing, header and data units 
+* [X] Support FITS files that may not fit in memory (iterator, possibility to seek directly to a specific pixel index/row)
+* [X] Async reading (requires to read the whole data. Seeking is not possible)
+* [X] Parsing of comments, history, continued, cards.
+* [X] Keep all the cards in the original order
+* [X] Basic support of Bintable
+* [X] Tiled image convention for storing compressed images in FITS binary tables
+    - [X] Compression supported, GZIP, GZIP2 and RICE on u8, i16, i32 and f32.
+    - [ ] H_compress and PLI0 are not supported
+    - [X] Dithering techniques for floating point images. Not well tested (test samples are welcome)
+    - [ ] `NULL_PIXEL_MASK` column and `ZMASKCMP` keyword is not supported
+* [ ] FITS writer/serializer
+* [ ] ASCII table extension parsing
+* [ ] Tile-compressed in binary table files (https://fits.gsfc.nasa.gov/registry/tilecompression.html). Only RICE and GZIP supported
+* [X] Support of multiple HDU. Image and binary tables extension support. Provide an idiomatic Rust iterator over the list of HDU.
+* [X] WCS parsing, see [wcs-rs](https://github.com/cds-astro/wcs-rs)
 
-Example
-----------
-
-For files that can fit in memory
-
-```rust
-use std::fs::File;
-use std::io::Cursor;
-use std::io::Read;
-use fitsrs::{Fits, HDU};
-use fitsrs::{hdu::header::Xtension, hdu::data::Data};
-
-let mut f = File::open("samples/fits.gsfc.nasa.gov/EUVE.fits").unwrap();
-
-let mut buf = Vec::new();
-f.read_to_end(&mut buf).unwrap();
-let reader = Cursor::new(&buf[..]);
+> [!WARNING]
+> Features not done are not planned to be done. A work for supporting them can be done only if people have use cases for those i.e. it is only at user's requests. The FITS standard and its conventions are massive and it is a huge work to support all the cases and sub cases.
 
-let mut hdu_list = Fits::from_reader(reader);
 
-// Access the HDU extensions
-while let Some(Ok(hdu)) = hdu_list.next() {
-    match hdu {
-        HDU::Primary(_) => (),
-        HDU::XImage(hdu) => {
-            let xtension = hdu.get_header().get_xtension();
+License
+-------
 
-            let naxis1 = *xtension.get_naxisn(1).unwrap() as usize;
-            let naxis2 = *xtension.get_naxisn(2).unwrap() as usize;
+fitsrs has the double the MIT/Apache-2.0 license.
 
-            let num_pixels = naxis2 * naxis1;
+It uses code adapted from the famous [CFITSIO](https://github.com/HEASARC/cfitsio/blob/main/licenses/License.txt) library.Especially the RICE compression/decompression source code has been ported from the original cfitsio [code](https://github.com/HEASARC/cfitsio/blob/main/ricecomp.c) to Rust.
 
-            match hdu.get_data(&mut hdu_list) {
-                Data::U8(mem) => assert_eq!(num_pixels, mem.len()),
-                Data::I16(mem) => assert_eq!(num_pixels, mem.len()),
-                Data::I32(mem) => assert_eq!(num_pixels, mem.len()),
-                Data::I64(mem) => assert_eq!(num_pixels, mem.len()),
-                Data::F32(mem) => assert_eq!(num_pixels, mem.len()),
-                Data::F64(mem) => assert_eq!(num_pixels, mem.len()),
-            }
-        },
-        HDU::XBinaryTable(hdu) => {
-            let num_bytes = hdu.get_header()
-                .get_xtension()
-                .get_num_bytes_data_block();
-
-            match hdu.get_data(&mut hdu_list) {
-                Data::U8(mem) => assert_eq!(num_bytes as usize, mem.len()),
-                _ => unreachable!()
-            }
-        },
-        HDU::XASCIITable(hdu) => {
-            let num_bytes = hdu.get_header()
-                .get_xtension()
-                .get_num_bytes_data_block();
-
-            match hdu.get_data(&mut hdu_list) {
-                Data::U8(mem) => assert_eq!(num_bytes as usize, mem.len()),
-                _ => unreachable!()
-            }
-        },
-    }
-}
-```
-
-For BufReader
+Example
+----------
 
 ```rust
 use std::fs::File;
 use std::io::Cursor;
-use fitsrs::hdu::HDU;
-use fitsrs::hdu::data::DataIter;
-use fitsrs::fits::Fits;
-use fitsrs::hdu::header::Xtension;
+use fitsrs::{Fits, ImageData, HDU, hdu::header::Xtension};
 
 use std::io::{BufReader, Read};
 
@@ -137,49 +90,52 @@ while let Some(Ok(hdu)) = hdu_list.next() {
 
             let num_pixels = (naxis2 * naxis1) as usize;
 
-            match hdu.get_data(&mut hdu_list) {
-                DataIter::U8(it) => {
+            match hdu_list.get_data(hdu) {
+                ImageData::U8(it) => {
                     let data = it.collect::<Vec<_>>();
                     assert_eq!(num_pixels, data.len())
                 },
-                DataIter::I16(it) => {
+                ImageData::I16(it) => {
                     let data = it.collect::<Vec<_>>();
                     assert_eq!(num_pixels, data.len())
                 },
-                DataIter::I32(it) => {
+                ImageData::I32(it) => {
                     let data = it.collect::<Vec<_>>();
                     assert_eq!(num_pixels, data.len())
                 },
-                DataIter::I64(it) => {
+                ImageData::I64(it) => {
                     let data = it.collect::<Vec<_>>();
                     assert_eq!(num_pixels, data.len())
                 },
-                DataIter::F32(it) => {
+                ImageData::F32(it) => {
                     let data = it.collect::<Vec<_>>();
                     assert_eq!(num_pixels, data.len())
                 },
-                DataIter::F64(it) => {
+                ImageData::F64(it) => {
                     let data = it.collect::<Vec<_>>();
                     assert_eq!(num_pixels, data.len())
                 },
             }
         },
         HDU::XBinaryTable(hdu) => {
-            let num_bytes = hdu.get_header()
+            let num_rows = hdu.get_header()
                 .get_xtension()
-                .get_num_bytes_data_block();
+                .get_num_rows();
 
-            let it_bytes = hdu.get_data(&mut hdu_list);
-            let data = it_bytes.collect::<Vec<_>>();
-            assert_eq!(num_bytes as usize, data.len());
+            let rows = hdu_list.get_data(hdu)
+                .row_iter()
+                .collect::<Vec<_>>();
+
+            assert_eq!(num_rows, rows.len());
         },
         HDU::XASCIITable(hdu) => {
             let num_bytes = hdu.get_header()
                 .get_xtension()
                 .get_num_bytes_data_block();
 
-            let it_bytes = hdu.get_data(&mut hdu_list);
-            let data = it_bytes.collect::<Vec<_>>();
+            let data = hdu_list.get_data(hdu)
+                .collect::<Vec<_>>();
+
             assert_eq!(num_bytes as usize, data.len());
         },
     }
@@ -217,7 +173,7 @@ async fn parse_fits_async() {
 
                 let num_pixels = naxis2 * naxis1;
 
-                match hdu.get_data(&mut hdu_list) {
+                match hdu_list.get_data(hdu) {
                     Stream::U8(st) => {
                         let data = st.collect::<Vec<_>>().await;
                         assert_eq!(num_pixels, data.len())
@@ -249,7 +205,7 @@ async fn parse_fits_async() {
                     .get_xtension()
                     .get_num_bytes_data_block();
 
-                let it_bytes = hdu.get_data(&mut hdu_list);
+                let it_bytes = hdu_list.get_data(hdu);
                 let data = it_bytes.collect::<Vec<_>>().await;
                 assert_eq!(num_bytes as usize, data.len());
             },
@@ -258,7 +214,7 @@ async fn parse_fits_async() {
                     .get_xtension()
                     .get_num_bytes_data_block();
 
-                let it_bytes = xhdu.get_data(&mut hdu_list);
+                let it_bytes = hdu_list.get_data(hdu);
                 let data = it_bytes.collect::<Vec<_>>().await;
                 assert_eq!(num_bytes as usize, data.len());
             },

fitswasm/src/lib.rs

@@ -4,7 +4,7 @@ extern crate wasm_bindgen;
 // see https://rustwasm.github.io/wasm-bindgen/
 use wasm_bindgen::prelude::*;
 
-use fitsrs::{BitpixValue, DataType, FITSHeaderKeyword, FITSKeywordValue, Fits};
+use fitsrs::{Bitpix, DataType, FITSHeaderKeyword, FITSKeywordValue, Fits};
 use js_sys::{Float32Array, Float64Array, Int16Array, Int32Array, Uint8Array};
 
 #[wasm_bindgen(js_name = read)]
@@ -46,12 +46,12 @@ pub fn read(bytes: Vec<u8>) -> Result<js_sys::Object, JsValue> {
             FITSHeaderKeyword::Simple => (String::from("SIMPLE"), JsValue::undefined()),
             FITSHeaderKeyword::Bitpix(b) => {
                 let value = match b {
-                    BitpixValue::F32 => -32,
-                    BitpixValue::F64 => -64,
-                    BitpixValue::U8 => 8,
-                    BitpixValue::I16 => 16,
-                    BitpixValue::I32 => 32,
-                    BitpixValue::I64 => 64,
+                    Bitpix::F32 => -32,
+                    Bitpix::F64 => -64,
+                    Bitpix::U8 => 8,
+                    Bitpix::I16 => 16,
+                    Bitpix::I32 => 32,
+                    Bitpix::I64 => 64,
                 };
 
                 (String::from("BITPIX"), JsValue::from_f64(value as f64))