Skip to content

Commit

Permalink
Merge pull request #141 from DimitriPapadopoulos/pedantic
Browse files Browse the repository at this point in the history 
Pedantic suggestions
  • Loading branch information
@kiyo-masui
kiyo-masui authored Sep 26, 2024
2 parents dbe3938 + a3d3d15 commit 4e37682
Show file tree
Hide file tree
Showing 11 changed files with 66 additions and 61 deletions.
28 changes: 14 additions & 14 deletions bitshuffle/ext.pyx
View file
Original file line number Diff line number Diff line change
Expand Up @@ -494,7 +494,7 @@ IF ZSTD_SUPPORT:
@cython.wraparound(False)
def compress_zstd(np.ndarray arr not None, int block_size=0, int comp_lvl=1):
"""Bitshuffle then compress an array using ZSTD.
Parameters
----------
arr : numpy array
Expand All @@ -504,14 +504,14 @@ IF ZSTD_SUPPORT:
automatically.
comp_lvl : positive integer
Compression level applied by ZSTD
Returns
-------
out : array with np.uint8 data type
Buffer holding compressed data.
"""

cdef int ii, size, itemsize, count=0
shape = (arr.shape[i] for i in range(arr.ndim))
if not arr.flags['C_CONTIGUOUS']:
Expand All @@ -520,12 +520,12 @@ IF ZSTD_SUPPORT:
size = arr.size
dtype = arr.dtype
itemsize = dtype.itemsize

max_out_size = bshuf_compress_zstd_bound(size, itemsize, block_size)

cdef np.ndarray out
out = np.empty(max_out_size, dtype=np.uint8)

cdef np.ndarray[dtype=np.uint8_t, ndim=1, mode="c"] arr_flat
arr_flat = arr.view(np.uint8).ravel()
cdef np.ndarray[dtype=np.uint8_t, ndim=1, mode="c"] out_flat
Expand All @@ -540,12 +540,12 @@ IF ZSTD_SUPPORT:
excp = RuntimeError(msg % count, count)
raise excp
return out[:count]

@cython.boundscheck(False)
@cython.wraparound(False)
def decompress_zstd(np.ndarray arr not None, shape, dtype, int block_size=0):
"""Decompress a buffer using ZSTD then bitunshuffle it yielding an array.
Parameters
----------
arr : numpy array
Expand All @@ -559,24 +559,24 @@ IF ZSTD_SUPPORT:
block_size : positive integer
Block size in number of elements. Must match value used for
compression.
Returns
-------
out : numpy array with shape *shape* and data type *dtype*
Decompressed data.
"""

cdef int ii, size, itemsize, count=0
if not arr.flags['C_CONTIGUOUS']:
msg = "Input array must be C-contiguous."
raise ValueError(msg)
size = np.prod(shape)
itemsize = dtype.itemsize

cdef np.ndarray out
out = np.empty(tuple(shape), dtype=dtype)

cdef np.ndarray[dtype=np.uint8_t, ndim=1, mode="c"] arr_flat
arr_flat = arr.view(np.uint8).ravel()
cdef np.ndarray[dtype=np.uint8_t, ndim=1, mode="c"] out_flat
Expand Down
2 changes: 0 additions & 2 deletions bitshuffle/h5.pyx
View file
Original file line number Diff line number Diff line change
Expand Up @@ -231,5 +231,3 @@ def create_bitshuffle_compressed_dataset(parent, name, shape, dtype,
filter_opts=filter_opts, maxshape=maxshape,
fillvalue=fillvalue, track_times=track_times)
return dset_id


13 changes: 9 additions & 4 deletions src/bitshuffle.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -23,12 +23,17 @@


// Macros.
#define CHECK_ERR_FREE_LZ(count, buf) if (count < 0) { \
free(buf); return count - 1000; }
#define CHECK_ERR_FREE_LZ(count, buf) \
do { \
if ((count) < 0) { \
free(buf); \
return (count) - 1000; \
} \
} while (0)


/* Bitshuffle and compress a single block. */
int64_t bshuf_compress_lz4_block(ioc_chain *C_ptr, \
int64_t bshuf_compress_lz4_block(ioc_chain *C_ptr,
const size_t size, const size_t elem_size, const int option) {

int64_t nbytes, count;
Expand Down Expand Up @@ -115,7 +120,7 @@ int64_t bshuf_decompress_lz4_block(ioc_chain *C_ptr,

#ifdef ZSTD_SUPPORT
/* Bitshuffle and compress a single block. */
int64_t bshuf_compress_zstd_block(ioc_chain *C_ptr, \
int64_t bshuf_compress_zstd_block(ioc_chain *C_ptr,
const size_t size, const size_t elem_size, const int comp_lvl) {

int64_t nbytes, count;
Expand Down
20 changes: 10 additions & 10 deletions src/bitshuffle_core.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -50,7 +50,7 @@ typedef size_t omp_size_t;
#endif

// Macros.
#define CHECK_MULT_EIGHT(n) if (n % 8) return -80;
#define CHECK_MULT_EIGHT(n) do { if ((n) % 8) return -80; } while (0)
#define MAX(X,Y) ((X) > (Y) ? (X) : (Y))


Expand Down Expand Up @@ -309,7 +309,7 @@ int64_t bshuf_trans_byte_bitrow_scal(const void* in, void* out, const size_t siz
for (jj = 0; jj < elem_size; jj++) {
for (ii = 0; ii < nbyte_row; ii++) {
for (kk = 0; kk < 8; kk++) {
out_b[ii * 8 * elem_size + jj * 8 + kk] = \
out_b[ii * 8 * elem_size + jj * 8 + kk] =
in_b[(jj * 8 + kk) * nbyte_row + ii];
}
}
Expand All @@ -319,7 +319,7 @@ int64_t bshuf_trans_byte_bitrow_scal(const void* in, void* out, const size_t siz


/* Shuffle bits within the bytes of eight element blocks. */
int64_t bshuf_shuffle_bit_eightelem_scal(const void* in, void* out, \
int64_t bshuf_shuffle_bit_eightelem_scal(const void* in, void* out,
const size_t size, const size_t elem_size) {

const char *in_b;
Expand Down Expand Up @@ -890,7 +890,7 @@ int64_t bshuf_shuffle_bit_eightelem_NEON(const void* in, void* out, const size_t
/* ---- Worker code that uses SSE2 ----
*
* The following code makes use of the SSE2 instruction set and specialized
* 16 byte registers. The SSE2 instructions are present on modern x86
* 16 byte registers. The SSE2 instructions are present on modern x86
* processors. The first Intel processor microarchitecture supporting SSE2 was
* Pentium 4 (2000).
*
Expand Down Expand Up @@ -1795,7 +1795,7 @@ int64_t bshuf_untrans_bit_elem_AVX512(const void* in, void* out, const size_t si

/* ---- Drivers selecting best instruction set at compile time. ---- */

int64_t bshuf_trans_bit_elem(const void* in, void* out, const size_t size,
int64_t bshuf_trans_bit_elem(const void* in, void* out, const size_t size,
const size_t elem_size) {

int64_t count;
Expand All @@ -1814,7 +1814,7 @@ int64_t bshuf_trans_bit_elem(const void* in, void* out, const size_t size,
}


int64_t bshuf_untrans_bit_elem(const void* in, void* out, const size_t size,
int64_t bshuf_untrans_bit_elem(const void* in, void* out, const size_t size,
const size_t elem_size) {

int64_t count;
Expand All @@ -1837,7 +1837,7 @@ int64_t bshuf_untrans_bit_elem(const void* in, void* out, const size_t size,

/* Wrap a function for processing a single block to process an entire buffer in
* parallel. */
int64_t bshuf_blocked_wrap_fun(bshufBlockFunDef fun, const void* in, void* out, \
int64_t bshuf_blocked_wrap_fun(bshufBlockFunDef fun, const void* in, void* out,
const size_t size, const size_t elem_size, size_t block_size, const int option) {

omp_size_t ii = 0;
Expand Down Expand Up @@ -1895,7 +1895,7 @@ int64_t bshuf_blocked_wrap_fun(bshufBlockFunDef fun, const void* in, void* out,


/* Bitshuffle a single block. */
int64_t bshuf_bitshuffle_block(ioc_chain *C_ptr, \
int64_t bshuf_bitshuffle_block(ioc_chain *C_ptr,
const size_t size, const size_t elem_size, const int option) {

size_t this_iter;
Expand All @@ -1904,7 +1904,7 @@ int64_t bshuf_bitshuffle_block(ioc_chain *C_ptr, \
int64_t count;



in = ioc_get_in(C_ptr, &this_iter);
ioc_set_next_in(C_ptr, &this_iter,
(void*) ((char*) in + size * elem_size));
Expand All @@ -1918,7 +1918,7 @@ int64_t bshuf_bitshuffle_block(ioc_chain *C_ptr, \


/* Bitunshuffle a single block. */
int64_t bshuf_bitunshuffle_block(ioc_chain* C_ptr, \
int64_t bshuf_bitunshuffle_block(ioc_chain* C_ptr,
const size_t size, const size_t elem_size, const int option) {


Expand Down
8 changes: 7 additions & 1 deletion src/bitshuffle_internals.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -38,7 +38,13 @@


// Macros.
#define CHECK_ERR_FREE(count, buf) if (count < 0) { free(buf); return count; }
#define CHECK_ERR_FREE(count, buf) \
do { \
if ((count) < 0) { \
free(buf); \
return (count); \
} \
} while (0)


#ifdef __cplusplus
Expand Down
26 changes: 13 additions & 13 deletions src/bshuf_h5filter.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -56,7 +56,7 @@ herr_t bshuf_h5_set_local(hid_t dcpl, hid_t type, hid_t space){

elem_size = H5Tget_size(type);
if(elem_size <= 0) {
PUSH_ERR("bshuf_h5_set_local", H5E_CALLBACK,
PUSH_ERR("bshuf_h5_set_local", H5E_CALLBACK,
"Invalid element size.");
return -1;
}
Expand All @@ -83,7 +83,7 @@ herr_t bshuf_h5_set_local(hid_t dcpl, hid_t type, hid_t space){
break;
#endif
default:
PUSH_ERR("bshuf_h5_set_local", H5E_CALLBACK,
PUSH_ERR("bshuf_h5_set_local", H5E_CALLBACK,
"Invalid bitshuffle compression.");
}
}
Expand All @@ -108,23 +108,23 @@ size_t bshuf_h5_filter(unsigned int flags, size_t cd_nelmts,
void *out_buf;

if (cd_nelmts < 3) {
PUSH_ERR("bshuf_h5_filter", H5E_CALLBACK,
PUSH_ERR("bshuf_h5_filter", H5E_CALLBACK,
"Not enough parameters.");
return 0;
}
elem_size = cd_values[2];
#ifdef ZSTD_SUPPORT
const int comp_lvl = cd_values[5];
const int comp_lvl = cd_values[5];
#endif

// User specified block size.
if (cd_nelmts > 3) block_size = cd_values[3];

if (block_size == 0) block_size = bshuf_default_block_size(elem_size);

#ifndef ZSTD_SUPPORT
if (cd_nelmts > 4 && (cd_values[4] == BSHUF_H5_COMPRESS_ZSTD)) {
PUSH_ERR("bshuf_h5_filter", H5E_CALLBACK,
PUSH_ERR("bshuf_h5_filter", H5E_CALLBACK,
"ZSTD compression filter chosen but ZSTD support not installed.");
return 0;
}
Expand All @@ -145,12 +145,12 @@ size_t bshuf_h5_filter(unsigned int flags, size_t cd_nelmts,
nbytes_uncomp = nbytes;
// Pick which compressions library to use
if(cd_values[4] == BSHUF_H5_COMPRESS_LZ4) {
buf_size_out = bshuf_compress_lz4_bound(nbytes_uncomp / elem_size,
buf_size_out = bshuf_compress_lz4_bound(nbytes_uncomp / elem_size,
elem_size, block_size) + 12;
}
#ifdef ZSTD_SUPPORT
else if (cd_values[4] == BSHUF_H5_COMPRESS_ZSTD) {
buf_size_out = bshuf_compress_zstd_bound(nbytes_uncomp / elem_size,
buf_size_out = bshuf_compress_zstd_bound(nbytes_uncomp / elem_size,
elem_size, block_size) + 12;
}
#endif
Expand All @@ -162,15 +162,15 @@ size_t bshuf_h5_filter(unsigned int flags, size_t cd_nelmts,

// TODO, remove this restriction by memcopying the extra.
if (nbytes_uncomp % elem_size) {
PUSH_ERR("bshuf_h5_filter", H5E_CALLBACK,
PUSH_ERR("bshuf_h5_filter", H5E_CALLBACK,
"Non integer number of elements.");
return 0;
}
size = nbytes_uncomp / elem_size;

out_buf = malloc(buf_size_out);
if (out_buf == NULL) {
PUSH_ERR("bshuf_h5_filter", H5E_CALLBACK,
PUSH_ERR("bshuf_h5_filter", H5E_CALLBACK,
"Could not allocate output buffer.");
return 0;
}
Expand Down Expand Up @@ -199,16 +199,16 @@ size_t bshuf_h5_filter(unsigned int flags, size_t cd_nelmts,
bshuf_write_uint32_BE((char*) out_buf + 8, block_size * elem_size);
if(cd_values[4] == BSHUF_H5_COMPRESS_LZ4) {
err = bshuf_compress_lz4(in_buf, (char*) out_buf + 12, size,
elem_size, block_size);
elem_size, block_size);
}
#ifdef ZSTD_SUPPORT
else if (cd_values[4] == BSHUF_H5_COMPRESS_ZSTD) {
err = bshuf_compress_zstd(in_buf, (char*) out_buf + 12, size,
elem_size, block_size, comp_lvl);
elem_size, block_size, comp_lvl);
}
#endif
nbytes_out = err + 12;
}
}
} else {
if (flags & H5Z_FLAG_REVERSE) {
// Bit unshuffle.
Expand Down
1 change: 0 additions & 1 deletion src/bshuf_h5plugin.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -16,4 +16,3 @@

H5PL_type_t H5PLget_plugin_type(void) {return H5PL_TYPE_FILTER;}
const void* H5PLget_plugin_info(void) {return bshuf_H5Filter;}

Loading

0 comments on commit 4e37682

Please sign in to comment.