Make cutout stream the result to save memory usage.

[ijiraq]

Using an HDUList in this way can consume significant memory.

cadctools/cadccutout/cadccutout/file_helpers/fits/fits_file_helper.py

Line 197 in 665fcb3

result_hdu_list = fits.HDUList([hdu_list[0]])

@andamian Using a Streaming HDU would help.

import numpy as np
from memory_profiler import profile



@profile
def do_cutout(f):
    fout = open('cutout.fits', 'wb')
    fits.PrimaryHDU(data=None, header=None).writeto(fout, output_verify='fix')
    fout.close()
    for i in range(1000):
        fout = open('cutout.fits', 'ab+')
        ext = np.random.choice(f[1:-1])
        x = int(np.random.random()*(ext.header['NAXIS1']) - 300)
        y = int( np.random.random()*(ext.header['NAXIS2']) - 300)
        header = ext.header
        header['NAXIS1']=300
        header['NAXIS2']=300
        shdu = fits.StreamingHDU(fout, header)
        shdu.write(ext.data[x:x+300, y:y+300])
        fout.close()

do_cutout(fits.open('1118695p.fits.fz', do_not_scale_image_data=True))

This approach uses about 100MB of RAM but does 1000 cutouts, which takes multiple GB to store.