alpha.cpp

/* -----------------------------------------------------------------------------

	Copyright (c) 2006 Simon Brown                          si@sjbrown.co.uk

	Permission is hereby granted, free of charge, to any person obtaining
	a copy of this software and associated documentation files (the 
	"Software"), to	deal in the Software without restriction, including
	without limitation the rights to use, copy, modify, merge, publish,
	distribute, sublicense, and/or sell copies of the Software, and to 
	permit persons to whom the Software is furnished to do so, subject to 
	the following conditions:

	The above copyright notice and this permission notice shall be included
	in all copies or substantial portions of the Software.

	THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
	OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF 
	MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
	IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY 
	CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, 
	TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE 
	SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
	
   -------------------------------------------------------------------------- */
   
#include "alpha.h"
#include <limits.h>
#include <algorithm>

namespace squish {

static int FloatToInt( float a, int limit )
{
	// use ANSI round-to-zero behaviour to get round-to-nearest
	int i = ( int )( a + 0.5f );

	// clamp to the limit
	if( i < 0 )
		i = 0;
	else if( i > limit )
		i = limit; 

	// done
	return i;
}

void CompressAlphaDxt3( u8 const* rgba, int mask, void* block )
{
	u8* bytes = reinterpret_cast< u8* >( block );
	
	// quantise and pack the alpha values pairwise
	for( int i = 0; i < 8; ++i )
	{
		// quantise down to 4 bits
		float alpha1 = ( float )rgba[8*i + 3] * ( 15.0f/255.0f );
		float alpha2 = ( float )rgba[8*i + 7] * ( 15.0f/255.0f );
		int quant1 = FloatToInt( alpha1, 15 );
		int quant2 = FloatToInt( alpha2, 15 );
		
		// set alpha to zero where masked
		int bit1 = 1 << ( 2*i );
		int bit2 = 1 << ( 2*i + 1 );
		if( ( mask & bit1 ) == 0 )
			quant1 = 0;
		if( ( mask & bit2 ) == 0 )
			quant2 = 0;

		// pack into the byte
		bytes[i] = ( u8 )( quant1 | ( quant2 << 4 ) );
	}
}

void DecompressAlphaDxt3( u8* rgba, void const* block )
{
	u8 const* bytes = reinterpret_cast< u8 const* >( block );
	
	// unpack the alpha values pairwise
	for( int i = 0; i < 8; ++i )
	{
		// quantise down to 4 bits
		u8 quant = bytes[i];
		
		// unpack the values
		u8 lo = quant & 0x0f;
		u8 hi = quant & 0xf0;

		// convert back up to bytes
		rgba[8*i + 3] = lo | ( lo << 4 );
		rgba[8*i + 7] = hi | ( hi >> 4 );
	}
}

static void FixRange( int& min, int& max, int steps )
{
	if( max - min < steps )
		max = std::min( min + steps, 255 );
	if( max - min < steps )
		min = std::max( 0, max - steps );
}

static int FitCodes( u8 const* rgba, int mask, u8 const* codes, u8* indices )
{
	// fit each alpha value to the codebook
	int err = 0;
	for( int i = 0; i < 16; ++i )
	{
		// check this pixel is valid
		int bit = 1 << i;
		if( ( mask & bit ) == 0 )
		{
			// use the first code
			indices[i] = 0;
			continue;
		}
		
		// find the least error and corresponding index
		int value = rgba[4*i + 3];
		int least = INT_MAX;
		int index = 0;
		for( int j = 0; j < 8; ++j )
		{
			// get the squared error from this code
			int dist = ( int )value - ( int )codes[j];
			dist *= dist;
			
			// compare with the best so far
			if( dist < least )
			{
				least = dist;
				index = j;
			}
		}
		
		// save this index and accumulate the error
		indices[i] = ( u8 )index;
		err += least;
	}
	
	// return the total error
	return err;
}

static void WriteAlphaBlock( int alpha0, int alpha1, u8 const* indices, void* block )
{
	u8* bytes = reinterpret_cast< u8* >( block );
	
	// write the first two bytes
	bytes[0] = ( u8 )alpha0;
	bytes[1] = ( u8 )alpha1;
	
	// pack the indices with 3 bits each
	u8* dest = bytes + 2;
	u8 const* src = indices;
	for( int i = 0; i < 2; ++i )
	{
		// pack 8 3-bit values
		int value = 0;
		for( int j = 0; j < 8; ++j )
		{
			int index = *src++;
			value |= ( index << 3*j );
		}
			
		// store in 3 bytes
		for( int j = 0; j < 3; ++j )
		{
			int byte = ( value >> 8*j ) & 0xff;
			*dest++ = ( u8 )byte;
		}
	}
}

static void WriteAlphaBlock5( int alpha0, int alpha1, u8 const* indices, void* block )
{
	// check the relative values of the endpoints
	if( alpha0 > alpha1 )
	{
		// swap the indices
		u8 swapped[16];
		for( int i = 0; i < 16; ++i )
		{
			u8 index = indices[i];
			if( index == 0 )
				swapped[i] = 1;
			else if( index == 1 )
				swapped[i] = 0;
			else if( index <= 5 )
				swapped[i] = 7 - index;
			else 
				swapped[i] = index;
		}
		
		// write the block
		WriteAlphaBlock( alpha1, alpha0, swapped, block );
	}
	else
	{
		// write the block
		WriteAlphaBlock( alpha0, alpha1, indices, block );
	}	
}

static void WriteAlphaBlock7( int alpha0, int alpha1, u8 const* indices, void* block )
{
	// check the relative values of the endpoints
	if( alpha0 < alpha1 )
	{
		// swap the indices
		u8 swapped[16];
		for( int i = 0; i < 16; ++i )
		{
			u8 index = indices[i];
			if( index == 0 )
				swapped[i] = 1;
			else if( index == 1 )
				swapped[i] = 0;
			else
				swapped[i] = 9 - index;
		}
		
		// write the block
		WriteAlphaBlock( alpha1, alpha0, swapped, block );
	}
	else
	{
		// write the block
		WriteAlphaBlock( alpha0, alpha1, indices, block );
	}	
}

void CompressAlphaDxt5( u8 const* rgba, int mask, void* block )
{
	// get the range for 5-alpha and 7-alpha interpolation
	int min5 = 255;
	int max5 = 0;
	int min7 = 255;
	int max7 = 0;
	for( int i = 0; i < 16; ++i )
	{
		// check this pixel is valid
		int bit = 1 << i;
		if( ( mask & bit ) == 0 )
			continue;

		// incorporate into the min/max
		int value = rgba[4*i + 3];
		if( value < min7 )
			min7 = value;
		if( value > max7 )
			max7 = value;
		if( value != 0 && value < min5 )
			min5 = value;
		if( value != 255 && value > max5 )
			max5 = value;
	}
	
	// handle the case that no valid range was found
	if( min5 > max5 )
		min5 = max5;
	if( min7 > max7 )
		min7 = max7;
		
	// fix the range to be the minimum in each case
	FixRange( min5, max5, 5 );
	FixRange( min7, max7, 7 );
	
	// set up the 5-alpha code book
	u8 codes5[8];
	codes5[0] = ( u8 )min5;
	codes5[1] = ( u8 )max5;
	for( int i = 1; i < 5; ++i )
		codes5[1 + i] = ( u8 )( ( ( 5 - i )*min5 + i*max5 )/5 );
	codes5[6] = 0;
	codes5[7] = 255;
	
	// set up the 7-alpha code book
	u8 codes7[8];
	codes7[0] = ( u8 )min7;
	codes7[1] = ( u8 )max7;
	for( int i = 1; i < 7; ++i )
		codes7[1 + i] = ( u8 )( ( ( 7 - i )*min7 + i*max7 )/7 );
		
	// fit the data to both code books
	u8 indices5[16];
	u8 indices7[16];
	int err5 = FitCodes( rgba, mask, codes5, indices5 );
	int err7 = FitCodes( rgba, mask, codes7, indices7 );
	
	// save the block with least error
	if( err5 <= err7 )
		WriteAlphaBlock5( min5, max5, indices5, block );
	else
		WriteAlphaBlock7( min7, max7, indices7, block );
}

void DecompressAlphaDxt5( u8* rgba, void const* block )
{
	// get the two alpha values
	u8 const* bytes = reinterpret_cast< u8 const* >( block );
	int alpha0 = bytes[0];
	int alpha1 = bytes[1];
	
	// compare the values to build the codebook
	u8 codes[8];
	codes[0] = ( u8 )alpha0;
	codes[1] = ( u8 )alpha1;
	if( alpha0 <= alpha1 )
	{
		// use 5-alpha codebook
		for( int i = 1; i < 5; ++i )
			codes[1 + i] = ( u8 )( ( ( 5 - i )*alpha0 + i*alpha1 )/5 );
		codes[6] = 0;
		codes[7] = 255;
	}
	else
	{
		// use 7-alpha codebook
		for( int i = 1; i < 7; ++i )
			codes[1 + i] = ( u8 )( ( ( 7 - i )*alpha0 + i*alpha1 )/7 );
	}
	
	// decode the indices
	u8 indices[16];
	u8 const* src = bytes + 2;
	u8* dest = indices;
	for( int i = 0; i < 2; ++i )
	{
		// grab 3 bytes
		int value = 0;
		for( int j = 0; j < 3; ++j )
		{
			int byte = *src++;
			value |= ( byte << 8*j );
		}
		
		// unpack 8 3-bit values from it
		for( int j = 0; j < 8; ++j )
		{
			int index = ( value >> 3*j ) & 0x7;
			*dest++ = ( u8 )index;
		}
	}
	
	// write out the indexed codebook values
	for( int i = 0; i < 16; ++i )
		rgba[4*i + 3] = codes[indices[i]];
}

} // namespace squish