Zlib compress/decompress > 64K buffer fails

Zlib compress/decompress > 64K buffer fails

I have taken the zlib compress/decompress pair out of zlib and I find that they work perfectly on buffer sizes up to 64K, but fail on anything larger. I tried doing the full initialization but found nothing had any impact. I just want to compress a buffer of memory and be able to decompress it, but 64K blocks create a lot of overhead for a 100M/sec application.

I'm using 6.0.0.127 samples, ipp 6.0.1.070 and compiling with VC2008 9.0.20729 sp.

Here is the compress/decomp and handler code

int compress2 (
Bytef *dest,
uLongf *destLen,
const Bytef *source,
uLong sourceLen,
int level)
{
z_stream stream;
int err;

stream.next_in = (Bytef*)source;
stream.avail_in = (uInt)sourceLen;

stream.next_out = dest;
stream.avail_out = (uInt)*destLen;
if ((uLong)stream.avail_out != *destLen)
return Z_BUF_ERROR;

stream.zalloc = (alloc_func)0;
stream.zfree = (free_func)0;
stream.opaque = (voidpf)0;

err = deflateInit2(&stream, level,
Z_DEFLATED, MAX_WBITS,
MAX_MEM_LEVEL, Z_DEFAULT_STRATEGY);

if (err != Z_OK)
return err;

err = deflate(&stream, Z_FINISH);

if (err != Z_STREAM_END)
{
printf("\nFAILED to completely compress data stream");
deflateEnd(&stream);
return err == Z_OK ? Z_BUF_ERROR : err;
}
*destLen = stream.total_out;

err = deflateEnd(&stream);
return err;
}

int uncompress (
Bytef *dest,
uLongf *destLen,
const Bytef *source,
uLong sourceLen)
{
z_stream stream;
int err;

stream.next_in = (Bytef*)source;
stream.avail_in = (uInt)sourceLen;

stream.next_out = dest;
stream.avail_out = (uInt)*destLen;
if ((uLong)stream.avail_out != *destLen)
return Z_BUF_ERROR;

stream.zalloc = (alloc_func)0;
stream.zfree = (free_func)0;
stream.opaque = (voidpf)0;

err = inflateInit(&stream);
if (err != Z_OK)
return err;

err = inflate(&stream, Z_FINISH);
if (err != Z_STREAM_END)
{
inflateEnd(&stream);
if (err == Z_NEED_DICT || (err == Z_BUF_ERROR && stream.avail_in == 0))
return Z_DATA_ERROR;
return err;
}

*destLen = stream.total_out;

err = inflateEnd(&stream);
return err;
}

static const int BUFSIZE = 1024*64; // any larger and it fails
void TEST(const char * pInFileName)
{
Ipp8u * src = ippsMalloc_8u(BUFSIZE);

uLong outputSize = compressBound(BUFSIZE);

outputSize += 4096;

Ipp8u * dst = ippsMalloc_8u(outputSize);

Ipp32u srcLen;
FILE * pIn;
fopen_s(&pIn, pInFileName, "rb");

unsigned long dstLen;

while ((srcLen = (Ipp32u)fread(src, 1, BUFSIZE, pIn)) > 0)
{

dstLen = outputSize;

int ret = compress2(dst, &dstLen, src, srcLen, Z_BEST_SPEED);
if (ret != Z_OK)
{
printf("\nFAILED %d\n", ret);
return;
}

Ipp8u * result = ippsMalloc_8u(BUFSIZE);
uLongf resultsize = BUFSIZE;
int r = uncompress(result, &resultsize, dst, dstLen);
int x = memcmp(result, src, srcLen);

/// FAILS not equal

}
fclose(pIn);

ippsFree(src);
ippsFree(dst);
}

5 posts / 0 new
Last post
For more complete information about compiler optimizations, see our Optimization Notice.

Please, modify line #755 of deflate.c
if( status == ippStsNoErr && ippflush == IppLZ77NoFlush) {
to
if( status == ippStsNoErr && strm->avail_in > 0) {

The problem must go away.

Regards,
Sergey

Regards,
Sergey

Quoting - robert-laumeyer
I have taken the zlib compress/decompress pair out of zlib and I find that they work perfectly on buffer sizes up to 64K, but fail on anything larger. I tried doing the full initialization but found nothing had any impact. I just want to compress a buffer of memory and be able to decompress it, but 64K blocks create a lot of overhead for a 100M/sec application.

I'm using 6.0.0.127 samples, ipp 6.0.1.070 and compiling with VC2008 9.0.20729 sp.

Here is the compress/decomp and handler code

int compress2 (
Bytef *dest,
uLongf *destLen,
const Bytef *source,
uLong sourceLen,
int level)
{
z_stream stream;
int err;

stream.next_in = (Bytef*)source;
stream.avail_in = (uInt)sourceLen;

stream.next_out = dest;
stream.avail_out = (uInt)*destLen;
if ((uLong)stream.avail_out != *destLen)
return Z_BUF_ERROR;

stream.zalloc = (alloc_func)0;
stream.zfree = (free_func)0;
stream.opaque = (voidpf)0;

err = deflateInit2(&stream, level,
Z_DEFLATED, MAX_WBITS,
MAX_MEM_LEVEL, Z_DEFAULT_STRATEGY);

if (err != Z_OK)
return err;

err = deflate(&stream, Z_FINISH);

if (err != Z_STREAM_END)
{
printf("nFAILED to completely compress data stream");
deflateEnd(&stream);
return err == Z_OK ? Z_BUF_ERROR : err;
}
*destLen = stream.total_out;

err = deflateEnd(&stream);
return err;
}

int uncompress (
Bytef *dest,
uLongf *destLen,
const Bytef *source,
uLong sourceLen)
{
z_stream stream;
int err;

stream.next_in = (Bytef*)source;
stream.avail_in = (uInt)sourceLen;

stream.next_out = dest;
stream.avail_out = (uInt)*destLen;
if ((uLong)stream.avail_out != *destLen)
return Z_BUF_ERROR;

stream.zalloc = (alloc_func)0;
stream.zfree = (free_func)0;
stream.opaque = (voidpf)0;

err = inflateInit(&stream);
if (err != Z_OK)
return err;

err = inflate(&stream, Z_FINISH);
if (err != Z_STREAM_END)
{
inflateEnd(&stream);
if (err == Z_NEED_DICT || (err == Z_BUF_ERROR && stream.avail_in == 0))
return Z_DATA_ERROR;
return err;
}

*destLen = stream.total_out;

err = inflateEnd(&stream);
return err;
}

static const int BUFSIZE = 1024*64; // any larger and it fails
void TEST(const char * pInFileName)
{
Ipp8u * src = ippsMalloc_8u(BUFSIZE);

uLong outputSize = compressBound(BUFSIZE);

outputSize += 4096;

Ipp8u * dst = ippsMalloc_8u(outputSize);

Ipp32u srcLen;
FILE * pIn;
fopen_s(&pIn, pInFileName, "rb");

unsigned long dstLen;

while ((srcLen = (Ipp32u)fread(src, 1, BUFSIZE, pIn)) > 0)
{

dstLen = outputSize;

int ret = compress2(dst, &dstLen, src, srcLen, Z_BEST_SPEED);
if (ret != Z_OK)
{
printf("nFAILED %dn", ret);
return;
}

Ipp8u * result = ippsMalloc_8u(BUFSIZE);
uLongf resultsize = BUFSIZE;
int r = uncompress(result, &resultsize, dst, dstLen);
int x = memcmp(result, src, srcLen);

/// FAILS not equal

}
fclose(pIn);

ippsFree(src);
ippsFree(dst);
}

The proposed fix of changing line 755 in deflate.c changes the behaviour but it stills fails to compress correctly.

Best Reply

Quoting - robert-laumeyer

The proposed fix of changing line 755 in deflate.c changes the behaviour but it stills fails to compress correctly.

Please, take deflate.c anddeflate.h They must be OK.

Regards,
Sergey

Quoting - sergey_kh

Please, take deflate.c anddeflate.h They must be OK.

That works! Thank you.

Leave a Comment

Please sign in to add a comment. Not a member? Join today