Skip to content

Commit

Permalink
fix a bug (when the error bound is pretty small, the compressed size …
Browse files Browse the repository at this point in the history 
…may be larger than the original data size, leading to core dump.
  • Loading branch information
Sheng Di committed Nov 13, 2018
1 parent 7beb8c9 commit ef62bf6
Show file tree
Hide file tree
Showing 7 changed files with 85 additions and 57 deletions.
2 changes: 1 addition & 1 deletion sz/include/sz_double.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -29,7 +29,7 @@ unsigned int optimize_intervals_double_1D_opt(double *oriData, size_t dataLength

TightDataPointStorageD* SZ_compress_double_1D_MDQ(double *oriData,
size_t dataLength, double realPrecision, double valueRangeSize, double medianValue_d);
void SZ_compress_args_double_StoreOriData(double* oriData, size_t dataLength, TightDataPointStorageD* tdps, unsigned char** newByteData, size_t *outSize);
void SZ_compress_args_double_StoreOriData(double* oriData, size_t dataLength, unsigned char** newByteData, size_t *outSize);

char SZ_compress_args_double_NoCkRngeNoGzip_1D(unsigned char** newByteData, double *oriData, size_t dataLength, double realPrecision, size_t *outSize, double valueRangeSize, double medianValue_d);

Expand Down
3 changes: 1 addition & 2 deletions sz/include/sz_float.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -34,8 +34,7 @@ unsigned int optimize_intervals_float_1D_opt(float *oriData, size_t dataLength,
TightDataPointStorageF* SZ_compress_float_1D_MDQ(float *oriData,
size_t dataLength, double realPrecision, float valueRangeSize, float medianValue_f);

void SZ_compress_args_float_StoreOriData(float* oriData, size_t dataLength, TightDataPointStorageF* tdps,
unsigned char** newByteData, size_t *outSize);
void SZ_compress_args_float_StoreOriData(float* oriData, size_t dataLength, unsigned char** newByteData, size_t *outSize);

char SZ_compress_args_float_NoCkRngeNoGzip_1D(unsigned char** newByteData, float *oriData,
size_t dataLength, double realPrecision, size_t *outSize, float valueRangeSize, float medianValue_f);
Expand Down
1 change: 1 addition & 0 deletions sz/src/sz.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -339,6 +339,7 @@ void *SZ_decompress(int dataType, unsigned char *bytes, size_t byteLength, size_
if(exe_params==NULL)
exe_params = (sz_exedata*)malloc(sizeof(sz_exedata));
memset(exe_params, 0, sizeof(sz_exedata));
exe_params->SZ_SIZE_TYPE = 8;

int x = 1;
char *y = (char*)&x;
Expand Down
53 changes: 34 additions & 19 deletions sz/src/sz_double.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -390,14 +390,13 @@ size_t dataLength, double realPrecision, double valueRangeSize, double medianVal
return tdps;
}

void SZ_compress_args_double_StoreOriData(double* oriData, size_t dataLength, TightDataPointStorageD* tdps,
unsigned char** newByteData, size_t *outSize)
{
void SZ_compress_args_double_StoreOriData(double* oriData, size_t dataLength, unsigned char** newByteData, size_t *outSize)
{
int doubleSize = sizeof(double);
size_t k = 0, i;
tdps->isLossless = 1;
size_t totalByteLength = 3 + MetaDataByteLength + exe_params->SZ_SIZE_TYPE + 1 + doubleSize*dataLength;
*newByteData = (unsigned char*)malloc(totalByteLength);
/*No need to malloc because newByteData should always already be allocated with no less totalByteLength.*/
//*newByteData = (unsigned char*)malloc(totalByteLength);

unsigned char dsLengthBytes[8];
for (i = 0; i < 3; i++)//3
Expand Down Expand Up @@ -454,8 +453,8 @@ size_t dataLength, double realPrecision, size_t *outSize, double valueRangeSize,

convertTDPStoFlatBytes_double(tdps, newByteData, outSize);

if(*outSize>dataLength*sizeof(double))
SZ_compress_args_double_StoreOriData(oriData, dataLength, tdps, newByteData, outSize);
if(*outSize>3 + MetaDataByteLength + exe_params->SZ_SIZE_TYPE + 1 + sizeof(double)*dataLength)
SZ_compress_args_double_StoreOriData(oriData, dataLength, newByteData, outSize);

free_TightDataPointStorageD(tdps);
return compressionType;
Expand Down Expand Up @@ -727,8 +726,8 @@ char SZ_compress_args_double_NoCkRngeNoGzip_2D(unsigned char** newByteData, doub

convertTDPStoFlatBytes_double(tdps, newByteData, outSize);

if(*outSize>dataLength*sizeof(double))
SZ_compress_args_double_StoreOriData(oriData, dataLength, tdps, newByteData, outSize);
if(*outSize>3 + MetaDataByteLength + exe_params->SZ_SIZE_TYPE + 1 + sizeof(double)*dataLength)
SZ_compress_args_double_StoreOriData(oriData, dataLength, newByteData, outSize);

free_TightDataPointStorageD(tdps);
return compressionType;
Expand Down Expand Up @@ -1117,8 +1116,8 @@ char SZ_compress_args_double_NoCkRngeNoGzip_3D(unsigned char** newByteData, doub

convertTDPStoFlatBytes_double(tdps, newByteData, outSize);

if(*outSize>dataLength*sizeof(double))
SZ_compress_args_double_StoreOriData(oriData, dataLength, tdps, newByteData, outSize);
if(*outSize>3 + MetaDataByteLength + exe_params->SZ_SIZE_TYPE + 1 + sizeof(double)*dataLength)
SZ_compress_args_double_StoreOriData(oriData, dataLength, newByteData, outSize);

free_TightDataPointStorageD(tdps);
return compressionType;
Expand Down Expand Up @@ -1460,8 +1459,8 @@ char SZ_compress_args_double_NoCkRngeNoGzip_4D(unsigned char** newByteData, doub
convertTDPStoFlatBytes_double(tdps, newByteData, outSize);

size_t dataLength = r1*r2*r3*r4;
if(*outSize>dataLength*sizeof(double))
SZ_compress_args_double_StoreOriData(oriData, dataLength, tdps, newByteData, outSize);
if(*outSize>3 + MetaDataByteLength + exe_params->SZ_SIZE_TYPE + 1 + sizeof(double)*dataLength)
SZ_compress_args_double_StoreOriData(oriData, dataLength, newByteData, outSize);

free_TightDataPointStorageD(tdps);
return 0;
Expand Down Expand Up @@ -1610,7 +1609,11 @@ int errBoundMode, double absErr_Bound, double relBoundRatio, double pwRelBoundRa
multisteps->compressionType = SZ_compress_args_double_NoCkRngeNoGzip_1D(&tmpByteData, oriData, r1, realPrecision, &tmpOutSize, valueRangeSize, medianValue);
else
#endif
SZ_compress_args_double_NoCkRngeNoGzip_1D(&tmpByteData, oriData, r1, realPrecision, &tmpOutSize, valueRangeSize, medianValue);
{
SZ_compress_args_double_NoCkRngeNoGzip_1D(&tmpByteData, oriData, r1, realPrecision, &tmpOutSize, valueRangeSize, medianValue);
if(tmpOutSize>=dataLength*sizeof(double) + 3 + MetaDataByteLength + exe_params->SZ_SIZE_TYPE + 1)
SZ_compress_args_double_StoreOriData(oriData, dataLength, &tmpByteData, &tmpOutSize);
}
}
else
if (r3==0)
Expand All @@ -1627,7 +1630,11 @@ int errBoundMode, double absErr_Bound, double relBoundRatio, double pwRelBoundRa
if(sz_with_regression == SZ_NO_REGRESSION)
SZ_compress_args_double_NoCkRngeNoGzip_2D(&tmpByteData, oriData, r2, r1, realPrecision, &tmpOutSize, valueRangeSize, medianValue);
else
tmpByteData = SZ_compress_double_2D_MDQ_nonblocked_with_blocked_regression(oriData, r2, r1, realPrecision, &tmpOutSize);
{
tmpByteData = SZ_compress_double_2D_MDQ_nonblocked_with_blocked_regression(oriData, r2, r1, realPrecision, &tmpOutSize);
if(tmpOutSize>=dataLength*sizeof(double) + 3 + MetaDataByteLength + exe_params->SZ_SIZE_TYPE + 1)
SZ_compress_args_double_StoreOriData(oriData, dataLength, &tmpByteData, &tmpOutSize);
}
}
}
else
Expand All @@ -1645,7 +1652,11 @@ int errBoundMode, double absErr_Bound, double relBoundRatio, double pwRelBoundRa
if(sz_with_regression == SZ_NO_REGRESSION)
SZ_compress_args_double_NoCkRngeNoGzip_3D(&tmpByteData, oriData, r3, r2, r1, realPrecision, &tmpOutSize, valueRangeSize, medianValue);
else
{
tmpByteData = SZ_compress_double_3D_MDQ_nonblocked_with_blocked_regression(oriData, r3, r2, r1, realPrecision, &tmpOutSize);
if(tmpOutSize>=dataLength*sizeof(double) + 3 + MetaDataByteLength + exe_params->SZ_SIZE_TYPE + 1)
SZ_compress_args_double_StoreOriData(oriData, dataLength, &tmpByteData, &tmpOutSize);
}
}


Expand All @@ -1665,7 +1676,11 @@ int errBoundMode, double absErr_Bound, double relBoundRatio, double pwRelBoundRa
if(sz_with_regression == SZ_NO_REGRESSION)
SZ_compress_args_double_NoCkRngeNoGzip_4D(&tmpByteData, oriData, r4, r3, r2, r1, realPrecision, &tmpOutSize, valueRangeSize, medianValue);
else
{
tmpByteData = SZ_compress_double_3D_MDQ_nonblocked_with_blocked_regression(oriData, r4*r3, r2, r1, realPrecision, &tmpOutSize);
if(tmpOutSize>=dataLength*sizeof(double) + 3 + MetaDataByteLength + exe_params->SZ_SIZE_TYPE + 1)
SZ_compress_args_double_StoreOriData(oriData, dataLength, &tmpByteData, &tmpOutSize);
}
}

}
Expand Down Expand Up @@ -1795,7 +1810,7 @@ size_t r1, size_t s1, size_t e1)

//TODO
// if(*outSize>dataLength*sizeof(double))
// SZ_compress_args_double_StoreOriData(oriData, dataLength, tdps, newByteData, outSize);
// SZ_compress_args_double_StoreOriData(oriData, dataLength, newByteData, outSize);

free_TightDataPointStorageD(tdps);
}
Expand All @@ -1822,7 +1837,7 @@ size_t r2, size_t r1, size_t s2, size_t s1, size_t e2, size_t e1)

//TODO
// if(*outSize>dataLength*sizeof(double))
// SZ_compress_args_double_StoreOriData(oriData, dataLength, tdps, newByteData, outSize);
// SZ_compress_args_double_StoreOriData(oriData, dataLength, newByteData, outSize);

free_TightDataPointStorageD(tdps);
}
Expand All @@ -1849,7 +1864,7 @@ size_t r3, size_t r2, size_t r1, size_t s3, size_t s2, size_t s1, size_t e3, siz

//TODO
// if(*outSize>dataLength*sizeof(double))
// SZ_compress_args_double_StoreOriData(oriData, dataLength, tdps, newByteData, outSize);
// SZ_compress_args_double_StoreOriData(oriData, dataLength, newByteData, outSize);

free_TightDataPointStorageD(tdps);
}
Expand All @@ -1876,7 +1891,7 @@ size_t r4, size_t r3, size_t r2, size_t r1, size_t s4, size_t s3, size_t s2, siz

//TODO
// if(*outSize>dataLength*sizeof(double))
// SZ_compress_args_double_StoreOriData(oriData, dataLength, tdps, newByteData, outSize);
// SZ_compress_args_double_StoreOriData(oriData, dataLength, newByteData, outSize);

free_TightDataPointStorageD(tdps);
}
Expand Down
16 changes: 8 additions & 8 deletions sz/src/sz_double_pwr.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1768,8 +1768,8 @@ size_t dataLength, double absErrBound, double relBoundRatio, double pwrErrRatio,

convertTDPStoFlatBytes_double(tdps, newByteData, outSize);

if(*outSize>dataLength*sizeof(double))
SZ_compress_args_double_StoreOriData(oriData, dataLength+2, tdps, newByteData, outSize);
if(*outSize>3 + MetaDataByteLength + exe_params->SZ_SIZE_TYPE + 1 + sizeof(double)*dataLength)
SZ_compress_args_double_StoreOriData(oriData, dataLength, newByteData, outSize);

free_TightDataPointStorageD(tdps);
}
Expand Down Expand Up @@ -1830,8 +1830,8 @@ void SZ_compress_args_double_NoCkRngeNoGzip_1D_pwr_pre_log(unsigned char** newBy
free(signs);

convertTDPStoFlatBytes_double(tdps, newByteData, outSize);
if(*outSize>dataLength*sizeof(double))
SZ_compress_args_double_StoreOriData(oriData, dataLength+2, tdps, newByteData, outSize);
if(*outSize>3 + MetaDataByteLength + exe_params->SZ_SIZE_TYPE + 1 + sizeof(double)*dataLength)
SZ_compress_args_double_StoreOriData(oriData, dataLength, newByteData, outSize);

free_TightDataPointStorageD(tdps);
}
Expand Down Expand Up @@ -1892,8 +1892,8 @@ void SZ_compress_args_double_NoCkRngeNoGzip_2D_pwr_pre_log(unsigned char** newBy
free(signs);

convertTDPStoFlatBytes_double(tdps, newByteData, outSize);
if(*outSize>dataLength*sizeof(double))
SZ_compress_args_double_StoreOriData(oriData, dataLength+2, tdps, newByteData, outSize);
if(*outSize>3 + MetaDataByteLength + exe_params->SZ_SIZE_TYPE + 1 + sizeof(double)*dataLength)
SZ_compress_args_double_StoreOriData(oriData, dataLength, newByteData, outSize);

free_TightDataPointStorageD(tdps);
}
Expand Down Expand Up @@ -1953,8 +1953,8 @@ void SZ_compress_args_double_NoCkRngeNoGzip_3D_pwr_pre_log(unsigned char** newBy
free(signs);

convertTDPStoFlatBytes_double(tdps, newByteData, outSize);
if(*outSize>dataLength*sizeof(double))
SZ_compress_args_double_StoreOriData(oriData, dataLength+2, tdps, newByteData, outSize);
if(*outSize>3 + MetaDataByteLength + exe_params->SZ_SIZE_TYPE + 1 + sizeof(double)*dataLength)
SZ_compress_args_double_StoreOriData(oriData, dataLength, newByteData, outSize);

free_TightDataPointStorageD(tdps);
}
51 changes: 32 additions & 19 deletions sz/src/sz_float.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -491,14 +491,13 @@ size_t dataLength, double realPrecision, float valueRangeSize, float medianValue
return tdps;
}

void SZ_compress_args_float_StoreOriData(float* oriData, size_t dataLength, TightDataPointStorageF* tdps,
unsigned char** newByteData, size_t *outSize)
{
void SZ_compress_args_float_StoreOriData(float* oriData, size_t dataLength, unsigned char** newByteData, size_t *outSize)
{
int floatSize=sizeof(float);
size_t k = 0, i;
tdps->isLossless = 1;
size_t totalByteLength = 3 + MetaDataByteLength + exe_params->SZ_SIZE_TYPE + 1 + floatSize*dataLength;
*newByteData = (unsigned char*)malloc(totalByteLength);
/*No need to malloc because newByteData should always already be allocated with no less totalByteLength.*/
//*newByteData = (unsigned char*)malloc(totalByteLength);

unsigned char dsLengthBytes[8];
for (i = 0; i < 3; i++)//3
Expand Down Expand Up @@ -555,8 +554,8 @@ size_t dataLength, double realPrecision, size_t *outSize, float valueRangeSize,

convertTDPStoFlatBytes_float(tdps, newByteData, outSize);

if(*outSize>dataLength*sizeof(float))
SZ_compress_args_float_StoreOriData(oriData, dataLength+2, tdps, newByteData, outSize);
if(*outSize>3 + MetaDataByteLength + exe_params->SZ_SIZE_TYPE + 1 + sizeof(float)*dataLength)
SZ_compress_args_float_StoreOriData(oriData, dataLength, newByteData, outSize);

free_TightDataPointStorageF(tdps);
return compressionType;
Expand Down Expand Up @@ -873,8 +872,8 @@ char SZ_compress_args_float_NoCkRngeNoGzip_2D(unsigned char** newByteData, float

convertTDPStoFlatBytes_float(tdps, newByteData, outSize);

if(*outSize>dataLength*sizeof(float))
SZ_compress_args_float_StoreOriData(oriData, dataLength, tdps, newByteData, outSize);
if(*outSize>3 + MetaDataByteLength + exe_params->SZ_SIZE_TYPE + 1 + sizeof(float)*dataLength)
SZ_compress_args_float_StoreOriData(oriData, dataLength, newByteData, outSize);

free_TightDataPointStorageF(tdps);

Expand Down Expand Up @@ -1373,8 +1372,8 @@ char SZ_compress_args_float_NoCkRngeNoGzip_3D(unsigned char** newByteData, float
if(tdps!=NULL)
{
convertTDPStoFlatBytes_float(tdps, newByteData, outSize);
if(*outSize>dataLength*sizeof(float))
SZ_compress_args_float_StoreOriData(oriData, dataLength, tdps, newByteData, outSize);
if(*outSize>3 + MetaDataByteLength + exe_params->SZ_SIZE_TYPE + 1 + sizeof(float)*dataLength)
SZ_compress_args_float_StoreOriData(oriData, dataLength, newByteData, outSize);
free_TightDataPointStorageF(tdps);
}

Expand Down Expand Up @@ -1716,8 +1715,8 @@ char SZ_compress_args_float_NoCkRngeNoGzip_4D(unsigned char** newByteData, float
convertTDPStoFlatBytes_float(tdps, newByteData, outSize);

int dataLength = r1*r2*r3*r4;
if(*outSize>dataLength*sizeof(float))
SZ_compress_args_float_StoreOriData(oriData, dataLength, tdps, newByteData, outSize);
if(*outSize>3 + MetaDataByteLength + exe_params->SZ_SIZE_TYPE + 1 + sizeof(float)*dataLength)
SZ_compress_args_float_StoreOriData(oriData, dataLength, newByteData, outSize);

free_TightDataPointStorageF(tdps);

Expand Down Expand Up @@ -1867,7 +1866,11 @@ int errBoundMode, double absErr_Bound, double relBoundRatio, double pwRelBoundRa
multisteps->compressionType = SZ_compress_args_float_NoCkRngeNoGzip_1D(&tmpByteData, oriData, r1, realPrecision, &tmpOutSize, valueRangeSize, medianValue);
else
#endif
SZ_compress_args_float_NoCkRngeNoGzip_1D(&tmpByteData, oriData, r1, realPrecision, &tmpOutSize, valueRangeSize, medianValue);
{
SZ_compress_args_float_NoCkRngeNoGzip_1D(&tmpByteData, oriData, r1, realPrecision, &tmpOutSize, valueRangeSize, medianValue);
if(tmpOutSize>=dataLength*sizeof(float) + 3 + MetaDataByteLength + exe_params->SZ_SIZE_TYPE + 1)
SZ_compress_args_float_StoreOriData(oriData, dataLength, &tmpByteData, &tmpOutSize);
}
}
else
if (r3==0)
Expand All @@ -1884,7 +1887,11 @@ int errBoundMode, double absErr_Bound, double relBoundRatio, double pwRelBoundRa
if(sz_with_regression == SZ_NO_REGRESSION)
SZ_compress_args_float_NoCkRngeNoGzip_2D(&tmpByteData, oriData, r2, r1, realPrecision, &tmpOutSize, valueRangeSize, medianValue);
else
{
tmpByteData = SZ_compress_float_2D_MDQ_nonblocked_with_blocked_regression(oriData, r2, r1, realPrecision, &tmpOutSize);
if(tmpOutSize>=dataLength*sizeof(float) + 3 + MetaDataByteLength + exe_params->SZ_SIZE_TYPE + 1)
SZ_compress_args_float_StoreOriData(oriData, dataLength, &tmpByteData, &tmpOutSize);
}
}
}
else
Expand Down Expand Up @@ -1913,6 +1920,8 @@ int errBoundMode, double absErr_Bound, double relBoundRatio, double pwRelBoundRa
{
tmpByteData = SZ_compress_float_3D_MDQ_decompression_random_access_with_blocked_regression(oriData, r3, r2, r1, realPrecision, &tmpOutSize);
}
if(tmpOutSize>=dataLength*sizeof(float) + 3 + MetaDataByteLength + exe_params->SZ_SIZE_TYPE + 1)
SZ_compress_args_float_StoreOriData(oriData, dataLength, &tmpByteData, &tmpOutSize);
#endif
}
}
Expand All @@ -1933,7 +1942,11 @@ int errBoundMode, double absErr_Bound, double relBoundRatio, double pwRelBoundRa
if(sz_with_regression == SZ_NO_REGRESSION)
SZ_compress_args_float_NoCkRngeNoGzip_4D(&tmpByteData, oriData, r4, r3, r2, r1, realPrecision, &tmpOutSize, valueRangeSize, medianValue);
else
tmpByteData = SZ_compress_float_3D_MDQ_nonblocked_with_blocked_regression(oriData, r4*r3, r2, r1, realPrecision, &tmpOutSize);
{
tmpByteData = SZ_compress_float_3D_MDQ_nonblocked_with_blocked_regression(oriData, r4*r3, r2, r1, realPrecision, &tmpOutSize);
if(tmpOutSize>=dataLength*sizeof(float) + 3 + MetaDataByteLength + exe_params->SZ_SIZE_TYPE + 1)
SZ_compress_args_float_StoreOriData(oriData, dataLength, &tmpByteData, &tmpOutSize);
}
}
}
else
Expand Down Expand Up @@ -2075,7 +2088,7 @@ size_t r1, size_t s1, size_t e1)

//TODO
// if(*outSize>dataLength*sizeof(float))
// SZ_compress_args_float_StoreOriData(oriData, dataLength, tdps, newByteData, outSize);
// SZ_compress_args_float_StoreOriData(oriData, dataLength, newByteData, outSize);

free_TightDataPointStorageF(tdps);
}
Expand All @@ -2102,7 +2115,7 @@ size_t r2, size_t r1, size_t s2, size_t s1, size_t e2, size_t e1)

//TODO
// if(*outSize>dataLength*sizeof(float))
// SZ_compress_args_float_StoreOriData(oriData, dataLength, tdps, newByteData, outSize);
// SZ_compress_args_float_StoreOriData(oriData, dataLength, newByteData, outSize);

free_TightDataPointStorageF(tdps);
}
Expand All @@ -2129,7 +2142,7 @@ size_t r3, size_t r2, size_t r1, size_t s3, size_t s2, size_t s1, size_t e3, siz

//TODO
// if(*outSize>dataLength*sizeof(float))
// SZ_compress_args_float_StoreOriData(oriData, dataLength, tdps, newByteData, outSize);
// SZ_compress_args_float_StoreOriData(oriData, dataLength, newByteData, outSize);

free_TightDataPointStorageF(tdps);
}
Expand All @@ -2156,7 +2169,7 @@ size_t r4, size_t r3, size_t r2, size_t r1, size_t s4, size_t s3, size_t s2, siz

//TODO
// if(*outSize>dataLength*sizeof(float))
// SZ_compress_args_float_StoreOriData(oriData, dataLength, tdps, newByteData, outSize);
// SZ_compress_args_float_StoreOriData(oriData, dataLength, newByteData, outSize);

free_TightDataPointStorageF(tdps);

Expand Down
Loading

0 comments on commit ef62bf6

Please sign in to comment.