diff --git a/example/sz.c b/example/sz.c index 2d5c7866..3c43864c 100644 --- a/example/sz.c +++ b/example/sz.c @@ -33,10 +33,11 @@ void usage() printf("* operation type:\n"); printf(" -z : the compression operation with an optionally specified output file.\n"); printf(" (the compressed file will be named as .sz if not specified)\n"); - printf(" -x : the decompression operation with an optionally specified output file.\n"); + printf(" -x : the decompression operation with an optionally specified output file\n"); printf(" (the decompressed file will be named as .out if not specified)\n"); printf(" -p: print meta data (configuration info)\n"); printf(" -h: print the help information\n"); + printf(" -v: print the version number\n"); printf("* data type:\n"); printf(" -f: single precision (float type)\n"); printf(" -d: double precision (double type)\n"); @@ -120,6 +121,9 @@ int main(int argc, char* argv[]) case 'h': usage(); exit(0); + case 'v': + printf("version: %d.%d.%d\n", SZ_VER_MAJOR, SZ_VER_MINOR, SZ_VER_BUILD); + exit(0); case 'b': binaryOutput = 1; break; diff --git a/sz/include/defines.h b/sz/include/defines.h index 6323d3fc..ef1f8dcc 100644 --- a/sz/include/defines.h +++ b/sz/include/defines.h @@ -13,7 +13,7 @@ #define SZ_VERNUM 0x0200 #define SZ_VER_MAJOR 2 #define SZ_VER_MINOR 1 -#define SZ_VER_BUILD 7 +#define SZ_VER_BUILD 8 #define SZ_VER_REVISION 0 #define PASTRI 103 @@ -93,11 +93,15 @@ #define GROUP_COUNT 16 //2^{16}=65536 -#define MetaDataByteLength 20 +#define MetaDataByteLength 28 +#define MetaDataByteLength_double 36 //meta data length for double type #define numOfBufferedSteps 1 //the number of time steps in the buffer #define GZIP_COMPRESSOR 0 //i.e., ZLIB_COMPRSSOR #define ZSTD_COMPRESSOR 1 + +#define PROTECT_VALUE_RANGE 0 + #endif /* _SZ_DEFINES_H */ diff --git a/sz/include/sz.h b/sz/include/sz.h index a05576cd..36b8a8f2 100644 --- a/sz/include/sz.h +++ b/sz/include/sz.h @@ -178,6 +178,9 @@ typedef struct sz_params int segment_size; //only used for 2D/3D data compression with pw_relBoundRatio (deprecated) int pwr_type; //only used for 2D/3D data compression with pw_relBoundRatio + float fmin, fmax; + double dmin, dmax; + int snapshotCmprStep; //perform single-snapshot-based compression if time_step == snapshotCmprStep int predictionMode; @@ -185,7 +188,8 @@ typedef struct sz_params int plus_bits; int randomAccess; - int withRegression; + int withRegression; + } sz_params; typedef struct sz_metadata diff --git a/sz/src/ByteToolkit.c b/sz/src/ByteToolkit.c index fed95037..c3d7ad1e 100644 --- a/sz/src/ByteToolkit.c +++ b/sz/src/ByteToolkit.c @@ -957,6 +957,18 @@ void convertSZParamsToBytes(sz_params* params, unsigned char* result) int32ToBytes_bigEndian(&result[16], params->max_quant_intervals); else int32ToBytes_bigEndian(&result[16], params->quantization_intervals); + + if(params->dataType==SZ_FLOAT) + { + floatToBytes(&result[20], params->fmin); + floatToBytes(&result[24], params->fmax); + } + else + { + doubleToBytes(&result[20], params->dmin); + doubleToBytes(&result[28], params->dmax); + } + } void convertBytesToSZParams(unsigned char* bytes, sz_params* params) @@ -1036,4 +1048,16 @@ void convertBytesToSZParams(unsigned char* bytes, sz_params* params) params->max_quant_intervals = 0; params->quantization_intervals = bytesToInt32_bigEndian(&bytes[16]); } + + if(params->dataType==SZ_FLOAT) + { + params->fmin = bytesToFloat(&bytes[20]); + params->fmax = bytesToFloat(&bytes[24]); + } + else if(params->dataType==SZ_DOUBLE) + { + params->dmin = bytesToDouble(&bytes[20]); + params->dmax = bytesToDouble(&bytes[28]); + } + } diff --git a/sz/src/TightDataPointStorageD.c b/sz/src/TightDataPointStorageD.c index 66491d18..039c253e 100644 --- a/sz/src/TightDataPointStorageD.c +++ b/sz/src/TightDataPointStorageD.c @@ -93,7 +93,7 @@ int new_TightDataPointStorageD_fromFlatBytes(TightDataPointStorageD **this, unsi } convertBytesToSZParams(&(flatBytes[index]), confparams_dec); - index += MetaDataByteLength; + index += MetaDataByteLength_double; int isRegression = (sameRByte >> 7) & 0x01; @@ -112,7 +112,7 @@ int new_TightDataPointStorageD_fromFlatBytes(TightDataPointStorageD **this, unsi else if(same==1) { (*this)->allSameData = 1; - //size_t exactMidBytesLength = sizeof(double);//flatBytesLength - 3 - 1 - MetaDataByteLength -exe_params->SZ_SIZE_TYPE; + //size_t exactMidBytesLength = sizeof(double);//flatBytesLength - 3 - 1 - MetaDataByteLength_double -exe_params->SZ_SIZE_TYPE; (*this)->exactMidBytes = &(flatBytes[index]); return errorBoundMode; } @@ -121,7 +121,7 @@ int new_TightDataPointStorageD_fromFlatBytes(TightDataPointStorageD **this, unsi if(isRegression == 1) { - (*this)->raBytes_size = flatBytesLength - 3 - 1 - MetaDataByteLength - exe_params->SZ_SIZE_TYPE; + (*this)->raBytes_size = flatBytesLength - 3 - 1 - MetaDataByteLength_double - exe_params->SZ_SIZE_TYPE; (*this)->raBytes = &(flatBytes[index]); return errorBoundMode; } @@ -222,7 +222,7 @@ int new_TightDataPointStorageD_fromFlatBytes(TightDataPointStorageD **this, unsi if ((*this)->rtypeArray != NULL) { - (*this)->residualMidBits_size = flatBytesLength - 3 - 1 - MetaDataByteLength - exe_params->SZ_SIZE_TYPE - 4 - radExpoL - segmentL - pwrErrBoundBytesL - 4 - 8 - 1 - 8 + (*this)->residualMidBits_size = flatBytesLength - 3 - 1 - MetaDataByteLength_double - exe_params->SZ_SIZE_TYPE - 4 - radExpoL - segmentL - pwrErrBoundBytesL - 4 - 8 - 1 - 8 - exe_params->SZ_SIZE_TYPE - exe_params->SZ_SIZE_TYPE - exe_params->SZ_SIZE_TYPE - minLogValueSize - exe_params->SZ_SIZE_TYPE - 8 - (*this)->rtypeArray_size - minLogValueSize - (*this)->typeArray_size - (*this)->leadNumArray_size - (*this)->exactMidBytes_size - pwrErrBoundBytes_size - 1 - 1; @@ -231,7 +231,7 @@ int new_TightDataPointStorageD_fromFlatBytes(TightDataPointStorageD **this, unsi } else { - (*this)->residualMidBits_size = flatBytesLength - 3 - 1 - MetaDataByteLength - exe_params->SZ_SIZE_TYPE - 4 - radExpoL - segmentL - pwrErrBoundBytesL - 4 - 8 - 1 - 8 + (*this)->residualMidBits_size = flatBytesLength - 3 - 1 - MetaDataByteLength_double - exe_params->SZ_SIZE_TYPE - 4 - radExpoL - segmentL - pwrErrBoundBytesL - 4 - 8 - 1 - 8 - exe_params->SZ_SIZE_TYPE - exe_params->SZ_SIZE_TYPE - exe_params->SZ_SIZE_TYPE - minLogValueSize - (*this)->typeArray_size - (*this)->leadNumArray_size - (*this)->exactMidBytes_size - pwrErrBoundBytes_size - 1 - 1; } @@ -395,7 +395,7 @@ void convertTDPStoBytes_double(TightDataPointStorageD* tdps, unsigned char* byte bytes[k++] = sameByte; //1 byte convertSZParamsToBytes(confparams_cpr, &(bytes[k])); - k = k + MetaDataByteLength; + k = k + MetaDataByteLength_double; for(i = 0;iSZ_SIZE_TYPE;i++)//ST: 4 or 8 bytes bytes[k++] = dsLengthBytes[i]; @@ -497,7 +497,7 @@ void convertTDPStoBytes_double_reserve(TightDataPointStorageD* tdps, unsigned ch bytes[k++] = sameByte; //1 convertSZParamsToBytes(confparams_cpr, &(bytes[k])); - k = k + MetaDataByteLength; + k = k + MetaDataByteLength_double; for(i = 0;iSZ_SIZE_TYPE;i++)//ST bytes[k++] = dsLengthBytes[i]; @@ -604,7 +604,7 @@ void convertTDPStoFlatBytes_double(TightDataPointStorageD *tdps, unsigned char** if(tdps->allSameData==1) { - size_t totalByteLength = 3 + 1 + MetaDataByteLength + exe_params->SZ_SIZE_TYPE + tdps->exactMidBytes_size; + size_t totalByteLength = 3 + 1 + MetaDataByteLength_double + exe_params->SZ_SIZE_TYPE + tdps->exactMidBytes_size; *bytes = (unsigned char *)malloc(sizeof(unsigned char)*totalByteLength); for (i = 0; i < 3; i++)//3 @@ -612,7 +612,7 @@ void convertTDPStoFlatBytes_double(TightDataPointStorageD *tdps, unsigned char** (*bytes)[k++] = sameByte; convertSZParamsToBytes(confparams_cpr, &((*bytes)[k])); - k = k + MetaDataByteLength; + k = k + MetaDataByteLength_double; for (i = 0; i < exe_params->SZ_SIZE_TYPE; i++) (*bytes)[k++] = dsLengthBytes[i]; @@ -636,7 +636,7 @@ void convertTDPStoFlatBytes_double(TightDataPointStorageD *tdps, unsigned char** minLogValueSize = 8; } - size_t totalByteLength = 3 + 1 + MetaDataByteLength + exe_params->SZ_SIZE_TYPE + 4 + radExpoL + segmentL + pwrBoundArrayL + 4 + 8 + 1 + 8 + size_t totalByteLength = 3 + 1 + MetaDataByteLength_double + exe_params->SZ_SIZE_TYPE + 4 + radExpoL + segmentL + pwrBoundArrayL + 4 + 8 + 1 + 8 + exe_params->SZ_SIZE_TYPE + exe_params->SZ_SIZE_TYPE + exe_params->SZ_SIZE_TYPE + minLogValueSize /*max absolute log value*/ + tdps->typeArray_size + tdps->leadNumArray_size @@ -678,14 +678,14 @@ void convertTDPStoFlatBytes_double_args(TightDataPointStorageD *tdps, unsigned c sameByte = (unsigned char) (sameByte | 0x08); //0000,1000, the 7th bit if(tdps->allSameData==1) { - size_t totalByteLength = 3 + 1 + MetaDataByteLength + exe_params->SZ_SIZE_TYPE + tdps->exactMidBytes_size; + size_t totalByteLength = 3 + 1 + MetaDataByteLength_double + exe_params->SZ_SIZE_TYPE + tdps->exactMidBytes_size; for (i = 0; i < 3; i++)//3 bytes[k++] = versionNumber[i]; bytes[k++] = sameByte; convertSZParamsToBytes(confparams_cpr, &(bytes[k])); - k = k + MetaDataByteLength; + k = k + MetaDataByteLength_double; for (i = 0; i < exe_params->SZ_SIZE_TYPE; i++) bytes[k++] = dsLengthBytes[i]; @@ -705,7 +705,7 @@ void convertTDPStoFlatBytes_double_args(TightDataPointStorageD *tdps, unsigned c pwrBoundArrayL = 4; } - size_t totalByteLength = 3 + 1 + MetaDataByteLength + exe_params->SZ_SIZE_TYPE+ 4 + radExpoL + segmentL + pwrBoundArrayL + 4 + 8 + 1 + 8 + size_t totalByteLength = 3 + 1 + MetaDataByteLength_double + exe_params->SZ_SIZE_TYPE+ 4 + radExpoL + segmentL + pwrBoundArrayL + 4 + 8 + 1 + 8 + exe_params->SZ_SIZE_TYPE + exe_params->SZ_SIZE_TYPE + exe_params->SZ_SIZE_TYPE + tdps->typeArray_size + tdps->leadNumArray_size + tdps->exactMidBytes_size + residualMidBitsLength + tdps->pwrErrBoundBytes_size; diff --git a/sz/src/conf.c b/sz/src/conf.c index a9bf4aff..44f9a6ec 100644 --- a/sz/src/conf.c +++ b/sz/src/conf.c @@ -168,7 +168,7 @@ int SZ_ReadConf(const char* sz_cfgFile) { else if(strcmp(sol_name, "SZ_Transpose")==0) confparams_cpr->sol_ID = SZ_Transpose; else{ - printf("[SZ] Error: wrong solution name (please check sz.config file)\n"); + printf("[SZ] Error: wrong solution name (please check sz.config file), sol=%s\n", sol_name); iniparser_freedict(ini); return SZ_NSCS; } diff --git a/sz/src/dataCompression.c b/sz/src/dataCompression.c index 52868e2a..a2003f90 100644 --- a/sz/src/dataCompression.c +++ b/sz/src/dataCompression.c @@ -695,7 +695,10 @@ int initRandomAccessBytes(unsigned char* raBytes) raBytes[k++] = sameByte; convertSZParamsToBytes(confparams_cpr, &(raBytes[k])); - k = k + MetaDataByteLength; + if(confparams_cpr->dataType==SZ_FLOAT) + k = k + MetaDataByteLength; + else if(confparams_cpr->dataType==SZ_DOUBLE) + k = k + MetaDataByteLength_double; return k; } diff --git a/sz/src/sz.c b/sz/src/sz.c index d4227fcb..53128d57 100644 --- a/sz/src/sz.c +++ b/sz/src/sz.c @@ -553,7 +553,10 @@ sz_metadata* SZ_getMetadata(unsigned char* bytes) if(confparams_dec!=NULL) free(confparams_dec); confparams_dec = params;*/ - index += MetaDataByteLength; + if(confparams_dec->dataType==SZ_FLOAT) + index += MetaDataByteLength; + else if(confparams_dec->dataType==SZ_DOUBLE) + index += MetaDataByteLength_double; if(confparams_dec->dataType!=SZ_FLOAT && confparams_dec->dataType!= SZ_DOUBLE) //if this type is an Int type index++; //jump to the dataLength info byte address @@ -591,7 +594,8 @@ sz_metadata* SZ_getMetadata(unsigned char* bytes) pwrErrBoundBytesL = 4; } - int offset_typearray = 3 + 1 + MetaDataByteLength + exe_params->SZ_SIZE_TYPE + 4 + radExpoL + segmentL + pwrErrBoundBytesL + 4 + (4 + confparams_dec->dataType*4) + 1 + 8 + int mdbl = confparams_dec->dataType==SZ_FLOAT?MetaDataByteLength:MetaDataByteLength_double; + int offset_typearray = 3 + 1 + mdbl + exe_params->SZ_SIZE_TYPE + 4 + radExpoL + segmentL + pwrErrBoundBytesL + 4 + (4 + confparams_dec->dataType*4) + 1 + 8 + exe_params->SZ_SIZE_TYPE + exe_params->SZ_SIZE_TYPE + exe_params->SZ_SIZE_TYPE + 4; defactoNBBins = bytesToInt_bigEndian(bytes+offset_typearray); } @@ -623,9 +627,13 @@ void SZ_printMetadata(sz_metadata* metadata) { case SZ_FLOAT: printf("Data type: \t FLOAT\n"); + printf("min value of raw data: \t %f\n", params->fmin); + printf("max value of raw data: \t %f\n", params->fmax); break; case SZ_DOUBLE: printf("Data type: \t DOUBLE\n"); + printf("min value of raw data: \t %f\n", params->dmin); + printf("max value of raw data: \t %f\n", params->dmax); break; case SZ_INT8: printf("Data type: \t INT8\n"); diff --git a/sz/src/sz_double.c b/sz/src/sz_double.c index 4de4027d..4de9fbdd 100644 --- a/sz/src/sz_double.c +++ b/sz/src/sz_double.c @@ -405,7 +405,7 @@ void SZ_compress_args_double_StoreOriData(double* oriData, size_t dataLength, un { int doubleSize = sizeof(double); size_t k = 0, i; - size_t totalByteLength = 3 + MetaDataByteLength + exe_params->SZ_SIZE_TYPE + 1 + doubleSize*dataLength; + size_t totalByteLength = 3 + MetaDataByteLength_double + exe_params->SZ_SIZE_TYPE + 1 + doubleSize*dataLength; /*No need to malloc because newByteData should always already be allocated with no less totalByteLength.*/ //*newByteData = (unsigned char*)malloc(totalByteLength); @@ -419,17 +419,17 @@ void SZ_compress_args_double_StoreOriData(double* oriData, size_t dataLength, un (*newByteData)[k++] = 80; //01010000: 01000000 indicates the SZ_SIZE_TYPE=8 convertSZParamsToBytes(confparams_cpr, &((*newByteData)[k])); - k = k + MetaDataByteLength; + k = k + MetaDataByteLength_double; sizeToBytes(dsLengthBytes,dataLength); for (i = 0; i < exe_params->SZ_SIZE_TYPE; i++)//ST: 4 or 8 (*newByteData)[k++] = dsLengthBytes[i]; if(sysEndianType==BIG_ENDIAN_SYSTEM) - memcpy((*newByteData)+4+MetaDataByteLength+exe_params->SZ_SIZE_TYPE, oriData, dataLength*doubleSize); + memcpy((*newByteData)+4+MetaDataByteLength_double+exe_params->SZ_SIZE_TYPE, oriData, dataLength*doubleSize); else { - unsigned char* p = (*newByteData)+4+MetaDataByteLength+exe_params->SZ_SIZE_TYPE; + unsigned char* p = (*newByteData)+4+MetaDataByteLength_double+exe_params->SZ_SIZE_TYPE; for(i=0;i3 + MetaDataByteLength + exe_params->SZ_SIZE_TYPE + 1 + sizeof(double)*dataLength) + if(*outSize>3 + MetaDataByteLength_double + exe_params->SZ_SIZE_TYPE + 1 + sizeof(double)*dataLength) SZ_compress_args_double_StoreOriData(oriData, dataLength, newByteData, outSize); free_TightDataPointStorageD(tdps); @@ -767,7 +767,7 @@ char SZ_compress_args_double_NoCkRngeNoGzip_2D(int cmprType, unsigned char** new convertTDPStoFlatBytes_double(tdps, newByteData, outSize); - if(*outSize>3 + MetaDataByteLength + exe_params->SZ_SIZE_TYPE + 1 + sizeof(double)*dataLength) + if(*outSize>3 + MetaDataByteLength_double + exe_params->SZ_SIZE_TYPE + 1 + sizeof(double)*dataLength) SZ_compress_args_double_StoreOriData(oriData, dataLength, newByteData, outSize); free_TightDataPointStorageD(tdps); @@ -1179,7 +1179,7 @@ char SZ_compress_args_double_NoCkRngeNoGzip_3D(int cmprType, unsigned char** new if(tdps!=NULL) { convertTDPStoFlatBytes_double(tdps, newByteData, outSize); - if(*outSize>3 + MetaDataByteLength + exe_params->SZ_SIZE_TYPE + 1 + sizeof(double)*dataLength) + if(*outSize>3 + MetaDataByteLength_double + exe_params->SZ_SIZE_TYPE + 1 + sizeof(double)*dataLength) SZ_compress_args_double_StoreOriData(oriData, dataLength, newByteData, outSize); free_TightDataPointStorageD(tdps); } @@ -1524,7 +1524,7 @@ char SZ_compress_args_double_NoCkRngeNoGzip_4D(unsigned char** newByteData, doub convertTDPStoFlatBytes_double(tdps, newByteData, outSize); size_t dataLength = r1*r2*r3*r4; - if(*outSize>3 + MetaDataByteLength + exe_params->SZ_SIZE_TYPE + 1 + sizeof(double)*dataLength) + if(*outSize>3 + MetaDataByteLength_double + exe_params->SZ_SIZE_TYPE + 1 + sizeof(double)*dataLength) SZ_compress_args_double_StoreOriData(oriData, dataLength, newByteData, outSize); free_TightDataPointStorageD(tdps); @@ -2446,7 +2446,7 @@ void SZ_compress_args_double_withinRange(unsigned char** newByteData, double *or //*newByteData = (unsigned char*)malloc(sizeof(unsigned char)*16); //for floating-point data (1+3+4+4) //memcpy(*newByteData, tmpByteData, 16); - *outSize = tmpOutSize;//12==3+1+8(double_size)+MetaDataByteLength + *outSize = tmpOutSize;//12==3+1+8(double_size)+MetaDataByteLength_double free_TightDataPointStorageD(tdps); } @@ -2540,7 +2540,9 @@ int errBoundMode, double absErr_Bound, double relBoundRatio, double pwRelBoundRa else min = computeRangeSize_double(oriData, dataLength, &valueRangeSize, &medianValue); double max = min+valueRangeSize; - + confparams_cpr->dmin = min; + confparams_cpr->dmax = max; + double realPrecision = 0; if(confparams_cpr->errorBoundMode==PSNR) @@ -2581,7 +2583,7 @@ int errBoundMode, double absErr_Bound, double relBoundRatio, double pwRelBoundRa #endif { SZ_compress_args_double_NoCkRngeNoGzip_1D(cmprType, &tmpByteData, oriData, r1, realPrecision, &tmpOutSize, valueRangeSize, medianValue); - if(tmpOutSize>=dataLength*sizeof(double) + 3 + MetaDataByteLength + exe_params->SZ_SIZE_TYPE + 1) + if(tmpOutSize>=dataLength*sizeof(double) + 3 + MetaDataByteLength_double + exe_params->SZ_SIZE_TYPE + 1) SZ_compress_args_double_StoreOriData(oriData, dataLength, &tmpByteData, &tmpOutSize); } } @@ -2607,7 +2609,7 @@ int errBoundMode, double absErr_Bound, double relBoundRatio, double pwRelBoundRa else { tmpByteData = SZ_compress_double_2D_MDQ_nonblocked_with_blocked_regression(oriData, r2, r1, realPrecision, &tmpOutSize); - if(tmpOutSize>=dataLength*sizeof(double) + 3 + MetaDataByteLength + exe_params->SZ_SIZE_TYPE + 1) + if(tmpOutSize>=dataLength*sizeof(double) + 3 + MetaDataByteLength_double + exe_params->SZ_SIZE_TYPE + 1) SZ_compress_args_double_StoreOriData(oriData, dataLength, &tmpByteData, &tmpOutSize); } } @@ -2634,7 +2636,7 @@ int errBoundMode, double absErr_Bound, double relBoundRatio, double pwRelBoundRa else { tmpByteData = SZ_compress_double_3D_MDQ_nonblocked_with_blocked_regression(oriData, r3, r2, r1, realPrecision, &tmpOutSize); - if(tmpOutSize>=dataLength*sizeof(double) + 3 + MetaDataByteLength + exe_params->SZ_SIZE_TYPE + 1) + if(tmpOutSize>=dataLength*sizeof(double) + 3 + MetaDataByteLength_double + exe_params->SZ_SIZE_TYPE + 1) SZ_compress_args_double_StoreOriData(oriData, dataLength, &tmpByteData, &tmpOutSize); } } @@ -2663,7 +2665,7 @@ int errBoundMode, double absErr_Bound, double relBoundRatio, double pwRelBoundRa else { tmpByteData = SZ_compress_double_3D_MDQ_nonblocked_with_blocked_regression(oriData, r4*r3, r2, r1, realPrecision, &tmpOutSize); - if(tmpOutSize>=dataLength*sizeof(double) + 3 + MetaDataByteLength + exe_params->SZ_SIZE_TYPE + 1) + if(tmpOutSize>=dataLength*sizeof(double) + 3 + MetaDataByteLength_double + exe_params->SZ_SIZE_TYPE + 1) SZ_compress_args_double_StoreOriData(oriData, dataLength, &tmpByteData, &tmpOutSize); } } @@ -5754,7 +5756,7 @@ unsigned char * SZ_compress_double_2D_MDQ_nonblocked_with_blocked_regression(dou unsigned char *treeBytes; unsigned int treeByteSize = convert_HuffTree_to_bytes_anyStates(huffmanTree, nodeCount, &treeBytes); - unsigned int meta_data_offset = 3 + 1 + MetaDataByteLength; + unsigned int meta_data_offset = 3 + 1 + MetaDataByteLength_double; // total size metadata # elements real precision intervals nodeCount huffman block index unpredicatable count mean unpred size elements unsigned char * result = (unsigned char *) calloc(meta_data_offset + exe_params->SZ_SIZE_TYPE + sizeof(double) + sizeof(int) + sizeof(int) + 5*treeByteSize + 3*num_blocks*sizeof(int) + num_blocks * sizeof(unsigned short) + num_blocks * sizeof(unsigned short) + num_blocks * sizeof(double) + total_unpred * sizeof(double) + num_elements * sizeof(int), 1); unsigned char * result_pos = result; @@ -6708,7 +6710,7 @@ unsigned char * SZ_compress_double_3D_MDQ_nonblocked_with_blocked_regression(dou unsigned char *treeBytes; unsigned int treeByteSize = convert_HuffTree_to_bytes_anyStates(huffmanTree, nodeCount, &treeBytes); - unsigned int meta_data_offset = 3 + 1 + MetaDataByteLength; + unsigned int meta_data_offset = 3 + 1 + MetaDataByteLength_double; // total size metadata # elements real precision intervals nodeCount huffman block index unpredicatable count mean unpred size elements unsigned char * result = (unsigned char *) calloc(meta_data_offset + exe_params->SZ_SIZE_TYPE + sizeof(double) + sizeof(int) + sizeof(int) + 5*treeByteSize + 4*num_blocks*sizeof(int)+ num_blocks * sizeof(unsigned short) + num_blocks * sizeof(unsigned short) + num_blocks * sizeof(double) + total_unpred * sizeof(double) + num_elements * sizeof(int), 1); unsigned char * result_pos = result; diff --git a/sz/src/sz_float.c b/sz/src/sz_float.c index ec488144..c0de5699 100644 --- a/sz/src/sz_float.c +++ b/sz/src/sz_float.c @@ -502,10 +502,10 @@ size_t dataLength, float realPrecision, float valueRangeSize, float medianValue_ realPrecision, medianValue, (char)reqLength, quantization_intervals, NULL, 0, 0); //sdi:Debug -/* int sum =0; + int sum =0; for(i=0;ifmin = min; + confparams_cpr->fmax = max; + double realPrecision = 0; if(confparams_cpr->errorBoundMode==PSNR) diff --git a/sz/src/szd_double.c b/sz/src/szd_double.c index 3f1f6338..40defe80 100644 --- a/sz/src/szd_double.c +++ b/sz/src/szd_double.c @@ -27,9 +27,9 @@ size_t cmpSize, int compressionType, double* hist_data) //unsigned char* tmpBytes; size_t targetUncompressSize = dataLength <<3; //i.e., *8 //tmpSize must be "much" smaller than dataLength - size_t i, tmpSize = 12+MetaDataByteLength+exe_params->SZ_SIZE_TYPE; + size_t i, tmpSize = 12+MetaDataByteLength_double+exe_params->SZ_SIZE_TYPE; unsigned char* szTmpBytes; - if(cmpSize!=12+4+MetaDataByteLength && cmpSize!=12+8+MetaDataByteLength) + if(cmpSize!=12+4+MetaDataByteLength_double && cmpSize!=12+8+MetaDataByteLength_double) { confparams_dec->losslessCompressor = is_lossless_compressed_data(cmpBytes, cmpSize); if(confparams_dec->szMode!=SZ_TEMPORAL_COMPRESSION) @@ -48,7 +48,7 @@ size_t cmpSize, int compressionType, double* hist_data) { if(targetUncompressSizelosslessCompressor, cmpBytes, (unsigned long)cmpSize, &szTmpBytes, (unsigned long)targetUncompressSize+4+MetaDataByteLength+exe_params->SZ_SIZE_TYPE); + tmpSize = sz_lossless_decompress(confparams_dec->losslessCompressor, cmpBytes, (unsigned long)cmpSize, &szTmpBytes, (unsigned long)targetUncompressSize+4+MetaDataByteLength_double+exe_params->SZ_SIZE_TYPE); //szTmpBytes = (unsigned char*)malloc(sizeof(unsigned char)*tmpSize); //memcpy(szTmpBytes, tmpBytes, tmpSize); //free(tmpBytes); //release useless memory @@ -64,7 +64,6 @@ size_t cmpSize, int compressionType, double* hist_data) szTmpBytes = cmpBytes; confparams_dec->sol_ID = szTmpBytes[4+14]; //szTmpBytes: version(3bytes), samebyte(1byte), [14]:sol_ID=SZ or SZ_Transpose - //TODO: convert szTmpBytes to double array. TightDataPointStorageD* tdps; int errBoundMode = new_TightDataPointStorageD_fromFlatBytes(&tdps, szTmpBytes, tmpSize); @@ -76,11 +75,11 @@ size_t cmpSize, int compressionType, double* hist_data) *newData = (double*)malloc(doubleSize*dataLength); if(sysEndianType==BIG_ENDIAN_SYSTEM) { - memcpy(*newData, szTmpBytes+4+MetaDataByteLength+exe_params->SZ_SIZE_TYPE, dataLength*doubleSize); + memcpy(*newData, szTmpBytes+4+MetaDataByteLength_double+exe_params->SZ_SIZE_TYPE, dataLength*doubleSize); } else { - unsigned char* p = szTmpBytes+4+MetaDataByteLength+exe_params->SZ_SIZE_TYPE; + unsigned char* p = szTmpBytes+4+MetaDataByteLength_double+exe_params->SZ_SIZE_TYPE; for(i=0;idmin; + double max = confparams_dec->dmax; + for(i=0;i= min) + continue; + if(v < min) + nd[i] = min; + else if(v > max) + nd[i] = max; + } + } + free_TightDataPointStorageD2(tdps); - if(confparams_dec->szMode!=SZ_BEST_SPEED && cmpSize!=12+MetaDataByteLength+exe_params->SZ_SIZE_TYPE) + if(confparams_dec->szMode!=SZ_BEST_SPEED && cmpSize!=12+MetaDataByteLength_double+exe_params->SZ_SIZE_TYPE) free(szTmpBytes); return status; } @@ -2946,6 +2962,7 @@ size_t decompressDataSeries_double_3D_RA_block(double * data, double mean, size_ void decompressDataSeries_double_2D_nonblocked_with_blocked_regression(double** data, size_t r1, size_t r2, unsigned char* comp_data, double* hist_data){ + printf("decompressDataSeries_double_2D_nonblocked_with_blocked_regression\n"); size_t dim0_offset = r2; size_t num_elements = r1 * r2; diff --git a/sz/src/szd_float.c b/sz/src/szd_float.c index f4428348..a8daf48b 100644 --- a/sz/src/szd_float.c +++ b/sz/src/szd_float.c @@ -18,6 +18,28 @@ #include "szd_float_ts.h" #include "utility.h" + +//struct timeval startTime_; +//struct timeval endTime_; /* Start and end times */ +//struct timeval costStart_; /*only used for recording the cost*/ +//double totalCost_ = 0; + +/*void cost_start_() +{ + totalCost_ = 0; + gettimeofday(&costStart_, NULL); +} + +void cost_end_() +{ + double elapsed; + struct timeval costEnd; + gettimeofday(&costEnd, NULL); + elapsed = ((costEnd.tv_sec*1000000+costEnd.tv_usec)-(costStart_.tv_sec*1000000+costStart_.tv_usec))/1000000.0; + totalCost_ += elapsed; +}*/ + + /** * * int compressionType: 1 (time-based compression) ; 0 (space-based compression) @@ -134,6 +156,26 @@ size_t cmpSize, int compressionType, float* hist_data) } } } + + //cost_start_(); + if(PROTECT_VALUE_RANGE) + { + float* nd = *newData; + float min = confparams_dec->fmin; + float max = confparams_dec->fmax; + for(i=0;i= min) + continue; + if(v < min) + nd[i] = min; + else if(v > max) + nd[i] = max; + } + } + //cost_end_(); + //printf("totalCost_=%f\n", totalCost_); free_TightDataPointStorageF2(tdps); if(confparams_dec->szMode!=SZ_BEST_SPEED && cmpSize!=8+MetaDataByteLength+exe_params->SZ_SIZE_TYPE) free(szTmpBytes);