32 if sys.version[0] ==
'2':
39 from daal.data_management
import Compressor_Zlib, Decompressor_Zlib
41 utils_folder = os.path.realpath(os.path.abspath(os.path.dirname(os.path.dirname(__file__))))
42 if utils_folder
not in sys.path:
43 sys.path.insert(0, utils_folder)
44 from utils
import getCRC32, readTextFile
46 datasetFileName = os.path.join(
'..',
'data',
'batch',
'logitboost_train.csv')
49 sendReceiveQueue = Queue.Queue()
51 maxDataBlockSize = 16384
53 def getUncompressedDataBlock(sentDataStream, availableDataSize):
54 cur_pos = sentDataStream.size - availableDataSize
57 if availableDataSize >= maxDataBlockSize:
58 return (sentDataStream[cur_pos:cur_pos + maxDataBlockSize], availableDataSize - maxDataBlockSize)
59 elif availableDataSize < maxDataBlockSize
and availableDataSize > 0:
60 return (sentDataStream[cur_pos:cur_pos + availableDataSize], 0)
64 def sendCompressedDataBlock(block):
65 currentBlock = np.copy(block)
67 sendReceiveQueue.put(currentBlock)
70 def receiveCompressedDataBlock():
72 if sendReceiveQueue.empty():
75 return np.copy(sendReceiveQueue.get())
78 def printCRC32(sentDataStream, receivedDataStream):
80 crcSentDataStream = getCRC32(sentDataStream)
81 crcReceivedDataStream = getCRC32(receivedDataStream)
83 print(
"\nCompression example program results:\n")
85 print(
"Input data checksum: 0x{:02X}".format(crcSentDataStream))
86 print(
"Received data checksum: 0x{:02X}".format(crcReceivedDataStream))
88 if sentDataStream.size != receivedDataStream.size:
89 print(
"ERROR: Received data size mismatches with the sent data size")
91 elif crcSentDataStream != crcReceivedDataStream:
92 print(
"ERROR: Received data CRC mismatches with the sent data CRC")
94 print(
"OK: Received data CRC matches with the sent data CRC")
97 if __name__ ==
"__main__":
100 sentDataStream = readTextFile(datasetFileName)
103 compressedDataBlock = np.empty(maxDataBlockSize, dtype=np.uint8)
104 receivedDataStream = np.empty(sentDataStream.size, dtype=np.uint8)
107 compressor = Compressor_Zlib()
110 (uncompressedDataBlock, availableDataSize) = getUncompressedDataBlock(sentDataStream, sentDataStream.size)
111 while uncompressedDataBlock
is not None:
113 compressor.setInputDataBlock(uncompressedDataBlock, 0)
118 compressor.run(compressedDataBlock, 0)
121 compressedDataView = compressedDataBlock[0:compressor.getUsedOutputDataBlockSize()]
124 sendCompressedDataBlock(compressedDataView)
127 if not compressor.isOutputDataBlockFull():
131 (uncompressedDataBlock, availableDataSize) = getUncompressedDataBlock(sentDataStream, availableDataSize)
134 decompressor = Decompressor_Zlib()
137 compressedDataBlock = receiveCompressedDataBlock()
140 while compressedDataBlock
is not None:
142 decompressor.setInputDataBlock(compressedDataBlock, 0)
145 decompressor.run(receivedDataStream, offset)
148 offset += decompressor.getUsedOutputDataBlockSize()
151 compressedDataBlock = receiveCompressedDataBlock()
154 printCRC32(sentDataStream, receivedDataStream)