[Fix] Correct Reporting of CTF Compressed Payload in Metadata

Author: MichaelLettrich

DataFormats/Detectors/Common/include/DetectorsCommonDataFormats/EncodedBlocks.h

@@ -929,9 +929,11 @@ CTFIOSize EncodedBlocks<H, N, W>::decode(D_IT dest,                        // it
   const auto& md = mMetadata[slot];
   LOGP(debug, "Slot{} | NStored={} Ndict={} nData={}, MD: messageLength:{} opt:{} min:{} max:{} offs:{} width:{} ", slot, block.getNStored(), block.getNDict(), block.getNData(), md.messageLength, (int)md.opt, md.min, md.max, md.literalsPackingOffset, md.literalsPackingWidth);
 
+  constexpr size_t word_size = sizeof(W);
+
   if (ansVersion == ANSVersionCompat) {
     if (!block.getNStored()) {
-      return {0, md.getUncompressedSize(), md.getCompressedSize()};
+      return {0, md.getUncompressedSize(), md.getCompressedSize() * word_size};
     }
     if (md.opt == Metadata::OptStore::EENCODE) {
       return decodeCompatImpl(dest, slot, decoderExt);
@@ -943,7 +945,7 @@ CTFIOSize EncodedBlocks<H, N, W>::decode(D_IT dest,                        // it
       return decodeUnpackImpl(dest, slot);
     }
     if (!block.getNStored()) {
-      return {0, md.getUncompressedSize(), md.getCompressedSize()};
+      return {0, md.getUncompressedSize(), md.getCompressedSize() * word_size};
     }
     if (md.opt == Metadata::OptStore::EENCODE) {
       return decodeRansV1Impl(dest, slot, decoderExt);
@@ -991,7 +993,7 @@ CTFIOSize EncodedBlocks<H, N, W>::decodeCompatImpl(dst_IT dstBegin, int slot, co
   } else {
     getDecoder().process(block.getData() + block.getNData(), dstBegin, md.messageLength, NDecoderStreams);
   }
-  return {0, md.getUncompressedSize(), md.getCompressedSize()};
+  return {0, md.getUncompressedSize(), md.getCompressedSize() * sizeof(W)};
 };
 
 template <typename H, int N, typename W>
@@ -1045,7 +1047,7 @@ CTFIOSize EncodedBlocks<H, N, W>::decodeRansV1Impl(dst_IT dstBegin, int slot, co
   } else {
     getDecoder().process(block.getData() + block.getNData(), dstBegin, md.messageLength, md.nStreams);
   }
-  return {0, md.getUncompressedSize(), md.getCompressedSize()};
+  return {0, md.getUncompressedSize(), md.getCompressedSize() * sizeof(W)};
 };
 
 template <typename H, int N, typename W>
@@ -1079,7 +1081,7 @@ CTFIOSize EncodedBlocks<H, N, W>::decodeUnpackImpl(dst_IT dest, int slot) const
   } else {
     rans::unpack(srcIt, messageLength, dest, packingWidth, offset);
   }
-  return {0, md.getUncompressedSize(), md.getCompressedSize()};
+  return {0, md.getUncompressedSize(), md.getCompressedSize() * sizeof(W)};
 };
 
 template <typename H, int N, typename W>
@@ -1098,7 +1100,7 @@ CTFIOSize EncodedBlocks<H, N, W>::decodeCopyImpl(dst_IT dest, int slot) const
   destPtr_t srcEnd = srcBegin + md.messageLength * sizeof(dest_t);
   std::copy(srcBegin, srcEnd, dest);
 
-  return {0, md.getUncompressedSize(), md.getCompressedSize()};
+  return {0, md.getUncompressedSize(), md.getCompressedSize() * sizeof(W)};
 };
 
 ///_____________________________________________________________________________
@@ -1268,7 +1270,7 @@ o2::ctf::CTFIOSize EncodedBlocks<H, N, W>::entropyCodeRANSCompat(const input_IT
                                                                                      dataSize,
                                                                                      nLiteralWords);
 
-  return {0, thisMetadata->getUncompressedSize(), thisMetadata->getCompressedSize()};
+  return {0, thisMetadata->getUncompressedSize(), thisMetadata->getCompressedSize() * sizeof(W)};
 }
 
 template <typename H, int N, typename W>
@@ -1349,7 +1351,7 @@ CTFIOSize EncodedBlocks<H, N, W>::encodeRANSV1External(const input_IT srcBegin,
                                                                                  dataSize,
                                                                                  literalsSize);
 
-  return {0, thisMetadata->getUncompressedSize(), thisMetadata->getCompressedSize()};
+  return {0, thisMetadata->getUncompressedSize(), thisMetadata->getCompressedSize() * sizeof(W)};
 };
 
 template <typename H, int N, typename W>
@@ -1458,7 +1460,7 @@ CTFIOSize EncodedBlocks<H, N, W>::encodeRANSV1Inplace(const input_IT srcBegin, c
                                                                                  dataSize,
                                                                                  literalsSize);
 
-  return {0, thisMetadata->getUncompressedSize(), thisMetadata->getCompressedSize()};
+  return {0, thisMetadata->getUncompressedSize(), thisMetadata->getCompressedSize() * sizeof(W)};
 }; // namespace ctf
 
 template <typename H, int N, typename W>
@@ -1491,7 +1493,7 @@ o2::ctf::CTFIOSize EncodedBlocks<H, N, W>::pack(const input_IT srcBegin, const i
   }
 
   LOGP(debug, "StoreData {} bytes, offs: {}:{}", packedSize * sizeof(storageBuffer_t), thisBlock->getOffsData(), thisBlock->getOffsData() + packedSize * sizeof(storageBuffer_t));
-  return {0, thisMetadata->getUncompressedSize(), thisMetadata->getCompressedSize()};
+  return {0, thisMetadata->getUncompressedSize(), thisMetadata->getCompressedSize() * sizeof(W)};
 };
 
 template <typename H, int N, typename W>
@@ -1513,7 +1515,7 @@ o2::ctf::CTFIOSize EncodedBlocks<H, N, W>::store(const input_IT srcBegin, const
 
   *thisMetadata = detail::makeMetadataStore<input_t, storageBuffer_t>(messageLength, opt, nBufferElems);
 
-  return {0, thisMetadata->getUncompressedSize(), thisMetadata->getCompressedSize()};
+  return {0, thisMetadata->getUncompressedSize(), thisMetadata->getCompressedSize() * sizeof(W)};
 };
 
 /// create a special EncodedBlocks containing only dictionaries made from provided vector of frequency tables

DataFormats/Detectors/Common/include/DetectorsCommonDataFormats/Metadata.h

@@ -37,7 +37,7 @@ struct Metadata {
   size_t nLiterals = 0;              // Number of samples that were stored as literals.
   uint8_t messageWordSize = 0;       // size in Bytes of a symbol in the encoded message.
   uint8_t coderType = 0;             // what type of CTF Coder is used? (32 vs 64 bit coders).
-  uint8_t streamSize = 0;            // how many Bytes is the rANS encoder emmiting during a stream-out step.
+  uint8_t streamSize = 0;            // number of Bytes emmitted during rANS stream out (ransCompat) or lower renorming bound (ransV1).
   uint8_t probabilityBits = 0;       // The encoder renormed the distribution of source symbols to sum up to 2^probabilityBits.
   OptStore opt = OptStore::EENCODE;  // The type of storage operation that was conducted.
   int32_t min = 0;                   // min symbol of the source dataset.
@@ -48,8 +48,21 @@ struct Metadata {
   int nDataWords = 0;                // Amount of words used to store the actual data.
   int nLiteralWords = 0;             // Amount of words used to store literal (incompressible) samples.
 
+  /**
+   * @brief Uncompressed size of stored data in bytes
+   *
+   * @return size_t Uncompressed size in bytes
+   */
   size_t getUncompressedSize() const { return messageLength * messageWordSize; }
-  size_t getCompressedSize() const { return (nDictWords + nDataWords + nLiteralWords) * streamSize; }
+
+  /**
+   * @brief Size of the stored, compressed data in multiples of the underlying buffer word size
+   *
+   * @return size_t The size in multiples of the underlying buffer word size
+   * @warning This size is in number of words of the underlying storage buffer.
+   * Multiply with the size of the storage buffer type to get the correct size in bytes.
+   */
+  size_t getCompressedSize() const { return nDictWords + nDataWords + nLiteralWords; }
   void clear()
   {
     nStreams = 0;