metaUtils: make the (de)compression chunk size settable, and cover the trailer case

src/metaUtils.cxx

@@ -57,7 +57,22 @@ bool META_DEBUG = false;
 
 static char MET_SeperatorChar = '=';
 
-constexpr static std::streamoff MET_MaxChunkSize = 1024 * 1024 * 1024;
+static std::streamoff MET_MaxChunkSize = 1024 * 1024 * 1024;
+
+void
+MET_SetMaxChunkSize(std::streamoff chunkSize)
+{
+  if (chunkSize > 0)
+  {
+    MET_MaxChunkSize = chunkSize;
+  }
+}
+
+std::streamoff
+MET_GetMaxChunkSize()
+{
+  return MET_MaxChunkSize;
+}
 
 MET_FieldRecordType *
 MET_GetFieldRecord(const char * _fieldName, std::vector<MET_FieldRecordType *> * _fields)

src/metaUtils.h

@@ -339,6 +339,15 @@ MET_PerformCompression(const unsigned char * source,
                        std::streamoff *      compressedDataSize,
                        int                   compressionLevel);
 
+// Size of the input and output pieces the (de)compression loops work in.
+METAIO_EXPORT
+void
+MET_SetMaxChunkSize(std::streamoff chunkSize);
+
+METAIO_EXPORT
+std::streamoff
+MET_GetMaxChunkSize();
+
 METAIO_EXPORT
 bool
 MET_PerformUncompression(const unsigned char * sourceCompressed,

src/tests/CMakeLists.txt

@@ -28,3 +28,4 @@ MetaAddTest(testMeta11Form)
 MetaAddTest(testMeta12Array)
 MetaAddTest(testMeta13ImageList)
 MetaAddTest(testMeta14ImageCompressed)
+MetaAddTest(testMeta15UncompressChunkBoundary)

src/tests/testMeta15UncompressChunkBoundary.cxx

@@ -0,0 +1,84 @@
+#include <cstring>
+#include <iostream>
+#include <vector>
+
+#include <metaUtils.h>
+
+// A chunk boundary that falls inside the gzip trailer leaves the trailer in
+// the next input piece after the output buffer is already full.
+static int
+TestTrailerInLaterChunk(const std::vector<unsigned char> & raw, const std::vector<unsigned char> & compressed)
+{
+  const std::streamoff savedChunkSize = MET_GetMaxChunkSize();
+  MET_SetMaxChunkSize(static_cast<std::streamoff>(compressed.size()) - 4);
+
+  std::vector<unsigned char> destination(raw.size(), 0);
+  const bool                 accepted = MET_PerformUncompression(compressed.data(),
+                                                 static_cast<std::streamoff>(compressed.size()),
+                                                 destination.data(),
+                                                 static_cast<std::streamoff>(raw.size()));
+  MET_SetMaxChunkSize(savedChunkSize);
+
+  if (!accepted)
+  {
+    std::cerr << "FAILED: valid stream rejected when the trailer lands in a later input chunk\n";
+    return 1;
+  }
+  if (destination != raw)
+  {
+    std::cerr << "FAILED: decompressed content does not match the original\n";
+    return 1;
+  }
+  return 0;
+}
+
+static int
+TestCorruptTrailerStillRejected(const std::vector<unsigned char> & raw, std::vector<unsigned char> compressed)
+{
+  compressed[compressed.size() - 1] ^= 0xFF;
+
+  std::vector<unsigned char> destination(raw.size(), 0);
+  std::cerr << "--- expect an uncompression failure message below ---\n";
+  const bool accepted = MET_PerformUncompression(compressed.data(),
+                                                 static_cast<std::streamoff>(compressed.size()),
+                                                 destination.data(),
+                                                 static_cast<std::streamoff>(raw.size()));
+  if (accepted)
+  {
+    std::cerr << "FAILED: stream with a corrupt CRC trailer was accepted\n";
+    return 1;
+  }
+  return 0;
+}
+
+int
+main(int, char *[])
+{
+  std::vector<unsigned char> raw(64 * 1024);
+  for (size_t i = 0; i < raw.size(); ++i)
+  {
+    raw[i] = static_cast<unsigned char>((i * 7 + (i >> 3)) & 0xFF);
+  }
+
+  std::streamoff  compressedSize = 0;
+  unsigned char * compressedBuffer =
+    MET_PerformCompression(raw.data(), static_cast<std::streamoff>(raw.size()), &compressedSize, 6);
+  if (compressedBuffer == nullptr || compressedSize <= 8)
+  {
+    std::cerr << "FAILED: compression did not produce a usable stream\n";
+    delete[] compressedBuffer;
+    return 1;
+  }
+  const std::vector<unsigned char> compressed(compressedBuffer, compressedBuffer + compressedSize);
+  delete[] compressedBuffer;
+
+  int result = 0;
+  result += TestTrailerInLaterChunk(raw, compressed);
+  result += TestCorruptTrailerStillRejected(raw, compressed);
+
+  if (result == 0)
+  {
+    std::cout << "testMeta15UncompressChunkBoundary passed\n";
+  }
+  return result;
+}