Merge PR #249, Update Inflater dynamic header reader to support partial reads

* Use IList<byte> for building Huffman Trees
* Permits using ArraySegment instead of copying the source array
* Uses Enumerable state machine
* Skips two array copies by using ArraySegment
* Throw usable exceptions when invalid values are being read
* Fixes #253

Author: piksel

src/ICSharpCode.SharpZipLib/Zip/Compression/Inflater.cs

@@ -429,7 +429,7 @@ private bool Decode()
 							mode = DECODE_HUFFMAN;
 							break;
 						case DeflaterConstants.DYN_TREES:
-							dynHeader = new InflaterDynHeader();
+							dynHeader = new InflaterDynHeader(input);
 							mode = DECODE_DYN_HEADER;
 							break;
 						default:
@@ -470,12 +470,12 @@ private bool Decode()
 					}
 
 				case DECODE_DYN_HEADER:
-					if (!dynHeader.Decode(input)) {
+					if (!dynHeader.AttemptRead()) {
 						return false;
 					}
 
-					litlenTree = dynHeader.BuildLitLenTree();
-					distTree = dynHeader.BuildDistTree();
+					litlenTree = dynHeader.LiteralLengthTree;
+					distTree = dynHeader.DistanceTree;
 					mode = DECODE_HUFFMAN;
 					goto case DECODE_HUFFMAN; // fall through
 

src/ICSharpCode.SharpZipLib/Zip/Compression/InflaterDynHeader.cs

@@ -1,4 +1,5 @@
 using System;
+using System.Collections.Generic;
 using ICSharpCode.SharpZipLib.Zip.Compression.Streams;
 
 namespace ICSharpCode.SharpZipLib.Zip.Compression
@@ -7,100 +8,148 @@ class InflaterDynHeader
 	{
 		#region Constants
 
-		static readonly int[] BL_ORDER =
-		{ 16, 17, 18, 0, 8, 7, 9, 6, 10, 5, 11, 4, 12, 3, 13, 2, 14, 1, 15 };
+		// maximum number of literal/length codes
+		const int LITLEN_MAX = 286;
+
+		// maximum number of distance codes
+		const int DIST_MAX = 30;
+
+		// maximum data code lengths to read
+		const int CODELEN_MAX = LITLEN_MAX + DIST_MAX;
+
+		// maximum meta code length codes to read
+		const int META_MAX = 19;
+
+		static readonly int[] MetaCodeLengthIndex =
+			{ 16, 17, 18, 0, 8, 7, 9, 6, 10, 5, 11, 4, 12, 3, 13, 2, 14, 1, 15 };
+
 		#endregion
 
-		public bool Decode(StreamManipulator input)
+		/// <summary>
+		/// Continue decoding header from <see cref="input"/> until more bits are needed or decoding has been completed
+		/// </summary>
+		/// <returns>Returns whether decoding could be completed</returns>
+		public bool AttemptRead()
+			=> !state.MoveNext() || state.Current;
+
+		public InflaterDynHeader(StreamManipulator input)
 		{
-			try
+			this.input = input;
+			stateMachine = CreateStateMachine();
+			state = stateMachine.GetEnumerator();
+		}
+
+		private IEnumerable<bool> CreateStateMachine()
+		{
+
+			// Read initial code length counts from header
+			while (!input.TryGetBits(5, ref litLenCodeCount, 257)) yield return false;
+			while (!input.TryGetBits(5, ref distanceCodeCount, 1)) yield return false;
+			while (!input.TryGetBits(4, ref metaCodeCount, 4)) yield return false;
+			var dataCodeCount = litLenCodeCount + distanceCodeCount;
+
+			if (litLenCodeCount > LITLEN_MAX) throw new ValueOutOfRangeException(nameof(litLenCodeCount));
+			if (distanceCodeCount > DIST_MAX) throw new ValueOutOfRangeException(nameof(distanceCodeCount));
+			if (metaCodeCount > META_MAX) throw new ValueOutOfRangeException(nameof(metaCodeCount));
+
+			// Load code lengths for the meta tree from the header bits
+			for (int i=0; i < metaCodeCount; i++)
+			{
+				while (!input.TryGetBits(3, ref codeLengths, MetaCodeLengthIndex[i])) yield return false;
+			}
+
+			var metaCodeTree = new InflaterHuffmanTree(codeLengths);
+
+			// Decompress the meta tree symbols into the data table code lengths
+			int index = 0;
+			while (index < dataCodeCount)
 			{
-				lnum = input.GrabBits(5) + 257;
-				dnum = input.GrabBits(5) + 1;
-				blnum = input.GrabBits(4) + 4;
-				num = lnum + dnum;
+				byte codeLength;
+				int symbol;
 
-				lengths = new byte[19];
+				while ((symbol = metaCodeTree.GetSymbol(input)) < 0) yield return false;
 
-				for (int i = 0; i < blnum; i++)
+				if (symbol < 16)
 				{
-					lengths[BL_ORDER[i]] = (byte)input.GrabBits(3, true);
+					// append literal code length
+					codeLengths[index++] = (byte)symbol;
 				}
-				blTree = new InflaterHuffmanTree(lengths);
-				lengths = new byte[num];
-
-				int index = 0;
-				while (index < lnum + dnum)
+				else
 				{
-					byte len;
-
-					int symbol = blTree.GetSymbol(input);
-					if (symbol < 0)
-						return false;
-					if (symbol < 16)
-						lengths[index++] = (byte)symbol;
-					else
+					int repeatCount = 0;
+
+					if (symbol == 16) // Repeat last code length 3..6 times
+					{
+
+						if (index == 0)
+							throw new StreamDecodingException("Cannot repeat previous code length when no other code length has been read");
+
+						codeLength = codeLengths[index - 1];
+
+						// 2 bits + 3, [3..6]
+						while(!input.TryGetBits(2, ref repeatCount, 3)) yield return false;
+					}
+					else if (symbol == 17) // Repeat zero 3..10 times
 					{
-						len = 0;
-						if (symbol == 16)
-						{
-							if (index == 0)
-								return false;   // No last length!
-							len = lengths[index - 1];
-							symbol = input.GrabBits(2, true) + 3;
-						}
-						else if (symbol == 17)
-						{
-							// repeat zero 3..10 times
-							symbol = input.GrabBits(3, true) + 3;
-						}
-						else
-						{
-							// (symbol == 18), repeat zero 11..138 times
-							symbol = input.GrabBits(7, true) + 11;
-						}
-
-						if (index + symbol > lnum + dnum)
-							return false; // too many lengths!
-
-						// repeat last or zero symbol times
-						while (symbol-- > 0)
-							lengths[index++] = len;
+						codeLength = 0;
+
+						// 3 bits + 3, [3..10]
+						while (!input.TryGetBits(3, ref repeatCount, 3)) yield return false;
 					}
-				}
+					else // (symbol == 18), Repeat zero 11..138 times
+					{
+						codeLength = 0;
 
-				if (lengths[256] == 0)
-					return false; // No end-of-block code!
+						// 7 bits + 11, [11..138]
+						while (!input.TryGetBits(7, ref repeatCount, 11)) yield return false;
+					}
 
-				return true;
-			}
-			catch (Exception x)
-			{
-				return false;
+					if (index + repeatCount > dataCodeCount)
+						throw new StreamDecodingException("Cannot repeat code lengths past total number of data code lengths");
+
+					while (repeatCount-- > 0)
+						codeLengths[index++] = codeLength;
+				}
 			}
-		}
 
-		public InflaterHuffmanTree BuildLitLenTree()
-		{
-			byte[] litlenLens = new byte[lnum];
-			Array.Copy(lengths, 0, litlenLens, 0, lnum);
-			return new InflaterHuffmanTree(litlenLens);
-		}
+			if (codeLengths[256] == 0)
+				throw new StreamDecodingException("Inflater dynamic header end-of-block code missing");
 
-		public InflaterHuffmanTree BuildDistTree()
-		{
-			byte[] distLens = new byte[dnum];
-			Array.Copy(lengths, lnum, distLens, 0, dnum);
-			return new InflaterHuffmanTree(distLens);
+			litLenTree = new InflaterHuffmanTree(new ArraySegment<byte>(codeLengths, 0, litLenCodeCount));
+			distTree = new InflaterHuffmanTree(new ArraySegment<byte>(codeLengths, litLenCodeCount, distanceCodeCount));
+
+			yield return true;
 		}
 
+		/// <summary>
+		/// Get literal/length huffman tree, must not be used before <see cref="AttemptRead"/> has returned true
+		/// </summary>
+		/// <exception cref="StreamDecodingException">If hader has not been successfully read by the state machine</exception>
+		public InflaterHuffmanTree LiteralLengthTree
+			=> litLenTree ?? throw new StreamDecodingException("Header properties were accessed before header had been successfully read");
+
+		/// <summary>
+		/// Get distance huffman tree, must not be used before <see cref="AttemptRead"/> has returned true
+		/// </summary>
+		/// <exception cref="StreamDecodingException">If hader has not been successfully read by the state machine</exception>
+		public InflaterHuffmanTree DistanceTree
+			=> distTree ?? throw new StreamDecodingException("Header properties were accessed before header had been successfully read");
+
 		#region Instance Fields
-		byte[] lengths;
 
-		InflaterHuffmanTree blTree;
+		private readonly StreamManipulator input;
+		private readonly IEnumerator<bool> state;
+		private readonly IEnumerable<bool> stateMachine;
+
+		private byte[] codeLengths = new byte[CODELEN_MAX];
+
+		private InflaterHuffmanTree litLenTree;
+		private InflaterHuffmanTree distTree;
+
+		int litLenCodeCount, distanceCodeCount, metaCodeCount;
 
-		int lnum, dnum, blnum, num;
 		#endregion
 
 	}
+
 }

src/ICSharpCode.SharpZipLib/Zip/Compression/InflaterHuffmanTree.cs

@@ -1,4 +1,5 @@
 using System;
+using System.Collections.Generic;
 using ICSharpCode.SharpZipLib.Zip.Compression.Streams;
 
 namespace ICSharpCode.SharpZipLib.Zip.Compression
@@ -63,18 +64,18 @@ static InflaterHuffmanTree()
 		/// <param name = "codeLengths">
 		/// the array of code lengths
 		/// </param>
-		public InflaterHuffmanTree(byte[] codeLengths)
+		public InflaterHuffmanTree(IList<byte> codeLengths)
 		{
 			BuildTree(codeLengths);
 		}
 		#endregion
 
-		void BuildTree(byte[] codeLengths)
+		void BuildTree(IList<byte> codeLengths)
 		{
 			int[] blCount = new int[MAX_BITLEN + 1];
 			int[] nextCode = new int[MAX_BITLEN + 1];
 
-			for (int i = 0; i < codeLengths.Length; i++) {
+			for (int i = 0; i < codeLengths.Count; i++) {
 				int bits = codeLengths[i];
 				if (bits > 0) {
 					blCount[bits]++;
@@ -115,7 +116,7 @@ void BuildTree(byte[] codeLengths)
 				}
 			}
 
-			for (int i = 0; i < codeLengths.Length; i++) {
+			for (int i = 0; i < codeLengths.Count; i++) {
 				int bits = codeLengths[i];
 				if (bits == 0) {
 					continue;
@@ -154,36 +155,49 @@ void BuildTree(byte[] codeLengths)
 		public int GetSymbol(StreamManipulator input)
 		{
 			int lookahead, symbol;
-			if ((lookahead = input.PeekBits(9)) >= 0) {
-				if ((symbol = tree[lookahead]) >= 0) {
+			if ((lookahead = input.PeekBits(9)) >= 0)
+			{
+				if ((symbol = tree[lookahead]) >= 0)
+				{
 					input.DropBits(symbol & 15);
 					return symbol >> 4;
 				}
 				int subtree = -(symbol >> 4);
 				int bitlen = symbol & 15;
-				if ((lookahead = input.PeekBits(bitlen)) >= 0) {
+				if ((lookahead = input.PeekBits(bitlen)) >= 0)
+				{
 					symbol = tree[subtree | (lookahead >> 9)];
 					input.DropBits(symbol & 15);
 					return symbol >> 4;
-				} else {
+				}
+				else
+				{
 					int bits = input.AvailableBits;
 					lookahead = input.PeekBits(bits);
 					symbol = tree[subtree | (lookahead >> 9)];
-					if ((symbol & 15) <= bits) {
+					if ((symbol & 15) <= bits)
+					{
 						input.DropBits(symbol & 15);
 						return symbol >> 4;
-					} else {
+					}
+					else
+					{
 						return -1;
 					}
 				}
-			} else {
+			}
+			else // Less than 9 bits
+			{
 				int bits = input.AvailableBits;
 				lookahead = input.PeekBits(bits);
 				symbol = tree[lookahead];
-				if (symbol >= 0 && (symbol & 15) <= bits) {
+				if (symbol >= 0 && (symbol & 15) <= bits)
+				{
 					input.DropBits(symbol & 15);
 					return symbol >> 4;
-				} else {
+				}
+				else
+				{
 					return -1;
 				}
 			}

src/ICSharpCode.SharpZipLib/Zip/Compression/Streams/StreamManipulator.cs

@@ -42,16 +42,38 @@ public int PeekBits(int bitCount)
 		}
 
 		/// <summary>
-		/// Grabs the next n bits from the input and throws if <paramref name="allowZero"/> is false and the result is 0.
+		/// Tries to grab the next <paramref name="bitCount"/> bits from the input and 
+		/// sets <paramref name="output"/> to the value, adding <paramref name="outputOffset"/>.
 		/// </summary>
-		public int GrabBits(int bitCount, bool allowZero = false)
+		/// <returns>true if enough bits could be read, otherwise false</returns>
+		public bool TryGetBits(int bitCount, ref int output, int outputOffset = 0)
 		{
-			var val = PeekBits(bitCount);
-			if (!allowZero && val == 0)
-				throw new SharpZipBaseException(bitCount + "-bit value cannot be zero");
+			var bits = PeekBits(bitCount);
+			if (bits < 0)
+			{
+				return false;
+			}
+			output = bits + outputOffset;
 			DropBits(bitCount);
-			return val;
-		} 
+			return true;
+		}
+
+		/// <summary>
+		/// Tries to grab the next <paramref name="bitCount"/> bits from the input and 
+		/// sets <paramref name="output"/> to the value, adding <paramref name="outputOffset"/>.
+		/// </summary>
+		/// <returns>true if enough bits could be read, otherwise false</returns>
+		public bool TryGetBits(int bitCount, ref byte[] array, int index)
+		{
+			var bits = PeekBits(bitCount);
+			if (bits < 0)
+			{
+				return false;
+			}
+			array[index] = (byte)bits;
+			DropBits(bitCount);
+			return true;
+		}
 
 		/// <summary>
 		/// Drops the next n bits from the input.  You should have called PeekBits