fix: Fix download flow, sparse conversion & parsing

13584452567 · 13584452567 · commit 35f6d1c3d748 · 2026-03-07T12:52:18.000+08:00
Add tests for download error cases and sparse flashing; validate device-accepted download size and fail early on mismatches. Reuse buffers (ArrayPool/stacked transfer buffer) to reduce allocations and avoid extra copies during chunked downloads; improve stream read/write error handling and CRC parsing with clearer error responses. Implement RAW-&gt;sparse conversion for oversized images by writing a temp raw file (when needed), converting to sparse, and flashing via sparse protocol; ensure temp cleanup and stream repositioning. Make UdpTransport append packet bytes without extra ToArray allocations. Harden HandleResponse parsing: unify delimiter handling, use prefix helper checks, accumulate TEXT blocks in a StringBuilder, and return collected text with final responses.
diff --git a/FirmwareKit.Comm.Fastboot.Tests/FastbootProtocolTests.cs b/FirmwareKit.Comm.Fastboot.Tests/FastbootProtocolTests.cs
@@ -271,16 +271,37 @@ public void HandleResponse_FragmentedDataPrefix_Succeeds()
         public void DownloadDataStream_PrematureEof_FailsEarly()
         {
             var transport = new MockTransport();
-            transport.EnqueueResponse("DATA00000010");
+            transport.EnqueueResponse("OKAYno");
+            for (int i = 0; i < 4; i++)
+            {
+                transport.EnqueueResponse("DATA00000010");
+            }
             var util = new FastbootDriver(transport);
 
             using var shortStream = new MemoryStream(new byte[4]);
             var response = util.DownloadData(shortStream, 16);
 
             Assert.Equal(FastbootState.Fail, response.Result);
+            Assert.Contains("Max retries exceeded", response.Response);
             Assert.Contains("stream ended early", response.Response);
         }
 
+        [Fact]
+        public void DownloadDataStream_DataSizeMismatch_Fails()
+        {
+            var transport = new MockTransport();
+            transport.EnqueueResponse("OKAYno");
+            transport.EnqueueResponse("DATA00000004");
+            var util = new FastbootDriver(transport);
+
+            using var stream = new MemoryStream(new byte[8]);
+            var response = util.DownloadData(stream, 8);
+
+            Assert.Equal(FastbootState.Fail, response.Result);
+            Assert.Contains("download size mismatch", response.Response);
+            Assert.DoesNotContain("Short write", response.Response);
+        }
+
         [Fact]
         public void DownloadDataBytes_ZeroLength_Fails()
         {
@@ -323,6 +344,35 @@ public void DownloadDataBytes_ShortWrite_Fails()
             Assert.Contains("Short write", response.Response);
         }
 
+        [Fact]
+        public void DownloadDataBytes_DataSizeMismatch_Fails()
+        {
+            var transport = new MockTransport();
+            transport.EnqueueResponse("DATA00000004");
+            var util = new FastbootDriver(transport);
+
+            var response = util.DownloadData(new byte[8]);
+
+            Assert.Equal(FastbootState.Fail, response.Result);
+            Assert.Contains("download size mismatch", response.Response);
+        }
+
+        [Fact]
+        public void DownloadDataStream_MalformedCrcResponse_Fails()
+        {
+            var transport = new MockTransport();
+            transport.EnqueueResponse("OKAYyes");
+            transport.EnqueueResponse("DATA00000004");
+            transport.EnqueueResponse("OKAY0xGGGGGGGG");
+            var util = new FastbootDriver(transport);
+
+            using var stream = new MemoryStream(new byte[] { 1, 2, 3, 4 });
+            var response = util.DownloadData(stream, 4);
+
+            Assert.Equal(FastbootState.Fail, response.Result);
+            Assert.Contains("invalid CRC response", response.Response);
+        }
+
         [Fact]
         public void UploadData_SegmentedRead_Succeeds()
         {
@@ -398,5 +448,27 @@ public void FlashImage_Stream_NonSparse_RewindsBeforeDownload()
             Assert.Contains("flash:boot", transport.Commands);
             Assert.Equal(imageBytes, transport.DownloadPayload.ToArray());
         }
+
+        [Fact]
+        public void FlashUnsparseImage_OversizedRaw_IsConvertedToSparseAndFlashed()
+        {
+            byte[] imageBytes = new byte[128];
+            var transport = new ProtocolDownloadCaptureTransport(new Dictionary<string, string>(StringComparer.OrdinalIgnoreCase)
+            {
+                ["getvar:has-slot:boot"] = "OKAYno",
+                ["getvar:is-logical:boot"] = "OKAYno",
+                ["getvar:max-download-size"] = "OKAY0x40",
+                ["flash:boot"] = "OKAY"
+            });
+
+            var util = new FastbootDriver(transport);
+
+            using var stream = new MemoryStream(imageBytes);
+            var response = util.FlashUnsparseImage("boot", stream, stream.Length);
+
+            Assert.Equal(FastbootState.Success, response.Result);
+            Assert.Contains(transport.Commands, c => c.StartsWith("download:", StringComparison.OrdinalIgnoreCase));
+            Assert.Contains("flash:boot", transport.Commands);
+        }
     }
 }
diff --git a/FirmwareKit.Comm.Fastboot/Backend/Network/UdpTransport.cs b/FirmwareKit.Comm.Fastboot/Backend/Network/UdpTransport.cs
@@ -142,7 +142,10 @@ private byte[] SendSinglePacket(PacketId id, ushort seq, PacketFlag flag, byte[]
 
                         if (rxPacket.Length > HeaderSize)
                         {
-                            fullResponse.AddRange(rxPacket.AsSpan(HeaderSize).ToArray());
+                            for (int j = HeaderSize; j < rxPacket.Length; j++)
+                            {
+                                fullResponse.Add(rxPacket[j]);
+                            }
                         }
 
                         gotValidResponse = true;
diff --git a/FirmwareKit.Comm.Fastboot/Command/InternalDownloadDataBytes.cs b/FirmwareKit.Comm.Fastboot/Command/InternalDownloadDataBytes.cs
@@ -21,17 +21,25 @@ public FastbootResponse DownloadData(byte[] data)
         FastbootResponse response = RawCommand("download:" + data.Length.ToString("x8"));
         if (response.Result != FastbootState.Data)
             return response;
+        if (response.DataSize != data.Length)
+        {
+            return new FastbootResponse
+            {
+                Result = FastbootState.Fail,
+                Response = $"download size mismatch: requested {data.Length}, device accepted {response.DataSize}"
+            };
+        }
 
         long bytesWritten = 0;
         int length = data.Length;
+        byte[] transferBuffer = new byte[Math.Min(OnceSendDataSize, length)];
 
         while (bytesWritten < length)
         {
             int toWrite = (int)Math.Min(OnceSendDataSize, length - bytesWritten);
-            byte[] chunk = new byte[toWrite];
-            Array.Copy(data, bytesWritten, chunk, 0, toWrite);
+            Buffer.BlockCopy(data, (int)bytesWritten, transferBuffer, 0, toWrite);
 
-            long written = Transport.Write(chunk, toWrite);
+            long written = Transport.Write(transferBuffer, toWrite);
             if (written != toWrite)
             {
                 return new FastbootResponse { Result = FastbootState.Fail, Response = $"Short write: {written}/{toWrite}" };
diff --git a/FirmwareKit.Comm.Fastboot/Command/InternalDownloadDataStream.cs b/FirmwareKit.Comm.Fastboot/Command/InternalDownloadDataStream.cs
@@ -1,5 +1,6 @@
 
 using Force.Crc32;
+using System.Buffers;
 
 namespace FirmwareKit.Comm.Fastboot;
 
@@ -63,33 +64,47 @@ private FastbootResponse DownloadDataInternal(Stream stream, long length, bool o
         FastbootResponse response = RawCommand("download:" + length.ToString("x8"));
         if (response.Result != FastbootState.Data)
             return response;
+        if (response.DataSize != length)
+        {
+            return new FastbootResponse
+            {
+                Result = FastbootState.Fail,
+                Response = $"download size mismatch: requested {length}, device accepted {response.DataSize}"
+            };
+        }
 
-        byte[] buffer = new byte[OnceSendDataSize];
+        byte[] buffer = ArrayPool<byte>.Shared.Rent(OnceSendDataSize);
         long bytesWritten = 0;
         uint crc = 0;
-
-        while (bytesWritten < length)
+        try
         {
-            int toRead = (int)Math.Min(OnceSendDataSize, length - bytesWritten);
-            int readSize = stream.Read(buffer, 0, toRead);
-            if (readSize <= 0)
+            while (bytesWritten < length)
             {
-                throw new Exception("stream ended early: " + bytesWritten + "/" + length);
-            }
+                int toRead = (int)Math.Min(OnceSendDataSize, length - bytesWritten);
+                int readSize = stream.Read(buffer, 0, toRead);
+                if (readSize <= 0)
+                {
+                    throw new Exception("stream ended early: " + bytesWritten + "/" + length);
+                }
 
-            if (useCrc)
-            {
-                crc = Crc32Algorithm.Append(crc, buffer, 0, readSize);
-            }
+                if (useCrc)
+                {
+                    crc = Crc32Algorithm.Append(crc, buffer, 0, readSize);
+                }
 
-            long written = Transport.Write(buffer, readSize);
-            if (written != readSize)
-            {
-                throw new Exception("Short write: " + written + "/" + readSize);
+                long written = Transport.Write(buffer, readSize);
+                if (written != readSize)
+                {
+                    throw new Exception("Short write: " + written + "/" + readSize);
+                }
+                bytesWritten += written;
+                if (onEvent)
+                    NotifyProgress(bytesWritten, length);
             }
-            bytesWritten += written;
-            if (onEvent)
-                NotifyProgress(bytesWritten, length);
+        }
+        finally
+        {
+            ArrayPool<byte>.Shared.Return(buffer);
         }
 
         var finalRes = HandleResponse();
@@ -107,7 +122,14 @@ private FastbootResponse DownloadDataInternal(Stream stream, long length, bool o
                         throw new Exception("CRC Mismatch: Device 0x" + deviceCrc.ToString("x8") + " != Host 0x" + crc.ToString("x8"));
                     }
                 }
-                catch (FormatException) { }
+                catch (FormatException)
+                {
+                    return new FastbootResponse
+                    {
+                        Result = FastbootState.Fail,
+                        Response = "invalid CRC response from device: " + resp
+                    };
+                }
             }
         }
 
diff --git a/FirmwareKit.Comm.Fastboot/Command/InternalFlashUnsparseImage.cs b/FirmwareKit.Comm.Fastboot/Command/InternalFlashUnsparseImage.cs
@@ -1,5 +1,7 @@
 
 using FirmwareKit.Sparse.Core;
+using FirmwareKit.Sparse.Utils;
+using System.Buffers;
 
 
 namespace FirmwareKit.Comm.Fastboot;
@@ -49,29 +51,81 @@ public FastbootResponse FlashUnsparseImage(string partition, Stream stream, long
 
         if (length > maxDownloadSize)
         {
-            FastbootDebug.Log($"Large image detected, splitting RAW: {length} > {maxDownloadSize}");
-            NotifyCurrentStep($"{partition} ({length} bytes) is larger than max-download-size ({maxDownloadSize}). Splitting RAW image...");
-            // Manually split RAW stream into sparse-like chunks for download
-            long bytesWritten = 0;
-            int count = 1;
-            long totalChunks = (length + maxDownloadSize - 1) / maxDownloadSize;
-
-            while (bytesWritten < length)
+            NotifyCurrentStep($"{partition} ({length} bytes) exceeds max-download-size ({maxDownloadSize}). Converting RAW image to sparse...");
+            string tempDir = Path.Combine(Path.GetTempPath(), "fastboot_sparse_" + Guid.NewGuid().ToString("N"));
+            string sparsePath = Path.Combine(tempDir, "input.sparse.img");
+
+            try
             {
-                long toWrite = Math.Min(maxDownloadSize, length - bytesWritten);
-                NotifyCurrentStep($"Sending {partition} RAW chunk {count}/{totalChunks} ({toWrite} bytes)");
+                Directory.CreateDirectory(tempDir);
 
-                // Create a sub-view of the RAW stream
-                using var subStream = new SubStream(stream, bytesWritten, toWrite);
-                DownloadData(subStream, toWrite).ThrowIfError();
+                string? sourceRawPath = null;
+                bool canUseFileDirectly = stream is FileStream fs &&
+                                          fs.CanSeek &&
+                                          originalPos == 0 &&
+                                          length == fs.Length;
 
-                NotifyCurrentStep($"Flashing {partition} RAW chunk {count}/{totalChunks}");
-                RawCommand("flash:" + partition).ThrowIfError();
+                if (canUseFileDirectly)
+                {
+                    sourceRawPath = ((FileStream)stream).Name;
+                    FastbootDebug.Log($"Converting RAW via direct source file: {sourceRawPath}");
+                }
+                else
+                {
+                    string rawPath = Path.Combine(tempDir, "input.raw");
+                    sourceRawPath = rawPath;
+                    if (stream.CanSeek)
+                    {
+                        stream.Seek(originalPos, SeekOrigin.Begin);
+                    }
 
-                bytesWritten += toWrite;
-                count++;
+                    using var ofs = File.Create(rawPath);
+                    byte[] copyBuffer = ArrayPool<byte>.Shared.Rent(1024 * 1024);
+                    try
+                    {
+                        long remaining = length;
+                        while (remaining > 0)
+                        {
+                            int toRead = (int)Math.Min(copyBuffer.Length, remaining);
+                            int read = stream.Read(copyBuffer, 0, toRead);
+                            if (read <= 0)
+                            {
+                                return new FastbootResponse
+                                {
+                                    Result = FastbootState.Fail,
+                                    Response = $"failed to read source RAW stream while converting to sparse: {length - remaining}/{length}"
+                                };
+                            }
+                            ofs.Write(copyBuffer, 0, read);
+                            remaining -= read;
+                        }
+                    }
+                    finally
+                    {
+                        ArrayPool<byte>.Shared.Return(copyBuffer);
+                    }
+                }
+
+                SparseImageConverter.ConvertRawToSparse(sourceRawPath!, sparsePath, 4096);
+                NotifyCurrentStep($"Converted RAW image to sparse. Flashing {partition} with sparse protocol...");
+                return FlashSparseImage(partition, sparsePath);
+            }
+            catch (Exception ex)
+            {
+                return new FastbootResponse
+                {
+                    Result = FastbootState.Fail,
+                    Response = "raw-to-sparse conversion failed: " + ex.Message
+                };
+            }
+            finally
+            {
+                try { if (Directory.Exists(tempDir)) Directory.Delete(tempDir, true); } catch { }
+                if (stream.CanSeek)
+                {
+                    try { stream.Seek(originalPos, SeekOrigin.Begin); } catch { }
+                }
             }
-            return new FastbootResponse { Result = FastbootState.Success };
         }
 
         // AVB Footer logic
diff --git a/FirmwareKit.Comm.Fastboot/Command/InternalHandleResponse.cs b/FirmwareKit.Comm.Fastboot/Command/InternalHandleResponse.cs

Original file line number	Diff line number	Diff line change
`@@ -142,7 +142,10 @@ private byte[] SendSinglePacket(PacketId id, ushort seq, PacketFlag flag, byte[]`
`142`	`142`
`143`	`143`	`if (rxPacket.Length > HeaderSize)`
`144`	`144`	`{`
`145`		`- fullResponse.AddRange(rxPacket.AsSpan(HeaderSize).ToArray());`
	`145`	`+ for (int j = HeaderSize; j < rxPacket.Length; j++)`
	`146`	`+ {`
	`147`	`+ fullResponse.Add(rxPacket[j]);`
	`148`	`+ }`
`146`	`149`	`}`
`147`	`150`
`148`	`151`	`gotValidResponse = true;`