MessagePack-Java Vulnerable to Remote Denial of Service via Malicious .msgpack Model File Triggering Unbounded EXT Payload Allocation

@@ -26,6 +26,8 @@
 import java.io.Closeable;
 import java.io.IOException;
 import java.math.BigInteger;
+import java.util.ArrayList;
+import java.util.List;
 import java.nio.ByteBuffer;
 import java.nio.CharBuffer;
 import java.nio.charset.CharacterCodingException;
@@ -150,6 +152,14 @@ public class MessageUnpacker
 {
     private static final MessageBuffer EMPTY_BUFFER = MessageBuffer.wrap(new byte[0]);
 
+    /**
+     * Threshold for switching from upfront allocation to gradual allocation.
+     * Payloads up to this size use efficient upfront allocation.
+     * Payloads exceeding this size use gradual allocation to detect malicious files
+     * that declare large payload sizes but contain little actual data.
+     */
+    private static final int GRADUAL_ALLOCATION_THRESHOLD = 64 * 1024 * 1024;  // 64 MB
+
     private final boolean allowReadingStringAsBinary;
     private final boolean allowReadingBinaryAsString;
     private final CodingErrorAction actionOnMalformedString;
@@ -1637,18 +1647,85 @@ public void readPayload(byte[] dst)
      * This method allocates a new byte array and consumes specified amount of bytes into the byte array.
      *
      * <p>
-     * This method is equivalent to <code>readPayload(new byte[length])</code>.
+     * For sizes up to {@link #GRADUAL_ALLOCATION_THRESHOLD}, this method uses efficient upfront allocation.
+     * For larger sizes, it uses gradual allocation to protect against malicious files that declare
+     * large payload sizes but contain little actual data.
      *
      * @param length number of bytes to be read
      * @return the new byte array
      * @throws IOException when underlying input throws IOException
+     * @throws MessageSizeException when the input ends before the declared size is reached (for large payloads)
      */
     public byte[] readPayload(int length)
             throws IOException
     {
-        byte[] newArray = new byte[length];
-        readPayload(newArray);
-        return newArray;
+        if (length <= GRADUAL_ALLOCATION_THRESHOLD) {
+            // Small/moderate size: use efficient upfront allocation
+            byte[] newArray = new byte[length];
+            readPayload(newArray);
+            return newArray;
+        }
+
+        // Large declared size: use gradual allocation to protect against malicious files
+        return readPayloadGradually(length);
+    }
+
+    /**
+     * Read payload gradually, allocating memory only as data becomes available.
+     * This method protects against malicious files that declare large payload sizes
+     * but contain little actual data.
+     *
+     * @param declaredLength the declared payload length
+     * @return the payload bytes
+     * @throws IOException when underlying input throws IOException
+     * @throws MessageSizeException when the input ends before the declared size is reached
+     */
+    private byte[] readPayloadGradually(int declaredLength)
+            throws IOException
+    {
+        List<byte[]> chunks = new ArrayList<>();
+        int totalRead = 0;
+        int remaining = declaredLength;
+
+        while (remaining > 0) {
+            int bufferRemaining = buffer.size() - position;
+            if (bufferRemaining == 0) {
+                // Need more data from input
+                MessageBuffer next = in.next();
+                if (next == null) {
+                    // Input ended before we read the declared size
+                    throw new MessageSizeException(
+                            String.format("Payload declared %,d bytes but input ended after %,d bytes",
+                                    declaredLength, totalRead),
+                            declaredLength);
+                }
+                totalReadBytes += buffer.size();
+                buffer = next;
+                position = 0;
+                bufferRemaining = buffer.size();
+            }
+
+            int toRead = Math.min(remaining, bufferRemaining);
+            byte[] chunk = new byte[toRead];
+            buffer.getBytes(position, chunk, 0, toRead);
+            chunks.add(chunk);
+            totalRead += toRead;
+            position += toRead;
+            remaining -= toRead;
+        }
+
+        // All data verified to exist - combine chunks into result
+        if (chunks.size() == 1) {
+            return chunks.get(0);  // Common case: single chunk, no copy needed
+        }
+
+        byte[] result = new byte[declaredLength];
+        int offset = 0;
+        for (byte[] chunk : chunks) {
+            System.arraycopy(chunk, 0, result, offset, chunk.length);
+            offset += chunk.length;
+        }
+        return result;
     }
 
     /**

@@ -0,0 +1,116 @@
+package org.msgpack.core
+
+import wvlet.airspec.AirSpec
+
+import java.nio.ByteBuffer
+
+class PayloadSizeLimitTest extends AirSpec:
+
+  test("detects malicious EXT32 file with huge declared size but tiny actual data") {
+    // Craft a malicious EXT32 header:
+    // 0xC9 = EXT32 format
+    // 4 bytes = declared length (100MB, which exceeds 64MB threshold)
+    // 1 byte = extension type
+    // followed by only 1 byte of actual data (not 100MB)
+    val declaredLength = 100 * 1024 * 1024  // 100 MB
+    val buffer = ByteBuffer.allocate(7)  // header(1) + length(4) + type(1) + data(1)
+    buffer.put(0xC9.toByte)  // EXT32 format code
+    buffer.putInt(declaredLength)  // declared length (big-endian)
+    buffer.put(0x01.toByte)  // extension type
+    buffer.put(0x41.toByte)  // only 1 byte of actual data
+    val maliciousData = buffer.array()
+
+    val unpacker = MessagePack.newDefaultUnpacker(maliciousData)
+
+    // Should throw MessageSizeException because the declared size exceeds actual data
+    intercept[MessageSizeException] {
+      unpacker.unpackValue()
+    }
+  }
+
+  test("detects malicious BIN32 file with huge declared size but tiny actual data") {
+    // Craft a malicious BIN32 header:
+    // 0xC6 = BIN32 format
+    // 4 bytes = declared length (100MB, which exceeds 64MB threshold)
+    // followed by only 1 byte of actual data (not 100MB)
+    val declaredLength = 100 * 1024 * 1024  // 100 MB
+    val buffer = ByteBuffer.allocate(6)  // header(1) + length(4) + data(1)
+    buffer.put(0xC6.toByte)  // BIN32 format code
+    buffer.putInt(declaredLength)  // declared length (big-endian)
+    buffer.put(0x41.toByte)  // only 1 byte of actual data
+    val maliciousData = buffer.array()
+
+    val unpacker = MessagePack.newDefaultUnpacker(maliciousData)
+
+    // Should throw MessageSizeException because the declared size exceeds actual data
+    intercept[MessageSizeException] {
+      unpacker.unpackValue()
+    }
+  }
+
+  test("legitimate extension data works correctly") {
+    val packer = MessagePack.newDefaultBufferPacker()
+    val testData = Array.fill[Byte](1000)(0x42)
+    packer.packExtensionTypeHeader(0x01.toByte, testData.length)
+    packer.writePayload(testData)
+    val msgpack = packer.toByteArray
+
+    val unpacker = MessagePack.newDefaultUnpacker(msgpack)
+    val value = unpacker.unpackValue()
+
+    assert(value.isExtensionValue)
+    assert(value.asExtensionValue().getData.length == 1000)
+  }
+
+  test("legitimate binary data works correctly") {
+    val packer = MessagePack.newDefaultBufferPacker()
+    val testData = Array.fill[Byte](1000)(0x42)
+    packer.packBinaryHeader(testData.length)
+    packer.writePayload(testData)
+    val msgpack = packer.toByteArray
+
+    val unpacker = MessagePack.newDefaultUnpacker(msgpack)
+    val value = unpacker.unpackValue()
+
+    assert(value.isBinaryValue)
+    assert(value.asBinaryValue().asByteArray().length == 1000)
+  }
+
+  test("readPayload directly with malicious size throws exception") {
+    // Test readPayload(int) directly with a malicious input
+    val declaredLength = 100 * 1024 * 1024  // 100 MB
+    val buffer = ByteBuffer.allocate(6)  // header(1) + length(4) + data(1)
+    buffer.put(0xC6.toByte)  // BIN32 format code
+    buffer.putInt(declaredLength)  // declared length (big-endian)
+    buffer.put(0x41.toByte)  // only 1 byte of actual data
+    val maliciousData = buffer.array()
+
+    val unpacker = MessagePack.newDefaultUnpacker(maliciousData)
+
+    // First, unpack the binary header to get the declared length
+    val len = unpacker.unpackBinaryHeader()
+    assert(len == declaredLength)
+
+    // Then try to read the payload - should throw exception
+    intercept[MessageSizeException] {
+      unpacker.readPayload(len)
+    }
+  }
+
+  test("small payloads under threshold work with upfront allocation") {
+    // Payloads under 64MB should use the efficient upfront allocation path
+    val packer = MessagePack.newDefaultBufferPacker()
+    val testData = Array.fill[Byte](10000)(0x42)  // 10KB, well under threshold
+    packer.packBinaryHeader(testData.length)
+    packer.writePayload(testData)
+    val msgpack = packer.toByteArray
+
+    val unpacker = MessagePack.newDefaultUnpacker(msgpack)
+    val len = unpacker.unpackBinaryHeader()
+    val payload = unpacker.readPayload(len)
+
+    assert(payload.length == 10000)
+    assert(payload.forall(_ == 0x42.toByte))
+  }
+
+end PayloadSizeLimitTest