From fc9abffe0c36c60916ff95aaeda381773f6ebc25 Mon Sep 17 00:00:00 2001 From: jack Date: Sun, 12 Jul 2026 10:29:30 -0400 Subject: [PATCH] Bound pre-auth compressed payload expansion --- .../android/protocol/BinaryProtocol.kt | 56 ++- .../android/protocol/CompressionUtil.kt | 121 ++++-- .../android/protocol/BinaryProtocolTest.kt | 359 ++++++++++++++++++ docs/file_transfer.md | 20 + 4 files changed, 509 insertions(+), 47 deletions(-) diff --git a/app/src/main/java/com/bitchat/android/protocol/BinaryProtocol.kt b/app/src/main/java/com/bitchat/android/protocol/BinaryProtocol.kt index a952d5fa..6b1b5325 100644 --- a/app/src/main/java/com/bitchat/android/protocol/BinaryProtocol.kt +++ b/app/src/main/java/com/bitchat/android/protocol/BinaryProtocol.kt @@ -324,21 +324,36 @@ object BinaryProtocol { } } - fun decode(data: ByteArray): BitchatPacket? { + fun decode(data: ByteArray): BitchatPacket? = + decode(data, CompressionUtil::decompress) + + /** Test seam used to prove rejected expansion sizes never reach inflation. */ + internal fun decodeForTesting( + data: ByteArray, + decompress: (ByteArray, Int) -> ByteArray? + ): BitchatPacket? = decode(data, decompress) + + private fun decode( + data: ByteArray, + decompress: (ByteArray, Int) -> ByteArray? + ): BitchatPacket? { // Try decode as-is first (robust when padding wasn't applied) - iOS fix - decodeCore(data)?.let { return it } + decodeCore(data, decompress)?.let { return it } // If that fails, try after removing padding val unpadded = MessagePadding.unpad(data) if (unpadded.contentEquals(data)) return null // No padding was removed, already failed - return decodeCore(unpadded) + return decodeCore(unpadded, decompress) } /** * Core decoding implementation used by decode() with and without padding removal - iOS fix */ - private fun decodeCore(raw: ByteArray): BitchatPacket? { + private fun decodeCore( + raw: ByteArray, + decompress: (ByteArray, Int) -> ByteArray? + ): BitchatPacket? { try { if (raw.size < HEADER_SIZE_V1 + SENDER_ID_SIZE) return null @@ -435,23 +450,42 @@ object BinaryProtocol { } else { buffer.getShort().toUShort().toInt() } + + val maxExpandedSize = com.bitchat.android.util.AppConstants.Protocol.MAX_PAYLOAD_LENGTH + if (originalSize <= 0 || originalSize > maxExpandedSize) { + Log.w( + "BinaryProtocol", + "Expanded payload size $originalSize is outside the allowed range 1..$maxExpandedSize" + ) + return null + } // Compressed payload val compressedSize = payloadLength.toInt() - lengthFieldBytes + if (compressedSize == 0) { + Log.w("BinaryProtocol", "Compressed payload has no deflate bytes") + return null + } val compressedPayload = ByteArray(compressedSize) buffer.get(compressedPayload) // Security check: Compression bomb protection - if (compressedSize > 0) { - val ratio = originalSize.toDouble() / compressedSize.toDouble() - if (ratio > 50_000.0) { - Log.w("BinaryProtocol", "🚫 Suspicious compression ratio: ${ratio}:1") - return null - } + val ratio = originalSize.toDouble() / compressedSize.toDouble() + if (ratio > 50_000.0) { + Log.w("BinaryProtocol", "🚫 Suspicious compression ratio: ${ratio}:1") + return null } // Decompress - CompressionUtil.decompress(compressedPayload, originalSize) ?: return null + val expandedPayload = decompress(compressedPayload, originalSize) ?: return null + if (expandedPayload.size != originalSize) { + Log.w( + "BinaryProtocol", + "Expanded payload size ${expandedPayload.size} did not match declared size $originalSize" + ) + return null + } + expandedPayload } else { val payloadBytes = ByteArray(payloadLength.toInt()) buffer.get(payloadBytes) diff --git a/app/src/main/java/com/bitchat/android/protocol/CompressionUtil.kt b/app/src/main/java/com/bitchat/android/protocol/CompressionUtil.kt index e8b59254..34e583fb 100644 --- a/app/src/main/java/com/bitchat/android/protocol/CompressionUtil.kt +++ b/app/src/main/java/com/bitchat/android/protocol/CompressionUtil.kt @@ -2,6 +2,7 @@ package com.bitchat.android.protocol import android.util.Log import java.io.ByteArrayOutputStream +import java.util.zip.DataFormatException import java.util.zip.Deflater import java.util.zip.Inflater @@ -11,6 +12,10 @@ import java.util.zip.Inflater */ object CompressionUtil { private const val COMPRESSION_THRESHOLD = com.bitchat.android.util.AppConstants.Protocol.COMPRESSION_THRESHOLD_BYTES // bytes - same as iOS + + // Inflation allocates the full declared output buffer. Keep that allocation single-flight so + // concurrent packets cannot multiply the bounded per-packet memory cost. + private val decompressionLock = Any() /** * Helper to check if compression is worth it - exact same logic as iOS @@ -73,49 +78,93 @@ object CompressionUtil { * iOS COMPRESSION_ZLIB produces raw deflate data (no headers) */ fun decompress(compressedData: ByteArray, originalSize: Int): ByteArray? { - // iOS COMPRESSION_ZLIB produces raw deflate format (no headers) - try { - val inflater = Inflater(true) // true = raw deflate, no headers - inflater.setInput(compressedData) - - val decompressedBuffer = ByteArray(originalSize) - val actualSize = inflater.inflate(decompressedBuffer) - inflater.end() - - // Verify decompressed size matches expected (same validation as iOS) - return if (actualSize == originalSize) { - decompressedBuffer - } else if (actualSize > 0) { - // Handle case where actual size is different - decompressedBuffer.copyOfRange(0, actualSize) - } else { + val maxExpandedSize = com.bitchat.android.util.AppConstants.Protocol.MAX_PAYLOAD_LENGTH + if (compressedData.isEmpty()) { + Log.w("CompressionUtil", "Refusing an empty compressed payload") + return null + } + if (originalSize <= 0 || originalSize > maxExpandedSize) { + Log.w( + "CompressionUtil", + "Refusing expanded payload size $originalSize outside 1..$maxExpandedSize" + ) + return null + } + + return synchronized(decompressionLock) { + // iOS COMPRESSION_ZLIB produces raw deflate format (no headers). + val rawResult = try { + inflateExact(compressedData, originalSize, nowrap = true) + } catch (e: Exception) { + Log.d( + "CompressionUtil", + "Raw deflate decompression failed: ${e.message}" + ) null } - } catch (e: Exception) { - Log.d("CompressionUtil", "Raw deflate decompression failed: ${e.message}, trying with zlib headers...") - - // Fallback: try with zlib headers in case of mixed usage - try { - val inflater = Inflater(false) // false = expect zlib headers - inflater.setInput(compressedData) - - val decompressedBuffer = ByteArray(originalSize) - val actualSize = inflater.inflate(decompressedBuffer) - inflater.end() - - return if (actualSize == originalSize) { - decompressedBuffer - } else if (actualSize > 0) { - decompressedBuffer.copyOfRange(0, actualSize) - } else { + + if (rawResult != null) { + rawResult + } else { + // Fallback after either a format error or an incomplete/wrong-sized raw stream: + // accept the zlib-wrapped form used by some older/mixed clients, but only when it + // independently satisfies the same exact-size and complete-stream checks. + try { + inflateExact(compressedData, originalSize, nowrap = false) + } catch (fallbackException: Exception) { + Log.e( + "CompressionUtil", + "Both raw deflate and zlib decompression failed: ${fallbackException.message}" + ) null } - } catch (fallbackException: Exception) { - Log.e("CompressionUtil", "Both raw deflate and zlib decompression failed: ${fallbackException.message}") - return null } } } + + /** + * Inflate one complete stream into exactly [originalSize] bytes. + * + * A full output buffer alone is not success: an attacker can under-declare a larger stream so + * the first inflate call fills the buffer while [Inflater.finished] remains false. Conversely, + * a truncated or over-declared stream can produce a non-empty prefix. Both forms are rejected, + * as are trailing bytes after the compressed stream. + * + * [DataFormatException] is deliberately allowed to escape so the caller can try the legacy + * zlib-wrapped format. Size/completion mismatches return null; the fallback must then prove the + * same bytes are a complete, exact-sized zlib stream before they can be accepted. + */ + @Throws(DataFormatException::class) + private fun inflateExact( + compressedData: ByteArray, + originalSize: Int, + nowrap: Boolean + ): ByteArray? { + val inflater = Inflater(nowrap) + return try { + inflater.setInput(compressedData) + val output = ByteArray(originalSize) + var written = 0 + + while (written < originalSize) { + val count = inflater.inflate(output, written, originalSize - written) + if (count == 0) break + written += count + } + + if (written != originalSize) return null + + // Give Inflater one byte of room to consume the end marker. Any produced byte proves + // the declared size was smaller than the actual expansion. + val overflowProbe = ByteArray(1) + if (inflater.inflate(overflowProbe) != 0) return null + + if (!inflater.finished() || inflater.remaining != 0) return null + output + } finally { + inflater.end() + } + } /** * Test function to verify deflate compression works correctly diff --git a/app/src/test/java/com/bitchat/android/protocol/BinaryProtocolTest.kt b/app/src/test/java/com/bitchat/android/protocol/BinaryProtocolTest.kt index c1b2327b..fb6e03bb 100644 --- a/app/src/test/java/com/bitchat/android/protocol/BinaryProtocolTest.kt +++ b/app/src/test/java/com/bitchat/android/protocol/BinaryProtocolTest.kt @@ -1,5 +1,6 @@ package com.bitchat.android.protocol +import com.bitchat.android.model.BitchatFilePacket import org.junit.Assert.assertEquals import org.junit.Assert.assertArrayEquals import org.junit.Assert.assertFalse @@ -7,9 +8,11 @@ import org.junit.Assert.assertNotNull import org.junit.Assert.assertNull import org.junit.Assert.assertTrue import org.junit.Test +import java.io.ByteArrayOutputStream import java.nio.ByteBuffer import java.nio.ByteOrder import java.util.Random +import java.util.zip.Deflater class BinaryProtocolTest { @@ -987,6 +990,309 @@ class BinaryProtocolTest { assertNull("v2 compression bomb (ratio > 50,000:1) must be rejected", result) } + @Test + fun `v2 expanded payload at exact maximum passes bound without allocating output`() { + val max = com.bitchat.android.util.AppConstants.Protocol.MAX_PAYLOAD_LENGTH + val raw = compressedPacket( + version = 2u, + originalSize = max, + compressedData = ByteArray(256) { it.toByte() } + ) + var calls = 0 + + val result = BinaryProtocol.decodeForTesting(raw) { _, requestedSize -> + calls += 1 + assertEquals(max, requestedSize) + null // Prove the boundary reached this seam without allocating a 10 MiB result. + } + + assertNull("The test decompressor deliberately returns no payload", result) + assertEquals(1, calls) + } + + @Test + fun `v2 expanded payload above maximum never reaches decompressor`() { + val max = com.bitchat.android.util.AppConstants.Protocol.MAX_PAYLOAD_LENGTH + val raw = compressedPacket( + version = 2u, + originalSize = max + 1, + compressedData = ByteArray(256) { it.toByte() } + ) + var calls = 0 + + val result = BinaryProtocol.decodeForTesting(raw) { _, _ -> + calls += 1 + byteArrayOf(0x42) + } + + assertNull("An oversized expansion must be rejected before inflation", result) + assertEquals("The decompressor must not be invoked", 0, calls) + } + + @Test + fun `v2 negative expanded payload never reaches decompressor`() { + val raw = compressedPacket( + version = 2u, + originalSize = -1, + compressedData = byteArrayOf(0x03) + ) + var calls = 0 + + val result = BinaryProtocol.decodeForTesting(raw) { _, _ -> + calls += 1 + byteArrayOf(0x42) + } + + assertNull("A negative expansion must be rejected before inflation", result) + assertEquals("The decompressor must not be invoked", 0, calls) + } + + @Test + fun `zero expanded payload never reaches decompressor`() { + val raw = compressedPacket( + version = 2u, + originalSize = 0, + compressedData = rawDeflate(ByteArray(0)) + ) + var calls = 0 + + val result = BinaryProtocol.decodeForTesting(raw) { _, _ -> + calls += 1 + ByteArray(0) + } + + assertNull("Compressed zero-length payloads are non-canonical and must be rejected", result) + assertEquals("The decompressor must not be invoked", 0, calls) + } + + @Test + fun `empty compressed body never reaches decompressor`() { + val raw = compressedPacket( + version = 2u, + originalSize = 128, + compressedData = ByteArray(0) + ) + var calls = 0 + + val result = BinaryProtocol.decodeForTesting(raw) { _, _ -> + calls += 1 + ByteArray(128) + } + + assertNull("A compressed payload must contain deflate bytes", result) + assertEquals("The decompressor must not be invoked", 0, calls) + } + + @Test + fun `v1 unsigned maximum expanded size reaches decompressor`() { + val originalSize = 0xFFFF + val raw = compressedPacket( + version = 1u, + originalSize = originalSize, + compressedData = byteArrayOf(0x01, 0x02) + ) + var calls = 0 + + val result = BinaryProtocol.decodeForTesting(raw) { _, requestedSize -> + calls += 1 + assertEquals(originalSize, requestedSize) + null + } + + assertNull("The test decompressor deliberately returns no payload", result) + assertEquals(1, calls) + } + + @Test + fun `compression utility rejects invalid expansion sizes directly`() { + val max = com.bitchat.android.util.AppConstants.Protocol.MAX_PAYLOAD_LENGTH + + assertNull(CompressionUtil.decompress(ByteArray(0), 1)) + assertNull(CompressionUtil.decompress(rawDeflate(ByteArray(0)), 0)) + assertNull(CompressionUtil.decompress(byteArrayOf(0x03), -1)) + assertNull(CompressionUtil.decompress(byteArrayOf(0x03), max + 1)) + } + + @Test + fun `raw deflate expands only when size and stream completion are exact`() { + val payload = ByteArray(4_096) { index -> (index % 17).toByte() } + val compressed = rawDeflate(payload) + + val decoded = BinaryProtocol.decode( + compressedPacket(version = 2u, originalSize = payload.size, compressedData = compressed) + ) + + assertNotNull(decoded) + assertArrayEquals(payload, decoded!!.payload) + } + + @Test + fun `zlib wrapped payload remains compatible when size and stream completion are exact`() { + val payload = ByteArray(4_096) { index -> (index % 23).toByte() } + val compressed = zlibDeflate(payload) + + val decoded = BinaryProtocol.decode( + compressedPacket(version = 2u, originalSize = payload.size, compressedData = compressed) + ) + + assertNotNull(decoded) + assertArrayEquals(payload, decoded!!.payload) + } + + @Test + fun `under-declared zlib expansion is rejected by fallback`() { + val payload = ByteArray(4_096) { 0x51 } + val compressed = zlibDeflate(payload) + + val decoded = BinaryProtocol.decode( + compressedPacket(version = 2u, originalSize = 128, compressedData = compressed) + ) + + assertNull("Zlib fallback must reject output beyond the declaration", decoded) + } + + @Test + fun `over-declared zlib expansion is rejected by fallback`() { + val payload = ByteArray(128) { 0x52 } + val compressed = zlibDeflate(payload) + + val decoded = BinaryProtocol.decode( + compressedPacket(version = 2u, originalSize = 256, compressedData = compressed) + ) + + assertNull("Zlib fallback must reject output shorter than the declaration", decoded) + } + + @Test + fun `truncated zlib stream is rejected by fallback`() { + val payload = ByteArray(4_096) { index -> (index % 29).toByte() } + val compressed = zlibDeflate(payload) + val truncated = compressed.copyOf(compressed.size - 1) + + val decoded = BinaryProtocol.decode( + compressedPacket(version = 2u, originalSize = payload.size, compressedData = truncated) + ) + + assertNull("Zlib fallback must require the stream end marker and checksum", decoded) + } + + @Test + fun `zlib stream with trailing bytes is rejected by fallback`() { + val payload = ByteArray(4_096) { index -> (index % 13).toByte() } + val compressedWithTrailingByte = zlibDeflate(payload) + byteArrayOf(0x00) + + val decoded = BinaryProtocol.decode( + compressedPacket( + version = 2u, + originalSize = payload.size, + compressedData = compressedWithTrailingByte + ) + ) + + assertNull("Zlib fallback must consume the complete input and nothing more", decoded) + } + + @Test + fun `under-declared raw expansion is rejected even when output buffer fills`() { + val payload = ByteArray(4_096) { 0x41 } + val compressed = rawDeflate(payload) + + val decoded = BinaryProtocol.decode( + compressedPacket(version = 2u, originalSize = 128, compressedData = compressed) + ) + + assertNull("Inflater must be finished, not merely fill the declared buffer", decoded) + } + + @Test + fun `over-declared raw expansion is rejected instead of returning a prefix`() { + val payload = ByteArray(128) { 0x42 } + val compressed = rawDeflate(payload) + + val decoded = BinaryProtocol.decode( + compressedPacket(version = 2u, originalSize = 256, compressedData = compressed) + ) + + assertNull("The expanded byte count must equal the declaration", decoded) + } + + @Test + fun `truncated raw stream is rejected even if all declared bytes were emitted`() { + val payload = ByteArray(4_096) { index -> (index % 31).toByte() } + val compressed = rawDeflate(payload) + val truncated = compressed.copyOf(compressed.size - 1) + + val decoded = BinaryProtocol.decode( + compressedPacket(version = 2u, originalSize = payload.size, compressedData = truncated) + ) + + assertNull("A stream without its end marker must not be accepted", decoded) + } + + @Test + fun `raw stream with trailing bytes is rejected`() { + val payload = ByteArray(4_096) { index -> (index % 19).toByte() } + val compressedWithTrailingByte = rawDeflate(payload) + byteArrayOf(0x00) + + val decoded = BinaryProtocol.decode( + compressedPacket( + version = 2u, + originalSize = payload.size, + compressedData = compressedWithTrailingByte + ) + ) + + assertNull("Trailing bytes after a complete stream must not be accepted", decoded) + } + + @Test + fun `decoder rejects a decompressor result shorter than its declaration`() { + val raw = compressedPacket( + version = 2u, + originalSize = 128, + compressedData = ByteArray(16) { it.toByte() } + ) + + val decoded = BinaryProtocol.decodeForTesting(raw) { _, _ -> byteArrayOf(0x01) } + + assertNull("BinaryProtocol must independently enforce the declared expanded size", decoded) + } + + @Test + fun `legacy 11 MiB public file transfer is explicitly rejected by bounded decoder`() { + val content = ByteArray(11 * 1024 * 1024) { 0x41 } + val filePayload = BitchatFilePacket( + fileName = "legacy-11m.bin", + fileSize = content.size.toLong(), + mimeType = "application/octet-stream", + content = content + ).encode() + assertNotNull(filePayload) + + val encoded = BinaryProtocol.encode( + BitchatPacket( + version = 2u, + type = MessageType.FILE_TRANSFER.value, + senderID = hexToBytes(senderHex), + recipientID = SpecialRecipients.BROADCAST, + timestamp = fixedTimestamp, + payload = filePayload!!, + ttl = 5u + ), + padding = false + ) + assertNotNull("A legacy sender can produce the highly-compressible wire packet", encoded) + assertTrue( + "The compressed wire body remains below the normal 10 MiB input cap", + encoded!!.size < com.bitchat.android.util.AppConstants.Protocol.MAX_PAYLOAD_LENGTH + ) + + assertNull( + "The uniform bound intentionally rejects this legacy expansion until streaming admission exists", + BinaryProtocol.decode(encoded) + ) + } + /** * Compression bomb is rejected * @@ -1163,6 +1469,59 @@ class BinaryProtocolTest { return result } + private fun compressedPacket( + version: UByte, + originalSize: Int, + compressedData: ByteArray, + type: UByte = MessageType.MESSAGE.value + ): ByteArray { + val originalSizeFieldBytes = if (version >= 2u.toUByte()) 4 else 2 + val payloadLength = originalSizeFieldBytes + compressedData.size + val headerSize = if (version >= 2u.toUByte()) 16 else 14 + val buffer = ByteBuffer.allocate(headerSize + 8 + payloadLength).apply { + order(ByteOrder.BIG_ENDIAN) + put(version.toByte()) + put(type.toByte()) + put(5.toByte()) + putLong(fixedTimestamp.toLong()) + put(BinaryProtocol.Flags.IS_COMPRESSED.toByte()) + if (version >= 2u.toUByte()) { + putInt(payloadLength) + } else { + putShort(payloadLength.toShort()) + } + put(hexToBytes(senderHex)) + if (version >= 2u.toUByte()) { + putInt(originalSize) + } else { + putShort(originalSize.toShort()) + } + put(compressedData) + } + return buffer.array() + } + + private fun rawDeflate(data: ByteArray): ByteArray = deflate(data, nowrap = true) + + private fun zlibDeflate(data: ByteArray): ByteArray = deflate(data, nowrap = false) + + private fun deflate(data: ByteArray, nowrap: Boolean): ByteArray { + val deflater = Deflater(Deflater.DEFAULT_COMPRESSION, nowrap) + return try { + deflater.setInput(data) + deflater.finish() + val output = ByteArrayOutputStream() + val buffer = ByteArray(1_024) + while (!deflater.finished()) { + val count = deflater.deflate(buffer) + output.write(buffer, 0, count) + } + output.toByteArray() + } finally { + deflater.end() + } + } + private fun makePacket( version: UByte = 1u, type: UByte = MessageType.MESSAGE.value, diff --git a/docs/file_transfer.md b/docs/file_transfer.md index 01b3e6f7..fc0d8b68 100644 --- a/docs/file_transfer.md +++ b/docs/file_transfer.md @@ -91,6 +91,26 @@ PayloadLength: 4 bytes (big-endian, max ~4 GiB) - Clients sending file transfers should preferentially use v2 format. - Fragmentation still applies: large files are split into fragments that fit within BLE MTU constraints (~128 KiB per fragment). +#### Compressed expansion rollout gate (draft/HOLD) + +Android's bounded decoder applies the same 10 MiB expanded-payload ceiling to every outer +message type. It also requires a non-empty compressed body, an exact declared output size, and a +complete deflate stream. The `FILE_TRANSFER (0x22)` byte cannot safely grant a larger ceiling: it is +attacker-controlled before packet signature verification, and the current receive pipeline must +inflate before it can perform that verification. + +This intentionally means a legacy Android sender can produce a highly compressible public file +between 10 MiB and the UI's 50 MiB send limit that the bounded decoder rejects. The hardening must +therefore remain a rollout HOLD rather than silently ship as backward compatible. Grandfathering +50 MiB also is not safe after only a type check: inflation allocates the declared payload and +`BitchatFilePacket.decode` currently copies the content again, creating a greater than 100 MiB peak +for a maximum-size transfer. + +Before enabling that legacy range, receive processing needs an authenticated admission decision +made before large allocation plus streaming inflation/TLV parsing into a bounded temporary file (or +another ownership-preserving design that avoids the second full-size copy). The sender limit and a +wire capability/version transition must then be coordinated so old and new clients fail predictably. + ### 1.3 File Transfer TLV payload (BitchatFilePacket) The file payload is a TLV structure with mixed length field sizes to support large contents efficiently.