mirror of
https://github.com/permissionlesstech/bitchat-android.git
synced 2026-07-27 00:45:27 +00:00
Align compressed payload send and receive bounds (#736)
* Align compressed payload send and receive bounds * Preserve ambiguous raw deflate compatibility * Pool decompression by memory budget
This commit is contained in:
a1denvalu3
@@ -183,7 +183,6 @@ object BinaryProtocol {
|
||||
private const val SENDER_ID_SIZE = 8
|
||||
private const val RECIPIENT_ID_SIZE = 8
|
||||
private const val SIGNATURE_SIZE = 64
|
||||
|
||||
object Flags {
|
||||
const val HAS_RECIPIENT: UByte = 0x01u
|
||||
const val HAS_SIGNATURE: UByte = 0x02u
|
||||
@@ -200,6 +199,15 @@ object BinaryProtocol {
|
||||
|
||||
fun encode(packet: BitchatPacket, padding: Boolean = true): ByteArray? {
|
||||
try {
|
||||
val maxPayloadLength = com.bitchat.android.util.AppConstants.Protocol.MAX_PAYLOAD_LENGTH
|
||||
if (packet.payload.size > maxPayloadLength) {
|
||||
Log.w(
|
||||
"BinaryProtocol",
|
||||
"Cannot encode payload ${packet.payload.size} above receiver limit $maxPayloadLength"
|
||||
)
|
||||
return null
|
||||
}
|
||||
|
||||
// Try to compress payload if beneficial
|
||||
var payload = packet.payload
|
||||
var originalPayloadSize: Int? = null
|
||||
@@ -325,7 +333,7 @@ object BinaryProtocol {
|
||||
}
|
||||
|
||||
fun decode(data: ByteArray): BitchatPacket? =
|
||||
decode(data, CompressionUtil::decompress)
|
||||
decode(data, CompressionUtil::decompressWithResourcesReserved)
|
||||
|
||||
/** Test seam used to prove rejected expansion sizes never reach inflation. */
|
||||
internal fun decodeForTesting(
|
||||
@@ -466,9 +474,6 @@ object BinaryProtocol {
|
||||
Log.w("BinaryProtocol", "Compressed payload has no deflate bytes")
|
||||
return null
|
||||
}
|
||||
val compressedPayload = ByteArray(compressedSize)
|
||||
buffer.get(compressedPayload)
|
||||
|
||||
// Security check: Compression bomb protection
|
||||
val ratio = originalSize.toDouble() / compressedSize.toDouble()
|
||||
if (ratio > 50_000.0) {
|
||||
@@ -476,8 +481,14 @@ object BinaryProtocol {
|
||||
return null
|
||||
}
|
||||
|
||||
// Decompress
|
||||
val expandedPayload = decompress(compressedPayload, originalSize) ?: return null
|
||||
// Reserve the compressed copy plus expanded output before either allocation.
|
||||
// Small packets share the memory pool; packets wait only while its budget is full.
|
||||
val resourceBytes = compressedSize.toLong() + originalSize.toLong()
|
||||
val expandedPayload = CompressionUtil.withDecompressionResources(resourceBytes) {
|
||||
val compressedPayload = ByteArray(compressedSize)
|
||||
buffer.get(compressedPayload)
|
||||
decompress(compressedPayload, originalSize)
|
||||
} ?: return null
|
||||
if (expandedPayload.size != originalSize) {
|
||||
Log.w(
|
||||
"BinaryProtocol",
|
||||
|
||||
@@ -13,9 +13,7 @@ import java.util.zip.Inflater
|
||||
object CompressionUtil {
|
||||
private const val COMPRESSION_THRESHOLD = com.bitchat.android.util.AppConstants.Protocol.COMPRESSION_THRESHOLD_BYTES // bytes - same as iOS
|
||||
|
||||
// Inflation allocates the full declared output buffer. Keep that allocation single-flight so
|
||||
// concurrent packets cannot multiply the bounded per-packet memory cost.
|
||||
private val decompressionLock = Any()
|
||||
private val decompressionPool = DecompressionResourcePool.forRuntime()
|
||||
|
||||
/**
|
||||
* Helper to check if compression is worth it - exact same logic as iOS
|
||||
@@ -78,50 +76,83 @@ object CompressionUtil {
|
||||
* iOS COMPRESSION_ZLIB produces raw deflate data (no headers)
|
||||
*/
|
||||
fun decompress(compressedData: ByteArray, originalSize: Int): ByteArray? {
|
||||
if (!isValidRequest(compressedData, originalSize)) return null
|
||||
return withDecompressionResources(originalSize.toLong()) {
|
||||
decompressWithResourcesReserved(compressedData, originalSize)
|
||||
}
|
||||
}
|
||||
|
||||
internal fun <T> withDecompressionResources(bytes: Long, block: () -> T): T? =
|
||||
decompressionPool.withReservation(bytes, block)
|
||||
|
||||
/**
|
||||
* Inflate after the caller has reserved all packet-specific allocations.
|
||||
* This avoids nested acquisition when BinaryProtocol reserves both its input copy and output.
|
||||
*/
|
||||
internal fun decompressWithResourcesReserved(
|
||||
compressedData: ByteArray,
|
||||
originalSize: Int
|
||||
): ByteArray? {
|
||||
if (!isValidRequest(compressedData, originalSize)) return null
|
||||
return decompressExact(compressedData, originalSize)
|
||||
}
|
||||
|
||||
private fun isValidRequest(compressedData: ByteArray, originalSize: Int): Boolean {
|
||||
val maxExpandedSize = com.bitchat.android.util.AppConstants.Protocol.MAX_PAYLOAD_LENGTH
|
||||
if (compressedData.isEmpty()) {
|
||||
Log.w("CompressionUtil", "Refusing an empty compressed payload")
|
||||
return null
|
||||
return false
|
||||
}
|
||||
if (originalSize <= 0 || originalSize > maxExpandedSize) {
|
||||
Log.w(
|
||||
"CompressionUtil",
|
||||
"Refusing expanded payload size $originalSize outside 1..$maxExpandedSize"
|
||||
)
|
||||
return null
|
||||
return false
|
||||
}
|
||||
return true
|
||||
}
|
||||
|
||||
return synchronized(decompressionLock) {
|
||||
// iOS COMPRESSION_ZLIB produces raw deflate format (no headers).
|
||||
val rawResult = try {
|
||||
inflateExact(compressedData, originalSize, nowrap = true)
|
||||
} catch (e: Exception) {
|
||||
Log.d(
|
||||
"CompressionUtil",
|
||||
"Raw deflate decompression failed: ${e.message}"
|
||||
)
|
||||
private fun decompressExact(compressedData: ByteArray, originalSize: Int): ByteArray? {
|
||||
return if (looksLikeZlib(compressedData)) {
|
||||
// A raw stream can coincidentally begin with a valid-looking zlib header. The
|
||||
// header therefore only determines which format to try first; any non-exact zlib
|
||||
// result must still fall back to raw under the same size/completion bounds.
|
||||
val zlibResult = try {
|
||||
inflateExact(compressedData, originalSize, nowrap = false)
|
||||
} catch (zlibException: DataFormatException) {
|
||||
null
|
||||
}
|
||||
|
||||
if (rawResult != null) {
|
||||
rawResult
|
||||
if (zlibResult != null) {
|
||||
zlibResult
|
||||
} else {
|
||||
// Fallback after either a format error or an incomplete/wrong-sized raw stream:
|
||||
// accept the zlib-wrapped form used by some older/mixed clients, but only when it
|
||||
// independently satisfies the same exact-size and complete-stream checks.
|
||||
try {
|
||||
inflateExact(compressedData, originalSize, nowrap = false)
|
||||
} catch (fallbackException: Exception) {
|
||||
Log.e(
|
||||
"CompressionUtil",
|
||||
"Both raw deflate and zlib decompression failed: ${fallbackException.message}"
|
||||
)
|
||||
inflateExact(compressedData, originalSize, nowrap = true)
|
||||
} catch (rawException: DataFormatException) {
|
||||
Log.d("CompressionUtil", "Invalid zlib/raw deflate stream")
|
||||
null
|
||||
}
|
||||
}
|
||||
} else {
|
||||
try {
|
||||
inflateExact(compressedData, originalSize, nowrap = true)
|
||||
} catch (rawException: DataFormatException) {
|
||||
Log.d("CompressionUtil", "Invalid raw deflate stream")
|
||||
null
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/** RFC 1950 header check used to avoid speculative double inflation. */
|
||||
private fun looksLikeZlib(data: ByteArray): Boolean {
|
||||
if (data.size < 2) return false
|
||||
val cmf = data[0].toInt() and 0xFF
|
||||
val flg = data[1].toInt() and 0xFF
|
||||
return (cmf and 0x0F) == 8 &&
|
||||
(cmf ushr 4) <= 7 &&
|
||||
((cmf shl 8) or flg) % 31 == 0
|
||||
}
|
||||
|
||||
/**
|
||||
* Inflate one complete stream into exactly [originalSize] bytes.
|
||||
*
|
||||
|
||||
@@ -0,0 +1,73 @@
|
||||
package com.bitchat.android.protocol
|
||||
|
||||
import java.util.concurrent.Semaphore
|
||||
import java.util.concurrent.TimeUnit
|
||||
import kotlin.math.ceil
|
||||
|
||||
/**
|
||||
* Fair, weighted admission control for decompression allocations.
|
||||
*
|
||||
* Permits represent memory rather than workers: small packets can proceed concurrently while
|
||||
* near-limit packets consume most of the budget. Callers must reserve before allocating any
|
||||
* packet-specific compressed copy or expanded output.
|
||||
*/
|
||||
internal class DecompressionResourcePool(
|
||||
budgetBytes: Long,
|
||||
private val unitBytes: Int,
|
||||
private val waitTimeoutMs: Long
|
||||
) {
|
||||
private val totalPermits = (budgetBytes / unitBytes).toInt().coerceAtLeast(1)
|
||||
private val permits = Semaphore(totalPermits, true)
|
||||
|
||||
fun <T> withReservation(bytes: Long, block: () -> T): T? {
|
||||
val requiredPermits = permitsFor(bytes)
|
||||
val acquired = try {
|
||||
permits.tryAcquire(requiredPermits, waitTimeoutMs, TimeUnit.MILLISECONDS)
|
||||
} catch (_: InterruptedException) {
|
||||
Thread.currentThread().interrupt()
|
||||
false
|
||||
}
|
||||
if (!acquired) return null
|
||||
|
||||
return try {
|
||||
block()
|
||||
} finally {
|
||||
permits.release(requiredPermits)
|
||||
}
|
||||
}
|
||||
|
||||
internal fun permitsFor(bytes: Long): Int =
|
||||
ceil(bytes.coerceAtLeast(1).toDouble() / unitBytes.toDouble())
|
||||
.toInt()
|
||||
.coerceAtMost(totalPermits)
|
||||
|
||||
internal val availablePermits: Int
|
||||
get() = permits.availablePermits()
|
||||
|
||||
companion object {
|
||||
private const val DEFAULT_UNIT_BYTES = 256 * 1024
|
||||
private const val DEFAULT_WAIT_TIMEOUT_MS = 1_000L
|
||||
private const val HEAP_BUDGET_DIVISOR = 8L
|
||||
private const val MAX_BUDGET_BYTES = 64L * 1024 * 1024
|
||||
|
||||
fun forRuntime(
|
||||
maxHeapBytes: Long = Runtime.getRuntime().maxMemory(),
|
||||
maxPacketResourceBytes: Long =
|
||||
2L * com.bitchat.android.util.AppConstants.Protocol.MAX_PAYLOAD_LENGTH
|
||||
): DecompressionResourcePool {
|
||||
val budget = recommendedBudgetBytes(maxHeapBytes, maxPacketResourceBytes)
|
||||
return DecompressionResourcePool(
|
||||
budgetBytes = budget,
|
||||
unitBytes = DEFAULT_UNIT_BYTES,
|
||||
waitTimeoutMs = DEFAULT_WAIT_TIMEOUT_MS
|
||||
)
|
||||
}
|
||||
|
||||
internal fun recommendedBudgetBytes(
|
||||
maxHeapBytes: Long,
|
||||
maxPacketResourceBytes: Long
|
||||
): Long = (maxHeapBytes / HEAP_BUDGET_DIVISOR)
|
||||
.coerceAtLeast(maxPacketResourceBytes)
|
||||
.coerceAtMost(MAX_BUDGET_BYTES.coerceAtLeast(maxPacketResourceBytes))
|
||||
}
|
||||
}
|
||||
@@ -23,7 +23,7 @@ class MediaSendingManager(
|
||||
get() = getMeshService()
|
||||
companion object {
|
||||
private const val TAG = "MediaSendingManager"
|
||||
private const val MAX_FILE_SIZE = com.bitchat.android.util.AppConstants.Media.MAX_FILE_SIZE_BYTES // 50MB limit
|
||||
private const val MAX_FILE_SIZE = com.bitchat.android.util.AppConstants.Media.MAX_FILE_SIZE_BYTES
|
||||
}
|
||||
|
||||
// Track in-flight transfer progress: transferId -> messageId and reverse
|
||||
|
||||
@@ -129,7 +129,9 @@ object AppConstants {
|
||||
}
|
||||
|
||||
object Media {
|
||||
const val MAX_FILE_SIZE_BYTES: Long = 50L * 1024 * 1024
|
||||
// A file is currently encoded into one protocol payload before BLE fragmentation.
|
||||
// Reserve room for maximum filename/MIME TLVs and encryption envelope overhead.
|
||||
const val MAX_FILE_SIZE_BYTES: Long = (10L * 1024 * 1024) - (132L * 1024)
|
||||
}
|
||||
|
||||
object Services {
|
||||
|
||||
@@ -1139,6 +1139,18 @@ class BinaryProtocolTest {
|
||||
assertArrayEquals(payload, decoded!!.payload)
|
||||
}
|
||||
|
||||
@Test
|
||||
fun `raw deflate with zlib-looking prefix falls back after non-exact zlib parse`() {
|
||||
val payload = ByteArray(29) { index -> (index + 1).toByte() }
|
||||
val compressed = byteArrayOf(
|
||||
0x08, // non-final raw stored block; also zlib CMF
|
||||
0x1d, 0x00, // LEN = 29; 0x08 0x1d passes the RFC 1950 header check
|
||||
0xe2.toByte(), 0xff.toByte() // one's complement of LEN
|
||||
) + payload + byteArrayOf(0x03, 0x00) // final empty fixed-Huffman block
|
||||
|
||||
assertArrayEquals(payload, CompressionUtil.decompress(compressed, payload.size))
|
||||
}
|
||||
|
||||
@Test
|
||||
fun `under-declared zlib expansion is rejected by fallback`() {
|
||||
val payload = ByteArray(4_096) { 0x51 }
|
||||
@@ -1259,7 +1271,7 @@ class BinaryProtocolTest {
|
||||
}
|
||||
|
||||
@Test
|
||||
fun `legacy 11 MiB public file transfer is explicitly rejected by bounded decoder`() {
|
||||
fun `new sender refuses legacy 11 MiB public file transfer before transmission`() {
|
||||
val content = ByteArray(11 * 1024 * 1024) { 0x41 }
|
||||
val filePayload = BitchatFilePacket(
|
||||
fileName = "legacy-11m.bin",
|
||||
@@ -1281,15 +1293,9 @@ class BinaryProtocolTest {
|
||||
),
|
||||
padding = false
|
||||
)
|
||||
assertNotNull("A legacy sender can produce the highly-compressible wire packet", encoded)
|
||||
assertTrue(
|
||||
"The compressed wire body remains below the normal 10 MiB input cap",
|
||||
encoded!!.size < com.bitchat.android.util.AppConstants.Protocol.MAX_PAYLOAD_LENGTH
|
||||
)
|
||||
|
||||
assertNull(
|
||||
"The uniform bound intentionally rejects this legacy expansion until streaming admission exists",
|
||||
BinaryProtocol.decode(encoded)
|
||||
"Sender and receiver must enforce the same expanded-payload ceiling",
|
||||
encoded
|
||||
)
|
||||
}
|
||||
|
||||
|
||||
@@ -0,0 +1,117 @@
|
||||
package com.bitchat.android.protocol
|
||||
|
||||
import org.junit.Assert.assertEquals
|
||||
import org.junit.Assert.assertFalse
|
||||
import org.junit.Assert.assertNull
|
||||
import org.junit.Assert.assertTrue
|
||||
import org.junit.Test
|
||||
import java.util.concurrent.CountDownLatch
|
||||
import java.util.concurrent.Executors
|
||||
import java.util.concurrent.TimeUnit
|
||||
|
||||
class DecompressionResourcePoolTest {
|
||||
@Test
|
||||
fun `small reservations run concurrently and next waits only when budget is full`() {
|
||||
val pool = DecompressionResourcePool(
|
||||
budgetBytes = 2_000,
|
||||
unitBytes = 1_000,
|
||||
waitTimeoutMs = 2_000
|
||||
)
|
||||
val executor = Executors.newFixedThreadPool(3)
|
||||
val entered = CountDownLatch(2)
|
||||
val release = CountDownLatch(1)
|
||||
val thirdEntered = CountDownLatch(1)
|
||||
|
||||
try {
|
||||
repeat(2) {
|
||||
executor.submit {
|
||||
pool.withReservation(1_000) {
|
||||
entered.countDown()
|
||||
release.await()
|
||||
}
|
||||
}
|
||||
}
|
||||
assertTrue(entered.await(1, TimeUnit.SECONDS))
|
||||
|
||||
executor.submit {
|
||||
pool.withReservation(1_000) {
|
||||
thirdEntered.countDown()
|
||||
}
|
||||
}
|
||||
assertFalse("third reservation must wait while budget is full", thirdEntered.await(100, TimeUnit.MILLISECONDS))
|
||||
|
||||
release.countDown()
|
||||
assertTrue("third reservation must proceed after release", thirdEntered.await(1, TimeUnit.SECONDS))
|
||||
} finally {
|
||||
release.countDown()
|
||||
executor.shutdownNow()
|
||||
}
|
||||
}
|
||||
|
||||
@Test
|
||||
fun `timed admission drops work instead of waiting indefinitely`() {
|
||||
val pool = DecompressionResourcePool(
|
||||
budgetBytes = 1_000,
|
||||
unitBytes = 1_000,
|
||||
waitTimeoutMs = 50
|
||||
)
|
||||
val entered = CountDownLatch(1)
|
||||
val release = CountDownLatch(1)
|
||||
val executor = Executors.newSingleThreadExecutor()
|
||||
|
||||
try {
|
||||
executor.submit {
|
||||
pool.withReservation(1_000) {
|
||||
entered.countDown()
|
||||
release.await()
|
||||
}
|
||||
}
|
||||
assertTrue(entered.await(1, TimeUnit.SECONDS))
|
||||
assertNull(pool.withReservation(1_000) { "unexpected" })
|
||||
} finally {
|
||||
release.countDown()
|
||||
executor.shutdownNow()
|
||||
}
|
||||
}
|
||||
|
||||
@Test
|
||||
fun `permits are released when decode throws`() {
|
||||
val pool = DecompressionResourcePool(2_000, 1_000, 50)
|
||||
|
||||
try {
|
||||
pool.withReservation(2_000) { error("boom") }
|
||||
} catch (_: IllegalStateException) {
|
||||
// Expected.
|
||||
}
|
||||
|
||||
assertEquals(2, pool.availablePermits)
|
||||
assertEquals("ok", pool.withReservation(2_000) { "ok" })
|
||||
}
|
||||
|
||||
@Test
|
||||
fun `runtime budget is based on heap memory and always admits one maximum packet`() {
|
||||
val maxPacketResources = 20L * 1024 * 1024
|
||||
|
||||
assertEquals(
|
||||
maxPacketResources,
|
||||
DecompressionResourcePool.recommendedBudgetBytes(
|
||||
maxHeapBytes = 64L * 1024 * 1024,
|
||||
maxPacketResourceBytes = maxPacketResources
|
||||
)
|
||||
)
|
||||
assertEquals(
|
||||
32L * 1024 * 1024,
|
||||
DecompressionResourcePool.recommendedBudgetBytes(
|
||||
maxHeapBytes = 256L * 1024 * 1024,
|
||||
maxPacketResourceBytes = maxPacketResources
|
||||
)
|
||||
)
|
||||
assertEquals(
|
||||
64L * 1024 * 1024,
|
||||
DecompressionResourcePool.recommendedBudgetBytes(
|
||||
maxHeapBytes = 2L * 1024 * 1024 * 1024,
|
||||
maxPacketResourceBytes = maxPacketResources
|
||||
)
|
||||
)
|
||||
}
|
||||
}
|
||||
@@ -237,7 +237,7 @@ class FileTransferTest {
|
||||
|
||||
// Given: Large file size (simulated)
|
||||
val largeFileSize = 100L * 1024 * 1024 // 100MB
|
||||
val maxAllowedSize = 50L * 1024 * 1024 // 50MB
|
||||
val maxAllowedSize = com.bitchat.android.util.AppConstants.Media.MAX_FILE_SIZE_BYTES
|
||||
|
||||
// When: Checking if file can be transferred
|
||||
val isAllowed = largeFileSize <= maxAllowedSize
|
||||
|
||||
@@ -99,10 +99,11 @@ complete deflate stream. The `FILE_TRANSFER (0x22)` byte cannot safely grant a l
|
||||
attacker-controlled before packet signature verification, and the current receive pipeline must
|
||||
inflate before it can perform that verification.
|
||||
|
||||
This intentionally means a legacy Android sender can produce a highly compressible public file
|
||||
between 10 MiB and the UI's 50 MiB send limit that the bounded decoder rejects. The hardening must
|
||||
therefore remain a rollout HOLD rather than silently ship as backward compatible. Grandfathering
|
||||
50 MiB also is not safe after only a type check: inflation allocates the declared payload and
|
||||
New Android senders cap files just below 10 MiB (reserving envelope overhead) and refuse to encode
|
||||
any payload above the receiver ceiling.
|
||||
Legacy Android senders can still produce a highly compressible public file between 10 MiB and their
|
||||
50 MiB UI limit that the bounded decoder rejects. Grandfathering 50 MiB is not safe after only a type
|
||||
check: inflation allocates the declared payload and
|
||||
`BitchatFilePacket.decode` currently copies the content again, creating a greater than 100 MiB peak
|
||||
for a maximum-size transfer.
|
||||
|
||||
|
||||
Reference in New Issue
Block a user