diff --git a/common/src/androidTest/java/com/shilapi/xcertplay/web/NativeVideoDecoderTest.kt b/common/src/androidTest/java/com/shilapi/xcertplay/web/NativeVideoDecoderTest.kt index fd58b78b..4a7525c3 100644 --- a/common/src/androidTest/java/com/shilapi/xcertplay/web/NativeVideoDecoderTest.kt +++ b/common/src/androidTest/java/com/shilapi/xcertplay/web/NativeVideoDecoderTest.kt @@ -81,7 +81,12 @@ class NativeVideoDecoderTest { val latch = java.util.concurrent.CountDownLatch(3) val frames = java.util.concurrent.LinkedBlockingQueue() val errors = java.util.concurrent.LinkedBlockingQueue() - val screen = com.shilapi.xcertplay.airplay.ScreenStream(key, {}, { NativeVideoDecoder(it, platform = true) }) + val loopback = java.net.InetAddress.getByName("127.0.0.1") + val screen = com.shilapi.xcertplay.airplay.ScreenStream( + key = key, + expectedPeer = loopback, + decoderFactory = { NativeVideoDecoder(it, platform = true) }, + ) val port = screen.listen(object : com.shilapi.xcertplay.airplay.ScreenStream.Listener { override fun onFrame(frame: com.shilapi.xcertplay.media.CompressedVideoFrame, pts: Long, receivedAtNs: Long, decryptNs: Long) { frames.offer(frame.toByteArray()); latch.countDown() @@ -89,7 +94,7 @@ class NativeVideoDecoderTest { override fun onClosed(cause: Throwable?) { if (cause != null) errors.offer(cause) } }) try { - java.net.Socket("127.0.0.1", port).use { socket -> + java.net.Socket(loopback, port).use { socket -> val expected = listOf(plaintext(8192), plaintext(128), plaintext(16384)) expected.forEachIndexed { index, plain -> val length = plain.size + 16 diff --git a/common/src/main/java/com/shilapi/xcertplay/projection/CarPlaySourceSession.kt b/common/src/main/java/com/shilapi/xcertplay/projection/CarPlaySourceSession.kt index a3e26786..e5ef155f 100644 --- a/common/src/main/java/com/shilapi/xcertplay/projection/CarPlaySourceSession.kt +++ b/common/src/main/java/com/shilapi/xcertplay/projection/CarPlaySourceSession.kt @@ -38,10 +38,8 @@ internal class CarPlaySourceSession(private val status: () -> String?) : Project override val mediaOutput get() = sink override fun touch(contacts: List): Boolean { if (contacts.any { it.id !in 0..1 }) return false - // The CarPlay HID device has fixed slots, even when the first finger is released. - return controller?.sendTouch((0..1).map { id -> - contacts.find { it.id == id } ?: ProjectionContact(id, 0.0, 0.0, false) - }) ?: false + // The controller preserves pointer identity and releases missing fingers at their last position. + return controller?.sendTouch(contacts) ?: false } override fun cancelInput(reason: ProjectionCancelReason) = touch(emptyList()) override fun setNightMode(night: Boolean) = diff --git a/common/src/main/java/moe/feng/wheelplay/CarPlayHostActivity.kt b/common/src/main/java/moe/feng/wheelplay/CarPlayHostActivity.kt index cdb7f59d..53048a0c 100644 --- a/common/src/main/java/moe/feng/wheelplay/CarPlayHostActivity.kt +++ b/common/src/main/java/moe/feng/wheelplay/CarPlayHostActivity.kt @@ -861,6 +861,9 @@ class CarPlayHostActivity : ComponentActivity() { } } + // DiagnosticRedactor discards protocol traces; do not build them on the send path. + override val wantsTrace: Boolean get() = false + override fun onDebugLog(message: String) { if (DiagnosticRedactor.redact(message) == null) return runOnUiThread { diff --git a/common/src/test/java/com/shilapi/xcertplay/projection/CarPlayTouchPipelineTest.kt b/common/src/test/java/com/shilapi/xcertplay/projection/CarPlayTouchPipelineTest.kt new file mode 100644 index 00000000..88be48b6 --- /dev/null +++ b/common/src/test/java/com/shilapi/xcertplay/projection/CarPlayTouchPipelineTest.kt @@ -0,0 +1,85 @@ +package com.shilapi.xcertplay.projection + +import com.shilapi.xcertplay.airplay.* +import com.shilapi.xcertplay.orchestration.* +import com.shilapi.xcertplay.transport.Iap2IdentificationConfig +import java.io.ByteArrayOutputStream +import java.io.OutputStream +import java.net.Socket +import java.util.concurrent.CountDownLatch +import java.util.concurrent.ExecutorService +import java.util.concurrent.TimeUnit +import org.junit.Assert.* +import org.junit.Test +import org.junit.runner.RunWith +import org.robolectric.RobolectricTestRunner +import org.robolectric.RuntimeEnvironment +import org.robolectric.annotation.Config + +/** Exercise the production projection adapter as well as its controller/HID serialization. */ +@RunWith(RobolectricTestRunner::class) +@Config(sdk = [29], manifest = Config.NONE) +class CarPlayTouchPipelineTest { + @Test fun singleFingerDragCoalescesAndCancelReleasesAtItsLastPosition() { + val entered = CountDownLatch(1) + val release = CountDownLatch(1) + val bytes = ByteArrayOutputStream() + val wire = object : OutputStream() { + override fun write(b: Int) = throw UnsupportedOperationException() + override fun write(b: ByteArray, off: Int, len: Int) { + entered.countDown() + check(release.await(5, TimeUnit.SECONDS)) + bytes.write(b, off, len) + } + } + val config = AirPlayConfig("test", "test", "", "1", AirPlayDisplayConfig(800, 480)) + val session = AirPlaySession(Socket(), config, AirPlayIdentity.generate(), PairingStore(), null, + object : AirPlaySessionListener {}, object : AirPlayMediaHandler {}) + val controller = CarPlayController( + RuntimeEnvironment.getApplication(), + CarPlayRuntimeConfig(mfiTarget = MfiTarget.LOCAL, transport = CarPlayTransport.WIRED, + identification = Iap2IdentificationConfig(name = "test", modelIdentifier = "test", manufacturer = "test", + serialNumber = "test", firmwareVersion = "1", hardwareVersion = "1", carPlayUsbInterfaceNumber = 3)), + config, AirPlayIdentity.generate(), PairingStore(), object : AirPlaySessionListener {}, object : AirPlayMediaHandler {}, {}, + ) + set(session, "eventSocket", object : Socket() { override fun getOutputStream() = wire }) + set(session, "eventCipher", ControlCipher(ByteArray(32) { 1 }, ByteArray(32) { 2 })) + set(controller, "activeSession", session) + val projection = CarPlaySourceSession { null }.apply { bind(controller, null) } + try { + // Pointer 1 arriving by itself also verifies that the adapter does not fill pointer 0 with an up. + assertTrue(projection.touch(listOf(ProjectionContact(1, .1, .2, true)))) + assertTrue(entered.await(5, TimeUnit.SECONDS)) + repeat(300) { index -> + assertTrue(projection.touch(listOf(ProjectionContact(1, .2 + index * .001, .3, true)))) + } + assertTrue(projection.touch(emptyList())) + release.countDown() + val executor = controller.javaClass.getDeclaredField("touchExecutor").apply { isAccessible = true } + .get(controller) as ExecutorService + executor.submit {}.get(5, TimeUnit.SECONDS) + val cipher = ControlCipher(ByteArray(32) { 2 }, ByteArray(32) { 1 }) + val requests = RtspMessage.parseMessages(cipher.decrypt(bytes.toByteArray()).data).messages + assertEquals("adapter prevented move coalescing", 3, requests.size) + val reports = requests.map { (BplistCodec.decode(it.body) as Map<*, *>)["hidReport"] as ByteArray } + assertEquals(1, reports[0][1].toInt()) + assertEquals(1, reports[1][1].toInt()) + assertEquals(0, reports[2][1].toInt()) + assertEquals(399, coordinate(reports[1], 2)) + assertEquals(144, coordinate(reports[1], 4)) + assertArrayEquals("cancel moved the finger to the origin", reports[1].copyOfRange(2, 6), reports[2].copyOfRange(2, 6)) + } finally { + release.countDown() + session.close() + controller.close() + controller.awaitClosed(2_000) + } + } + + private fun coordinate(report: ByteArray, offset: Int) = + (report[offset].toInt() and 0xff) or ((report[offset + 1].toInt() and 0xff) shl 8) + + private fun set(target: Any, name: String, value: Any) { + target.javaClass.getDeclaredField(name).apply { isAccessible = true }.set(target, value) + } +} diff --git a/docs/UPSTREAM.md b/docs/UPSTREAM.md index 76add674..5bf7d8a5 100644 --- a/docs/UPSTREAM.md +++ b/docs/UPSTREAM.md @@ -87,3 +87,16 @@ Material Components for Android 1.13.0(Apache-2.0),使用官方 Material 3 WheelPlay 实现固定 1:1 选区、图片平移/双指缩放、无障碍移动与缩放、横竖屏布局及重置; 原图坐标与窗口尺寸解耦,加载和保存由 ViewModel 管理。仅输出车厂图标所需的 1024 × 1024 PNG,不引入 crop-kit/JitPack 运行时依赖。 + +## CarPlay 协议首批同步(2026-11-15) + +基于 WheelPlay `1c0c2b88`,对照 [DiPlay `615b32ce`](https://github.com/shihabal3amri/DiPlay/tree/615b32ce0b40cd0b99ce94ba23a512001553684f) 选择性移植以下修复。这里只表示列出的补丁已适配,不表示已经合并 DiPlay 的全部协议或媒体实现。 + +| 范围 | 上游提交 | WheelPlay 适配 | +| --- | --- | --- | +| 麦克风 | [`c2afb171`](https://github.com/shihabal3amri/DiPlay/commit/c2afb171)、[`9647d689`](https://github.com/shihabal3amri/DiPlay/commit/9647d689)、[`8d5fd4b2`](https://github.com/shihabal3amri/DiPlay/commit/8d5fd4b2) | 按手机的 input/上行端口识别 App 录音;非通话输入流在没有下行时等待 500 ms 开麦;未明确协商 framesPerPacket 的 PCM 采用适合 1500 字节 IPv6 MTU 的包长。保留原有 Opus RTP 时钟及浏览器麦克风路由。 | +| 触控 | [`4b5e0bb7`](https://github.com/shihabal3amri/DiPlay/commit/4b5e0bb7)、[`48c97171`](https://github.com/shihabal3amri/DiPlay/commit/48c97171)、[`ad03bf44`](https://github.com/shihabal3amri/DiPlay/commit/ad03bf44)、[`47b39a5c`](https://github.com/shihabal3amri/DiPlay/commit/47b39a5c)、[`50957d41`](https://github.com/shihabal3amri/DiPlay/commit/50957d41) | 事件 TCP_NODELAY、固定手指槽位和原位释放、合并同组手指的待发 MOVE、延迟构造 TRACE。投影适配器直接传递触点,由会话分配槽位并补发原位释放;复制输入列表并在执行时检查会话归属;实际界面声明不接收会被 DiagnosticRedactor 丢弃的 TRACE。 | +| 对端校验 | [`aeb1d733`](https://github.com/shihabal3amri/DiPlay/commit/aeb1d733) | 视频、事件和 iAP TCP 连接仅接受控制连接的同一 IP;NTP 响应仅由指定对端调整时钟;拒绝缺少认证标签的视频帧。保留原生视频解密/帧对象接口。 | +| 解析边界 | [`1759a981`](https://github.com/shihabal3amri/DiPlay/commit/1759a981)、[`a83284c5`](https://github.com/shihabal3amri/DiPlay/commit/a83284c5) 中的 iAP 包长修复 | NTB16 限制条目数、累计复制字节及 NDP 重访;iAP 非法包长立即报错。沿用 WheelPlay 既有增量 RTSP 解码器、读取期限、认证门禁和 bplist 展开预算。 | + +延迟开麦使用每流独立的启动门闩,启动与替换、TEARDOWN、会话关闭串行化;取消时移除计时任务,避免旧回调重新打开麦克风。通话仍等待首个下行包,明确的 input=false 不启动 App 录音。音频流 audioType 隔离、有线启动恢复和 type-103 buffered audio 留待后续批次。 diff --git a/docs/VALIDATION.md b/docs/VALIDATION.md index 31e2a953..a6772146 100644 --- a/docs/VALIDATION.md +++ b/docs/VALIDATION.md @@ -557,3 +557,40 @@ run-1 捕获的第 12 个访问单元(索引 11)为: - `npm run test:browser:media`:真实 Chromium 在生产 CSP 下通过 seek 能力切换、最后一次拖动位置(42.123 秒 → 42123 毫秒)、暂停状态保持、切歌后的旧 seek 回调隔离,以及既有画面/触控/重连检查。回调由测试脚本触发,未验证车机的实体按键或系统进度条 UI。 - `:mobile:lintDebug`:0 错误、18 警告;`git diff --check` 通过。 - 未安装或操作真实车机/iPhone。手机音乐应用是否执行跳转、进度回报及本地/浏览器输出的声音连续性仍需实机验收。本次复用现有 CarPlay 音频链路,未根据猜测注入 AirPlay 缓冲清空命令。 + +## DiPlay CarPlay 协议首批同步(2026-11-15) + +从 WheelPlay `1c0c2b88` 新建 `feat/sync-diplay-carplay-batch1`,按 [来源记录](UPSTREAM.md#carplay-协议首批同步2026-11-15) 选择性同步麦克风、触控、对端校验和解析边界修复。 + +回归覆盖: + +- App 录音按 input/上行端口启用;无下行时启动一次,通话仍等待下行;替换、TEARDOWN、会话关闭取消延迟开麦。PCM 默认包长连同 IPv6/UDP 头不超过 1500 字节;显式 framesPerPacket 及 Opus RTP 时钟继续按协商值处理。 +- 阻塞事件 socket 下合并 500 次 MOVE,保留按下/抬起及多指边界;切换会话丢弃旧待发输入,拒绝执行器关闭后的提交,复制调用方触点列表。生产投影适配器直接进入同一链路,300 次拖动在阻塞解除后只产生按下、最终移动、原位释放三份加密 HID 报文。 +- 视频、事件和 iAP TCP 拒绝其他 IP,拒绝者不占用正常对端连接;覆盖 IPv4、IPv6 及映射地址。NTP 只接受指定对端的时钟响应;视频帧缺少认证标签时不进入解码器。 +- NTB16 拒绝 NDP 循环、过多条目及超预算重复复制,保留正常 NDP 链与非零缓冲偏移;iAP 拒绝非法长度并保留跨加密帧的增量重组。既有 pair-verify、RTSP 读取期限、bplist 展开预算和浏览器媒体路由测试继续参与全量回归。 + +验证命令: + +```sh +./gradlew :mobile:assembleDebug :mobile:lintDebug +./gradlew :shared:testDebugUnitTest :common:testDebugUnitTest +``` + +结果:shared 全量 556 项、common 全量 545 项测试全部通过,均无失败、错误或跳过。Debug APK 构建通过;`lintDebug` 为 0 errors、24 warnings;`git diff --check` 通过。 + +本轮未连接 iPhone 或车机,未测量实际触控延迟或验证真实 App 录音;上游设备报告不作为 WheelPlay 真机验证结果。 + +## 首批同步的 CI 仪器测试编译补验(2026-11-15) + +[PR #61 首次构建](https://github.com/fython/wheelplay/actions/runs/43681265193/job/132712212254) 在 `:common:compileDebugAndroidTestKotlin` 失败:`ScreenStream` 增加必填的 `expectedPeer` 后,`NativeVideoDecoderTest` 仍使用旧的位置参数,导致两个 lambda 分别被解析为错误的参数类型。此前的 APK/lint 和 JVM 单元测试验证没有包含仪器测试 APK 的编译。 + +测试现在通过命名参数提供预期的 IPv4 loopback 对端和原生解码器工厂,并使用同一个地址建立发送 socket。先在本地运行 `:common:compileDebugAndroidTestKotlin` 复现两处类型错误,再按 `.github/workflows/debug-build.yml` 的完整构建任务补验,全部通过: + +```sh +./gradlew --build-cache :projection-plugin-api:assembleRelease \ + :projection-plugin-sample:lintDebug :projection-plugin-sample:assembleDebug \ + :common:assembleDebugAndroidTest :mobile:lintDebug :mobile:assembleDebug \ + :mobile:assembleDebugAndroidTest +``` + +结果包含插件 SDK、示例 APK、mobile APK、common/mobile 仪器测试 APK 及两项 lint;这是编译和打包验证,仪器测试的 Android 运行结果由远端 CI 单独记录。 diff --git a/shared/src/main/java/com/shilapi/xcertplay/airplay/AirPlayHid.kt b/shared/src/main/java/com/shilapi/xcertplay/airplay/AirPlayHid.kt index 3616bbcd..447f5c69 100644 --- a/shared/src/main/java/com/shilapi/xcertplay/airplay/AirPlayHid.kt +++ b/shared/src/main/java/com/shilapi/xcertplay/airplay/AirPlayHid.kt @@ -20,7 +20,7 @@ object AirPlayHid { const val MEDIA_HID_UID = 0x2a2a2a2c const val TELEPHONY_HID_UID = 0x2a2a2a2d - private const val TOUCH_CONTACTS = 2 + internal const val TOUCH_CONTACTS = 2 private const val BYTES_PER_FINGER = 6 fun touchHidDevice(xMax: Int, yMax: Int, displayUuid: String): Map = @@ -35,12 +35,13 @@ object AirPlayHid { fun telephonyHidDevice(displayUuid: String): Map = hidDeviceEntry(TELEPHONY_HID_UID, "xcertplay Telephony", telephonyDescriptor, displayUuid) + /** Contacts use the stable slot ids assigned by AirPlayTouchSlots. */ fun touchReport(contacts: List): ByteArray { val report = ByteArray(BYTES_PER_FINGER * TOUCH_CONTACTS) for (slot in 0 until TOUCH_CONTACTS) { val offset = slot * BYTES_PER_FINGER report[offset] = slot.toByte() - val contact = contacts.getOrNull(slot) ?: continue + val contact = contacts.firstOrNull { it.id == slot } ?: continue report[offset + 1] = if (contact.down) 0x01 else 0x00 writeU16Le(report, offset + 2, Math.round(contact.x.coerceAtLeast(0.0)).toInt()) writeU16Le(report, offset + 4, Math.round(contact.y.coerceAtLeast(0.0)).toInt()) diff --git a/shared/src/main/java/com/shilapi/xcertplay/airplay/AirPlaySession.kt b/shared/src/main/java/com/shilapi/xcertplay/airplay/AirPlaySession.kt index b753395c..02e01bdd 100644 --- a/shared/src/main/java/com/shilapi/xcertplay/airplay/AirPlaySession.kt +++ b/shared/src/main/java/com/shilapi/xcertplay/airplay/AirPlaySession.kt @@ -33,6 +33,8 @@ interface AirPlaySessionListener { fun onDeviceInfo(session: AirPlaySession, info: AirPlayDeviceInfo) {} fun onHostUiRequested(session: AirPlaySession) {} fun onCommand(session: AirPlaySession, type: String, params: Map) {} + /** False avoids constructing packet hex dumps that this listener would discard. */ + val wantsTrace: Boolean get() = true fun onDebugLog(message: String) {} } @@ -71,6 +73,7 @@ class AirPlaySession( internal val activeStreams = linkedSetOf() private val closed = AtomicBoolean(false) + internal val isClosed: Boolean get() = closed.get() private val notified = AtomicBoolean(false) private var eventServer: ServerSocket? = null @Volatile private var eventSocket: Socket? = null @@ -79,6 +82,8 @@ class AirPlaySession( private var pendingNightMode: Boolean? = null private val firstTouchSendLogged = AtomicBoolean(false) private val touchSendFailureLogged = AtomicBoolean(false) + // sendTouch is confined to the controller's single touch executor. + private val touchSlots = AirPlayTouchSlots(AirPlayHid.TOUCH_CONTACTS) private val ntp = NtpClock() private var keepAliveSocket: DatagramSocket? = null private var keepAliveThread: Thread? = null @@ -104,7 +109,7 @@ class AirPlaySession( if (!closed.get()) listener.onVideoFrameRendered(this) } - internal fun logTrace(message: String) = trace(message) + internal fun logTrace(message: String) = trace { message } fun start() { Thread(::runControl, "airplay-control").apply { @@ -138,7 +143,7 @@ class AirPlaySession( "Content-Type: $PLIST_CONTENT_TYPE\r\n" + "Content-Length: ${body.size}\r\n" + "CSeq: $eventCseq\r\n\r\n" - trace("airplay event tx headers=$head bodyHex=${body.toHex()}") + trace { "airplay event tx headers=$head bodyHex=${body.toHex()}" } return try { val bytes = cipher.encrypt(head.toByteArray(Charsets.US_ASCII) + body) val output = socket.getOutputStream() @@ -153,7 +158,7 @@ class AirPlaySession( } fun sendTouch(contacts: List): Boolean { - val scaled = contacts.map { + val scaled = touchSlots.assign(contacts).map { it.copy(x = it.x * config.main.widthPixels, y = it.y * config.main.heightPixels) } val report = AirPlayHid.touchReport(scaled) @@ -299,10 +304,10 @@ class AirPlaySession( "airplay rx ${request.method} ${request.path} cseq=$cseq body=${request.body.size}", showInDebugOverlay, ) - trace( + trace { "airplay control rx headers=${request.headers} " + - "bodyHex=${request.body.toHex()}", - ) + "bodyHex=${request.body.toHex()}" + } val response = try { handle(request) } catch (error: Exception) { @@ -318,7 +323,7 @@ class AirPlaySession( showInDebugOverlay, ) val wire = RtspMessage.buildResponse(request, response) - trace("airplay control tx wireHex=${wire.toHex()}") + trace { "airplay control tx wireHex=${wire.toHex()}" } output.write(cipher?.encrypt(wire) ?: wire) if (cipher == null && pairVerify.controlKeys != null) { val keys = pairVerify.controlKeys!! @@ -432,9 +437,10 @@ class AirPlaySession( } } - private fun trace(message: String) { + private inline fun trace(message: () -> String) { + if (!listener.wantsTrace) return try { - listener.onDebugLog("TRACE $message") + listener.onDebugLog("TRACE ${message()}") } catch (error: Exception) { Log.w(TAG, "trace log callback failed", error) } @@ -453,7 +459,7 @@ class AirPlaySession( val responseStreams = handleStreams(streams) debugLog("airplay SETUP response streams=$responseStreams") val body = BplistCodec.encode(linkedMapOf("streams" to responseStreams)) - trace("airplay SETUP response bplistHex=${body.toHex()}") + trace { "airplay SETUP response bplistHex=${body.toHex()}" } return RtspMessage.Response(headers = mapOf("Content-Type" to PLIST_CONTENT_TYPE), body = body) } @@ -562,7 +568,7 @@ class AirPlaySession( "airplay TEARDOWN types=$types activeBefore=$activeStreams " + "body=${request.body.size} bytes payload=$decodedBody", ) - trace("airplay TEARDOWN raw${request.body.size}Hex=${request.body.toHex()}") + trace { "airplay TEARDOWN raw${request.body.size}Hex=${request.body.toHex()}" } if (types.isEmpty()) { activeStreams.toList().forEach { media.onTeardown(this, it) } @@ -626,9 +632,11 @@ class AirPlaySession( private fun acceptEvent(server: ServerSocket) { try { - val socket = server.accept() + val expected = peerAddress ?: throw java.io.IOException("control peer unknown") + val socket = server.acceptFrom(expected) { debugLog("airplay event rejected sender outside the control session") } socket.setSoLinger(true, 0) - debugLog("airplay event connection accepted from ${socket.remoteSocketAddress}") + socket.tcpNoDelay = true + debugLog("airplay event connection accepted from ${socket.remoteSocketAddress} noDelay=${socket.tcpNoDelay}") eventSocket = socket val shared = pairVerify.shared if (shared == null) { @@ -691,11 +699,11 @@ class AirPlaySession( "airplay event rx ${message.method} ${message.path} cseq=${message.headers["cseq"] ?: "-"} body=${message.body.size}", ) val response = RtspMessage.buildResponse(message, RtspMessage.Response(status = 200)) - trace( + trace { "airplay event rx headers=${message.headers} " + - "bodyHex=${message.body.toHex()}", - ) - trace("airplay event tx wireHex=${response.toHex()}") + "bodyHex=${message.body.toHex()}" + } + trace { "airplay event tx wireHex=${response.toHex()}" } synchronized(eventWriteLock) { output.write(cipher.encrypt(response)) output.flush() diff --git a/shared/src/main/java/com/shilapi/xcertplay/airplay/AirPlayTcpAccepted.kt b/shared/src/main/java/com/shilapi/xcertplay/airplay/AirPlayTcpAccepted.kt index 9284e315..324bd904 100644 --- a/shared/src/main/java/com/shilapi/xcertplay/airplay/AirPlayTcpAccepted.kt +++ b/shared/src/main/java/com/shilapi/xcertplay/airplay/AirPlayTcpAccepted.kt @@ -2,6 +2,8 @@ package com.shilapi.xcertplay.airplay import java.net.InetAddress import java.net.NetworkInterface +import java.net.ServerSocket +import java.net.Socket import java.util.concurrent.atomic.AtomicLong /** 每次绑定生成独立身份,防止旧 accept 和旧清理作用于新 listener。 */ @@ -19,3 +21,13 @@ data class AirPlayTcpAccepted( internal fun isInternalAirPlayPeer(peer: InetAddress, local: InetAddress): Boolean = peer.isLoopbackAddress || peer == local || runCatching { NetworkInterface.getByInetAddress(peer) != null }.getOrDefault(true) + +/** Accept data connections only from the control-session peer, closing strangers unread. */ +internal fun ServerSocket.acceptFrom(expected: InetAddress, onRejected: (InetAddress) -> Unit = {}): Socket { + while (true) { + val socket = accept() + if (socket.inetAddress == expected) return socket + safeClose(socket) + onRejected(socket.inetAddress) + } +} diff --git a/shared/src/main/java/com/shilapi/xcertplay/airplay/AirPlayTouchSlots.kt b/shared/src/main/java/com/shilapi/xcertplay/airplay/AirPlayTouchSlots.kt new file mode 100644 index 00000000..ea3ae0a1 --- /dev/null +++ b/shared/src/main/java/com/shilapi/xcertplay/airplay/AirPlayTouchSlots.kt @@ -0,0 +1,35 @@ +package com.shilapi.xcertplay.airplay + +/** + * Keeps each finger in one touch report slot from press to release (#444). Contacts arrive with + * Android pointer ids, which stay with a finger while the pointer index of the others changes. A + * finger that is gone, and every finger when the list is empty, is released where it last was: a + * release at (0,0) drags it to the corner first. A finger beyond the free slots is left out. + */ +internal class AirPlayTouchSlots(private val slots: Int) { + private val pointers = arrayOfNulls(slots) + private val held = arrayOfNulls(slots) + + /** The contacts to report, each with its slot as id; empty slots are left out. */ + fun assign(contacts: List): List { + val report = arrayOfNulls(slots) + for (contact in contacts) { + val slot = pointers.indexOf(contact.id).takeIf { it >= 0 } + ?: pointers.indexOf(null).takeIf { it >= 0 && contact.down } + ?: continue + pointers[slot] = contact.id + report[slot] = contact.copy(id = slot) + } + for (slot in 0 until slots) { + if (report[slot] == null) report[slot] = held[slot]?.copy(down = false) + val sent = report[slot] + if (sent?.down == true) { + held[slot] = sent + } else { + held[slot] = null + pointers[slot] = null + } + } + return report.filterNotNull() + } +} diff --git a/shared/src/main/java/com/shilapi/xcertplay/airplay/CarPlayMediaEngine.kt b/shared/src/main/java/com/shilapi/xcertplay/airplay/CarPlayMediaEngine.kt index 2c8ae453..220d3271 100644 --- a/shared/src/main/java/com/shilapi/xcertplay/airplay/CarPlayMediaEngine.kt +++ b/shared/src/main/java/com/shilapi/xcertplay/airplay/CarPlayMediaEngine.kt @@ -11,6 +11,8 @@ import java.net.Inet6Address import java.net.InetAddress import java.math.BigInteger import java.util.concurrent.ConcurrentHashMap +import java.util.concurrent.ScheduledThreadPoolExecutor +import java.util.concurrent.TimeUnit /** Rendering seam for the decrypted CarPlay media streams. */ interface MediaSink { @@ -36,8 +38,8 @@ interface MediaSink { /** * Concrete [AirPlayMediaHandler] that binds the screen, audio and iAP2 DataStream ports, - * decrypts their payloads, and hands decoded media to a [MediaSink]. Telephony and speech - * streams can additionally return a PCM microphone uplink through the sink. + * decrypts their payloads, and hands decoded media to a [MediaSink]. Streams that request input + * can additionally return a microphone uplink through the sink. */ class CarPlayMediaEngine( private val sink: MediaSink, @@ -64,9 +66,20 @@ class CarPlayMediaEngine( val handler: (BlockingDuplexByteStream) -> Boolean, ) + private data class PendingMicrophone(val session: AirPlaySession, val start: MicrophoneStart) + private val streams = ConcurrentHashMap() private val audioMeta = ConcurrentHashMap() - private val pendingMicrophone = ConcurrentHashMap() + // The sink owns one microphone per stream type. Serialize delayed starts with replacement/teardown. + private val microphoneLock = Any() + private val pendingMicrophone = mutableMapOf() + private val microphoneScheduler by lazy { + ScheduledThreadPoolExecutor(1) { task -> Thread(task, "carplay-mic-start").apply { isDaemon = true } }.apply { + removeOnCancelPolicy = true + setKeepAliveTime(5, TimeUnit.SECONDS) + allowCoreThreadTimeOut(true) + } + } private val audioCaptures = ConcurrentHashMap() private val pendingIapTunnels = ConcurrentHashMap() @Volatile private var iapTunnelHandler: ((BlockingDuplexByteStream) -> Boolean)? = null @@ -77,9 +90,10 @@ class CarPlayMediaEngine( override fun onScreen(session: AirPlaySession, type: Int, stream: Map): Int? { val key = outputKey(session, stream) ?: return null + val peer = session.remoteAddress ?: return null val streamKey = StreamKey(session, type) Log.i(TAG, "airplay screen key connectionID=${unsignedPlistDecimal(stream["streamConnectionID"])}") - val screen = ScreenStream(key, session::logDebug, sink::createVideoFrameDecoder) + val screen = ScreenStream(key, peer, session::logDebug, sink::createVideoFrameDecoder) sink.setVideoDiagnosticHandler(type) { if (it == "first frame rendered") session.videoFrameRendered() session.logDebug("Video: $it") @@ -118,7 +132,7 @@ class CarPlayMediaEngine( streams.remove(streamKey)?.close() audioMeta.remove(type) audioCaptures.remove(type)?.close() - if (pendingMicrophone.remove(type) != null) sink.onMicrophoneStopped(type) + synchronized(microphoneLock) { stopMicrophone(type) } sink.onAudioStopped(type) val key = outputKey(session, stream) ?: return null @@ -138,10 +152,11 @@ class CarPlayMediaEngine( val latencyMs = (stream["audioLatencyMs"] as? Number)?.toInt() ?: 0 val meta = AudioMeta(type, format, connectionId, latencyMs) val microphone = microphoneConfig(session, type, stream, format) - if (microphone != null) pendingMicrophone[type] = microphone + val pending = microphone?.let { config -> + PendingMicrophone(session, MicrophoneStart { sink.onMicrophoneStarted(type, config) }) + } val capture = audioCaptureDirectory?.let { AudioPacketCapture(it, type) } - if (capture != null) audioCaptures[type] = capture val audio = AudioStream(key, type, session::logDebug) val (dataPort, controlPort) = audio.listen( object : AudioStream.Listener { @@ -149,7 +164,11 @@ class CarPlayMediaEngine( meta.firstSample = firstSample meta.originNs = nanoTime() sink.onAudioStarted(type, format, firstSample) - microphone?.let { sink.onMicrophoneStarted(type, it) } + synchronized(microphoneLock) { + if (!session.isClosed && pending != null && pendingMicrophone[type] === pending) { + pending.start.onDownlink() + } + } } override fun onRtp(rtp: ByteArray, sample: Int) = @@ -165,8 +184,29 @@ class CarPlayMediaEngine( } }, ) - streams[streamKey] = audio - audioMeta[type] = meta + synchronized(microphoneLock) { + if (session.isClosed) { + audio.close() + capture?.close() + pending?.start?.close() + return null + } + streams[streamKey] = audio + audioMeta[type] = meta + if (capture != null) audioCaptures[type] = capture + if (pending != null) { + pendingMicrophone[type] = pending + if (MicrophoneStart.startsWithoutDownlink(audioType)) { + pending.start.scheduleSilentStart(microphoneScheduler, MicrophoneStart.SILENT_START_MILLIS) { startNow -> + synchronized(microphoneLock) { + if (!session.isClosed && pendingMicrophone[type] === pending && startNow()) { + session.logDebug("Audio: microphone started without downlink type=$type audioType=$audioType") + } + } + } + } + } + } return linkedMapOf( "type" to type, "dataPort" to dataPort, @@ -179,6 +219,7 @@ class CarPlayMediaEngine( val uuid = (stream["clientTypeUUID"] as? String)?.uppercase() ?: return null if (uuid != IAP_DATASTREAM_UUID) return null val shared = session.sharedSecret ?: return null + val peer = session.remoteAddress ?: return null val seed = unsignedPlistDecimal(stream["seed"]) ?: return null session.logDebug( "AirPlay iAP SETUP uuid=$uuid seed=$seed " + @@ -192,6 +233,7 @@ class CarPlayMediaEngine( ) val tunnel = IapTunnel( readKey = key, + expectedPeer = peer, bindAddress = session.localAddress ?: when (session.remoteAddress) { is Inet6Address -> InetAddress.getByName("::") @@ -284,7 +326,9 @@ class CarPlayMediaEngine( override fun onTeardown(session: AirPlaySession, type: Int) { if (type == STREAM_TYPE_DATA) clearPendingIapTunnel(session) - if (pendingMicrophone.remove(type) != null) sink.onMicrophoneStopped(type) + synchronized(microphoneLock) { + if (pendingMicrophone[type]?.session === session) stopMicrophone(type) + } audioMeta.remove(type) audioCaptures.remove(type)?.close() sink.onAudioStopped(type) @@ -294,17 +338,26 @@ class CarPlayMediaEngine( override fun onSessionClosed(session: AirPlaySession) { clearPendingIapTunnel(session) + synchronized(microphoneLock) { + pendingMicrophone.filterValues { it.session === session }.keys.toList().forEach(::stopMicrophone) + } val sessionStreams = streams.keys.filter { it.session === session } sessionStreams .filter { isScreenStreamType(it.type) } .forEach { sink.onScreenStreamActive(it.type, false) } sessionStreams.forEach { streams.remove(it)?.close() } audioMeta.clear() - pendingMicrophone.clear() audioCaptures.values.forEach(AudioPacketCapture::close) audioCaptures.clear() } + /** Must hold microphoneLock, including the sink callback, to fence already-running timer tasks. */ + private fun stopMicrophone(type: Int) { + val previous = pendingMicrophone.remove(type) ?: return + previous.start.close() + sink.onMicrophoneStopped(type) + } + private fun replacePendingIapTunnel(session: AirPlaySession, next: PendingIapTunnel) { val previous = pendingIapTunnels.put(session, next) previous?.bridge?.close() @@ -331,7 +384,7 @@ class CarPlayMediaEngine( format: AudioFormat, ): MicrophoneConfig? { if (!microphoneEnabled || type != STREAM_TYPE_MAIN_AUDIO) return null - if (format.audioType != "telephony" && format.audioType != "speechrecognition") return null + if (!wantsMicrophone(format.audioType, stream)) return null val port = (stream["dataPort"] as? Number)?.toInt() ?: return null if (port !in 1..65535) return null val host = session.remoteAddress ?: return null @@ -343,7 +396,7 @@ class CarPlayMediaEngine( } else if (framesPerPacket > 0) { Math.round(framesPerPacket * 1000.0 / format.sampleRate).toInt().coerceIn(5, 60) } else { - 20 + MicrophoneConfig.pcmFrameMillis(format.sampleRate, format.channels) } val opusBitrate = when { formatBits and OPUS_48K != 0L -> 96_000 @@ -383,17 +436,28 @@ class CarPlayMediaEngine( private fun isScreenStreamType(type: Int): Boolean = type == STREAM_TYPE_MAIN_SCREEN || type == STREAM_TYPE_ALT_SCREEN - private companion object { - const val TAG = "xcertplay-usb" - const val STREAM_TYPE_MAIN_SCREEN = 110 - const val STREAM_TYPE_ALT_SCREEN = 111 - const val STREAM_TYPE_MAIN_AUDIO = 100 - const val STREAM_TYPE_DATA = 130 - const val DATASTREAM_OUTPUT_KEY = "DataStream-Output-Encryption-Key" - const val DATASTREAM_INPUT_KEY = "DataStream-Input-Encryption-Key" - const val IAP_DATASTREAM_UUID = "E9459FD0-BCAD-4C45-820F-1E72447EF2F2" - const val OPUS_24K = 0x20000000L - const val OPUS_48K = 0x40000000L + internal companion object { + /** App audio also carries navigation, so record it only when the phone requests input. */ + fun wantsMicrophone(audioType: String, stream: Map): Boolean = when (audioType) { + "telephony", "speechrecognition" -> true + "default", "compatibility" -> when (val input = stream["input"]) { + is Boolean -> input + is Number -> input.toInt() != 0 + else -> (stream["dataPort"] as? Number)?.toInt()?.let { it in 1..65535 } == true + } + else -> false + } + + private const val TAG = "xcertplay-usb" + private const val STREAM_TYPE_MAIN_SCREEN = 110 + private const val STREAM_TYPE_ALT_SCREEN = 111 + private const val STREAM_TYPE_MAIN_AUDIO = 100 + private const val STREAM_TYPE_DATA = 130 + private const val DATASTREAM_OUTPUT_KEY = "DataStream-Output-Encryption-Key" + private const val DATASTREAM_INPUT_KEY = "DataStream-Input-Encryption-Key" + private const val IAP_DATASTREAM_UUID = "E9459FD0-BCAD-4C45-820F-1E72447EF2F2" + private const val OPUS_24K = 0x20000000L + private const val OPUS_48K = 0x40000000L } } diff --git a/shared/src/main/java/com/shilapi/xcertplay/airplay/IapTunnel.kt b/shared/src/main/java/com/shilapi/xcertplay/airplay/IapTunnel.kt index 230b96fa..787b1925 100644 --- a/shared/src/main/java/com/shilapi/xcertplay/airplay/IapTunnel.kt +++ b/shared/src/main/java/com/shilapi/xcertplay/airplay/IapTunnel.kt @@ -20,6 +20,7 @@ import java.util.concurrent.atomic.AtomicLong */ class IapTunnel( private val readKey: ByteArray, + private val expectedPeer: InetAddress, bindAddress: InetAddress = InetAddress.getByName("0.0.0.0"), ) : Closeable { interface Listener { @@ -119,6 +120,11 @@ class IapTunnel( safeClose(accepted) return } + if (accepted.inetAddress != expectedPeer) { + safeClose(accepted) + listener.onDebug("AirPlay iAP tunnel rejected sender outside the control session") + continue + } accepted.setSoLinger(true, 0) socket = accepted readCounter.set(0) @@ -184,7 +190,7 @@ class IapTunnel( var offset = 0 while (buffer.size - offset >= PACKAGE_HEADER_LEN) { val size = readU32Be(buffer, offset) - if (size < PACKAGE_HEADER_LEN || size > MAX_PACKAGE) break + require(size in PACKAGE_HEADER_LEN..MAX_PACKAGE) { "bad iAP package size $size" } if (buffer.size - offset < size) break val messageType = readU32Be(buffer, offset + MESSAGE_TYPE_OFFSET) if (messageType == MSG_TYPE_COMM) { diff --git a/shared/src/main/java/com/shilapi/xcertplay/airplay/MicrophonePacketizer.kt b/shared/src/main/java/com/shilapi/xcertplay/airplay/MicrophonePacketizer.kt index 693e70ea..22bb8d00 100644 --- a/shared/src/main/java/com/shilapi/xcertplay/airplay/MicrophonePacketizer.kt +++ b/shared/src/main/java/com/shilapi/xcertplay/airplay/MicrophonePacketizer.kt @@ -14,8 +14,13 @@ data class MicrophoneConfig( val key: ByteArray, val codec: AudioCodecKind = AudioCodecKind.LPCM, val bitrate: Int? = null, - /** Negotiated CarPlay RTP clock; Opus capture/encoding remains at 48 kHz. */ - val opusClockRate: Int = 48_000, + /** + * The CarPlay clock for an Opus stream. Capture and encoding remain at 48 kHz. + * Captured Siri packets for audioFormat 0x20000000 advance by 480 per 20 ms and calls + * (0x40000000) by 960, so the clock follows the format's rate; 0x10000000 counts at 16 kHz on that + * rule. The report's "Audio: rtp clock" line measures the iPhone's own clock for each stream. + */ + val opusClockRate: Int = OPUS_CAPTURE_RATE, ) { val samplesPerPacket: Int get() = if (codec == AudioCodecKind.OPUS) { @@ -24,22 +29,45 @@ data class MicrophoneConfig( maxOf(1, sampleRate * frameMillis / 1000) } - val frameBytes: Int - get() = samplesPerPacket * channels * 2 + /** The clock the RTP timestamps count in: the Opus format's rate, or the PCM sample rate. */ + val rtpClockRate: Int + get() = if (codec == AudioCodecKind.OPUS) opusClockRate else sampleRate + /** How far each packet moves the RTP timestamp, on the negotiated clock. */ val rtpSamplesPerPacket: Int - get() = if (codec == AudioCodecKind.OPUS) opusClockRate * 20 / 1000 else samplesPerPacket + get() = if (codec == AudioCodecKind.OPUS) opusClockRate * OPUS_FRAME_MILLIS / 1000 else samplesPerPacket + + val frameBytes: Int + get() = samplesPerPacket * channels * 2 companion object { private const val OPUS_SAMPLES_PER_PACKET = 960 + private const val OPUS_CAPTURE_RATE = 48_000 + private const val OPUS_FRAME_MILLIS = 20 - /** DiPlay #295/#547: CarPlay timestamps follow its 16/24/48 kHz format bits. */ + /** The negotiated Opus format's rate: 48, 24 or 16 kHz; 48 kHz when no Opus bit is set. */ fun opusClockRate(formatBits: Long): Int = when { formatBits and 0x40000000L != 0L -> 48_000 formatBits and 0x20000000L != 0L -> 24_000 formatBits and 0x10000000L != 0L -> 16_000 - else -> 48_000 + else -> OPUS_CAPTURE_RATE } + + /** + * Milliseconds of PCM per packet when the iPhone names no framesPerPacket: the longest of 20, 10 and 5 ms whose + * sealed packet still fits one 1,500-byte link MTU with its IPv6 and UDP headers. 20 ms of 48 kHz mono is a + * 1,956-byte packet that leaves as IP fragments. Over USB every 48 kHz recording in the #1094 reports came out + * silent while 24 kHz Siri packets (996 bytes) worked, and wireless sends small Opus packets instead (#718, + * #1017). 48 kHz mono therefore sends 10 ms, the 480 samples a 24 kHz Siri packet already carries. + */ + fun pcmFrameMillis(sampleRate: Int, channels: Int): Int = PCM_FRAME_MILLIS.firstOrNull { millis -> + MicrophonePacketizer.SEAL_OVERHEAD + sampleRate * millis / 1000 * channels * 2 <= MAX_UDP_PAYLOAD + } ?: PCM_FRAME_MILLIS.last() + + private val PCM_FRAME_MILLIS = listOf(20, 10, 5) + + /** A 1,500-byte link MTU less the 40-byte IPv6 and 8-byte UDP headers. */ + private const val MAX_UDP_PAYLOAD = 1500 - 40 - 8 } } @@ -61,6 +89,9 @@ object MicrophonePacketizer { const val TAG_LEN = 16 const val NONCE_LEN = 8 + /** Bytes a sealed packet adds to its samples: the RTP header, the Poly1305 tag and the nonce. */ + const val SEAL_OVERHEAD = RTP_HEADER_LEN + TAG_LEN + NONCE_LEN + fun sealPacket( key: ByteArray, payloadType: Int, diff --git a/shared/src/main/java/com/shilapi/xcertplay/airplay/MicrophoneStart.kt b/shared/src/main/java/com/shilapi/xcertplay/airplay/MicrophoneStart.kt new file mode 100644 index 00000000..8a028fef --- /dev/null +++ b/shared/src/main/java/com/shilapi/xcertplay/airplay/MicrophoneStart.kt @@ -0,0 +1,56 @@ +package com.shilapi.xcertplay.airplay + +import java.util.concurrent.Future +import java.util.concurrent.ScheduledExecutorService +import java.util.concurrent.TimeUnit +import java.util.concurrent.atomic.AtomicBoolean + +/** + * Starts one stream's microphone at most once: at the first downlink packet, or [SILENT_START_MILLIS] + * after SETUP when the iPhone asks for input but plays nothing. An app voice message (Douyin, #566) + * opens a speech-recognition stream without downlink audio, and capture waited for a packet that + * never came. Calls still wait for their downlink, preserving their existing capture startup order. + */ +internal class MicrophoneStart(private val start: () -> Unit) : java.io.Closeable { + private val claimed = AtomicBoolean() + @Volatile private var silentStart: Future<*>? = null + + /** The first downlink packet arrived. */ + fun onDownlink() { + silentStart?.cancel(false) + if (claimed.compareAndSet(false, true)) start() + } + + override fun close() { + claimed.set(true) + silentStart?.cancel(false) + } + + /** + * Starts capture after [delayMillis] unless the downlink started it first. [whileCurrent] runs the + * given start only while the stream still owns its microphone; the start returns true when it + * started capture. + */ + fun scheduleSilentStart( + scheduler: ScheduledExecutorService, + delayMillis: Long, + whileCurrent: (startNow: () -> Boolean) -> Unit, + ) { + silentStart = scheduler.schedule({ + whileCurrent { + if (claimed.compareAndSet(false, true)) { + start() + true + } else { + false + } + } + }, delayMillis, TimeUnit.MILLISECONDS) + } + + companion object { + const val SILENT_START_MILLIS = 500L + + fun startsWithoutDownlink(audioType: String): Boolean = audioType != "telephony" + } +} diff --git a/shared/src/main/java/com/shilapi/xcertplay/airplay/NtpClock.kt b/shared/src/main/java/com/shilapi/xcertplay/airplay/NtpClock.kt index 297ad998..44d55e99 100644 --- a/shared/src/main/java/com/shilapi/xcertplay/airplay/NtpClock.kt +++ b/shared/src/main/java/com/shilapi/xcertplay/airplay/NtpClock.kt @@ -116,7 +116,7 @@ class NtpClock : Closeable { private fun handleMessage(message: ByteArray, source: InetSocketAddress) { when (message[1].toInt() and 0xff) { PT_REQUEST -> respondToRequest(message, source) - PT_RESPONSE -> handleResponse(message) + PT_RESPONSE -> if (source.address == peer?.address) handleResponse(message) } } diff --git a/shared/src/main/java/com/shilapi/xcertplay/airplay/ScreenStream.kt b/shared/src/main/java/com/shilapi/xcertplay/airplay/ScreenStream.kt index 98187674..3988b43f 100644 --- a/shared/src/main/java/com/shilapi/xcertplay/airplay/ScreenStream.kt +++ b/shared/src/main/java/com/shilapi/xcertplay/airplay/ScreenStream.kt @@ -22,7 +22,12 @@ typealias VideoCodec = com.shilapi.xcertplay.media.VideoCodec * (avcC/hvcC) or a ChaCha20-Poly1305 sealed VideoFrame. The key is the DataStream output key * and the per-frame nonce is an 8-byte little-endian counter. */ -class ScreenStream(private val key: ByteArray, private val onDiagnostic: (String) -> Unit = {}, private val decoderFactory: ((ByteArray) -> VideoFrameDecoder?)? = null) : Closeable { +class ScreenStream( + private val key: ByteArray, + private val expectedPeer: InetAddress, + private val onDiagnostic: (String) -> Unit = {}, + private val decoderFactory: ((ByteArray) -> VideoFrameDecoder?)? = null, +) : Closeable { interface Listener { fun onFrame(frame: CompressedVideoFrame, pts: Long, receivedAtNs: Long, decryptNs: Long) = onFrame(frame.toByteArray(), pts, receivedAtNs, decryptNs) fun onCodec(codec: VideoCodec) {} @@ -60,7 +65,7 @@ class ScreenStream(private val key: ByteArray, private val onDiagnostic: (String private fun accept(bound: ServerSocket) { try { - val accepted = bound.accept() + val accepted = bound.acceptFrom(expectedPeer) { onDiagnostic("Video: rejected sender outside the control session") } socket = accepted run(accepted) } catch (error: Exception) { @@ -106,18 +111,18 @@ class ScreenStream(private val key: ByteArray, private val onDiagnostic: (String private fun onMessage(header: ByteArray, body: ByteArray, bodySize: Int) { when (header[OPCODE_OFFSET].toInt() and 0xff) { OP_VIDEO_FRAME -> { + if (bodySize < ScreenCodec.TAG_SIZE) return val receivedAtNs = System.nanoTime() val decoder = frameDecoder - if (decoder != null && bodySize >= ScreenCodec.TAG_SIZE) { + if (decoder != null) { val frame = decoder.decode(frameCounter.get(), header, body, bodySize) frameCounter.incrementAndGet() val decryptNs = System.nanoTime() - receivedAtNs frame.use { listener.onFrame(it, ScreenCodec.presentationTimeUs(header), receivedAtNs, decryptNs) } } else { val exact = if (body.size == bodySize) body else body.copyOf(bodySize) - val payload = if (exact.size >= ScreenCodec.TAG_SIZE) { - ScreenCodec.decryptFrame(key, frameCounter.get(), header, exact).also { frameCounter.incrementAndGet() } - } else exact + val payload = ScreenCodec.decryptFrame(key, frameCounter.get(), header, exact) + frameCounter.incrementAndGet() val decryptNs = System.nanoTime() - receivedAtNs HeapVideoFrame(ScreenCodec.lengthPrefixedToAnnexB(payload)).use { listener.onFrame(it, ScreenCodec.presentationTimeUs(header), receivedAtNs, decryptNs) diff --git a/shared/src/main/java/com/shilapi/xcertplay/media/MicrophoneUplink.kt b/shared/src/main/java/com/shilapi/xcertplay/media/MicrophoneUplink.kt index 98e34584..ef90505e 100644 --- a/shared/src/main/java/com/shilapi/xcertplay/media/MicrophoneUplink.kt +++ b/shared/src/main/java/com/shilapi/xcertplay/media/MicrophoneUplink.kt @@ -20,7 +20,7 @@ import java.util.concurrent.atomic.AtomicBoolean * Captures one PCM microphone stream and sends it back to the phone as sealed CarPlay RTP. * * The recorder runs only while the matching audio stream is active, so callers start this after - * the first downlink audio packet and close it on stream teardown. + * the first downlink audio packet (or the input-only startup delay) and close it on stream teardown. */ internal class MicrophoneUplink( private val config: MicrophoneConfig, diff --git a/shared/src/main/java/com/shilapi/xcertplay/orchestration/CarPlayController.kt b/shared/src/main/java/com/shilapi/xcertplay/orchestration/CarPlayController.kt index 3f269a7f..8f3b675a 100644 --- a/shared/src/main/java/com/shilapi/xcertplay/orchestration/CarPlayController.kt +++ b/shared/src/main/java/com/shilapi/xcertplay/orchestration/CarPlayController.kt @@ -266,6 +266,8 @@ class CarPlayController( } private val sessionListener = object : AirPlaySessionListener { + override val wantsTrace: Boolean get() = uiListener?.wantsTrace == true + override fun onSessionActive(session: AirPlaySession) { if (activeSession !== session) BydNavigationOutputs.start(appContext) activeSession = session @@ -383,11 +385,44 @@ class CarPlayController( } } + private class PendingMove(val session: AirPlaySession, var contacts: List) + private val touchLock = Any() + private var lastTouch: List? = null + private var lastTouchSession: AirPlaySession? = null + private var pendingMove: PendingMove? = null + + /** Merge waiting moves of the same fingers; preserve every press and release. */ fun sendTouch(contacts: List): Boolean { if (closed) return false val session = activeSession ?: return false + val snapshot = contacts.toList() return try { - touchExecutor.execute { session.sendTouch(contacts) } + synchronized(touchLock) { + val isMove = lastTouchSession === session && snapshot.isNotEmpty() && snapshot.all { it.down } && + lastTouch?.filter { it.down }?.map { it.id }?.toSet() == snapshot.map { it.id }.toSet() + val pending = pendingMove + if (isMove && pending != null && pending.session === session) { + pending.contacts = snapshot + } else if (!isMove) { + touchExecutor.execute { + if (!closed && activeSession === session) session.sendTouch(snapshot) + } + pendingMove = null + } else { + val move = PendingMove(session, snapshot) + touchExecutor.execute { + val latest = synchronized(touchLock) { + if (pendingMove === move) pendingMove = null + move.contacts + } + if (!closed && activeSession === session) session.sendTouch(latest) + } + // Publish only after execute accepted it, while the task still waits for touchLock. + pendingMove = move + } + lastTouch = snapshot + lastTouchSession = session + } true } catch (_: Exception) { false diff --git a/shared/src/main/java/com/shilapi/xcertplay/transport/Ntb16Codec.kt b/shared/src/main/java/com/shilapi/xcertplay/transport/Ntb16Codec.kt index 00cf2bef..2787b581 100644 --- a/shared/src/main/java/com/shilapi/xcertplay/transport/Ntb16Codec.kt +++ b/shared/src/main/java/com/shilapi/xcertplay/transport/Ntb16Codec.kt @@ -16,6 +16,9 @@ object Ntb16Codec { private const val DATAGRAM_INDEX = 28 private const val USB_PACKET_SIZE = 512 + // Same bound as the bridge's frame queue, which fails the session past it. + private const val MAX_ENTRIES_PER_BLOCK = 256 + /** Wraps one Ethernet frame in one NTB16 block, padding an exact 512-byte boundary. */ fun build(frame: ByteArray, sequence: Int): ByteArray { require(frame.isNotEmpty()) { "frame must not be empty" } @@ -47,7 +50,9 @@ object Ntb16Codec { * Extracts the Ethernet frames carried by one NTB16 block. * * Wire garbage returns an empty list instead of throwing; malformed entries are skipped where - * a bounds check fails. A step guard stops a malformed NDP pointer chain from cycling. + * a bounds check fails. A block that repeats an NDP offset, lists more than + * [MAX_ENTRIES_PER_BLOCK] entries or yields more datagram bytes than its own length (entries + * that overlap) is dropped whole. */ fun parse(block: ByteArray): List = parse(block, 0, block.size) @@ -61,10 +66,12 @@ object Ntb16Codec { val datagrams = ArrayList(1) var ndpOffset = offset + readU16(block, offset + 10) - var hops = 0 - val maxHops = maxOf(1, length / 4) - while (ndpOffset != 0 && hops < maxHops) { + val visitedNdps = HashSet() + var entries = 0 + var totalBytes = 0 + while (ndpOffset != 0) { if (ndpOffset < offset || ndpOffset + 12 > end) break + if (!visitedNdps.add(ndpOffset)) return emptyList() if (readU32(block, ndpOffset) and 0x00ffffff != NDP16_SIG and 0x00ffffff) break val ndpLength = readU16(block, ndpOffset + 4) val nextNdp = readU16(block, ndpOffset + 6) @@ -74,15 +81,17 @@ object Ntb16Codec { val datagramIndex = readU16(block, entry) val datagramLength = readU16(block, entry + 2) if (datagramIndex == 0 || datagramLength == 0) break + if (++entries > MAX_ENTRIES_PER_BLOCK) return emptyList() val datagramStart = offset + datagramIndex val datagramEnd = datagramStart + datagramLength if (datagramStart >= offset && datagramStart <= end && datagramEnd <= end) { + totalBytes += datagramLength + if (totalBytes > length) return emptyList() datagrams.add(block.copyOfRange(datagramStart, datagramEnd)) } entry += 4 } ndpOffset = if (nextNdp == 0) 0 else offset + nextNdp - hops++ } return datagrams } diff --git a/shared/src/test/java/com/shilapi/xcertplay/airplay/AirPlayEventPeerTest.kt b/shared/src/test/java/com/shilapi/xcertplay/airplay/AirPlayEventPeerTest.kt new file mode 100644 index 00000000..39d4b412 --- /dev/null +++ b/shared/src/test/java/com/shilapi/xcertplay/airplay/AirPlayEventPeerTest.kt @@ -0,0 +1,76 @@ +package com.shilapi.xcertplay.airplay + +import java.net.InetAddress +import java.net.ServerSocket +import java.net.Socket +import java.util.concurrent.CopyOnWriteArrayList +import org.junit.Assert.assertTrue +import org.junit.Assert.fail +import org.junit.Test +import org.junit.runner.RunWith +import org.robolectric.RobolectricTestRunner +import org.robolectric.annotation.Config + +@RunWith(RobolectricTestRunner::class) +@Config(sdk = [29], manifest = Config.NONE) +class AirPlayEventPeerTest { + @org.junit.Before fun needsIpv6Loopback() { + org.junit.Assume.assumeTrue("no IPv6 loopback", runCatching { + java.net.ServerSocket(0, 1, java.net.InetAddress.getByName("::1")).use { true } + }.getOrDefault(false)) + } + + private val logs = CopyOnWriteArrayList() + + private fun openEvent(session: AirPlaySession): Int { + val method = AirPlaySession::class.java.getDeclaredMethod("openEvent") + method.isAccessible = true + return method.invoke(session) as Int + } + + /** A session whose control peer is 127.0.0.1. */ + private fun withSession(block: (AirPlaySession) -> Unit) { + ServerSocket(0, 1, InetAddress.getByName("127.0.0.1")).use { control -> + val client = Socket(InetAddress.getByName("127.0.0.1"), control.localPort) + val accepted = control.accept() + val session = AirPlaySession( + socket = client, + config = AirPlayConfig( + deviceName = "test", + deviceId = "02:00:00:00:00:02", + btMac = "02:00:00:00:00:01", + sourceVersion = "1.0", + main = AirPlayDisplayConfig(widthPixels = 800, heightPixels = 480), + ), + identity = AirPlayIdentity.generate(), + pairings = PairingStore(), + mfi = null, + listener = object : AirPlaySessionListener { + override fun onDebugLog(message: String) { logs += message } + }, + media = object : AirPlayMediaHandler {}, + ) + try { block(session) } finally { session.close(); accepted.close() } + } + } + + @Test + fun aStrangerCannotTakeTheEventSlot() = withSession { session -> + val port = openEvent(session) + Socket(InetAddress.getByName("::1"), port).use { Thread.sleep(500) } + try { + Socket(InetAddress.getByName("127.0.0.1"), port).use { Thread.sleep(500) } + } catch (error: java.net.ConnectException) { + fail("control peer could not connect after a stranger took the slot: $error") + } + assertTrue("stranger was accepted: $logs", logs.none { it.contains("accepted from /0:0:0:0:0:0:0:1") }) + assertTrue("control peer was not accepted: $logs", logs.any { it.contains("accepted from /127.0.0.1") }) + } + + @Test + fun theControlPeerIsAccepted() = withSession { session -> + val port = openEvent(session) + Socket(InetAddress.getByName("127.0.0.1"), port).use { Thread.sleep(500) } + assertTrue("control peer was not accepted: $logs", logs.any { it.contains("accepted from /127.0.0.1") }) + } +} diff --git a/shared/src/test/java/com/shilapi/xcertplay/airplay/AirPlayIapTunnelStreamTest.kt b/shared/src/test/java/com/shilapi/xcertplay/airplay/AirPlayIapTunnelStreamTest.kt index dcd4f4b0..c25a3b42 100644 --- a/shared/src/test/java/com/shilapi/xcertplay/airplay/AirPlayIapTunnelStreamTest.kt +++ b/shared/src/test/java/com/shilapi/xcertplay/airplay/AirPlayIapTunnelStreamTest.kt @@ -13,7 +13,7 @@ class AirPlayIapTunnelStreamTest { val session = testSession() val bridge = AirPlayIapTunnelStream( session = session, - tunnel = IapTunnel(ByteArray(32)), + tunnel = IapTunnel(ByteArray(32), InetAddress.getByName("127.0.0.1")), sendCommand = { data, _ -> sent += data.copyOf() true @@ -41,6 +41,7 @@ class AirPlayIapTunnelStreamTest { session = testSession(), tunnel = IapTunnel( readKey = ByteArray(32), + expectedPeer = InetAddress.getByName("::1"), bindAddress = InetAddress.getByName("::"), ), sendCommand = { _, _ -> true }, diff --git a/shared/src/test/java/com/shilapi/xcertplay/airplay/AirPlayTouchSlotsTest.kt b/shared/src/test/java/com/shilapi/xcertplay/airplay/AirPlayTouchSlotsTest.kt new file mode 100644 index 00000000..da0ab71a --- /dev/null +++ b/shared/src/test/java/com/shilapi/xcertplay/airplay/AirPlayTouchSlotsTest.kt @@ -0,0 +1,57 @@ +package com.shilapi.xcertplay.airplay + +import org.junit.Assert.* +import org.junit.Test + +class AirPlayTouchSlotsTest { + // #444, as a pinch on a Galaxy Note20 sent it: the first finger lifts while the second stays. + // main then pressed slot 0 again at the second finger and released slot 1 at (0,0). + @Test fun theFingerThatStaysKeepsItsSlotAndTheOtherIsReleasedWhereItWas() { + val slots = AirPlayTouchSlots(2) + assertEquals(listOf(c(0, .6, .5)), slots.assign(listOf(c(0, .6, .5)))) + assertEquals(listOf(c(0, .6, .5), c(1, .4, .3)), slots.assign(listOf(c(0, .6, .5), c(1, .4, .3)))) + // ACTION_POINTER_UP for pointer 0. + assertEquals(listOf(c(0, .55, .45, down = false), c(1, .42, .35)), + slots.assign(listOf(c(0, .55, .45, down = false), c(1, .42, .35)))) + // Pointer 1 is now at index 0 and moves on alone. + assertEquals(listOf(c(1, .43, .36)), slots.assign(listOf(c(1, .43, .36)))) + assertEquals(listOf(c(1, .43, .36, down = false)), slots.assign(listOf(c(1, .43, .36, down = false)))) + assertEquals(emptyList(), slots.assign(emptyList())) + } + + // Opening the settings menu or DiPlay's home, and the settings swipe, send an empty list. + @Test fun anEmptyListReleasesEveryFingerWhereItWas() { + val slots = AirPlayTouchSlots(2) + slots.assign(listOf(c(3, .1, .2), c(5, .7, .8))) + assertEquals(listOf(c(0, .1, .2, down = false), c(1, .7, .8, down = false)), slots.assign(emptyList())) + assertEquals(emptyList(), slots.assign(emptyList())) + } + + @Test fun aFingerMissingFromTheNextEventIsReleasedWhereItWas() { + val slots = AirPlayTouchSlots(2) + slots.assign(listOf(c(0, .1, .2), c(1, .7, .8))) + assertEquals(listOf(c(0, .15, .25), c(1, .7, .8, down = false)), slots.assign(listOf(c(0, .15, .25)))) + } + + @Test fun aThirdFingerWaitsForAFreeSlot() { + val slots = AirPlayTouchSlots(2) + slots.assign(listOf(c(0, .1, .1), c(1, .2, .2))) + assertEquals(listOf(c(0, .1, .1), c(1, .2, .2)), + slots.assign(listOf(c(0, .1, .1), c(1, .2, .2), c(2, .3, .3)))) + assertEquals(listOf(c(0, .1, .1, down = false), c(1, .2, .2)), + slots.assign(listOf(c(0, .1, .1, down = false), c(1, .2, .2), c(2, .3, .3)))) + assertEquals(listOf(c(0, .3, .3), c(1, .2, .2)), slots.assign(listOf(c(1, .2, .2), c(2, .3, .3)))) + } + + @Test fun aReleaseOfAFingerThatWasNeverReportedIsLeftOut() { + assertEquals(emptyList(), AirPlayTouchSlots(2).assign(listOf(c(4, .5, .5, down = false)))) + } + + @Test fun eachContactGoesInTheSlotItsIdNames() { + // Slot 0 up at (0,0); slot 1 down at (300,400), coordinates little-endian. + assertArrayEquals(byteArrayOf(0, 0, 0, 0, 0, 0, 1, 1, 0x2c, 0x01, 0x90.toByte(), 0x01), + AirPlayHid.touchReport(listOf(AirPlayContact(1, 300.0, 400.0, true)))) + } + + private fun c(id: Int, x: Double, y: Double, down: Boolean = true) = AirPlayContact(id, x, y, down) +} diff --git a/shared/src/test/java/com/shilapi/xcertplay/airplay/AirPlayTouchTraceCostTest.kt b/shared/src/test/java/com/shilapi/xcertplay/airplay/AirPlayTouchTraceCostTest.kt new file mode 100644 index 00000000..de6a533c --- /dev/null +++ b/shared/src/test/java/com/shilapi/xcertplay/airplay/AirPlayTouchTraceCostTest.kt @@ -0,0 +1,73 @@ +package com.shilapi.xcertplay.airplay + +import java.io.ByteArrayOutputStream +import java.net.Socket +import org.junit.Assert.assertEquals +import org.junit.Assert.assertTrue +import org.junit.Test +import org.junit.runner.RunWith +import org.robolectric.RobolectricTestRunner +import org.robolectric.annotation.Config + +/** Every touch report is a POST /command; its TRACE text is a hex dump that nobody may want. */ +@RunWith(RobolectricTestRunner::class) +@Config(sdk = [29]) +class AirPlayTouchTraceCostTest { + private class Recorder(private val wants: Boolean) : AirPlaySessionListener { + val traces = mutableListOf() + override val wantsTrace: Boolean get() = wants + override fun onDebugLog(message: String) { if (message.startsWith("TRACE ")) traces += message } + } + + @Test fun listenerThatDeclinesTraceCostsNoTraceTextForADrag() { + val listener = Recorder(wants = false) + drag(listener, MOVES) + assertEquals("trace chars built for ${MOVES + 2} touch events", 0, listener.traces.sumOf { it.length }) + } + + @Test fun listenerThatWantsTraceStillGetsTheSameHexDump() { + val listener = Recorder(wants = true) + drag(listener, MOVES) + assertEquals(MOVES + 2, listener.traces.size) + val first = listener.traces.first() + assertTrue(first, first.startsWith("TRACE airplay event tx headers=POST /command RTSP/1.0\r\n")) + assertTrue(first, first.contains("CSeq: 1\r\n\r\n bodyHex=")) + val hex = first.substringAfter("bodyHex=") + val body = ByteArray(hex.length / 2) { hex.substring(it * 2, it * 2 + 2).toInt(16).toByte() } + val command = BplistCodec.decode(body) as Map<*, *> + assertEquals("hidSendReport", command["type"]) + assertTrue(listener.traces.last().contains("CSeq: ${MOVES + 2}\r\n")) + } + + private fun drag(listener: AirPlaySessionListener, moves: Int) { + val session = session(listener) + try { + session.sendTouch(listOf(AirPlayContact(1, 0.5, 0.5, true))) + repeat(moves) { session.sendTouch(listOf(AirPlayContact(1, 0.5 + it * 0.001, 0.5, true))) } + session.sendTouch(emptyList()) + } finally { + session.close() + } + } + + private fun session(listener: AirPlaySessionListener): AirPlaySession { + val session = AirPlaySession( + Socket(), + AirPlayConfig("test", "02:00:00:00:00:02", "02:00:00:00:00:01", "1.0", main = AirPlayDisplayConfig(800, 480)), + AirPlayIdentity.generate(), PairingStore(), null, listener, object : AirPlayMediaHandler {}, + ) + val sink = object : Socket() { + private val out = ByteArrayOutputStream() + override fun getOutputStream() = out + } + set(session, "eventSocket", sink) + set(session, "eventCipher", ControlCipher(ByteArray(32) { 1 }, ByteArray(32) { 2 })) + return session + } + + private fun set(session: AirPlaySession, name: String, value: Any) { + AirPlaySession::class.java.getDeclaredField(name).apply { isAccessible = true }.set(session, value) + } + + private companion object { const val MOVES = 98 } +} diff --git a/shared/src/test/java/com/shilapi/xcertplay/airplay/AppMicrophoneRequestTest.kt b/shared/src/test/java/com/shilapi/xcertplay/airplay/AppMicrophoneRequestTest.kt new file mode 100644 index 00000000..536b37f9 --- /dev/null +++ b/shared/src/test/java/com/shilapi/xcertplay/airplay/AppMicrophoneRequestTest.kt @@ -0,0 +1,33 @@ +package com.shilapi.xcertplay.airplay + +import org.junit.Assert.assertFalse +import org.junit.Assert.assertTrue +import org.junit.Test + +class AppMicrophoneRequestTest { + private fun wants(audioType: String, vararg entries: Pair) = + CarPlayMediaEngine.wantsMicrophone(audioType, mapOf(*entries)) + + @Test fun callsAndSiriAlwaysRecord() { + assertTrue(wants("telephony")) + assertTrue(wants("speechrecognition")) + } + + @Test fun appAudioRecordsOnlyWhenTheIphoneAsksForInput() { + // A WhatsApp voice note arrives as app audio with an uplink port. + assertTrue(wants("default", "dataPort" to 50_000)) + assertTrue(wants("compatibility", "dataPort" to 50_000)) + assertTrue(wants("default", "input" to true)) + assertTrue(wants("default", "input" to 1L, "dataPort" to 50_000)) + // Guidance prompts use the same audio type without input. + assertFalse(wants("default")) + assertFalse(wants("default", "dataPort" to 0)) + assertFalse(wants("default", "input" to false, "dataPort" to 50_000)) + assertFalse(wants("default", "input" to 0, "dataPort" to 50_000)) + } + + @Test fun mediaAndAlertsNeverRecord() { + assertFalse(wants("media", "dataPort" to 50_000)) + assertFalse(wants("alert", "input" to true, "dataPort" to 50_000)) + } +} diff --git a/shared/src/test/java/com/shilapi/xcertplay/airplay/CarPlayMicrophoneLifecycleTest.kt b/shared/src/test/java/com/shilapi/xcertplay/airplay/CarPlayMicrophoneLifecycleTest.kt new file mode 100644 index 00000000..38f191c0 --- /dev/null +++ b/shared/src/test/java/com/shilapi/xcertplay/airplay/CarPlayMicrophoneLifecycleTest.kt @@ -0,0 +1,114 @@ +package com.shilapi.xcertplay.airplay + +import java.net.DatagramPacket +import java.net.DatagramSocket +import java.net.InetAddress +import java.net.ServerSocket +import java.net.Socket +import java.util.concurrent.LinkedBlockingQueue +import java.util.concurrent.TimeUnit +import org.junit.Assert.* +import org.junit.Test +import org.junit.runner.RunWith +import org.robolectric.RobolectricTestRunner +import org.robolectric.annotation.Config + +@RunWith(RobolectricTestRunner::class) +@Config(sdk = [29], manifest = Config.NONE) +class CarPlayMicrophoneLifecycleTest { + private class Rig : java.io.Closeable { + private val loopback = InetAddress.getByName("127.0.0.1") + private val server = ServerSocket(0, 1, loopback) + private val client = Socket(loopback, server.localPort) + val starts = LinkedBlockingQueue() + val stops = LinkedBlockingQueue() + val engine = CarPlayMediaEngine(object : MediaSink { + override fun onMicrophoneStarted(type: Int, config: MicrophoneConfig) { starts.add(config) } + override fun onMicrophoneStopped(type: Int) { stops.add(type) } + }, microphoneEnabled = true) + val shared = ByteArray(32) { 4 } + val session = AirPlaySession( + server.accept(), AirPlayConfig("test", "test", "", "1", AirPlayDisplayConfig(800, 480)), + AirPlayIdentity.generate(), PairingStore(), null, object : AirPlaySessionListener {}, engine, + ).also { + // Synthetic verified state: tests exercise SETUP/media, never real authentication assets. + it.pairVerify.javaClass.getDeclaredField("sharedSecret").apply { isAccessible = true }.set(it.pairVerify, shared) + it.pairVerify.javaClass.getDeclaredField("verified").apply { isAccessible = true }.setBoolean(it.pairVerify, true) + } + + fun setup(audioType: String = "default", vararg extra: Pair): Map { + val stream = mapOf("audioType" to audioType, "audioFormat" to 0x4000L, + "streamConnectionID" to 42L, "dataPort" to 50000) + extra + return requireNotNull(engine.onAudio(session, 100, stream)) + } + + fun downlink(response: Map) { + val key = AirPlayCrypto.hkdfSha512(shared, "DataStream-Salt42".asciiBytes(), + "DataStream-Output-Encryption-Key".asciiBytes(), 32) + val packet = MicrophonePacketizer.sealPacket(key, 100, MicrophoneCounters(), byteArrayOf(0, 0), 1) + DatagramSocket().use { it.send(DatagramPacket(packet, packet.size, loopback, (response["dataPort"] as Number).toInt())) } + } + + override fun close() { + session.close() + client.close() + server.close() + } + } + + @Test fun appVoiceNoteStartsWithoutDownlinkAndUsesMtuSizedPcm() = Rig().use { rig -> + rig.setup() + val config = rig.starts.poll(3, TimeUnit.SECONDS) + assertNotNull("no capture without downlink", config) + assertEquals("default", config!!.audioType) + assertEquals(48000, config.sampleRate) + assertEquals(10, config.frameMillis) + assertEquals(480, config.rtpSamplesPerPacket) + } + + @Test fun aDownlinkAndTheTimerCannotStartTwoMicrophones() = Rig().use { rig -> + val response = rig.setup("speechrecognition") + rig.downlink(response) + assertNotNull(rig.starts.poll(3, TimeUnit.SECONDS)) + assertNull("timer started capture twice", rig.starts.poll(800, TimeUnit.MILLISECONDS)) + } + + @Test fun callsStillWaitForTheirDownlink() = Rig().use { rig -> + val response = rig.setup("telephony") + assertNull(rig.starts.poll(800, TimeUnit.MILLISECONDS)) + rig.downlink(response) + assertNotNull(rig.starts.poll(3, TimeUnit.SECONDS)) + } + + @Test fun teardownCancelsDelayedCaptureAndStopsItsMicrophone() = Rig().use { rig -> + rig.setup() + rig.engine.onTeardown(rig.session, 100) + assertEquals(100, rig.stops.poll(1, TimeUnit.SECONDS)) + assertNull("capture restarted after TEARDOWN", rig.starts.poll(800, TimeUnit.MILLISECONDS)) + } + + @Test fun replacementCancelsTheOldTimerAndDoesNotRecordNavigation() = Rig().use { rig -> + rig.setup() + rig.setup("default", "input" to false) + assertEquals(100, rig.stops.poll(1, TimeUnit.SECONDS)) + assertNull("replaced recording started during navigation", rig.starts.poll(800, TimeUnit.MILLISECONDS)) + } + + @Test fun sessionCloseCancelsDelayedCapture() = Rig().use { rig -> + rig.setup() + rig.session.close() + assertEquals(100, rig.stops.poll(1, TimeUnit.SECONDS)) + assertNull("capture restarted after close", rig.starts.poll(800, TimeUnit.MILLISECONDS)) + } + + @Test fun explicitPacketDurationAndOpusClockArePreserved() = Rig().use { rig -> + rig.setup("speechrecognition", "framesPerPacket" to 960) + assertEquals(20, rig.starts.poll(3, TimeUnit.SECONDS)!!.frameMillis) + rig.setup("speechrecognition", "audioFormat" to 0x20000000L) + val opus = rig.starts.poll(3, TimeUnit.SECONDS)!! + assertEquals(AudioCodecKind.OPUS, opus.codec) + assertEquals(960, opus.samplesPerPacket) + assertEquals(480, opus.rtpSamplesPerPacket) + assertEquals(20, opus.frameMillis) + } +} diff --git a/shared/src/test/java/com/shilapi/xcertplay/airplay/IapPackageBoundsTest.kt b/shared/src/test/java/com/shilapi/xcertplay/airplay/IapPackageBoundsTest.kt new file mode 100644 index 00000000..a4e9e2a4 --- /dev/null +++ b/shared/src/test/java/com/shilapi/xcertplay/airplay/IapPackageBoundsTest.kt @@ -0,0 +1,85 @@ +package com.shilapi.xcertplay.airplay + +import java.net.InetAddress +import java.net.Socket +import java.util.concurrent.CountDownLatch +import java.util.concurrent.TimeUnit +import org.junit.Assert.assertArrayEquals +import org.junit.Assert.assertTrue +import org.junit.Test + +class IapPackageBoundsTest { + private val key = ByteArray(32) { 7 } + private val loopback = InetAddress.getByName("127.0.0.1") + + private fun frame(counter: Long, plain: ByteArray): ByteArray { + val aad = byteArrayOf(plain.size.toByte(), (plain.size shr 8).toByte()) + return aad + AirPlayCrypto.chachaSeal(key, AirPlayCrypto.nonce64(counter), plain, aad) + } + + private fun header(size: Int, type: String): ByteArray { + val h = ByteArray(32) + h[0] = (size ushr 24).toByte(); h[1] = (size ushr 16).toByte() + h[2] = (size ushr 8).toByte(); h[3] = size.toByte() + type.toByteArray(Charsets.US_ASCII).copyInto(h, 16) + return h + } + + @Test fun iapTunnelClosesOnInvalidPackageLengthsWithoutWaitingForTheBody() { + for (size in listOf(0, 31, -1, Int.MAX_VALUE)) { + val closed = CountDownLatch(1) + val tunnel = IapTunnel(key, expectedPeer = loopback, bindAddress = loopback) + var failure: Throwable? = null + val port = tunnel.listen(object : IapTunnel.Listener { + override fun onClosed(cause: Throwable?) { failure = cause; closed.countDown() } + }) + try { + Socket(loopback, port).use { client -> + client.getOutputStream().apply { write(frame(0, header(size, "comm"))); flush() } + assertTrue("tunnel still open after invalid package size $size", closed.await(3, TimeUnit.SECONDS)) + assertTrue("wrong terminal failure: $failure", failure is IllegalArgumentException) + } + } finally { tunnel.close() } + } + } + + @Test fun iapTunnelDeliversRealPackage() { + val got = CountDownLatch(1) + var body = ByteArray(0) + val tunnel = IapTunnel(key, expectedPeer = loopback, bindAddress = loopback) + val port = tunnel.listen(object : IapTunnel.Listener { + override fun onIap(bytes: ByteArray) { body = bytes; got.countDown() } + }) + try { + Socket(loopback, port).use { client -> + val payload = byteArrayOf(1, 2, 3) + client.getOutputStream().apply { write(frame(0, header(32 + payload.size, "comm") + payload)); flush() } + assertTrue(got.await(3, TimeUnit.SECONDS)) + assertArrayEquals(payload, body) + } + } finally { + tunnel.close() + } + } + + @Test fun aPartialPackageCanFinishInTheNextEncryptedFrame() { + val got = CountDownLatch(1) + var body = ByteArray(0) + val tunnel = IapTunnel(key, expectedPeer = loopback, bindAddress = loopback) + val port = tunnel.listen(object : IapTunnel.Listener { + override fun onIap(bytes: ByteArray) { body = bytes; got.countDown() } + }) + try { + Socket(loopback, port).use { client -> + val payload = byteArrayOf(1, 2, 3) + val pack = header(32 + payload.size, "comm") + payload + client.getOutputStream().apply { write(frame(0, pack.copyOf(33))); flush() } + org.junit.Assert.assertFalse("partial body was delivered", got.await(200, TimeUnit.MILLISECONDS)) + client.getOutputStream().apply { write(frame(1, pack.copyOfRange(33, pack.size))); flush() } + assertTrue(got.await(3, TimeUnit.SECONDS)) + assertArrayEquals(payload, body) + } + } finally { tunnel.close() } + } + +} diff --git a/shared/src/test/java/com/shilapi/xcertplay/airplay/IapTunnelPeerTest.kt b/shared/src/test/java/com/shilapi/xcertplay/airplay/IapTunnelPeerTest.kt new file mode 100644 index 00000000..c93a658a --- /dev/null +++ b/shared/src/test/java/com/shilapi/xcertplay/airplay/IapTunnelPeerTest.kt @@ -0,0 +1,66 @@ +package com.shilapi.xcertplay.airplay + +import java.net.InetAddress +import java.net.Socket +import java.util.concurrent.LinkedBlockingQueue +import java.util.concurrent.TimeUnit +import org.junit.Assert.assertArrayEquals +import org.junit.Assert.assertNull +import org.junit.Test + +class IapTunnelPeerTest { + private val key = ByteArray(32) { (it + 1).toByte() } + private val iap = LinkedBlockingQueue() + private val opens = LinkedBlockingQueue() + private val closes = LinkedBlockingQueue() + + private val listener = object : IapTunnel.Listener { + override fun onOpen(remoteAddress: String?) { opens.add(remoteAddress) } + override fun onIap(bytes: ByteArray) { iap.add(bytes) } + override fun onClosed(cause: Throwable?) { closes.add(cause?.toString() ?: "peer EOF") } + } + + private fun frame(counter: Long, plain: ByteArray): ByteArray { + val aad = byteArrayOf(plain.size.toByte(), (plain.size shr 8).toByte()) + return aad + AirPlayCrypto.chachaSeal(key, AirPlayCrypto.nonce64(counter), plain, aad) + } + + private fun commPackage(body: ByteArray): ByteArray { + val size = 32 + body.size + val header = ByteArray(32) + header[3] = size.toByte() + "comm".toByteArray(Charsets.US_ASCII).copyInto(header, 16) + return header + body + } + + private fun send(port: Int, bytes: ByteArray) { + Socket(InetAddress.getLoopbackAddress(), port).use { socket -> + socket.getOutputStream().write(bytes) + socket.getOutputStream().flush() + Thread.sleep(SETTLE_MILLIS) + } + } + + @Test + fun aStrangersBadFrameDoesNotEndTheTunnel() { + val tunnel = IapTunnel(key, InetAddress.getByAddress(byteArrayOf(127, 0, 0, 2))) + try { + val port = tunnel.listen(listener) + send(port, byteArrayOf(32, 0) + ByteArray(32 + 16) { 9 }) + assertNull("tunnel closed by stranger: ${closes.peek()}", closes.peek()) + assertNull("stranger was announced as the peer: ${opens.peek()}", opens.peek()) + } finally { tunnel.close() } + } + + @Test + fun theControlPeersSealedPackageIsDelivered() { + val tunnel = IapTunnel(key, InetAddress.getLoopbackAddress()) + try { + val port = tunnel.listen(listener) + send(port, frame(0, commPackage(byteArrayOf(1, 2, 3)))) + assertArrayEquals(byteArrayOf(1, 2, 3), iap.poll(3, TimeUnit.SECONDS)) + } finally { tunnel.close() } + } + + private companion object { const val SETTLE_MILLIS = 700L } +} diff --git a/shared/src/test/java/com/shilapi/xcertplay/airplay/MicrophonePacketizerTest.kt b/shared/src/test/java/com/shilapi/xcertplay/airplay/MicrophonePacketizerTest.kt index bc3e57e9..18c0d8cd 100644 --- a/shared/src/test/java/com/shilapi/xcertplay/airplay/MicrophonePacketizerTest.kt +++ b/shared/src/test/java/com/shilapi/xcertplay/airplay/MicrophonePacketizerTest.kt @@ -1,6 +1,7 @@ package com.shilapi.xcertplay.airplay import org.junit.Assert.assertArrayEquals +import org.junit.Assert.assertTrue import org.junit.Assert.assertEquals import org.junit.Test @@ -49,4 +50,36 @@ class MicrophonePacketizerTest { ), ) } + + @Test + fun pcmPacketsFitOneLinkMtuWithTheirIpv6AndUdpHeaders() { + // #1094: 20 ms of 48 kHz mono sealed to 1,956 bytes, which leaves as IP fragments; those recordings were silent. + assertEquals(10, MicrophoneConfig.pcmFrameMillis(48_000, 1)) + assertEquals(20, MicrophoneConfig.pcmFrameMillis(24_000, 1)) + assertEquals(20, MicrophoneConfig.pcmFrameMillis(16_000, 1)) + assertEquals(5, MicrophoneConfig.pcmFrameMillis(48_000, 2)) + + for ((sampleRate, channels) in listOf(48_000 to 1, 48_000 to 2, 44_100 to 1, 24_000 to 1, 16_000 to 1, 8_000 to 1)) { + val millis = MicrophoneConfig.pcmFrameMillis(sampleRate, channels) + val pcm = config(AudioCodecKind.LPCM, sampleRate = sampleRate).copy(channels = channels, frameMillis = millis) + val packet = MicrophonePacketizer.sealPacket(ByteArray(32), 100, MicrophoneCounters(), ByteArray(pcm.frameBytes), + pcm.samplesPerPacket) + assertTrue("$sampleRate Hz x$channels: ${packet.size} bytes", 40 + 8 + packet.size <= 1500) + } + // A 24 kHz Siri packet and a 48 kHz one now carry the same 480 samples. + assertEquals(480, config(AudioCodecKind.LPCM, sampleRate = 48_000).copy(frameMillis = 10).samplesPerPacket) + } + + private fun config(codec: AudioCodecKind, sampleRate: Int = 48_000, opusClockRate: Int = 48_000) = MicrophoneConfig( + audioType = "speechrecognition", + sampleRate = sampleRate, + channels = 1, + payloadType = 100, + frameMillis = 20, + host = java.net.InetAddress.getLoopbackAddress(), + port = 1, + key = ByteArray(32), + codec = codec, + opusClockRate = opusClockRate, + ) } diff --git a/shared/src/test/java/com/shilapi/xcertplay/airplay/MicrophoneStartTest.kt b/shared/src/test/java/com/shilapi/xcertplay/airplay/MicrophoneStartTest.kt new file mode 100644 index 00000000..bfa27535 --- /dev/null +++ b/shared/src/test/java/com/shilapi/xcertplay/airplay/MicrophoneStartTest.kt @@ -0,0 +1,60 @@ +package com.shilapi.xcertplay.airplay + +import java.util.concurrent.Executors +import java.util.concurrent.TimeUnit +import java.util.concurrent.atomic.AtomicInteger +import org.junit.Assert.assertEquals +import org.junit.Assert.assertFalse +import org.junit.Assert.assertTrue +import org.junit.Test + +class MicrophoneStartTest { + private val starts = AtomicInteger() + private val gate = MicrophoneStart { starts.incrementAndGet() } + private val scheduler = Executors.newSingleThreadScheduledExecutor() + + @Test fun aVoiceMessageWithoutDownlinkAudioStartsTheMicrophoneOnce() { + var started: Boolean? = null + gate.scheduleSilentStart(scheduler, 0) { startNow -> started = startNow() } + drain() + + assertEquals(true, started) + assertEquals(1, starts.get()) + gate.onDownlink() + assertEquals(1, starts.get()) + } + + @Test fun theFirstDownlinkPacketStartsCaptureAndCancelsTheSilentStart() { + val checks = AtomicInteger() + gate.scheduleSilentStart(scheduler, 60_000) { startNow -> + checks.incrementAndGet() + startNow() + } + + gate.onDownlink() + gate.onDownlink() + drain() + + assertEquals(1, starts.get()) + assertEquals(0, checks.get()) + } + + @Test fun aStreamTornDownBeforeTheDelayNeverStartsLate() { + gate.scheduleSilentStart(scheduler, 0) { /* TEARDOWN already removed the stream's microphone. */ } + drain() + + assertEquals(0, starts.get()) + } + + @Test fun callsWaitForTheirDownlinkAndOtherInputStreamsDoNot() { + assertFalse(MicrophoneStart.startsWithoutDownlink("telephony")) + assertTrue(MicrophoneStart.startsWithoutDownlink("speechrecognition")) + assertTrue(MicrophoneStart.startsWithoutDownlink("default")) + } + + private fun drain() { + // Delayed tasks still run after shutdown; cancelled ones are dropped. + scheduler.shutdown() + assertTrue(scheduler.awaitTermination(5, TimeUnit.SECONDS)) + } +} diff --git a/shared/src/test/java/com/shilapi/xcertplay/airplay/NtpClockPeerTest.kt b/shared/src/test/java/com/shilapi/xcertplay/airplay/NtpClockPeerTest.kt new file mode 100644 index 00000000..f1d2788b --- /dev/null +++ b/shared/src/test/java/com/shilapi/xcertplay/airplay/NtpClockPeerTest.kt @@ -0,0 +1,81 @@ +package com.shilapi.xcertplay.airplay + +import java.net.DatagramPacket +import java.net.DatagramSocket +import java.net.InetAddress +import java.net.InetSocketAddress +import java.nio.ByteBuffer +import org.junit.Assert.assertTrue +import org.junit.Test + +class NtpClockPeerTest { + @org.junit.Before fun needsIpv6Loopback() { + org.junit.Assume.assumeTrue("no IPv6 loopback", runCatching { + java.net.ServerSocket(0, 1, java.net.InetAddress.getByName("::1")).use { true } + }.getOrDefault(false)) + } + + private val iphone = InetAddress.getByName("::1") + private val stranger = InetAddress.getByName("127.0.0.1") + + private fun skewedResponse(request: ByteArray): ByteArray { + val t1 = ByteBuffer.wrap(request).getLong(TRANSMIT) + val response = ByteArray(32) + response[0] = 0x80.toByte() + response[1] = 211.toByte() + val buffer = ByteBuffer.wrap(response) + buffer.putLong(ORIGINATE, t1) + buffer.putLong(RECEIVE, t1 + SKEW) + buffer.putLong(TRANSMIT, t1 + SKEW) + return response + } + + /** Seconds the synced clock is ahead of this host's wall clock. */ + private fun driftSeconds(ntp: NtpClock): Long { + val synced = ntp.syncedNtp().shiftRight(32).toLong() + return synced - (System.currentTimeMillis() / 1000 + NTP_EPOCH_OFFSET) + } + + private fun runExchange(replyFrom: InetAddress): Long { + val ntp = NtpClock() + val phone = DatagramSocket(InetSocketAddress(iphone, 0)) + val replier = DatagramSocket(InetSocketAddress(replyFrom, 0)) + try { + phone.soTimeout = 3000 + val port = ntp.listen() + ntp.start(iphone, phone.localPort) + repeat(2) { + val request = DatagramPacket(ByteArray(64), 64) + phone.receive(request) + val response = skewedResponse(request.data.copyOf(32)) + replier.send(DatagramPacket(response, response.size, replyFrom, port)) + } + Thread.sleep(300) + return driftSeconds(ntp) + } finally { + ntp.close() + phone.close() + replier.close() + } + } + + @Test + fun aStrangersResponseCannotMoveTheClock() { + val drift = runExchange(replyFrom = stranger) + assertTrue("clock skewed by a non-peer source: drift=${drift}s", Math.abs(drift) < 60) + } + + @Test + fun thePhonesResponseStillMovesTheClock() { + val drift = runExchange(replyFrom = iphone) + assertTrue("clock did not follow the peer: drift=${drift}s", drift > 3000) + } + + private companion object { + const val ORIGINATE = 8 + const val RECEIVE = 16 + const val TRANSMIT = 24 + const val SKEW = 3600L shl 32 + const val NTP_EPOCH_OFFSET = 2_208_988_800L + } +} diff --git a/shared/src/test/java/com/shilapi/xcertplay/airplay/ScreenStreamPeerTest.kt b/shared/src/test/java/com/shilapi/xcertplay/airplay/ScreenStreamPeerTest.kt new file mode 100644 index 00000000..947abdd5 --- /dev/null +++ b/shared/src/test/java/com/shilapi/xcertplay/airplay/ScreenStreamPeerTest.kt @@ -0,0 +1,138 @@ +package com.shilapi.xcertplay.airplay + +import java.net.InetAddress +import java.net.Socket +import java.util.concurrent.LinkedBlockingQueue +import java.util.concurrent.TimeUnit +import org.junit.Assert.assertArrayEquals +import org.junit.Assert.assertNotNull +import org.junit.Assert.assertNull +import org.junit.Test +import org.junit.runner.RunWith +import org.robolectric.RobolectricTestRunner +import org.robolectric.annotation.Config + +@RunWith(RobolectricTestRunner::class) +@Config(sdk = [29], manifest = Config.NONE) +class ScreenStreamPeerTest { + @org.junit.Before fun needsIpv6Loopback() { + org.junit.Assume.assumeTrue("no IPv6 loopback", runCatching { + java.net.ServerSocket(0, 1, java.net.InetAddress.getByName("::1")).use { true } + }.getOrDefault(false)) + } + + private val key = ByteArray(32) { it.toByte() } + private val frames = LinkedBlockingQueue() + private val configs = LinkedBlockingQueue() + private val closes = LinkedBlockingQueue() + + private val listener = object : ScreenStream.Listener { + override fun onFrame(naluBytes: ByteArray) { frames.add(naluBytes) } + override fun onConfig(codecData: ByteArray) { configs.add(codecData) } + override fun onClosed(cause: Throwable?) { closes.add(cause?.toString() ?: "peer EOF") } + } + + private fun header(opcode: Int, bodySize: Int) = ByteArray(128).also { + it[0] = bodySize.toByte() + it[1] = (bodySize shr 8).toByte() + it[OPCODE] = opcode.toByte() + } + + private fun sealedFrame(counter: Long, nal: ByteArray): ByteArray { + val plain = byteArrayOf(0, 0, 0, nal.size.toByte()) + nal + val head = header(0, plain.size + 16) + return head + AirPlayCrypto.chachaSeal(key, AirPlayCrypto.nonce64(counter), plain, head) + } + + private fun send(port: Int, bytes: ByteArray) { + Socket(InetAddress.getLoopbackAddress(), port).use { socket -> + socket.getOutputStream().write(bytes) + socket.getOutputStream().flush() + Thread.sleep(SETTLE_MILLIS) + } + } + + @Test + fun aStrangersBadFrameDoesNotEndTheSession() { + val stream = ScreenStream(key, InetAddress.getByAddress(byteArrayOf(127, 0, 0, 2))) + try { + val port = stream.listen(listener) + send(port, header(0, 32) + ByteArray(32) { 7 }) + assertNull("session closed by stranger: ${closes.peek()}", closes.peek()) + } finally { stream.close() } + } + + @Test + fun aStrangersConfigIsNotDelivered() { + val stream = ScreenStream(key, InetAddress.getByAddress(byteArrayOf(127, 0, 0, 2))) + try { + val port = stream.listen(listener) + val avcc = byteArrayOf(1, 0x64, 0, 0x1f, 0xff.toByte(), 0xe1.toByte(), 0, 4, 0x67, 0x64, 0, 0x1f) + send(port, header(1, avcc.size) + avcc) + assertNull("stranger config reached the decoder: ${configs.peek()?.size} bytes", configs.peek()) + } finally { stream.close() } + } + + @Test + fun aFrameShorterThanTheTagIsDropped() { + val stream = ScreenStream(key, InetAddress.getLoopbackAddress()) + try { + val port = stream.listen(listener) + send(port, header(0, 8) + byteArrayOf(0, 0, 0, 4, 1, 2, 3, 4)) + assertNull("unsealed frame reached the decoder: ${frames.peek()?.size} bytes", frames.peek()) + } finally { stream.close() } + } + + @Test + fun theControlPeersSealedFrameIsDelivered() { + val stream = ScreenStream(key, InetAddress.getLoopbackAddress()) + try { + val port = stream.listen(listener) + send(port, sealedFrame(0, byteArrayOf(0x65, 0x11))) + assertArrayEquals(byteArrayOf(0, 0, 0, 1, 0x65, 0x11), frames.poll(3, TimeUnit.SECONDS)) + } finally { stream.close() } + } + + @Test + fun theControlPeersConfigIsStillDelivered() { + val stream = ScreenStream(key, InetAddress.getLoopbackAddress()) + try { + val port = stream.listen(listener) + val avcc = byteArrayOf(1, 0x64, 0, 0x1f, 0xff.toByte(), 0xe1.toByte(), 0, 4, 0x67, 0x64, 0, 0x1f) + send(port, header(1, avcc.size) + avcc) + assertNotNull(configs.poll(3, TimeUnit.SECONDS)) + } finally { stream.close() } + } + + @Test + fun aStrangerHoldingAnOpenConnectionDoesNotBlockTheControlPeer() { + val ipv6Peer = InetAddress.getByName("::1") + val stream = ScreenStream(key, ipv6Peer) + try { + val port = stream.listen(listener) + Socket(InetAddress.getByName("127.0.0.1"), port).use { + Thread.sleep(SETTLE_MILLIS) + Socket(ipv6Peer, port).use { peer -> + peer.getOutputStream().write(sealedFrame(0, byteArrayOf(0x65, 0x22))) + assertArrayEquals(byteArrayOf(0, 0, 0, 1, 0x65, 0x22), frames.poll(3, TimeUnit.SECONDS)) + } + } + } finally { stream.close() } + } + + @Test + fun anIpv4MappedControlPeerIsMatchedToItsIpv4Connection() { + val mapped = InetAddress.getByAddress(ByteArray(10) + byteArrayOf(-1, -1, 127, 0, 0, 1)) + val stream = ScreenStream(key, mapped) + try { + val port = stream.listen(listener) + send(port, sealedFrame(0, byteArrayOf(0x65, 0x33))) + assertArrayEquals(byteArrayOf(0, 0, 0, 1, 0x65, 0x33), frames.poll(3, TimeUnit.SECONDS)) + } finally { stream.close() } + } + + private companion object { + const val OPCODE = 4 + const val SETTLE_MILLIS = 700L + } +} diff --git a/shared/src/test/java/com/shilapi/xcertplay/orchestration/CarPlayControllerTouchQueueTest.kt b/shared/src/test/java/com/shilapi/xcertplay/orchestration/CarPlayControllerTouchQueueTest.kt new file mode 100644 index 00000000..eb229e39 --- /dev/null +++ b/shared/src/test/java/com/shilapi/xcertplay/orchestration/CarPlayControllerTouchQueueTest.kt @@ -0,0 +1,265 @@ +package com.shilapi.xcertplay.orchestration + +import com.shilapi.xcertplay.airplay.* +import com.shilapi.xcertplay.transport.Iap2IdentificationConfig +import java.io.ByteArrayOutputStream +import java.io.OutputStream +import java.net.Socket +import java.util.concurrent.CountDownLatch +import java.util.concurrent.TimeUnit +import org.junit.Assert.assertEquals +import org.junit.Assert.assertFalse +import org.junit.Assert.assertTrue +import org.junit.Test +import org.junit.runner.RunWith +import org.robolectric.RobolectricTestRunner +import org.robolectric.RuntimeEnvironment +import org.robolectric.annotation.Config + +/** The touch executor runs one bplist encode, one AEAD seal and one socket write per queued report. */ +@RunWith(RobolectricTestRunner::class) +@Config(sdk = [29], manifest = Config.NONE) +class CarPlayControllerTouchQueueTest { + /** Counts writes; the first one blocks until [release] when [stall] is set. */ + private class Wire(stall: Boolean) : OutputStream() { + val bytes = ByteArrayOutputStream() + val started = CountDownLatch(1) + private val release = CountDownLatch(if (stall) 1 else 0) + @Volatile var writes = 0 + fun release() = release.countDown() + override fun write(b: Int) = throw UnsupportedOperationException() + override fun write(b: ByteArray, off: Int, len: Int) { + started.countDown() + check(release.await(20, TimeUnit.SECONDS)) { "test never released the stalled write" } + synchronized(bytes) { bytes.write(b, off, len) } + writes++ + } + } + + private class Rig(stall: Boolean) { + val wire = Wire(stall) + val controller = CarPlayController( + RuntimeEnvironment.getApplication(), + CarPlayRuntimeConfig( + mfiTarget = MfiTarget.LOCAL, transport = CarPlayTransport.WIRED, + identification = Iap2IdentificationConfig(name = "test", modelIdentifier = "test", manufacturer = "test", + serialNumber = "test", firmwareVersion = "1", hardwareVersion = "1", carPlayUsbInterfaceNumber = 3), + ), + AirPlayConfig("test", "test", "", "1", AirPlayDisplayConfig(800, 480)), + AirPlayIdentity.generate(), PairingStore(), object : AirPlaySessionListener {}, object : AirPlayMediaHandler {}, {}, + ) + val session = AirPlaySession( + Socket(), AirPlayConfig("test", "test", "", "1", AirPlayDisplayConfig(800, 480)), + AirPlayIdentity.generate(), PairingStore(), null, object : AirPlaySessionListener {}, object : AirPlayMediaHandler {}, + ) + + init { + val socket = object : Socket() { override fun getOutputStream() = wire } + field(session, "eventSocket", socket) + field(session, "eventCipher", ControlCipher(ByteArray(32) { 1 }, ByteArray(32) { 2 })) + field(controller, "activeSession", session) + } + + fun close() { + wire.release() + session.close() + controller.close() + controller.awaitClosed(2_000) + } + + /** Reports in send order, one string per report: the slots that are down or have a position. */ + fun reports(): List { + val decoder = ControlCipher(ByteArray(32) { 2 }, ByteArray(32) { 1 }) + val plain = decoder.decrypt(synchronized(wire.bytes) { wire.bytes.toByteArray() }).data + return RtspMessage.parseMessages(plain).messages.map { request -> + val report = (BplistCodec.decode(request.body) as Map<*, *>)["hidReport"] as ByteArray + (0 until AirPlayHid.TOUCH_CONTACTS).mapNotNull { slot -> + val o = slot * 6 + val x = (report[o + 2].toInt() and 0xff) or ((report[o + 3].toInt() and 0xff) shl 8) + val y = (report[o + 4].toInt() and 0xff) or ((report[o + 5].toInt() and 0xff) shl 8) + if (report[o + 1].toInt() == 0 && x == 0 && y == 0) null else "$slot:${report[o + 1]}@$x,$y" + }.joinToString(" ") + } + } + + fun awaitWrites(count: Int) { + val deadline = System.nanoTime() + TimeUnit.SECONDS.toNanos(20) + while (wire.writes < count && System.nanoTime() < deadline) Thread.sleep(5) + } + + fun settle(): Int { + val executor = controller.javaClass.getDeclaredField("touchExecutor").apply { isAccessible = true } + .get(controller) as java.util.concurrent.ExecutorService + executor.submit {}.get(5, TimeUnit.SECONDS) + return wire.writes + } + + fun replaceSession(): Pair { + val nextWire = Wire(stall = false) + val next = AirPlaySession( + Socket(), AirPlayConfig("test", "test", "", "1", AirPlayDisplayConfig(800, 480)), + AirPlayIdentity.generate(), PairingStore(), null, object : AirPlaySessionListener {}, object : AirPlayMediaHandler {}, + ) + field(next, "eventSocket", object : Socket() { override fun getOutputStream() = nextWire }) + field(next, "eventCipher", ControlCipher(ByteArray(32) { 1 }, ByteArray(32) { 2 })) + field(controller, "activeSession", next) + return next to nextWire + } + + private fun field(target: Any, name: String, value: Any) { + target.javaClass.getDeclaredField(name).apply { isAccessible = true }.set(target, value) + } + } + + private fun finger(id: Int, step: Int, down: Boolean = true) = + AirPlayContact(id, 0.1 + step * 0.001, 0.2 + id * 0.1, down) + + private fun px(contact: AirPlayContact) = "${Math.round(contact.x * 800).toInt()},${Math.round(contact.y * 480).toInt()}" + + @Test fun stalledSocketDoesNotQueueEveryMove() { + val rig = Rig(stall = true) + try { + rig.controller.sendTouch(listOf(finger(7, 0))) + assertTrue(rig.wire.started.await(20, TimeUnit.SECONDS)) + repeat(MOVES) { rig.controller.sendTouch(listOf(finger(7, it + 1))) } + rig.wire.release() + val executed = rig.settle() + assertTrue("$executed touch reports ran after a stall of ${MOVES + 1} events", executed <= 3) + } finally { + rig.close() + } + } + + @Test fun finalPositionIsAlwaysDelivered() { + val rig = Rig(stall = true) + try { + rig.controller.sendTouch(listOf(finger(7, 0))) + assertTrue(rig.wire.started.await(20, TimeUnit.SECONDS)) + repeat(MOVES) { rig.controller.sendTouch(listOf(finger(7, it + 1))) } + rig.wire.release() + rig.settle() + val reports = rig.reports() + assertEquals("0:1@${px(finger(7, 0))}", reports.first()) + assertEquals("0:1@${px(finger(7, MOVES))}", reports.last()) + } finally { + rig.close() + } + } + + @Test fun downUpAndSecondFingerKeepTheirOrderWhileMovesCollapse() { + val rig = Rig(stall = true) + try { + val first = listOf(finger(7, 0)) + rig.controller.sendTouch(first) + assertTrue(rig.wire.started.await(20, TimeUnit.SECONDS)) + repeat(20) { rig.controller.sendTouch(listOf(finger(7, 1 + it))) } + val twoDown = listOf(finger(7, 21), finger(3, 21)) + rig.controller.sendTouch(twoDown) + repeat(20) { rig.controller.sendTouch(listOf(finger(7, 22 + it), finger(3, 22 + it))) } + val secondUp = listOf(finger(7, 42), finger(3, 42, down = false)) + rig.controller.sendTouch(secondUp) + repeat(20) { rig.controller.sendTouch(listOf(finger(7, 43 + it))) } + val allUp = listOf(finger(7, 63, down = false)) + rig.controller.sendTouch(allUp) + rig.wire.release() + rig.settle() + assertEquals( + listOf( + "0:1@${px(finger(7, 0))}", + "0:1@${px(finger(7, 20))}", + "0:1@${px(finger(7, 21))} 1:1@${px(finger(3, 21))}", + "0:1@${px(finger(7, 41))} 1:1@${px(finger(3, 41))}", + "0:1@${px(finger(7, 42))} 1:0@${px(finger(3, 42))}", + "0:1@${px(finger(7, 62))}", + "0:0@${px(finger(7, 63))}", + ), + rig.reports(), + ) + } finally { + rig.close() + } + } + + @Test fun everyDownAndUpOfAStalledBurstOfTapsIsDelivered() { + val rig = Rig(stall = true) + try { + repeat(TAPS) { + rig.controller.sendTouch(listOf(finger(7, it))) + rig.controller.sendTouch(listOf(finger(7, it, down = false))) + } + rig.wire.release() + rig.awaitWrites(TAPS * 2) + assertEquals(TAPS * 2, rig.settle()) + } finally { + rig.close() + } + } + + @Test fun movesAreNotDroppedWhenTheSocketKeepsUp() { + val rig = Rig(stall = false) + try { + val events = listOf(finger(7, 0)) + (1..30).map { finger(7, it) } + finger(7, 31, down = false) + events.forEachIndexed { index, contact -> + rig.controller.sendTouch(listOf(contact)) + rig.awaitWrites(index + 1) + } + assertEquals(events.size, rig.settle()) + assertEquals(events.map { (if (it.down) "0:1@" else "0:0@") + px(it) }, rig.reports()) + } finally { + rig.close() + } + } + + @Test fun queuedMovesFromAnOldSessionCannotReachTheNewSession() { + val rig = Rig(stall = true) + var next: AirPlaySession? = null + try { + assertTrue(rig.controller.sendTouch(listOf(finger(7, 0)))) + assertTrue(rig.wire.started.await(5, TimeUnit.SECONDS)) + assertTrue(rig.controller.sendTouch(listOf(finger(7, 1)))) + val (newSession, newWire) = rig.replaceSession() + next = newSession + assertTrue(rig.controller.sendTouch(listOf(finger(7, 2)))) + rig.wire.release() + assertEquals("retired session sent a queued move", 1, rig.settle()) + assertEquals("new session lost its first press", 1, newWire.writes) + } finally { + next?.close() + rig.close() + } + } + + @Test fun queuedContactsAreAnImmutableSnapshot() { + val rig = Rig(stall = true) + try { + val contacts = mutableListOf(finger(7, 0)) + assertTrue(rig.controller.sendTouch(contacts)) + assertTrue(rig.wire.started.await(5, TimeUnit.SECONDS)) + contacts[0] = finger(7, 1) + assertTrue(rig.controller.sendTouch(contacts)) + contacts.clear() + rig.wire.release() + rig.settle() + assertEquals(listOf("0:1@${px(finger(7, 0))}", "0:1@${px(finger(7, 1))}"), rig.reports()) + } finally { rig.close() } + } + + private companion object { + const val MOVES = 500 + const val TAPS = 40 + } + + @Test fun movesAreRefusedWhenTheExecutorCannotRunThem() { + val rig = Rig(stall = false) + try { + assertTrue(rig.controller.sendTouch(listOf(finger(7, 0)))) + rig.awaitWrites(1) + rig.controller.javaClass.getDeclaredField("touchExecutor").apply { isAccessible = true } + .get(rig.controller).let { (it as java.util.concurrent.ExecutorService).shutdownNow() } + assertFalse("first move reported as sent", rig.controller.sendTouch(listOf(finger(7, 1)))) + assertFalse("later move merged into a move that never ran", rig.controller.sendTouch(listOf(finger(7, 2)))) + } finally { + rig.close() + } + } +} diff --git a/shared/src/test/java/com/shilapi/xcertplay/transport/Ntb16CodecTest.kt b/shared/src/test/java/com/shilapi/xcertplay/transport/Ntb16CodecTest.kt index d8787fed..f265c2e8 100644 --- a/shared/src/test/java/com/shilapi/xcertplay/transport/Ntb16CodecTest.kt +++ b/shared/src/test/java/com/shilapi/xcertplay/transport/Ntb16CodecTest.kt @@ -2,6 +2,7 @@ package com.shilapi.xcertplay.transport import org.junit.Assert.assertArrayEquals import org.junit.Assert.assertEquals +import org.junit.Assert.assertTrue import org.junit.Test class Ntb16CodecTest { @@ -29,4 +30,122 @@ class Ntb16CodecTest { fun datagramThatWouldOverflowBlockLengthIsRejected() { Ntb16Codec.build(ByteArray(65_508), 0x1234) } + + @Test + fun overlappingDatagramsCannotMultiplyTheBlockIntoGigabytes() { + val entries = 1_000 + val tableEnd = NDP_OFFSET + 8 + entries * 4 + val block = ByteArray(65_535) + header(block, NDP_OFFSET) + ndp(block, NDP_OFFSET, entries * 4 + 8, 0) + repeat(entries) { putEntry(block, NDP_OFFSET + 8 + it * 4, tableEnd, block.size - tableEnd) } + + val parsed = Ntb16Codec.parse(block) + + val total = parsed.sumOf { it.size.toLong() } + assertTrue("parsed ${parsed.size} datagrams totalling $total bytes from a ${block.size}-byte block", total <= block.size) + } + + @Test(timeout = 10_000) + fun ndpThatPointsBackAtItselfIsRejected() { + val block = ByteArray(4_096) + header(block, NDP_OFFSET) + ndp(block, NDP_OFFSET, 16, NDP_OFFSET) + putEntry(block, NDP_OFFSET + 8, 40, 20) + + val parsed = Ntb16Codec.parse(block) + + assertTrue("parsed ${parsed.size} datagrams from an NDP chain that loops", parsed.isEmpty()) + } + + @Test(timeout = 10_000) + fun twoNdpsThatPointAtEachOtherAreRejected() { + val second = NDP_OFFSET + 32 + val block = ByteArray(second + 32) + header(block, NDP_OFFSET) + ndp(block, NDP_OFFSET, 16, second) + putEntry(block, NDP_OFFSET + 8, second + 16, 8) + ndp(block, second, 16, NDP_OFFSET) // back to the first + putEntry(block, second + 8, second + 24, 8) + + val parsed = Ntb16Codec.parse(block) + + assertTrue("parsed ${parsed.size} datagrams from an NDP chain that loops", parsed.isEmpty()) + } + + @Test + fun theBridgeQueueLimitOfDatagramsPerBlockIsTheMostABlockMayHold() { + for ((entries, expected) in listOf(256 to 256, 257 to 0)) { + val dataStart = NDP_OFFSET + 8 + entries * 4 + 4 + val block = ByteArray(dataStart + entries * 4) + header(block, NDP_OFFSET) + ndp(block, NDP_OFFSET, 8 + entries * 4 + 4, 0) + repeat(entries) { putEntry(block, NDP_OFFSET + 8 + it * 4, dataStart + it * 4, 4) } + + assertEquals("$entries entries", expected, Ntb16Codec.parse(block).size) + } + } + + @Test + fun manyDistinctDatagramsInOneNdpAreAllReturned() { + val datagrams = 10 + val block = ByteArray(NDP_OFFSET + 8 + datagrams * 4 + 4 + datagrams * 20) + header(block, NDP_OFFSET) + ndp(block, NDP_OFFSET, 8 + datagrams * 4 + 4, 0) + val dataStart = NDP_OFFSET + 8 + datagrams * 4 + 4 + repeat(datagrams) { + block.fill((it + 1).toByte(), dataStart + it * 20, dataStart + (it + 1) * 20) + putEntry(block, NDP_OFFSET + 8 + it * 4, dataStart + it * 20, 20) + } + + val parsed = Ntb16Codec.parse(block) + + assertEquals(datagrams, parsed.size) + parsed.forEachIndexed { index, datagram -> assertArrayEquals(ByteArray(20) { (index + 1).toByte() }, datagram) } + } + + @Test + fun chainOfDistinctNdpsIsFollowed() { + val second = NDP_OFFSET + 32 + val block = ByteArray(second + 32) + header(block, NDP_OFFSET) + ndp(block, NDP_OFFSET, 16, second) + putEntry(block, NDP_OFFSET + 8, second + 16, 8) + ndp(block, second, 16, 0) + putEntry(block, second + 8, second + 24, 8) + block.fill(1, second + 16, second + 24) + block.fill(2, second + 24, second + 32) + + val parsed = Ntb16Codec.parse(block) + + assertEquals(2, parsed.size) + assertArrayEquals(ByteArray(8) { 1 }, parsed[0]) + assertArrayEquals(ByteArray(8) { 2 }, parsed[1]) + } + + private fun header(block: ByteArray, ndpIndex: Int) { + put(block, 0, Ntb16Codec.NTH16_SIG, 4) + put(block, 4, 12, 2) + put(block, 8, block.size, 2) + put(block, 10, ndpIndex, 2) + } + + private fun ndp(block: ByteArray, at: Int, length: Int, next: Int) { + put(block, at, Ntb16Codec.NDP16_SIG, 4) + put(block, at + 4, length, 2) + put(block, at + 6, next, 2) + } + + private fun putEntry(block: ByteArray, at: Int, index: Int, length: Int) { + put(block, at, index, 2) + put(block, at + 2, length, 2) + } + + private fun put(block: ByteArray, at: Int, value: Int, bytes: Int) { + repeat(bytes) { block[at + it] = (value ushr (8 * it)).toByte() } + } + + private companion object { + const val NDP_OFFSET = 12 + } }