ChibiOS USB driver: prevent deadlock with CONSOLE_ENABLE = yes (#12472)

Before this commit, attaching an ARM-based (i.e. ChibiOS-based) keyboard that
uses CONSOLE_ENABLE = yes and produces debug messages would deadlock the
keyboard unless one was running hid_listen.

With this commit, dead-locking writes to the queue are detected and prevented.

fixes #5631

1 files changed, 13 insertions, 1 deletions

diff --git a/tmk_core/protocol/chibios/usb_driver.c b/tmk_core/protocol/chibios/usb_driver.c
index cc0ce7600f..eb72f8ff6d 100644
--- a/tmk_core/protocol/chibios/usb_driver.c
+++ b/tmk_core/protocol/chibios/usb_driver.c
@@ -80,7 +80,19 @@ static bool qmkusb_start_receive(QMKUSBDriver *qmkusbp) {
  * Interface implementation.
  */
 
-static size_t _write(void *ip, const uint8_t *bp, size_t n) { return obqWriteTimeout(&((QMKUSBDriver *)ip)->obqueue, bp, n, TIME_INFINITE); }
+static size_t _write(void *ip, const uint8_t *bp, size_t n) {
+  output_buffers_queue_t *obqueue = &((QMKUSBDriver *)ip)->obqueue;
+  chSysLock();
+  const bool full = obqIsFullI(obqueue);
+  chSysUnlock();
+  if (full || bqIsSuspendedX(obqueue)) {
+    /* Discard any writes while the queue is suspended or full, i.e. the hidraw
+       interface is not open. If we tried to send with an infinite timeout, we
+       would deadlock the keyboard otherwise. */
+    return -1;
+  }
+  return obqWriteTimeout(obqueue, bp, n, TIME_INFINITE);
+}
 
 static size_t _read(void *ip, uint8_t *bp, size_t n) { return ibqReadTimeout(&((QMKUSBDriver *)ip)->ibqueue, bp, n, TIME_INFINITE); }