From c64ba938ef7d05b1efd79ed1cd738e8a59c54472 Mon Sep 17 00:00:00 2001 From: LeanBitLab Date: Sun, 15 Nov 2026 13:33:08 +0530 Subject: [PATCH] fix(voice): ignore the voice key while processing and trim silence from dictated pieces Pressing the voice key during Processing started a new session, which cancelled the pending recognition and dropped its text. Ignore it in that state; the cancel button on the strip still does it on purpose. In transcribe while speaking, the last piece included the whole silence timeout and a single noisy frame counted as speech, so Whisper invented trailing words. Cut the silence after the last speech (keeping 0.5 s of padding, at least 1 s) and drop pieces with under 300 ms of speech. --- .../java/helium314/keyboard/latin/LatinIME.kt | 6 ++- .../keyboard/latin/voice/VoiceInputManager.kt | 40 +++++++++++++++++-- .../latin/voice/VoicePiecewiseTest.kt | 37 +++++++++++++++++ 3 files changed, 78 insertions(+), 5 deletions(-) create mode 100644 app/src/test/java/helium314/keyboard/latin/voice/VoicePiecewiseTest.kt diff --git a/app/src/main/java/helium314/keyboard/latin/LatinIME.kt b/app/src/main/java/helium314/keyboard/latin/LatinIME.kt index 36dc0dd4e..a935dc69b 100644 --- a/app/src/main/java/helium314/keyboard/latin/LatinIME.kt +++ b/app/src/main/java/helium314/keyboard/latin/LatinIME.kt @@ -1652,7 +1652,11 @@ class LatinIME : InputMethodService(), VoiceConstants.VOICE_PROVIDER_OFFLINE, VoiceConstants.VOICE_PROVIDER_ONLINE -> { if (ContextCompat.checkSelfPermission(this, Manifest.permission.RECORD_AUDIO) == PackageManager.PERMISSION_GRANTED) { voiceInputManager?.let { vim -> - if (vim.isRecording()) vim.stopVoice() else vim.startVoice() + when (VoiceInputManager.voiceKeyActionFor(vim.getState(), vim.isRecording())) { + VoiceInputManager.Companion.VoiceKeyAction.STOP -> vim.stopVoice() + VoiceInputManager.Companion.VoiceKeyAction.START -> vim.startVoice() + VoiceInputManager.Companion.VoiceKeyAction.IGNORE -> Unit + } } } else { Toast.makeText(this, "Microphone permission required for voice input. Enable in Settings -> Voice Input", Toast.LENGTH_LONG).show() diff --git a/app/src/main/java/helium314/keyboard/latin/voice/VoiceInputManager.kt b/app/src/main/java/helium314/keyboard/latin/voice/VoiceInputManager.kt index e439e2bb4..328838c8a 100644 --- a/app/src/main/java/helium314/keyboard/latin/voice/VoiceInputManager.kt +++ b/app/src/main/java/helium314/keyboard/latin/voice/VoiceInputManager.kt @@ -570,6 +570,8 @@ class VoiceInputManager( val piece = if (piecewise) ByteArrayOutputStream() else null var pieceHasSpeech = false var pieceLastSpeech = 0L + var pieceSpeechMs = 0 + var pieceLastSpeechEnd = 0 // bytes of the piece up to the end of the last speech frame try { if (!isOnline && writePfd != null) { @@ -631,16 +633,20 @@ class VoiceInputManager( if (isSpeech) { pieceHasSpeech = true pieceLastSpeech = now + pieceSpeechMs += read / BYTES_PER_MS + pieceLastSpeechEnd = piece.size() } val pieceMs = piece.size() / BYTES_PER_MS val atPause = pieceHasSpeech && now - pieceLastSpeech >= PIECE_PAUSE_MS // a piece needs some seconds of speech: Whisper invents text for very short ones if ((atPause && pieceMs >= PIECE_MIN_MS) || pieceMs >= PIECE_MAX_MS) { - val bytes = piece.toByteArray() + val bytes = trimTrailingSilence(piece.toByteArray(), pieceLastSpeechEnd) piece.reset() - val hadSpeech = pieceHasSpeech + val enoughSpeech = pieceSpeechMs >= PIECE_MIN_SPEECH_MS pieceHasSpeech = false - if (hadSpeech) mainHandler.post { onPieceRecorded(bytes, generation) } + pieceSpeechMs = 0 + pieceLastSpeechEnd = 0 + if (enoughSpeech) mainHandler.post { onPieceRecorded(bytes, generation) } } } @@ -684,7 +690,8 @@ class VoiceInputManager( } finally { currentAudioLevel = 0f if (piece != null) { - val rest = if (pieceHasSpeech) piece.toByteArray() else null + // without the trailing silence and without a few noisy frames: Whisper invents text for both + val rest = if (pieceSpeechMs >= PIECE_MIN_SPEECH_MS) trimTrailingSilence(piece.toByteArray(), pieceLastSpeechEnd) else null mainHandler.post { onPiecewiseRecordingEnded(rest, generation) } } try { outputStream?.close() } catch (_: Exception) {} @@ -1368,6 +1375,31 @@ class VoiceInputManager( /** Whisper works on 30 s windows, cut before even without a pause */ private const val PIECE_MAX_MS = 25000 private const val PIECE_CONTEXT_CHARS = 200 + /** a piece with less speech than this is noise, not worth transcribing */ + private const val PIECE_MIN_SPEECH_MS = 300 + /** audio kept after the last speech of a piece */ + private const val PIECE_TRAIL_PAD_MS = 500 + /** never trim a piece below this, Whisper does better with a little context */ + private const val PIECE_MIN_KEEP_MS = 1000 + + /** Cuts the silence after the last speech off a piece, [lastSpeechEnd] is a byte offset into [audio]. */ + internal fun trimTrailingSilence(audio: ByteArray, lastSpeechEnd: Int): ByteArray { + val keep = (lastSpeechEnd + PIECE_TRAIL_PAD_MS * BYTES_PER_MS).coerceAtLeast(PIECE_MIN_KEEP_MS * BYTES_PER_MS) + val aligned = keep and 1.inv() // 16 bit samples + return if (aligned >= audio.size) audio else audio.copyOf(aligned) + } + + enum class VoiceKeyAction { START, STOP, IGNORE } + + /** + * What a press of the voice key does. While the final text is being recognized it does nothing: starting + * a new session would cancel this one and drop that text (the cancel button on the strip does it on purpose). + */ + internal fun voiceKeyActionFor(state: VoiceState, isRecording: Boolean): VoiceKeyAction = when { + isRecording -> VoiceKeyAction.STOP + state == VoiceState.PROCESSING_FINAL -> VoiceKeyAction.IGNORE + else -> VoiceKeyAction.START + } fun isBlockedEditor(info: EditorInfo?): Boolean { if (info == null) return false diff --git a/app/src/test/java/helium314/keyboard/latin/voice/VoicePiecewiseTest.kt b/app/src/test/java/helium314/keyboard/latin/voice/VoicePiecewiseTest.kt new file mode 100644 index 000000000..ace3d6ec4 --- /dev/null +++ b/app/src/test/java/helium314/keyboard/latin/voice/VoicePiecewiseTest.kt @@ -0,0 +1,37 @@ +// SPDX-License-Identifier: GPL-3.0-only +package helium314.keyboard.latin.voice + +import helium314.keyboard.latin.voice.VoiceInputManager.Companion.VoiceKeyAction +import helium314.keyboard.latin.voice.VoiceInputManager.VoiceState +import kotlin.test.Test +import kotlin.test.assertEquals +import kotlin.test.assertSame + +class VoicePiecewiseTest { + private val bytesPerMs = 32 // 16 kHz, 16 bit + + @Test fun trailingSilenceIsCutButAPaddingStays() { + val audio = ByteArray(8000 * bytesPerMs) // 8 s + val trimmed = VoiceInputManager.trimTrailingSilence(audio, 2000 * bytesPerMs) // speech ends at 2 s + assertEquals(2500 * bytesPerMs, trimmed.size) // 0.5 s padding + } + + @Test fun shortAudioAndShortSpeechKeepAtLeastASecond() { + val audio = ByteArray(5000 * bytesPerMs) + assertEquals(1000 * bytesPerMs, VoiceInputManager.trimTrailingSilence(audio, 100 * bytesPerMs).size) + val short = ByteArray(600 * bytesPerMs) + assertSame(short, VoiceInputManager.trimTrailingSilence(short, 100 * bytesPerMs)) // nothing to cut + } + + @Test fun trimmedAudioKeepsWholeSamples() { + val audio = ByteArray(100_000) + assertEquals(0, VoiceInputManager.trimTrailingSilence(audio, 40_001).size % 2) + } + + @Test fun voiceKeyIsIgnoredWhileProcessing() { + assertEquals(VoiceKeyAction.STOP, VoiceInputManager.voiceKeyActionFor(VoiceState.RECORDING, true)) + assertEquals(VoiceKeyAction.IGNORE, VoiceInputManager.voiceKeyActionFor(VoiceState.PROCESSING_FINAL, false)) + assertEquals(VoiceKeyAction.START, VoiceInputManager.voiceKeyActionFor(VoiceState.IDLE, false)) + assertEquals(VoiceKeyAction.START, VoiceInputManager.voiceKeyActionFor(VoiceState.ERROR, false)) + } +}