Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
6 changes: 5 additions & 1 deletion app/src/main/java/helium314/keyboard/latin/LatinIME.kt
Original file line number Diff line number Diff line change
Expand Up @@ -1652,7 +1652,11 @@ class LatinIME : InputMethodService(),
VoiceConstants.VOICE_PROVIDER_OFFLINE, VoiceConstants.VOICE_PROVIDER_ONLINE -> {
if (ContextCompat.checkSelfPermission(this, Manifest.permission.RECORD_AUDIO) == PackageManager.PERMISSION_GRANTED) {
voiceInputManager?.let { vim ->
if (vim.isRecording()) vim.stopVoice() else vim.startVoice()
when (VoiceInputManager.voiceKeyActionFor(vim.getState(), vim.isRecording())) {
VoiceInputManager.Companion.VoiceKeyAction.STOP -> vim.stopVoice()
VoiceInputManager.Companion.VoiceKeyAction.START -> vim.startVoice()
VoiceInputManager.Companion.VoiceKeyAction.IGNORE -> Unit
}
}
} else {
Toast.makeText(this, "Microphone permission required for voice input. Enable in Settings -> Voice Input", Toast.LENGTH_LONG).show()
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -570,6 +570,8 @@ class VoiceInputManager(
val piece = if (piecewise) ByteArrayOutputStream() else null
var pieceHasSpeech = false
var pieceLastSpeech = 0L
var pieceSpeechMs = 0
var pieceLastSpeechEnd = 0 // bytes of the piece up to the end of the last speech frame

try {
if (!isOnline && writePfd != null) {
Expand Down Expand Up @@ -631,16 +633,20 @@ class VoiceInputManager(
if (isSpeech) {
pieceHasSpeech = true
pieceLastSpeech = now
pieceSpeechMs += read / BYTES_PER_MS
pieceLastSpeechEnd = piece.size()
}
val pieceMs = piece.size() / BYTES_PER_MS
val atPause = pieceHasSpeech && now - pieceLastSpeech >= PIECE_PAUSE_MS
// a piece needs some seconds of speech: Whisper invents text for very short ones
if ((atPause && pieceMs >= PIECE_MIN_MS) || pieceMs >= PIECE_MAX_MS) {
val bytes = piece.toByteArray()
val bytes = trimTrailingSilence(piece.toByteArray(), pieceLastSpeechEnd)
piece.reset()
val hadSpeech = pieceHasSpeech
val enoughSpeech = pieceSpeechMs >= PIECE_MIN_SPEECH_MS
pieceHasSpeech = false
if (hadSpeech) mainHandler.post { onPieceRecorded(bytes, generation) }
pieceSpeechMs = 0
pieceLastSpeechEnd = 0
if (enoughSpeech) mainHandler.post { onPieceRecorded(bytes, generation) }
}
}

Expand Down Expand Up @@ -684,7 +690,8 @@ class VoiceInputManager(
} finally {
currentAudioLevel = 0f
if (piece != null) {
val rest = if (pieceHasSpeech) piece.toByteArray() else null
// without the trailing silence and without a few noisy frames: Whisper invents text for both
val rest = if (pieceSpeechMs >= PIECE_MIN_SPEECH_MS) trimTrailingSilence(piece.toByteArray(), pieceLastSpeechEnd) else null
mainHandler.post { onPiecewiseRecordingEnded(rest, generation) }
}
try { outputStream?.close() } catch (_: Exception) {}
Expand Down Expand Up @@ -1368,6 +1375,31 @@ class VoiceInputManager(
/** Whisper works on 30 s windows, cut before even without a pause */
private const val PIECE_MAX_MS = 25000
private const val PIECE_CONTEXT_CHARS = 200
/** a piece with less speech than this is noise, not worth transcribing */
private const val PIECE_MIN_SPEECH_MS = 300
/** audio kept after the last speech of a piece */
private const val PIECE_TRAIL_PAD_MS = 500
/** never trim a piece below this, Whisper does better with a little context */
private const val PIECE_MIN_KEEP_MS = 1000

/** Cuts the silence after the last speech off a piece, [lastSpeechEnd] is a byte offset into [audio]. */
internal fun trimTrailingSilence(audio: ByteArray, lastSpeechEnd: Int): ByteArray {
val keep = (lastSpeechEnd + PIECE_TRAIL_PAD_MS * BYTES_PER_MS).coerceAtLeast(PIECE_MIN_KEEP_MS * BYTES_PER_MS)
val aligned = keep and 1.inv() // 16 bit samples
return if (aligned >= audio.size) audio else audio.copyOf(aligned)
}

enum class VoiceKeyAction { START, STOP, IGNORE }

/**
* What a press of the voice key does. While the final text is being recognized it does nothing: starting
* a new session would cancel this one and drop that text (the cancel button on the strip does it on purpose).
*/
internal fun voiceKeyActionFor(state: VoiceState, isRecording: Boolean): VoiceKeyAction = when {
isRecording -> VoiceKeyAction.STOP
state == VoiceState.PROCESSING_FINAL -> VoiceKeyAction.IGNORE
else -> VoiceKeyAction.START
}

fun isBlockedEditor(info: EditorInfo?): Boolean {
if (info == null) return false
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,37 @@
// SPDX-License-Identifier: GPL-3.0-only
package helium314.keyboard.latin.voice

import helium314.keyboard.latin.voice.VoiceInputManager.Companion.VoiceKeyAction
import helium314.keyboard.latin.voice.VoiceInputManager.VoiceState
import kotlin.test.Test
import kotlin.test.assertEquals
import kotlin.test.assertSame

class VoicePiecewiseTest {
private val bytesPerMs = 32 // 16 kHz, 16 bit

@Test fun trailingSilenceIsCutButAPaddingStays() {
val audio = ByteArray(8000 * bytesPerMs) // 8 s
val trimmed = VoiceInputManager.trimTrailingSilence(audio, 2000 * bytesPerMs) // speech ends at 2 s
assertEquals(2500 * bytesPerMs, trimmed.size) // 0.5 s padding
}

@Test fun shortAudioAndShortSpeechKeepAtLeastASecond() {
val audio = ByteArray(5000 * bytesPerMs)
assertEquals(1000 * bytesPerMs, VoiceInputManager.trimTrailingSilence(audio, 100 * bytesPerMs).size)
val short = ByteArray(600 * bytesPerMs)
assertSame(short, VoiceInputManager.trimTrailingSilence(short, 100 * bytesPerMs)) // nothing to cut
}

@Test fun trimmedAudioKeepsWholeSamples() {
val audio = ByteArray(100_000)
assertEquals(0, VoiceInputManager.trimTrailingSilence(audio, 40_001).size % 2)
}

@Test fun voiceKeyIsIgnoredWhileProcessing() {
assertEquals(VoiceKeyAction.STOP, VoiceInputManager.voiceKeyActionFor(VoiceState.RECORDING, true))
assertEquals(VoiceKeyAction.IGNORE, VoiceInputManager.voiceKeyActionFor(VoiceState.PROCESSING_FINAL, false))
assertEquals(VoiceKeyAction.START, VoiceInputManager.voiceKeyActionFor(VoiceState.IDLE, false))
assertEquals(VoiceKeyAction.START, VoiceInputManager.voiceKeyActionFor(VoiceState.ERROR, false))
}
}
Loading