Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
@@ -0,0 +1,130 @@
// SPDX-License-Identifier: GPL-3.0-only
package helium314.keyboard.latin.utils

import android.content.ContentValues
import android.content.Context
import android.provider.UserDictionary
import java.io.BufferedReader
import java.io.Writer
import java.util.Locale

/**
* Reads and writes the personal dictionary, i.e. the words in Android's user dictionary, as a text file
* with one word per line (word, shortcut and locale separated by tabs, like the Gboard dictionary export).
* Used by the export and import of the personal dictionary screen and by the backup.
*/
object PersonalDictionaryTransfer {
/** Entry name of the personal dictionary in a backup */
const val BACKUP_ENTRY_NAME = "personal_dictionary.txt"

private const val HEADER_PREFIX = "# Gboard Dictionary format:"

/** [locale] is the locale as stored in the user dictionary (e.g. "en_US"), null for all languages */
data class Entry(val word: String, val shortcut: String?, val locale: String?)

/** Writes the header and the [entries], returns the number of entries */
fun write(entries: Sequence<Entry>, writer: Writer): Int {
writer.write("$HEADER_PREFIX word\tshortcut\tlocale\n")
var count = 0
for (entry in entries) {
writer.write("${entry.word}\t${entry.shortcut.orEmpty()}\t${entry.locale.orEmpty()}\n")
count++
}
return count
}

fun parse(reader: BufferedReader): List<Entry> {
val result = mutableListOf<Entry>()
var wordIndex = 0
var shortcutIndex = 1
var localeIndex = 2
while (true) {
val line = reader.readLine() ?: break
if (line.startsWith("#")) {
if (line.startsWith(HEADER_PREFIX)) {
val columnsStr = line.substring(HEADER_PREFIX.length)
val columns = (if (columnsStr.contains("\t")) columnsStr.split("\t") else columnsStr.split(Regex("\\s+")))
.map { it.trim() }
val word = columns.indexOf("word")
if (word != -1) {
wordIndex = word
shortcutIndex = columns.indexOf("shortcut")
localeIndex = columns.indexOf("language_tag").let { if (it == -1) columns.indexOf("locale") else it }
}
}
continue
}
val parts = line.split("\t")
val word = parts.getOrNull(wordIndex).orEmpty()
if (word.isBlank()) continue
val shortcut = if (shortcutIndex != -1) parts.getOrNull(shortcutIndex)?.ifBlank { null } else null
val locale = if (localeIndex != -1) normalizeLocale(parts.getOrNull(localeIndex)) else null
result.add(Entry(word, shortcut, locale))
}
return result
}

/**
* Locale as stored in the user dictionary: "en_US" is kept, a language tag like "en-US" is converted,
* and blank or "all" means all languages.
*/
fun normalizeLocale(raw: String?): String? {
val value = raw?.trim().orEmpty()
if (value.isEmpty() || value == "all") return null
if (value.contains('_')) return value // already in the form of Locale.toString
return try { Locale.forLanguageTag(value).toString().ifEmpty { null } } catch (_: Exception) { null }
}

/** All words of the user dictionary; the cursor can be null if there is no user dictionary */
fun readAll(context: Context): List<Entry> {
val cursor = context.contentResolver.query(
UserDictionary.Words.CONTENT_URI,
arrayOf(UserDictionary.Words.WORD, UserDictionary.Words.SHORTCUT, UserDictionary.Words.LOCALE),
null, null, UserDictionary.Words.DEFAULT_SORT_ORDER
) ?: return emptyList()
return cursor.use {
val wordIndex = it.getColumnIndexOrThrow(UserDictionary.Words.WORD)
val shortcutIndex = it.getColumnIndexOrThrow(UserDictionary.Words.SHORTCUT)
val localeIndex = it.getColumnIndexOrThrow(UserDictionary.Words.LOCALE)
buildList {
while (it.moveToNext()) {
val word = it.getString(wordIndex) ?: continue
add(Entry(word, it.getString(shortcutIndex)?.ifEmpty { null }, it.getString(localeIndex)?.ifEmpty { null }))
}
}
}
}

/** Adds [entries] to the user dictionary, without the ones that are in there already if [skipExisting]; returns the number added */
fun insert(context: Context, entries: List<Entry>, skipExisting: Boolean): Int {
val toAdd = if (skipExisting) {
val existing = readAll(context).toSet()
entries.filter { it !in existing }.distinct()
} else entries
if (toAdd.isEmpty()) return 0
val values = toAdd.map {
ContentValues(5).apply {
put(UserDictionary.Words.WORD, it.word)
put(UserDictionary.Words.FREQUENCY, 250)
put(UserDictionary.Words.LOCALE, it.locale)
put(UserDictionary.Words.APP_ID, 0)
put(UserDictionary.Words.SHORTCUT, it.shortcut)
}
}
return try {
context.contentResolver.bulkInsert(UserDictionary.Words.CONTENT_URI, values.toTypedArray())
} catch (e: Exception) {
Log.e("PersonalDictionary", "Bulk insert failed, falling back to one-by-one insert", e)
var added = 0
for (v in values) {
try {
context.contentResolver.insert(UserDictionary.Words.CONTENT_URI, v)
added++
} catch (ex: Exception) {
Log.w("PersonalDictionary", "Failed to add a word", ex)
}
}
added
}
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -45,6 +45,7 @@ import helium314.keyboard.latin.utils.Log
import helium314.keyboard.latin.utils.ProofreadService
import helium314.keyboard.latin.utils.SubtypeSettings
import helium314.keyboard.latin.utils.getActivity
import helium314.keyboard.latin.utils.PersonalDictionaryTransfer
import helium314.keyboard.latin.utils.prefs
import helium314.keyboard.latin.utils.protectedPrefs
import helium314.keyboard.settings.Setting
Expand Down Expand Up @@ -76,6 +77,8 @@ fun BackupRestorePreference(setting: Setting) {
BackupCategory.LAYOUTS,
BackupCategory.THEME_APPEARANCE,
BackupCategory.DICTIONARIES,
BackupCategory.PERSONAL_DICTIONARY,
BackupCategory.TEXT_EXPANDER,
BackupCategory.TYPING_HISTORY,
BackupCategory.CLIPBOARD,
BackupCategory.GENERAL_SETTINGS
Expand All @@ -99,6 +102,8 @@ fun BackupRestorePreference(setting: Setting) {
BackupCategory.LAYOUTS to R.string.backup_category_layouts,
BackupCategory.THEME_APPEARANCE to R.string.backup_category_theme,
BackupCategory.DICTIONARIES to R.string.backup_category_dictionaries,
BackupCategory.PERSONAL_DICTIONARY to R.string.backup_category_personal_dictionary,
BackupCategory.TEXT_EXPANDER to R.string.backup_category_text_expander,
BackupCategory.TYPING_HISTORY to R.string.backup_category_typing_history,
BackupCategory.CLIPBOARD to R.string.backup_category_clipboard,
BackupCategory.API_KEYS to R.string.backup_category_api_keys,
Expand Down Expand Up @@ -254,6 +259,17 @@ private fun backupLauncher(
}
}
}
if (selectedCategories.contains(BackupCategory.PERSONAL_DICTIONARY)) {
// the personal dictionary is in the system user dictionary, not in the files of the app
val words = PersonalDictionaryTransfer.readAll(ctx)
if (words.isNotEmpty()) {
zipStream.putNextEntry(ZipEntry(PersonalDictionaryTransfer.BACKUP_ENTRY_NAME))
val writer = zipStream.writer(Charsets.UTF_8) // not closed, this would close the zip
PersonalDictionaryTransfer.write(words.asSequence(), writer)
writer.flush()
zipStream.closeEntry()
}
}
if (selectedCategories.contains(BackupCategory.API_KEYS)) {
val secureSettings = ProofreadService(ctx).getSecureSettings()
if (secureSettings.isNotEmpty()) {
Expand Down Expand Up @@ -325,6 +341,8 @@ private fun restoreLauncher(
File(filesDir, "dicts").deleteRecursively()
File(filesDir, "blacklists").deleteRecursively()
File(deviceProtectedFilesDir, "blacklists").deleteRecursively()
}
if (selectedCategories.contains(BackupCategory.TEXT_EXPANDER)) {
File(filesDir, "text_expander_images").deleteRecursively()
File(deviceProtectedFilesDir, "text_expander_images").deleteRecursively()
helium314.keyboard.latin.utils.TextExpanderImageUtils.clearCache()
Expand Down Expand Up @@ -382,6 +400,12 @@ private fun restoreLauncher(
val restoredMap = readJsonLinesToMap(apiKeyLines)
ProofreadService(ctx).restoreSecureSettings(restoredMap)
}
} else if (entry.name == PersonalDictionaryTransfer.BACKUP_ENTRY_NAME) {
if (selectedCategories.contains(BackupCategory.PERSONAL_DICTIONARY)) {
// adds the words that are not in the user dictionary yet, existing words are kept
val words = PersonalDictionaryTransfer.parse(zip.reader(Charsets.UTF_8).buffered())
PersonalDictionaryTransfer.insert(ctx, words, skipExisting = true)
}
} else if (entry.name == PREFS_FILE_NAME) {
preferenceLines = String(zip.readBytes()).split("\n")
} else if (entry.name == PROTECTED_PREFS_FILE_NAME) {
Expand Down Expand Up @@ -588,13 +612,15 @@ enum class BackupCategory {
LAYOUTS,
THEME_APPEARANCE,
DICTIONARIES,
PERSONAL_DICTIONARY,
TEXT_EXPANDER,
TYPING_HISTORY,
CLIPBOARD,
API_KEYS,
GENERAL_SETTINGS
}

private fun getCategoryForPrefKey(key: String): BackupCategory {
internal fun getCategoryForPrefKey(key: String): BackupCategory {
if (key.startsWith("layout_")) return BackupCategory.LAYOUTS

val themeKeys = setOf(
Expand Down Expand Up @@ -623,7 +649,8 @@ private fun getCategoryForPrefKey(key: String): BackupCategory {
"add_to_personal_dictionary", "add_to_personal_dict_threshold",
"auto_correct_aggressiveness"
)
if (dictKeys.contains(key) || key.startsWith("pref_text_expander_")) return BackupCategory.DICTIONARIES
if (key.startsWith("pref_text_expander_")) return BackupCategory.TEXT_EXPANDER
if (dictKeys.contains(key)) return BackupCategory.DICTIONARIES

val clipboardKeys = setOf(
"enable_clipboard_history", "suggest_screenshots", "compress_screenshots",
Expand All @@ -644,10 +671,12 @@ internal fun getCategoryForFilePath(path: String): BackupCategory? {
}
if (path.startsWith("dicts${File.separator}") || path.startsWith("dicts/")
|| path.startsWith("blacklists${File.separator}") || path.startsWith("blacklists/")
|| path.startsWith("text_expander_images${File.separator}") || path.startsWith("text_expander_images/")
) {
return BackupCategory.DICTIONARIES
}
if (path.startsWith("text_expander_images${File.separator}") || path.startsWith("text_expander_images/")) {
return BackupCategory.TEXT_EXPANDER
}
if (path.startsWith("UserHistoryDictionary")) {
return BackupCategory.TYPING_HISTORY
}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -45,6 +45,7 @@ import androidx.compose.runtime.setValue
import androidx.compose.ui.res.painterResource
import helium314.keyboard.keyboard.KeyboardSwitcher
import helium314.keyboard.latin.utils.Log
import helium314.keyboard.latin.utils.PersonalDictionaryTransfer
import kotlinx.coroutines.CoroutineScope
import kotlinx.coroutines.Dispatchers
import kotlinx.coroutines.launch
Expand Down Expand Up @@ -202,75 +203,8 @@ private suspend fun importGboardDictionary(context: android.content.Context, uri
return addedCount
}

private fun parseAndInsert(context: android.content.Context, reader: BufferedReader): Int {
val valuesList = mutableListOf<ContentValues>()
var line: String?
var wordIndex = 0
var shortcutIndex = 1
var localeIndex = 2
while (reader.readLine().also { line = it } != null) {
val currentLine = line ?: continue
if (currentLine.startsWith("#")) {
val formatPrefix = "# Gboard Dictionary format:"
if (currentLine.startsWith(formatPrefix)) {
val columnsStr = currentLine.substring(formatPrefix.length)
val columns = if (columnsStr.contains("\t")) {
columnsStr.split("\t")
} else {
columnsStr.split(Regex("\\s+"))
}.map { it.trim() }
val wIdx = columns.indexOf("word")
val sIdx = columns.indexOf("shortcut")
val lIdx = columns.indexOf("language_tag").let { if (it == -1) columns.indexOf("locale") else it }
if (wIdx != -1) {
wordIndex = wIdx
shortcutIndex = sIdx
localeIndex = lIdx
}
}
continue
}
val parts = currentLine.split("\t")
if (parts.isNotEmpty()) {
val word = if (wordIndex in parts.indices) parts[wordIndex] else ""
if (word.isNotBlank()) {
val shortcut = if (shortcutIndex != -1 && shortcutIndex in parts.indices) parts[shortcutIndex].ifBlank { null } else null
val localeStr = if (localeIndex != -1 && localeIndex in parts.indices) parts[localeIndex].ifBlank { null } else null

val locale = if (localeStr != null && localeStr != "all") {
try { Locale.forLanguageTag(localeStr) } catch(_: Exception) { null }
} else null

val values = ContentValues(5).apply {
put(UserDictionary.Words.WORD, word)
put(UserDictionary.Words.FREQUENCY, 250)
put(UserDictionary.Words.LOCALE, locale?.toString())
put(UserDictionary.Words.APP_ID, 0)
put(UserDictionary.Words.SHORTCUT, shortcut)
}
valuesList.add(values)
}
}
}

if (valuesList.isEmpty()) return 0

return try {
context.contentResolver.bulkInsert(UserDictionary.Words.CONTENT_URI, valuesList.toTypedArray())
} catch (e: Exception) {
Log.e("ImportDict", "Bulk insert failed, falling back to one-by-one insert", e)
var successCount = 0
for (values in valuesList) {
try {
context.contentResolver.insert(UserDictionary.Words.CONTENT_URI, values)
successCount++
} catch (ex: Exception) {
Log.w("ImportDict", "Failed to add word ${values.getAsString(UserDictionary.Words.WORD)}", ex)
}
}
successCount
}
}
private fun parseAndInsert(context: android.content.Context, reader: BufferedReader): Int =
PersonalDictionaryTransfer.insert(context, PersonalDictionaryTransfer.parse(reader), skipExisting = false)

fun getSortedDictionaryLocales(): TreeSet<Locale> {
val sortedLocales = sortedSetOf<Locale>(compareBy { it.toLanguageTag().lowercase() })
Expand All @@ -293,27 +227,8 @@ fun getSortedDictionaryLocales(): TreeSet<Locale> {
}

private suspend fun exportAllDictionaries(context: android.content.Context, uri: android.net.Uri): Int = withContext(Dispatchers.IO) {
val cursor = context.contentResolver.query(
UserDictionary.Words.CONTENT_URI,
arrayOf(UserDictionary.Words.WORD, UserDictionary.Words.SHORTCUT, UserDictionary.Words.LOCALE),
null, null, UserDictionary.Words.DEFAULT_SORT_ORDER
) ?: return@withContext 0
var count = 0
context.contentResolver.openOutputStream(uri)?.bufferedWriter()?.use { writer ->
writer.write("# Gboard Dictionary format: word\tshortcut\tlocale\n")
val wordIndex = cursor.getColumnIndexOrThrow(UserDictionary.Words.WORD)
val shortcutIndex = cursor.getColumnIndexOrThrow(UserDictionary.Words.SHORTCUT)
val localeIndex = cursor.getColumnIndexOrThrow(UserDictionary.Words.LOCALE)
while (cursor.moveToNext()) {
val word = cursor.getString(wordIndex) ?: continue
val shortcut = cursor.getString(shortcutIndex).orEmpty()
val loc = cursor.getString(localeIndex).orEmpty()
writer.write("$word\t$shortcut\t$loc\n")
count++
}
}
cursor.close()
count
val entries = PersonalDictionaryTransfer.readAll(context)
context.contentResolver.openOutputStream(uri)?.bufferedWriter()?.use { PersonalDictionaryTransfer.write(entries.asSequence(), it) } ?: 0
}

@Preview
Expand Down
2 changes: 2 additions & 0 deletions app/src/main/res/values-ar/strings.xml
Original file line number Diff line number Diff line change
Expand Up @@ -1771,4 +1771,6 @@
<string name="custom_text_styles_title">أنماط النص المخصصة</string>
<string name="custom_text_styles_summary">نمط واحد لكل سطر. التنسيق: الاسم: بالنمط A-Z a-z 0-9 (52 أو 62 حرفًا).</string>
<string name="voice_transcribe_while_speaking_summary">انتظار أقصر للإملاءات الطويلة</string>
<string name="backup_category_personal_dictionary">القاموس الشخصي</string>
<string name="backup_category_text_expander">موسع النص</string>
</resources>
2 changes: 2 additions & 0 deletions app/src/main/res/values-be/strings.xml
Original file line number Diff line number Diff line change
Expand Up @@ -1748,4 +1748,6 @@
<string name="custom_text_styles_title">Карыстальніцкія стылі тэксту</string>
<string name="custom_text_styles_summary">Адзін стыль на радок. Фармат: Імя: у стылі A-Z a-z 0-9 (52 або 62 сімвалы).</string>
<string name="voice_transcribe_while_speaking_summary">Карацей чакаць доўгіх дыктовак</string>
<string name="backup_category_personal_dictionary">Асабісты слоўнік</string>
<string name="backup_category_text_expander">Расшыральнік тэксту</string>
</resources>
2 changes: 2 additions & 0 deletions app/src/main/res/values-bg/strings.xml
Original file line number Diff line number Diff line change
Expand Up @@ -1748,4 +1748,6 @@
<string name="custom_text_styles_title">Персонализирани текстови стилове</string>
<string name="custom_text_styles_summary">Един стил на ред. Формат: Име: стилизирано A-Z a-z 0-9 (52 или 62 знака).</string>
<string name="voice_transcribe_while_speaking_summary">По-кратко изчакване за дълги диктовки</string>
<string name="backup_category_personal_dictionary">Личен речник</string>
<string name="backup_category_text_expander">Текстов разширител</string>
</resources>
2 changes: 2 additions & 0 deletions app/src/main/res/values-cs/strings.xml
Original file line number Diff line number Diff line change
Expand Up @@ -1713,4 +1713,6 @@ Umožňuje lokální inferenci GGUF / llama.cpp na zařízení pro korektury a p
<string name="custom_text_styles_title">Vlastní styly textu</string>
<string name="custom_text_styles_summary">Jeden styl na řádek. Formát: Název: stylizovaný A-Z a-z 0-9 (52 nebo 62 znaků).</string>
<string name="voice_transcribe_while_speaking_summary">Kratší čekání na dlouhé diktáty</string>
<string name="backup_category_personal_dictionary">Osobní slovník</string>
<string name="backup_category_text_expander">Text Expander</string>
</resources>
Loading
Loading