Add the Urdu dictionaries, and suggest near words when nothing matches

Two Urdu sources join the Persian ones. Wiktionary's Urdu extract gives Urdu
headwords glossed in English, and Urdu Wiktionary itself gives definitions
written in Urdu — the only source here that does. The latter is thin, about
3,100 usable entries out of 31,000 pages since many are stubs, but for a word
it carries an Urdu reader is better served by it than by a translation into
English: عشق comes back as شدید جذبۂ محبت، گہری چاہت، محبت، پریم، پیار.

Every definition now names the dictionary and its language pair, and lays out
in the direction its own script reads, so an Urdu definition is right-aligned
beside a left-aligned English one.

When nothing matches, the sheet offers near words ranked by how many letters
they share with what was looked up, drawn from an index range scan on the
leading letters rather than a scan of the whole table. خودکامی, which has no
entry, offers خودکامه — the lemma it wants.

Measured honestly: the Urdu sources add little coverage over Persian — nine
words from the English-glossed extract, two from Urdu Wiktionary, against
1,285 from thirteen poems. They are here because an Urdu reader wants Urdu,
not because they widen the net.

The suggestion ranking is verified against the built database rather than only
on device: خودکامی → خودکامه, شیرازی → شیراز, مشکلها → مشکل. On an API 36
emulator all four sources answer عشق with their labels.

Known rough edge: affix stripping across four languages can mislead. ناولها
reaches ناول, the Urdu for "novel", which is not what Hafez meant. The matched
headword is always shown, so it is visible rather than silent.

Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>

app/src/main/java/com/ganjoor/android/data/Dictionary.kt

@@ -17,7 +17,7 @@ data class Definition(val word: String, val gloss: String, val source: String)
  */
 object Dictionary {
     /** Guards against a copy interrupted half-way leaving an unopenable file behind. */
-    private const val ASSET_BYTES = 22_298_624L
+    private const val ASSET_BYTES = 27742208L
 
     // Not a .gz: the build packager silently gunzips those and drops the extension, which left
     // the asset under a different name than the code was opening.
@@ -95,6 +95,47 @@ object Dictionary {
             }
         }
 
+    /**
+     * Headwords that look like [raw], for when nothing matched exactly — a misread letter, an
+     * unusual spelling, or a word the dictionary simply spells differently.
+     *
+     * Candidates come from an index range scan on the first letters, then are ranked by how
+     * many letters they share with the query. ponytail: shared letters rather than an edit
+     * distance, which would need the whole table scanned to be worth the extra precision.
+     */
+    suspend fun suggest(raw: String, limit: Int = 6): List<String> = withContext(Dispatchers.IO) {
+        val database = open() ?: return@withContext emptyList()
+        val word = normalise(raw).takeIf { it.length > 1 } ?: return@withContext emptyList()
+
+        // Widen the prefix until there is something to rank, but never scan the whole table.
+        val candidates = generateSequence(minOf(3, word.length - 1)) { (it - 1).takeIf { n -> n >= 1 } }
+            .map { prefixLength -> byPrefix(database, word.take(prefixLength)) }
+            .firstOrNull { it.size >= 3 }
+            ?: return@withContext emptyList()
+
+        candidates
+            .asSequence()
+            .filter { it.first != word }
+            .map { (normalised, display) -> display to letterOverlap(word, normalised) }
+            .filter { it.second > 0.45f }
+            .sortedByDescending { it.second }
+            .map { it.first }
+            .distinct()
+            .take(limit)
+            .toList()
+    }
+
+    /** Index range scan: everything whose normalised form starts with [prefix]. */
+    private fun byPrefix(database: SQLiteDatabase, prefix: String): List<Pair<String, String>> =
+        database.rawQuery(
+            "SELECT DISTINCT word, display FROM entry WHERE word >= ? AND word < ? LIMIT 400",
+            arrayOf(prefix, prefix + '\uFFFF'),
+        ).use { cursor ->
+            buildList {
+                while (cursor.moveToNext()) add(cursor.getString(0) to cursor.getString(1))
+            }
+        }
+
     private fun lemmas(database: SQLiteDatabase, form: String): List<String> =
         database.rawQuery(
             "SELECT lemma FROM form WHERE form = ? LIMIT 6",
@@ -152,6 +193,17 @@ internal fun normalise(text: String, keepZwnj: Boolean = false): String {
     return (if (keepZwnj) folded else folded.replace(ZWNJ.toString(), "")).trim()
 }
 
+/**
+ * How much of two words' letters coincide — the overlapping letters counted against the longer
+ * word, so مشکل and مشکلها score high while a word that merely starts the same does not.
+ */
+internal fun letterOverlap(a: String, b: String): Float {
+    if (a.isEmpty() || b.isEmpty()) return 0f
+    val remaining = b.toMutableList()
+    val shared = a.count { remaining.remove(it) }
+    return shared.toFloat() / maxOf(a.length, b.length)
+}
+
 /** The whole word surrounding [index], for turning a tap into something to look up. */
 internal fun wordAt(text: String, index: Int): String? {
     if (text.isEmpty()) return null

app/src/main/java/com/ganjoor/android/ui/AboutScreen.kt

@@ -64,11 +64,17 @@ private val CREDITS = listOf(
     ),
     Credit(
         "Wiktionary",
-        "Wiktionary contributors — the word definitions, and the inflected-form index that " +
-            "finds a conjugated verb's dictionary entry",
+        "Wiktionary contributors — Persian and Urdu definitions, and the inflected-form " +
+            "index that finds a conjugated verb's dictionary entry",
         "CC BY-SA 3.0. The bundled dictionary is therefore also CC BY-SA 3.0.",
         url = "https://en.wiktionary.org",
     ),
+    Credit(
+        "Urdu Wiktionary",
+        "Urdu Wiktionary contributors — the definitions written in Urdu rather than English",
+        "CC BY-SA 3.0. The bundled dictionary is therefore also CC BY-SA 3.0.",
+        url = "https://ur.wiktionary.org",
+    ),
     Credit(
         "Daneshjoo Dictionary",
         "Layered under Wiktionary for the words it doesn't carry",

app/src/main/java/com/ganjoor/android/ui/WordSheet.kt

@@ -7,17 +7,21 @@ import androidx.compose.foundation.layout.navigationBarsPadding
 import androidx.compose.foundation.layout.padding
 import androidx.compose.foundation.rememberScrollState
 import androidx.compose.foundation.verticalScroll
+import androidx.compose.foundation.layout.FlowRow
 import androidx.compose.material3.CircularProgressIndicator
 import androidx.compose.material3.ExperimentalMaterial3Api
 import androidx.compose.material3.HorizontalDivider
 import androidx.compose.material3.MaterialTheme
 import androidx.compose.material3.ModalBottomSheet
+import androidx.compose.material3.SuggestionChip
 import androidx.compose.material3.Text
 import androidx.compose.runtime.Composable
 import androidx.compose.runtime.CompositionLocalProvider
 import androidx.compose.runtime.getValue
 import androidx.compose.runtime.mutableStateOf
 import androidx.compose.runtime.produceState
+import androidx.compose.runtime.remember
+import androidx.compose.runtime.setValue
 import androidx.compose.ui.Modifier
 import androidx.compose.ui.platform.LocalLayoutDirection
 import androidx.compose.ui.res.stringResource
@@ -34,12 +38,39 @@ private fun LeftToRight(content: @Composable () -> Unit) {
     CompositionLocalProvider(LocalLayoutDirection provides LayoutDirection.Ltr, content = content)
 }
 
+/** Lays a definition out the way its own script reads. */
+@Composable
+private fun InDirectionOf(text: String, content: @Composable () -> Unit) {
+    val arabicScript = text.count { it in '\u0600'..'\u06FF' }
+    val latin = text.count { it in 'A'..'Z' || it in 'a'..'z' }
+    CompositionLocalProvider(
+        LocalLayoutDirection provides
+            if (arabicScript > latin) LayoutDirection.Rtl else LayoutDirection.Ltr,
+        content = content,
+    )
+}
+
+/** Which dictionary answered, and in which language pair. */
+private fun sourceLabel(source: String) = when (source) {
+    "wiktionary-fa" -> R.string.source_wiktionary
+    "wiktionary-ur" -> R.string.source_wiktionary_ur
+    "urwiktionary" -> R.string.source_urwiktionary
+    else -> R.string.source_daneshjoo
+}
+
 /** What the dictionary knows about a tapped word. */
 @OptIn(ExperimentalMaterial3Api::class)
 @Composable
 fun WordSheet(word: String, onDismiss: () -> Unit) {
     val prefs = LocalSettings.current.value
-    val definitions by produceState<List<Definition>?>(null, word) { value = Dictionary.lookup(word) }
+    // A suggestion replaces what is being looked up, so the sheet can be followed like a trail.
+    var current by remember(word) { mutableStateOf(word) }
+    val definitions by produceState<List<Definition>?>(null, current) {
+        value = Dictionary.lookup(current)
+    }
+    val suggestions by produceState(emptyList<String>(), current, definitions) {
+        value = if (definitions?.isEmpty() == true) Dictionary.suggest(current) else emptyList()
+    }
 
     ModalBottomSheet(onDismissRequest = onDismiss) {
         Column(
@@ -52,7 +83,7 @@ fun WordSheet(word: String, onDismiss: () -> Unit) {
         ) {
             // The headword in the reading font, at reading size: it is a line of poetry, after all.
             Text(
-                text = word,
+                text = current,
                 style = readingStyle(prefs.font, prefs.fontSize, prefs.fontWeight.weight),
                 modifier = Modifier.fillMaxWidth(),
             )
@@ -61,38 +92,52 @@ fun WordSheet(word: String, onDismiss: () -> Unit) {
             when {
                 definitions == null -> CircularProgressIndicator(Modifier.padding(vertical = 16.dp))
 
-                definitions!!.isEmpty() -> LeftToRight {
-                    Text(
-                        text = stringResource(R.string.no_definition),
-                        style = MaterialTheme.typography.bodyMedium,
-                        color = MaterialTheme.colorScheme.onSurfaceVariant,
-                        modifier = Modifier.fillMaxWidth(),
-                    )
+                definitions!!.isEmpty() -> Column(
+                    verticalArrangement = Arrangement.spacedBy(8.dp),
+                ) {
+                    LeftToRight {
+                        Text(
+                            text = stringResource(R.string.no_definition),
+                            style = MaterialTheme.typography.bodyMedium,
+                            color = MaterialTheme.colorScheme.onSurfaceVariant,
+                            modifier = Modifier.fillMaxWidth(),
+                        )
+                    }
+                    if (suggestions.isNotEmpty()) {
+                        Text(
+                            text = stringResource(R.string.did_you_mean),
+                            style = MaterialTheme.typography.titleSmall,
+                            color = MaterialTheme.colorScheme.primary,
+                        )
+                        FlowRow(horizontalArrangement = Arrangement.spacedBy(8.dp)) {
+                            suggestions.forEach { suggestion ->
+                                SuggestionChip(
+                                    onClick = { current = suggestion },
+                                    label = { Text(suggestion) },
+                                )
+                            }
+                        }
+                    }
                 }
 
                 else -> definitions!!.forEach { definition ->
                     Column(Modifier.fillMaxWidth()) {
                         // The headword actually matched, which may be the lemma rather than the
                         // word as it appears in the line. Persian, so it stays right-to-left.
-                        if (definition.word != word) {
+                        if (definition.word != current) {
                             Text(
                                 text = definition.word,
                                 style = MaterialTheme.typography.titleSmall,
                                 color = MaterialTheme.colorScheme.primary,
                             )
                         }
-                        // The definitions are English; right-aligning them reads badly.
-                        LeftToRight {
+                        // Most definitions are English and right-aligning them reads badly;
+                        // the Urdu ones are right-to-left like the rest of the app.
+                        InDirectionOf(definition.gloss) {
                             Column(Modifier.fillMaxWidth()) {
                                 Text(definition.gloss, style = MaterialTheme.typography.bodyMedium)
                                 Text(
-                                    text = stringResource(
-                                        if (definition.source == "wiktionary") {
-                                            R.string.source_wiktionary
-                                        } else {
-                                            R.string.source_daneshjoo
-                                        }
-                                    ),
+                                    text = stringResource(sourceLabel(definition.source)),
                                     style = MaterialTheme.typography.labelSmall,
                                     color = MaterialTheme.colorScheme.onSurfaceVariant,
                                 )

app/src/main/res/values-fa/strings.xml

@@ -65,4 +65,7 @@
     <string name="source_wiktionary">ویکی‌واژه — فارسی به انگلیسی (CC BY-SA 3.0)</string>
     <string name="source_daneshjoo">فرهنگ دانشجو — فارسی به انگلیسی</string>
     <string name="dictionary">لغت‌نامه</string>
+    <string name="source_wiktionary_ur">ویکی‌واژه — اردو به انگلیسی (CC BY-SA 3.0)</string>
+    <string name="source_urwiktionary">ویکی‌واژهٔ اردو — اردو به اردو (CC BY-SA 3.0)</string>
+    <string name="did_you_mean">واژه‌های نزدیک</string>
 </resources>

app/src/main/res/values-ur/strings.xml

@@ -65,4 +65,7 @@
     <string name="source_wiktionary">ویکی لغت — فارسی سے انگریزی (CC BY-SA 3.0)</string>
     <string name="source_daneshjoo">دانشجو لغت — فارسی سے انگریزی</string>
     <string name="dictionary">لغت نامہ</string>
+    <string name="source_wiktionary_ur">ویکی لغت — اردو سے انگریزی (CC BY-SA 3.0)</string>
+    <string name="source_urwiktionary">اردو ویکی لغت — اردو سے اردو (CC BY-SA 3.0)</string>
+    <string name="did_you_mean">ملتے جلتے الفاظ</string>
 </resources>

app/src/main/res/values/strings.xml

@@ -70,4 +70,7 @@
     <string name="source_wiktionary">Wiktionary — Persian to English (CC BY-SA 3.0)</string>
     <string name="source_daneshjoo">Daneshjoo — Persian to English</string>
     <string name="dictionary">Dictionary</string>
+    <string name="source_wiktionary_ur">Wiktionary — Urdu to English (CC BY-SA 3.0)</string>
+    <string name="source_urwiktionary">Urdu Wiktionary — Urdu to Urdu (CC BY-SA 3.0)</string>
+    <string name="did_you_mean">Similar words</string>
 </resources>

app/src/test/java/com/ganjoor/android/DictionaryTest.kt

@@ -1,6 +1,7 @@
 package com.ganjoor.android
 
 import com.ganjoor.android.data.affixes
+import com.ganjoor.android.data.letterOverlap
 import com.ganjoor.android.data.normalise
 import com.ganjoor.android.data.wordAt
 import org.junit.Assert.assertEquals
@@ -106,3 +107,31 @@ class PersianMorphologyTest {
         assertTrue(affixes("بها").none { it.length < 2 })
     }
 }
+
+class LetterOverlapTest {
+    @Test
+    fun `an identical word overlaps completely`() {
+        assertEquals(1f, letterOverlap("عشق", "عشق"), 0.001f)
+    }
+
+    @Test
+    fun `a suffixed form still scores high against its stem`() {
+        assertTrue(letterOverlap("مشکل", "مشکلها") > 0.6f)
+    }
+
+    @Test
+    fun `sharing only a first letter scores low`() {
+        assertTrue(letterOverlap("عشق", "عبادتگاه") < 0.4f)
+    }
+
+    @Test
+    fun `letters are counted once each, not by presence alone`() {
+        // ااا against ا shares one letter, not three
+        assertEquals(1f / 3f, letterOverlap("ااا", "ا"), 0.001f)
+    }
+
+    @Test
+    fun `an empty word never matches`() {
+        assertEquals(0f, letterOverlap("", "عشق"), 0.001f)
+    }
+}