Urdu-aware search without SQL Server full-text (#3)

- MakeTextSearchable: Urdu letter variants, Urdu full stop, Urdu diacritics, zwj/marks
- SearchLikePatterns: phrase or per-word LIKE patterns (escaped), normalised like stored text
- poem, similar-poem and comment search use LIKE (Linux SQL Server image has no full-text)

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

README.md

@@ -15,6 +15,7 @@ Diwan is a standalone project built on a fork of [GanjoorService](https://github
 - **Config fixes so env vars work:** the four places that read `appsettings.json` directly now also read environment variables. The JWT issuer follows `RSecurityBackend:ApplicationName` instead of the hard-coded "Diwan". `deploy/entrypoint.sh` copies the settings that RSecurityBackend reads only from `appsettings.json` (connection string, secret, app name, admin email) into the file at container start.
 - **Links:** `ganjoor.net` links to the site's own pages are now relative. Links to Diwan's other services (blog, audio, etc.) are left as they are.
 - **Locale:** `ur-PK`.
+- **Search without full-text:** the SQL Server Linux image has no full-text search, so poem, similar-poem and comment search use `LIKE` patterns (`LanguageUtils.SearchLikePatterns`). The normaliser handles Urdu letter variants (Arabic ي/ك/ه → ی/ک/ہ, ۂ/ۓ), the Urdu full stop and Urdu diacritics. After changing normalisation rules, rebuild stored search text with `POST /api/diwan/regenplaintext/0` (admin).
 
 ## Static reader (no server)
 

RMuseum/Services/Implementation/DiwanService.cs

@@ -1469,48 +1469,19 @@ namespace RMuseum.Services.Implementation
         /// <returns></returns>
         public async Task<RServiceResult<(PaginationMetadata PagingMeta, DiwanCommentFullViewModel[] Items)>> GetRecentComments(PagingParameterModel paging, Guid filterUserId, bool onlyPublished, bool onlyAwaiting = false, string term = null)
         {
-            string searchConditions = null;
+            string[] searchPatterns = LanguageUtils.SearchLikePatterns(term); // diwan: LIKE instead of full-text; empty term -> no filter
-            if (!string.IsNullOrEmpty(term))
-            {
-                /* You need to run this scripts manually on the database before using this method:
-                 CREATE FULLTEXT CATALOG [DiwanHtmlCommentTextCatalog] WITH ACCENT_SENSITIVITY = OFF AS DEFAULT
-                 
-                 CREATE FULLTEXT INDEX ON [dbo].[DiwanComments](
-                 [HtmlComment] LANGUAGE 'English')
-                 KEY INDEX [PK_DiwanComments]ON ([DiwanHtmlCommentTextCatalog], FILEGROUP [PRIMARY])
-                 WITH (CHANGE_TRACKING = AUTO, STOPLIST = SYSTEM)
-                */
-                term = term.Replace("‌", " ");//replace zwnj with space
-
-                if (term.IndexOf('"') == 0 && term.LastIndexOf('"') == (term.Length - 1))
-                {
-                    searchConditions = term.Replace("\"", "").Replace("'", "");
-                    searchConditions = $"\"{searchConditions}\"";
-                }
-                else
-                {
-                    string[] words = term.Replace("\"", "").Replace("'", "").Split(' ', StringSplitOptions.RemoveEmptyEntries);
-
-                    searchConditions = "";
-                    string emptyOrAnd = "";
-                    foreach (string word in words)
-                    {
-                        searchConditions += $" {emptyOrAnd} \"*{word}*\" ";
-                        emptyOrAnd = " AND ";
-                    }
-                }
-            }
 
+            var comments = _context.DiwanComments.AsQueryable();
+            foreach (var pattern in searchPatterns)
+                comments = comments.Where(c => EF.Functions.Like(c.HtmlComment, pattern));
             var source =
-                 from comment in _context.DiwanComments.Include(c => c.Poem).Include(c => c.User).Include(c => c.InReplyTo).ThenInclude(r => r.User)
+                 from comment in comments.Include(c => c.Poem).Include(c => c.User).Include(c => c.InReplyTo).ThenInclude(r => r.User)
                  where
                   ((comment.Status == PublishStatus.Published) || !onlyPublished)
                   &&
                   ((comment.Status == PublishStatus.Awaiting) || !onlyAwaiting)
                  &&
                  ((filterUserId == Guid.Empty) || (filterUserId != Guid.Empty && comment.UserId == filterUserId))
-                 &&
-                 (string.IsNullOrEmpty(searchConditions) || (!string.IsNullOrEmpty(searchConditions) && EF.Functions.Contains(comment.HtmlComment, searchConditions)))
                  orderby comment.CommentDate descending
                  select new DiwanCommentFullViewModel()
                  {
@@ -2673,30 +2644,12 @@ namespace RMuseum.Services.Implementation
                 catIdList.Add((int)catId);
                 await _populateCategoryChildren(_context, (int)catId, catIdList);
             }
-            string searchConditions = "";
+            string[] searchPatterns = LanguageUtils.SearchLikePatterns(term); // diwan: LIKE instead of full-text; empty term -> no filter
-            if (!string.IsNullOrEmpty(term))
+            var sections = _context.DiwanPoemSections.AsQueryable();
-            {
+            foreach (var pattern in searchPatterns)
-                term = term.Replace("‌", " ");//replace zwnj with space
+                sections = sections.Where(s => EF.Functions.Like(s.Poem.PlainText, pattern));
-                if (term.IndexOf('"') == 0 && term.LastIndexOf('"') == (term.Length - 1))
-                {
-                    searchConditions = term.Replace("\"", "").Replace("'", "");
-                    searchConditions = $"\"{searchConditions}\"";
-                }
-                else
-                {
-                    string[] words = term.Replace("\"", "").Replace("'", "").Split(' ', StringSplitOptions.RemoveEmptyEntries);
-
-                    searchConditions = "";
-                    string emptyOrAnd = "";
-                    foreach (string word in words)
-                    {
-                        searchConditions += $" {emptyOrAnd} \"*{word}*\" ";
-                        emptyOrAnd = " AND ";
-                    }
-                }
-            }
             var source =
-                _context.DiwanPoemSections.Include(s => s.Poem).Include(s => s.Poet).Include(s => s.DiwanMetre)
+                sections.Include(s => s.Poem).Include(s => s.Poet).Include(s => s.DiwanMetre)
                 .Where(s =>
                         (poetId == null || s.PoetId == poetId)
                         &&
@@ -2715,8 +2668,6 @@ namespace RMuseum.Services.Implementation
                         (s.CoupletsCount >= coupletCountsFrom)
                         &&
                         (coupletCountsTo == 0 || s.CoupletsCount <= coupletCountsTo)
-                        &&
-                        (searchConditions == "" || (searchConditions != "" && EF.Functions.Contains(s.Poem.PlainText, searchConditions)))
                         )
                 .OrderBy(p => p.Poet.BirthYearInLHijri).ThenBy(p => p.Poet.Nickname).ThenBy(p => p.SectionType).ThenBy(p => p.Poem.Id)
                 .Select
@@ -2894,24 +2845,7 @@ namespace RMuseum.Services.Implementation
             term = term.Replace("‌", " ");//replace zwnj with space
 
 
-            string searchConditions;
+            string[] searchPatterns = LanguageUtils.SearchLikePatterns(term); // diwan: LIKE instead of full-text
-            if (term.IndexOf('"') == 0 && term.LastIndexOf('"') == (term.Length - 1))
-            {
-                searchConditions = term.Replace("\"", "").Replace("'", "");
-                searchConditions = $"\"{searchConditions}\"";
-            }
-            else
-            {
-                string[] words = term.Replace("\"", "").Replace("'", "").Split(' ', StringSplitOptions.RemoveEmptyEntries);
-
-                searchConditions = "";
-                string emptyOrAnd = "";
-                foreach (string word in words)
-                {
-                    searchConditions += $" {emptyOrAnd} \"*{word}*\" ";
-                    emptyOrAnd = " AND ";
-                }
-            }
             if (poetId == null)
             {
                 catId = null;
@@ -2938,14 +2872,15 @@ namespace RMuseum.Services.Implementation
                 await _populateCategoryChildren(_context, (int)catId, catIdList);
             }
 
+            var poems = _context.DiwanPoems.AsQueryable();
+            foreach (var pattern in searchPatterns)
+                poems = poems.Where(p => EF.Functions.Like(p.PlainText, pattern));
             var source =
-                _context.DiwanPoems
+                poems
                 .Where(p =>
                         (catId == null || catIdList.Contains(p.CatId))
                         &&
                         (exceptPoetId.Length == 0 || !exceptPoetId.Contains(p.Cat.PoetId))
-                        &&
-                       EF.Functions.Contains(p.PlainText, searchConditions)
                         )
                 .Include(p => p.Cat).ThenInclude(c => c.Poet)
                 .OrderBy(p => p.Cat.Poet.BirthYearInLHijri).ThenBy(p => p.Cat.Poet.Nickname).ThenBy(p => p.Id)

RMuseum/Services/Implementation/LanguageUtils.cs

@@ -73,9 +73,38 @@ namespace RMuseum.Services.Implementation
                        .Replace("«", "")
                        .Replace("»", "")
                        .Replace("ْ", "")//sokoon
+                       // diwan: Urdu normalisation (same rules apply to stored text and search terms)
+                       .Replace("\u064A", "\u06CC")//arabic yeh -> farsi/urdu yeh
+                       .Replace("\u0649", "\u06CC")//alef maksura -> yeh
+                       .Replace("\u0643", "\u06A9")//arabic kaf -> keheh
+                       .Replace("\u06C2", "\u06C1")//heh goal with hamza -> heh goal
+                       .Replace("\u0647", "\u06C1")//arabic heh -> heh goal (do-chashmi heh U+06BE stays distinct)
+                       .Replace("\u06D3", "\u06D2")//yeh barree with hamza -> yeh barree
+                       .Replace("\u0623", "\u0627").Replace("\u0625", "\u0627")//alef with hamza -> alef
+                       .Replace("\u06D4", "")//urdu full stop
+                       .Replace("\u0670", "").Replace("\u0657", "").Replace("\u0658", "").Replace("\u0615", "")//urdu diacritics
+                       .Replace("\u200D", "").Replace("\u200F", "").Replace("\u200E", "")//zwj, rtl/ltr marks
                        ;
         }
 
+        /// <summary>
+        /// diwan: SQL LIKE patterns for a search term (replaces SQL Server full-text CONTAINS, which the
+        /// Linux SQL Server image does not ship). "quoted phrase" -> one pattern; otherwise one pattern per
+        /// word, all of which must match. Terms are normalised like stored PlainText.
+        /// </summary>
+        public static string[] SearchLikePatterns(string term)
+        {
+            if (string.IsNullOrWhiteSpace(term))
+                return Array.Empty<string>();
+            term = term.Trim();
+            bool phrase = term.Length > 1 && term.StartsWith("\"") && term.EndsWith("\"");
+            string normalised = MakeTextSearchable(term.ApplyCorrectYeKe().Replace("\"", "").Replace("'", ""));
+            string[] parts = phrase ? new[] { normalised.Trim() } : normalised.Split(' ', StringSplitOptions.RemoveEmptyEntries);
+            return parts.Where(p => p.Length > 0)
+                        .Select(p => "%" + p.Replace("[", "[[]").Replace("%", "[%]").Replace("_", "[_]") + "%")
+                        .ToArray();
+        }
+
         public static string CleanTextForTransileration(string text)
         {
             if (string.IsNullOrEmpty(text)) return text;