From 9bd445614e37fbf32c903581cc2202a280451bc3 Mon Sep 17 00:00:00 2001 From: Anas Rashid Date: Mon, 5 Oct 2026 00:09:09 +0200 Subject: [PATCH] Urdu-aware search without SQL Server full-text (#3) - MakeTextSearchable: Urdu letter variants, Urdu full stop, Urdu diacritics, zwj/marks - SearchLikePatterns: phrase or per-word LIKE patterns (escaped), normalised like stored text - poem, similar-poem and comment search use LIKE (Linux SQL Server image has no full-text) Co-Authored-By: Claude Opus 5.5 --- README.md | 1 + .../Services/Implementation/DivanService.cs | 95 +++---------------- .../Services/Implementation/LanguageUtils.cs | 29 ++++++ 3 files changed, 45 insertions(+), 80 deletions(-) diff --git a/README.md b/README.md index bffb34d3..afc69628 100644 --- a/README.md +++ b/README.md @@ -15,6 +15,7 @@ Divan is a standalone project built on a fork of [GanjoorService](https://github - **Config fixes so env vars work:** the four places that read `appsettings.json` directly now also read environment variables. The JWT issuer follows `RSecurityBackend:ApplicationName` instead of the hard-coded "Divan". `deploy/entrypoint.sh` copies the settings that RSecurityBackend reads only from `appsettings.json` (connection string, secret, app name, admin email) into the file at container start. - **Links:** `ganjoor.net` links to the site's own pages are now relative. Links to Divan's other services (blog, audio, etc.) are left as they are. - **Locale:** `ur-PK`. +- **Search without full-text:** the SQL Server Linux image has no full-text search, so poem, similar-poem and comment search use `LIKE` patterns (`LanguageUtils.SearchLikePatterns`). The normaliser handles Urdu letter variants (Arabic ي/ك/ه → ی/ک/ہ, ۂ/ۓ), the Urdu full stop and Urdu diacritics. After changing normalisation rules, rebuild stored search text with `POST /api/divan/regenplaintext/0` (admin). ## Static reader (no server) diff --git a/RMuseum/Services/Implementation/DivanService.cs b/RMuseum/Services/Implementation/DivanService.cs index 4147b70c..41a5c894 100644 --- a/RMuseum/Services/Implementation/DivanService.cs +++ b/RMuseum/Services/Implementation/DivanService.cs @@ -1469,48 +1469,19 @@ namespace RMuseum.Services.Implementation /// public async Task> GetRecentComments(PagingParameterModel paging, Guid filterUserId, bool onlyPublished, bool onlyAwaiting = false, string term = null) { - string searchConditions = null; - if (!string.IsNullOrEmpty(term)) - { - /* You need to run this scripts manually on the database before using this method: - CREATE FULLTEXT CATALOG [DivanHtmlCommentTextCatalog] WITH ACCENT_SENSITIVITY = OFF AS DEFAULT - - CREATE FULLTEXT INDEX ON [dbo].[DivanComments]( - [HtmlComment] LANGUAGE 'English') - KEY INDEX [PK_DivanComments]ON ([DivanHtmlCommentTextCatalog], FILEGROUP [PRIMARY]) - WITH (CHANGE_TRACKING = AUTO, STOPLIST = SYSTEM) - */ - term = term.Replace("‌", " ");//replace zwnj with space - - if (term.IndexOf('"') == 0 && term.LastIndexOf('"') == (term.Length - 1)) - { - searchConditions = term.Replace("\"", "").Replace("'", ""); - searchConditions = $"\"{searchConditions}\""; - } - else - { - string[] words = term.Replace("\"", "").Replace("'", "").Split(' ', StringSplitOptions.RemoveEmptyEntries); - - searchConditions = ""; - string emptyOrAnd = ""; - foreach (string word in words) - { - searchConditions += $" {emptyOrAnd} \"*{word}*\" "; - emptyOrAnd = " AND "; - } - } - } + string[] searchPatterns = LanguageUtils.SearchLikePatterns(term); // divan: LIKE instead of full-text; empty term -> no filter + var comments = _context.DivanComments.AsQueryable(); + foreach (var pattern in searchPatterns) + comments = comments.Where(c => EF.Functions.Like(c.HtmlComment, pattern)); var source = - from comment in _context.DivanComments.Include(c => c.Poem).Include(c => c.User).Include(c => c.InReplyTo).ThenInclude(r => r.User) + from comment in comments.Include(c => c.Poem).Include(c => c.User).Include(c => c.InReplyTo).ThenInclude(r => r.User) where ((comment.Status == PublishStatus.Published) || !onlyPublished) && ((comment.Status == PublishStatus.Awaiting) || !onlyAwaiting) && ((filterUserId == Guid.Empty) || (filterUserId != Guid.Empty && comment.UserId == filterUserId)) - && - (string.IsNullOrEmpty(searchConditions) || (!string.IsNullOrEmpty(searchConditions) && EF.Functions.Contains(comment.HtmlComment, searchConditions))) orderby comment.CommentDate descending select new DivanCommentFullViewModel() { @@ -2673,30 +2644,12 @@ namespace RMuseum.Services.Implementation catIdList.Add((int)catId); await _populateCategoryChildren(_context, (int)catId, catIdList); } - string searchConditions = ""; - if (!string.IsNullOrEmpty(term)) - { - term = term.Replace("‌", " ");//replace zwnj with space - if (term.IndexOf('"') == 0 && term.LastIndexOf('"') == (term.Length - 1)) - { - searchConditions = term.Replace("\"", "").Replace("'", ""); - searchConditions = $"\"{searchConditions}\""; - } - else - { - string[] words = term.Replace("\"", "").Replace("'", "").Split(' ', StringSplitOptions.RemoveEmptyEntries); - - searchConditions = ""; - string emptyOrAnd = ""; - foreach (string word in words) - { - searchConditions += $" {emptyOrAnd} \"*{word}*\" "; - emptyOrAnd = " AND "; - } - } - } + string[] searchPatterns = LanguageUtils.SearchLikePatterns(term); // divan: LIKE instead of full-text; empty term -> no filter + var sections = _context.DivanPoemSections.AsQueryable(); + foreach (var pattern in searchPatterns) + sections = sections.Where(s => EF.Functions.Like(s.Poem.PlainText, pattern)); var source = - _context.DivanPoemSections.Include(s => s.Poem).Include(s => s.Poet).Include(s => s.DivanMetre) + sections.Include(s => s.Poem).Include(s => s.Poet).Include(s => s.DivanMetre) .Where(s => (poetId == null || s.PoetId == poetId) && @@ -2715,8 +2668,6 @@ namespace RMuseum.Services.Implementation (s.CoupletsCount >= coupletCountsFrom) && (coupletCountsTo == 0 || s.CoupletsCount <= coupletCountsTo) - && - (searchConditions == "" || (searchConditions != "" && EF.Functions.Contains(s.Poem.PlainText, searchConditions))) ) .OrderBy(p => p.Poet.BirthYearInLHijri).ThenBy(p => p.Poet.Nickname).ThenBy(p => p.SectionType).ThenBy(p => p.Poem.Id) .Select @@ -2894,24 +2845,7 @@ namespace RMuseum.Services.Implementation term = term.Replace("‌", " ");//replace zwnj with space - string searchConditions; - if (term.IndexOf('"') == 0 && term.LastIndexOf('"') == (term.Length - 1)) - { - searchConditions = term.Replace("\"", "").Replace("'", ""); - searchConditions = $"\"{searchConditions}\""; - } - else - { - string[] words = term.Replace("\"", "").Replace("'", "").Split(' ', StringSplitOptions.RemoveEmptyEntries); - - searchConditions = ""; - string emptyOrAnd = ""; - foreach (string word in words) - { - searchConditions += $" {emptyOrAnd} \"*{word}*\" "; - emptyOrAnd = " AND "; - } - } + string[] searchPatterns = LanguageUtils.SearchLikePatterns(term); // divan: LIKE instead of full-text if (poetId == null) { catId = null; @@ -2938,14 +2872,15 @@ namespace RMuseum.Services.Implementation await _populateCategoryChildren(_context, (int)catId, catIdList); } + var poems = _context.DivanPoems.AsQueryable(); + foreach (var pattern in searchPatterns) + poems = poems.Where(p => EF.Functions.Like(p.PlainText, pattern)); var source = - _context.DivanPoems + poems .Where(p => (catId == null || catIdList.Contains(p.CatId)) && (exceptPoetId.Length == 0 || !exceptPoetId.Contains(p.Cat.PoetId)) - && - EF.Functions.Contains(p.PlainText, searchConditions) ) .Include(p => p.Cat).ThenInclude(c => c.Poet) .OrderBy(p => p.Cat.Poet.BirthYearInLHijri).ThenBy(p => p.Cat.Poet.Nickname).ThenBy(p => p.Id) diff --git a/RMuseum/Services/Implementation/LanguageUtils.cs b/RMuseum/Services/Implementation/LanguageUtils.cs index 3234d3ab..2c86d7a3 100644 --- a/RMuseum/Services/Implementation/LanguageUtils.cs +++ b/RMuseum/Services/Implementation/LanguageUtils.cs @@ -73,9 +73,38 @@ namespace RMuseum.Services.Implementation .Replace("«", "") .Replace("»", "") .Replace("ْ", "")//sokoon + // divan: Urdu normalisation (same rules apply to stored text and search terms) + .Replace("\u064A", "\u06CC")//arabic yeh -> farsi/urdu yeh + .Replace("\u0649", "\u06CC")//alef maksura -> yeh + .Replace("\u0643", "\u06A9")//arabic kaf -> keheh + .Replace("\u06C2", "\u06C1")//heh goal with hamza -> heh goal + .Replace("\u0647", "\u06C1")//arabic heh -> heh goal (do-chashmi heh U+06BE stays distinct) + .Replace("\u06D3", "\u06D2")//yeh barree with hamza -> yeh barree + .Replace("\u0623", "\u0627").Replace("\u0625", "\u0627")//alef with hamza -> alef + .Replace("\u06D4", "")//urdu full stop + .Replace("\u0670", "").Replace("\u0657", "").Replace("\u0658", "").Replace("\u0615", "")//urdu diacritics + .Replace("\u200D", "").Replace("\u200F", "").Replace("\u200E", "")//zwj, rtl/ltr marks ; } + /// + /// divan: SQL LIKE patterns for a search term (replaces SQL Server full-text CONTAINS, which the + /// Linux SQL Server image does not ship). "quoted phrase" -> one pattern; otherwise one pattern per + /// word, all of which must match. Terms are normalised like stored PlainText. + /// + public static string[] SearchLikePatterns(string term) + { + if (string.IsNullOrWhiteSpace(term)) + return Array.Empty(); + term = term.Trim(); + bool phrase = term.Length > 1 && term.StartsWith("\"") && term.EndsWith("\""); + string normalised = MakeTextSearchable(term.ApplyCorrectYeKe().Replace("\"", "").Replace("'", "")); + string[] parts = phrase ? new[] { normalised.Trim() } : normalised.Split(' ', StringSplitOptions.RemoveEmptyEntries); + return parts.Where(p => p.Length > 0) + .Select(p => "%" + p.Replace("[", "[[]").Replace("%", "[%]").Replace("_", "[_]") + "%") + .ToArray(); + } + public static string CleanTextForTransileration(string text) { if (string.IsNullOrEmpty(text)) return text;