From e63c8c5e9dd7520b6788b5ee515212363d165ffc Mon Sep 17 00:00:00 2001 From: Hamid Reza Mohammadi Date: Thu, 10 Sep 2026 13:03:20 +0330 Subject: [PATCH] semantic search --- GanjooRazor/APIRoot.cs | 25 ++++ GanjooRazor/Pages/SemanticSearch.cshtml | 140 ++++++++++++++++++ GanjooRazor/Pages/SemanticSearch.cshtml.cs | 69 +++++++++ GanjooRazor/appsettings.json | 1 + .../Controllers/SemanticSearchController.cs | 2 +- .../SemanticSearch/SemanticSearchDtos.cs | 40 +++++ .../Implementation/SemanticSearchService.cs | 110 +++++++++++--- RMuseum/Startup.cs | 5 + .../Utils/SemanticSearch/EmbeddingIndex.cs | 23 ++- .../SemanticSearch/LazyQueryScopeIndex.cs | 74 +++++++++ .../Utils/SemanticSearch/QueryScopeIndex.cs | 137 +++++++++++++++++ 11 files changed, 601 insertions(+), 25 deletions(-) create mode 100644 GanjooRazor/Pages/SemanticSearch.cshtml create mode 100644 GanjooRazor/Pages/SemanticSearch.cshtml.cs create mode 100644 RMuseum/Utils/SemanticSearch/LazyQueryScopeIndex.cs create mode 100644 RMuseum/Utils/SemanticSearch/QueryScopeIndex.cs diff --git a/GanjooRazor/APIRoot.cs b/GanjooRazor/APIRoot.cs index cd5502a9..845e7c1e 100644 --- a/GanjooRazor/APIRoot.cs +++ b/GanjooRazor/APIRoot.cs @@ -45,5 +45,30 @@ namespace GanjooRazor return _InternetUrl; } } + + private static string _semanticSearchUrl = ""; + + /// + /// Semantic search API endpoint — deliberately configurable separately from + /// Url/InternetUrl. Currently points at a physically separate domain/app pool + /// (ganjgah.ir) hosting the same RMuseum codebase, so that a problem with this one + /// feature (a native ONNX Runtime crash, a performance issue, anything) can't affect + /// api.ganjoor.net or the main site at all — proven necessary by an actual production + /// incident, not a hypothetical precaution. Point this at the main API root instead once + /// the feature has run stably on its own domain for a while. + /// + public static string SemanticSearchUrl + { + get + { + if (!string.IsNullOrEmpty(_semanticSearchUrl)) + return _semanticSearchUrl; + IConfigurationRoot configuration = new ConfigurationBuilder() + .SetBasePath(Directory.GetCurrentDirectory()).AddJsonFile("appsettings.json") + .Build(); + _semanticSearchUrl = configuration["SemanticSearchAPIRoot"]; + return _semanticSearchUrl; + } + } } } diff --git a/GanjooRazor/Pages/SemanticSearch.cshtml b/GanjooRazor/Pages/SemanticSearch.cshtml new file mode 100644 index 00000000..d0356b05 --- /dev/null +++ b/GanjooRazor/Pages/SemanticSearch.cshtml @@ -0,0 +1,140 @@ +@page +@model GanjooRazor.Pages.SemanticSearchModel +@{ + Layout = "_Layout"; + ViewData["Title"] = "جست‌وجوی معنایی"; +} +@section Head { + +} + +
+ +

جست‌وجوی معنایی

+

+ به‌جای جست‌وجوی کلمه‌به‌کلمه، توصیف کنید دنبال چه مضمونی هستید — مثلاً «شعری دربارهٔ بی‌وفایی دنیا». +
+ اگر نام یک سخنور یا یک بخش/کتاب خاص را هم در جمله بیاورید (مثلاً «در کدام شعر حافظ» یا «در کدام بخش شاهنامه»)، جست‌وجو فقط در همان محدوده انجام می‌شود. +

+ +
+ + +
+ + + +
+ +
+ +@section Scripts { + +} diff --git a/GanjooRazor/Pages/SemanticSearch.cshtml.cs b/GanjooRazor/Pages/SemanticSearch.cshtml.cs new file mode 100644 index 00000000..38cc4e3e --- /dev/null +++ b/GanjooRazor/Pages/SemanticSearch.cshtml.cs @@ -0,0 +1,69 @@ +using Microsoft.AspNetCore.Mvc; +using Microsoft.AspNetCore.Mvc.RazorPages; +using System; +using System.Net; +using System.Net.Http; +using System.Text; +using System.Threading.Tasks; + +namespace GanjooRazor.Pages +{ + /// + /// "find a poem about..." chatbox. Deliberately calls APIRoot.SemanticSearchUrl, NOT + /// APIRoot.Url/InternetUrl — see the comment on that property in APIRoot.cs. Currently a + /// physically separate domain/app pool (ganjgah.ir) from api.ganjoor.net, so a problem with + /// this one feature can't affect the main site or its API, a precaution proven necessary by + /// an actual production incident during development, not a hypothetical one. + /// + [IgnoreAntiforgeryToken(Order = 1001)] + public class SemanticSearchModel : PageModel + { + public void OnGet() + { + } + + /// + /// proxies the query to the semantic search API and passes its JSON straight back to the + /// page's own JS — no local re-modeling of the response, so there's no risk of a shape + /// mismatch between the two sides silently dropping a field + /// + public async Task OnPostSearchAsync(string query) + { + if (string.IsNullOrWhiteSpace(query)) + { + return BadRequest("لطفاً عبارتی برای جست‌وجو وارد کنید."); + } + + using (HttpClient client = new HttpClient()) + { + client.Timeout = TimeSpan.FromSeconds(30); // query-time ONNX inference is not instant - give it real room, but not unbounded + + var requestBody = "{\"query\":" + System.Text.Json.JsonSerializer.Serialize(query) + ",\"topK\":10}"; + var content = new StringContent(requestBody, Encoding.UTF8, "application/json"); + + HttpResponseMessage response; + try + { + response = await client.PostAsync($"{APIRoot.SemanticSearchUrl}/api/ganjoor/search/semantic", content); + } + catch (Exception) + { + // a network-level failure reaching the isolated semantic-search domain - + // exactly the kind of failure this domain-level isolation is meant to + // contain to just this feature, so report it plainly rather than let an + // unhandled exception propagate + return StatusCode((int)HttpStatusCode.ServiceUnavailable, + "در حال حاضر جست‌وجوی معنایی در دسترس نیست. لطفاً کمی بعد دوباره امتحان کنید."); + } + + string responseBody = await response.Content.ReadAsStringAsync(); + if (!response.IsSuccessStatusCode) + { + return StatusCode((int)response.StatusCode, responseBody); + } + + return Content(responseBody, "application/json"); + } + } + } +} diff --git a/GanjooRazor/appsettings.json b/GanjooRazor/appsettings.json index 2f676f05..a3ff5a4d 100644 --- a/GanjooRazor/appsettings.json +++ b/GanjooRazor/appsettings.json @@ -10,6 +10,7 @@ "TrackingScript": "", "APIRoot": "https://api.ganjoor.net", "GlobalAPIRoot": "https://api.ganjoor.net", + "SemanticSearchAPIRoot": "https://ganjgah.ir", "SiteUrl": "http://localhost:33081", "MockSpotify": "False", "Spotify": { diff --git a/RMuseum/Controllers/SemanticSearchController.cs b/RMuseum/Controllers/SemanticSearchController.cs index b3b2ea37..a29f3ff0 100644 --- a/RMuseum/Controllers/SemanticSearchController.cs +++ b/RMuseum/Controllers/SemanticSearchController.cs @@ -42,7 +42,7 @@ namespace RMuseum.Controllers { try { - var result = await _semanticSearchService.SearchAsync(request.Query, request.TopK); + var result = await _semanticSearchService.SearchAsync(request); return Ok(result); } catch (SemanticSearchUnavailableException exp) diff --git a/RMuseum/Models/Ganjoor/SemanticSearch/SemanticSearchDtos.cs b/RMuseum/Models/Ganjoor/SemanticSearch/SemanticSearchDtos.cs index 927e27ff..b7d46cc3 100644 --- a/RMuseum/Models/Ganjoor/SemanticSearch/SemanticSearchDtos.cs +++ b/RMuseum/Models/Ganjoor/SemanticSearch/SemanticSearchDtos.cs @@ -13,6 +13,28 @@ namespace RMuseum.Models.Ganjoor.SemanticSearch /// how many results to return; a sane default is applied server-side if omitted/invalid /// public int? TopK { get; set; } + + /// + /// Optional explicit scope, for a future UI (e.g. a poet picker) that wants to restrict + /// search without relying on auto-detection from the query text. Auto-detection (see + /// SemanticSearchService.DetectQueryScope) still runs even when these are set — an + /// explicit PoetId/CatId narrows further, it doesn't replace detection. + /// + public int? PoetId { get; set; } + public int? CatId { get; set; } + } + + public class SemanticSearchVerseDto + { + public int VOrder { get; set; } + + /// + /// matches GanjoorVerse.VersePosition.ToString() (e.g. "Right"/"Left") - same convention + /// already used by the main ganjoor-data export, so any existing hemistich-pairing + /// frontend code (see mini-ganjoor's renderVerses) can be reused as-is. + /// + public string Position { get; set; } + public string Text { get; set; } } public class SemanticSearchResultDto @@ -23,6 +45,12 @@ namespace RMuseum.Models.Ganjoor.SemanticSearch public string FullUrl { get; set; } public string PoemSummary { get; set; } + /// + /// A short preview from the poem's actual text - the first couple of couplets, in + /// original verse order. Not the whole poem; just enough for a result card. + /// + public List Verses { get; set; } = new List(); + /// /// cosine similarity, 0..1 for these normalized vectors (in practice results cluster in /// a narrower band - this is a relative ranking signal, not a calibrated probability) @@ -34,5 +62,17 @@ namespace RMuseum.Models.Ganjoor.SemanticSearch { public string Query { get; set; } public List Results { get; set; } = new List(); + + /// + /// If the query text was recognized as referring to a specific poet and/or + /// category/book (e.g. "در کدام شعر حافظ" -> حافظ, "در کدام بخش شاهنامه" -> شاهنامه), + /// search was restricted to that scope and these are populated so the UI can show the + /// user what was detected ("نتایج محدود به: حافظ") rather than silently filtering. + /// Null/null if nothing was detected (or an explicit PoetId/CatId narrowed things + /// without any name being recognized in the free text). + /// + public string DetectedPoetName { get; set; } + public string DetectedCategoryName { get; set; } } } + diff --git a/RMuseum/Services/Implementation/SemanticSearchService.cs b/RMuseum/Services/Implementation/SemanticSearchService.cs index a8ef6c3b..8faafc3b 100644 --- a/RMuseum/Services/Implementation/SemanticSearchService.cs +++ b/RMuseum/Services/Implementation/SemanticSearchService.cs @@ -11,7 +11,7 @@ namespace RMuseum.Services.Implementation { public interface ISemanticSearchService { - Task SearchAsync(string query, int? topK); + Task SearchAsync(SemanticSearchRequestDto request); } /// @@ -29,23 +29,31 @@ namespace RMuseum.Services.Implementation /// being constructed, taking down every endpoint under /api/ganjoor with a 503, not just /// semantic search. This class must never let a resource-loading failure become an unhandled /// exception that propagates past SearchAsync — see the catch below. + /// + /// LazyQueryScopeIndex is a SEPARATE lazy singleton from LazySemanticSearchResources on + /// purpose — poet/category name detection ("در کدام شعر حافظ") is a genuinely independent + /// concern from the embedding/model loading, with its own independent failure mode; a bug in + /// one must not be able to disable the other. /// public class SemanticSearchService : ISemanticSearchService { private const int DefaultTopK = 10; private const int MaxTopK = 50; + private const int PreviewVerseCount = 4; // ~2 couplets - enough for a result-card preview, not the whole poem private readonly LazySemanticSearchResources _resources; + private readonly LazyQueryScopeIndex _queryScopeIndex; - public SemanticSearchService(LazySemanticSearchResources resources) + public SemanticSearchService(LazySemanticSearchResources resources, LazyQueryScopeIndex queryScopeIndex) { _resources = resources; + _queryScopeIndex = queryScopeIndex; } - public async Task SearchAsync(string query, int? topK) + public async Task SearchAsync(SemanticSearchRequestDto request) { - if (string.IsNullOrWhiteSpace(query)) - throw new ArgumentException("query must not be empty", nameof(query)); + if (request == null || string.IsNullOrWhiteSpace(request.Query)) + throw new ArgumentException("query must not be empty", nameof(request)); if (!_resources.TryGetResources(out var embeddingIndex, out var queryEmbedder, out var error)) { @@ -53,20 +61,66 @@ namespace RMuseum.Services.Implementation "Semantic search is not available right now" + (error != null ? $": {error}" : ".")); } - int k = topK.GetValueOrDefault(DefaultTopK); + int k = request.TopK.GetValueOrDefault(DefaultTopK); if (k <= 0 || k > MaxTopK) k = DefaultTopK; - float[] queryVector = queryEmbedder.EmbedQuery(query); - List<(int PoemId, float Score)> topMatches = embeddingIndex.FindTopSimilar(queryVector, k); - - var response = new SemanticSearchResponseDto { Query = query }; + var response = new SemanticSearchResponseDto { Query = request.Query }; // a fresh, short-lived DbContext per call - same pattern GanjoorService's background // jobs already use throughout this codebase, since a scoped/request DbContext can't // be injected into this singleton service using (RMuseumDbContext context = new RMuseumDbContext(new DbContextOptions())) { + // An explicit request.PoetId/CatId (from a future UI, e.g. a poet picker) always + // wins; auto-detection only fills in whatever wasn't already specified. A + // detection failure here is swallowed on top of LazyQueryScopeIndex's own + // try/catch, as a second safety net - this is a nice-to-have, never worth + // failing the whole search over. + int? scopePoetId = request.PoetId; + int? scopeCatId = request.CatId; + try + { + var scopeIndex = await _queryScopeIndex.TryGetIndexAsync(context); + if (scopeIndex != null) + { + var detected = scopeIndex.DetectScope(request.Query); + if (detected.HasAny) + { + scopePoetId = scopePoetId ?? detected.PoetId; + scopeCatId = scopeCatId ?? detected.CatId; + response.DetectedPoetName = detected.PoetName; + response.DetectedCategoryName = detected.CategoryName; + } + } + } + catch (Exception) + { + // scope detection is best-effort only - fall through with no scope rather + // than fail the search + } + + HashSet allowedPoemIds = null; + if (scopeCatId.HasValue) + { + var ids = await context.GanjoorPoems.AsNoTracking() + .Where(p => p.CatId == scopeCatId.Value) + .Select(p => p.Id) + .ToListAsync(); + allowedPoemIds = new HashSet(ids); + } + else if (scopePoetId.HasValue) + { + var ids = await context.GanjoorPoems.AsNoTracking() + .Where(p => p.Cat.PoetId == scopePoetId.Value) + .Select(p => p.Id) + .ToListAsync(); + allowedPoemIds = new HashSet(ids); + } + + float[] queryVector = queryEmbedder.EmbedQuery(request.Query); + List<(int PoemId, float Score)> topMatches = embeddingIndex.FindTopSimilar(queryVector, k, allowedPoemIds); + var poemIds = topMatches.Select(m => m.PoemId).ToList(); var poemsById = await context.GanjoorPoems.AsNoTracking() .Where(p => poemIds.Contains(p.Id)) @@ -77,18 +131,34 @@ namespace RMuseum.Services.Implementation // a poem id present in the embedding index but missing from the live DB // (e.g. deleted/unpublished since the embeddings were generated) is silently // skipped rather than failing the whole search for everyone else's results - if (poemsById.TryGetValue(match.PoemId, out var poem)) + if (!poemsById.TryGetValue(match.PoemId, out var poem)) + continue; + + // one small, bounded query per result (topK is capped at 50) rather than + // loading every verse of every matched poem just to keep the first few - a + // poem can have hundreds of verses, no reason to pull all of them over a + // preview snippet + var verses = await context.GanjoorVerses.AsNoTracking() + .Where(v => v.PoemId == poem.Id) + .OrderBy(v => v.VOrder) + .Take(PreviewVerseCount) + .ToListAsync(); + + response.Results.Add(new SemanticSearchResultDto { - response.Results.Add(new SemanticSearchResultDto + PoemId = poem.Id, + Title = poem.Title, + FullTitle = poem.FullTitle, + FullUrl = poem.FullUrl, + PoemSummary = poem.PoemSummary, + Score = match.Score, + Verses = verses.Select(v => new SemanticSearchVerseDto { - PoemId = poem.Id, - Title = poem.Title, - FullTitle = poem.FullTitle, - FullUrl = poem.FullUrl, - PoemSummary = poem.PoemSummary, - Score = match.Score, - }); - } + VOrder = v.VOrder, + Position = v.VersePosition.ToString(), + Text = v.Text, + }).ToList(), + }); } } diff --git a/RMuseum/Startup.cs b/RMuseum/Startup.cs index de3ed703..7796e19b 100644 --- a/RMuseum/Startup.cs +++ b/RMuseum/Startup.cs @@ -336,6 +336,11 @@ namespace RMuseum // never throws; a failure there disables semantic search only. services.AddSingleton(); + // Separate lazy singleton from LazySemanticSearchResources - poet/category name + // detection ("در کدام شعر حافظ") is an independent concern with its own independent + // failure mode; a bug in one must not be able to disable the other. + services.AddSingleton(); + services.AddSingleton(); diff --git a/RMuseum/Utils/SemanticSearch/EmbeddingIndex.cs b/RMuseum/Utils/SemanticSearch/EmbeddingIndex.cs index 3ec8e792..3d7d724a 100644 --- a/RMuseum/Utils/SemanticSearch/EmbeddingIndex.cs +++ b/RMuseum/Utils/SemanticSearch/EmbeddingIndex.cs @@ -105,8 +105,14 @@ namespace RMuseum.Utils.SemanticSearch /// similarity. queryVector must already be the SAME dimension as this index and, for the /// score to mean what it claims (a true cosine similarity), should already be /// L2-normalized the same way the indexed vectors are — see QueryEmbedder. + /// + /// If allowedPoemIds is provided (non-null), only poems in that set are eligible — + /// everything else is scored as excluded and can never appear in the results, however + /// similar it might be. Used for scoped search ("در کدام شعر حافظ" -> restrict to + /// Hafez's poems) — still a full scan either way, just with cheap early-outs for + /// excluded rows, since this corpus is small enough that a full scan is fast regardless. /// - public List<(int PoemId, float Score)> FindTopSimilar(ReadOnlySpan queryVector, int topK) + public List<(int PoemId, float Score)> FindTopSimilar(ReadOnlySpan queryVector, int topK, ISet allowedPoemIds = null) { if (queryVector.Length != Metadata.Dimension) throw new ArgumentException( @@ -121,6 +127,12 @@ namespace RMuseum.Utils.SemanticSearch // this is genuinely computing cosine similarity, not just a raw dot product for (int i = 0; i < count; i++) { + if (allowedPoemIds != null && !allowedPoemIds.Contains(_poemIds[i])) + { + scores[i] = float.NegativeInfinity; // excluded from this search - will never sort into the results + continue; + } + float dot = 0f; int baseIdx = i * Metadata.Dimension; for (int d = 0; d < Metadata.Dimension; d++) @@ -135,10 +147,13 @@ namespace RMuseum.Utils.SemanticSearch // ~130k floats is already comfortably fast (low tens of ms) and simpler to trust Array.Sort(indices, (a, b) => scores[b].CompareTo(scores[a])); - var results = new List<(int, float)>(Math.Min(topK, count)); - for (int i = 0; i < Math.Min(topK, count); i++) + var results = new List<(int, float)>(); + for (int i = 0; i < count && results.Count < topK; i++) { - results.Add((_poemIds[indices[i]], scores[indices[i]])); + int idx = indices[i]; + if (float.IsNegativeInfinity(scores[idx])) + break; // sorted descending - everything from here on is also excluded + results.Add((_poemIds[idx], scores[idx])); } return results; } diff --git a/RMuseum/Utils/SemanticSearch/LazyQueryScopeIndex.cs b/RMuseum/Utils/SemanticSearch/LazyQueryScopeIndex.cs new file mode 100644 index 00000000..dbc474dc --- /dev/null +++ b/RMuseum/Utils/SemanticSearch/LazyQueryScopeIndex.cs @@ -0,0 +1,74 @@ +using Microsoft.Extensions.Logging; +using RMuseum.DbContext; +using System; +using System.Threading; +using System.Threading.Tasks; + +namespace RMuseum.Utils.SemanticSearch +{ + /// + /// Same "load once, never throw" pattern as LazySemanticSearchResources, but deliberately a + /// SEPARATE singleton with its own independent failure domain: if the poet/category name + /// lookup fails to load for any reason, that must only disable scope auto-detection + /// ("در کدام شعر حافظ" -> unscoped, searches everything) — it must never affect the embedding + /// index/model loading or plain (unscoped) search, which are a completely different concern. + /// + /// Uses a SemaphoreSlim rather than a plain `lock`, since the actual load is async (DB + /// queries) and you can't `await` inside a `lock` block. + /// + public class LazyQueryScopeIndex + { + private readonly SemaphoreSlim _semaphore = new SemaphoreSlim(1, 1); + private QueryScopeIndex _index; + private bool _attempted; + private readonly ILogger _logger; + + public LazyQueryScopeIndex(ILogger logger) + { + _logger = logger; + } + + /// + /// Loads (once) and returns the scope index, or null if loading failed or hasn't + /// succeeded yet — NEVER throws. The passed-in context is only actually used on the + /// first call that does real work; a context created by the caller for its own + /// SearchAsync call is reused here rather than this class creating its own, since it's + /// only needed for the one-time load. + /// + public async Task TryGetIndexAsync(RMuseumDbContext context) + { + if (_attempted) + return _index; + + await _semaphore.WaitAsync(); + try + { + if (_attempted) + return _index; + + try + { + _index = await QueryScopeIndex.LoadAsync(context); + _logger.LogInformation("Query scope index loaded."); + } + catch (Exception exp) + { + _index = null; + _logger.LogError(exp, + "Query scope index failed to load — poet/category auto-detection will be " + + "unavailable, but this must not affect plain search."); + } + finally + { + _attempted = true; + } + } + finally + { + _semaphore.Release(); + } + + return _index; + } + } +} diff --git a/RMuseum/Utils/SemanticSearch/QueryScopeIndex.cs b/RMuseum/Utils/SemanticSearch/QueryScopeIndex.cs new file mode 100644 index 00000000..da6d922a --- /dev/null +++ b/RMuseum/Utils/SemanticSearch/QueryScopeIndex.cs @@ -0,0 +1,137 @@ +using Microsoft.EntityFrameworkCore; +using RMuseum.DbContext; +using System.Collections.Generic; +using System.Linq; +using System.Threading.Tasks; + +namespace RMuseum.Utils.SemanticSearch +{ + public class PoetScopeEntry + { + public int PoetId { get; set; } + public string Nickname { get; set; } + } + + public class CategoryScopeEntry + { + public int CatId { get; set; } + public int PoetId { get; set; } + public string Title { get; set; } + } + + public class DetectedScope + { + public int? PoetId { get; set; } + public string PoetName { get; set; } + public int? CatId { get; set; } + public string CategoryName { get; set; } + + public bool HasAny => PoetId.HasValue || CatId.HasValue; + } + + /// + /// Detects when a free-text query names a specific poet and/or book/collection (e.g. + /// "در کدام شعر حافظ" -> حافظ, "در کدام بخش شاهنامه" -> شاهنامه) so search can be scoped to + /// just that poet/category instead of the whole corpus. + /// + /// Deliberately simple substring matching, not real NLP/NER — good enough for the common, + /// unambiguous case (a poet's distinctive nickname, or a book title effectively unique to one + /// poet, like شاهنامه), and safely conservative for the ambiguous case: a generic category + /// title shared by many poets (غزلیات appears for most of them) is only used as a scope if a + /// specific poet was ALSO named in the same query, narrowing which one is meant — otherwise + /// it's dropped rather than guessing which poet's غزلیات the person meant. + /// + public class QueryScopeIndex + { + private readonly List _poets; + private readonly List _categories; + + private QueryScopeIndex(List poets, List categories) + { + _poets = poets; + _categories = categories; + } + + public static async Task LoadAsync(RMuseumDbContext context) + { + var poets = await context.GanjoorPoets.AsNoTracking() + .Where(p => p.Published) + .Select(p => new PoetScopeEntry { PoetId = p.Id, Nickname = p.Nickname }) + .ToListAsync(); + + // "book"/"collection" level = direct children of a poet's own root category + // (شاهنامه, غزلیات, دیوان شمس, ...) - not deeper structural subsections, which would + // add a lot of short, generic, easily-false-positive titles to match against. + var rootCatIds = await context.GanjoorCategories.AsNoTracking() + .Where(c => c.ParentId == null) + .Select(c => c.Id) + .ToListAsync(); + var rootCatIdSet = new HashSet(rootCatIds); + + var categories = await context.GanjoorCategories.AsNoTracking() + .Where(c => c.ParentId != null) + .Select(c => new { c.Id, c.ParentId, c.PoetId, c.Title }) + .ToListAsync(); + + var scopedCategories = categories + .Where(c => rootCatIdSet.Contains(c.ParentId.Value)) + .Select(c => new CategoryScopeEntry { CatId = c.Id, PoetId = c.PoetId, Title = c.Title }) + .ToList(); + + return new QueryScopeIndex(poets, scopedCategories); + } + + public DetectedScope DetectScope(string query) + { + var result = new DetectedScope(); + if (string.IsNullOrWhiteSpace(query)) + return result; + + PoetScopeEntry bestPoet = null; + foreach (var poet in _poets) + { + if (string.IsNullOrEmpty(poet.Nickname)) + continue; + if (query.Contains(poet.Nickname) && (bestPoet == null || poet.Nickname.Length > bestPoet.Nickname.Length)) + { + bestPoet = poet; + } + } + + string bestCategoryTitle = null; + foreach (var cat in _categories) + { + if (string.IsNullOrEmpty(cat.Title)) + continue; + if (query.Contains(cat.Title) && (bestCategoryTitle == null || cat.Title.Length > bestCategoryTitle.Length)) + { + bestCategoryTitle = cat.Title; + } + } + + CategoryScopeEntry bestCategory = null; + if (bestCategoryTitle != null) + { + var matches = _categories.Where(c => c.Title == bestCategoryTitle).ToList(); + if (matches.Count == 1) + { + bestCategory = matches[0]; // unambiguous - only one category anywhere has this exact title + } + else if (bestPoet != null) + { + // ambiguous title (shared by multiple poets), but a specific poet was also + // named - use that poet's version of it, if they have one + bestCategory = matches.FirstOrDefault(c => c.PoetId == bestPoet.PoetId); + } + // else: ambiguous with no poet to disambiguate against - deliberately left null + // rather than guessing which poet's version was meant + } + + result.PoetId = bestPoet?.PoetId; + result.PoetName = bestPoet?.Nickname; + result.CatId = bestCategory?.CatId; + result.CategoryName = bestCategory?.Title; + return result; + } + } +}