semantic search
This commit is contained in:
parent
5c70e88b09
commit
e63c8c5e9d
@ -45,5 +45,30 @@ namespace GanjooRazor
|
||||
return _InternetUrl;
|
||||
}
|
||||
}
|
||||
|
||||
private static string _semanticSearchUrl = "";
|
||||
|
||||
/// <summary>
|
||||
/// Semantic search API endpoint — deliberately configurable separately from
|
||||
/// Url/InternetUrl. Currently points at a physically separate domain/app pool
|
||||
/// (ganjgah.ir) hosting the same RMuseum codebase, so that a problem with this one
|
||||
/// feature (a native ONNX Runtime crash, a performance issue, anything) can't affect
|
||||
/// api.ganjoor.net or the main site at all — proven necessary by an actual production
|
||||
/// incident, not a hypothetical precaution. Point this at the main API root instead once
|
||||
/// the feature has run stably on its own domain for a while.
|
||||
/// </summary>
|
||||
public static string SemanticSearchUrl
|
||||
{
|
||||
get
|
||||
{
|
||||
if (!string.IsNullOrEmpty(_semanticSearchUrl))
|
||||
return _semanticSearchUrl;
|
||||
IConfigurationRoot configuration = new ConfigurationBuilder()
|
||||
.SetBasePath(Directory.GetCurrentDirectory()).AddJsonFile("appsettings.json")
|
||||
.Build();
|
||||
_semanticSearchUrl = configuration["SemanticSearchAPIRoot"];
|
||||
return _semanticSearchUrl;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
140
GanjooRazor/Pages/SemanticSearch.cshtml
Normal file
140
GanjooRazor/Pages/SemanticSearch.cshtml
Normal file
@ -0,0 +1,140 @@
|
||||
@page
|
||||
@model GanjooRazor.Pages.SemanticSearchModel
|
||||
@{
|
||||
Layout = "_Layout";
|
||||
ViewData["Title"] = "جستوجوی معنایی";
|
||||
}
|
||||
@section Head {
|
||||
<meta name="description" content="یک شعر را با توصیف معنای آن پیدا کنید — نه با جستوجوی کلمهبهکلمه." />
|
||||
}
|
||||
|
||||
<div id="semantic-search-page" dir="rtl" style="max-width: 760px; margin: 0 auto; padding: 24px 16px 60px;">
|
||||
|
||||
<h1 style="text-align:center; color: var(--brand); -webkit-text-fill-color: var(--brand);">جستوجوی معنایی</h1>
|
||||
<p style="text-align:center; color: var(--text-muted); margin-bottom: 24px;">
|
||||
بهجای جستوجوی کلمهبهکلمه، توصیف کنید دنبال چه مضمونی هستید — مثلاً «شعری دربارهٔ بیوفایی دنیا».
|
||||
<br />
|
||||
<span style="font-size:0.85rem;">اگر نام یک سخنور یا یک بخش/کتاب خاص را هم در جمله بیاورید (مثلاً «در کدام شعر حافظ» یا «در کدام بخش شاهنامه»)، جستوجو فقط در همان محدوده انجام میشود.</span>
|
||||
</p>
|
||||
|
||||
<form id="semantic-search-form" style="display:flex; gap:8px; margin-bottom: 24px;">
|
||||
<input
|
||||
type="text"
|
||||
id="semantic-query"
|
||||
name="query"
|
||||
placeholder="مثلاً: شعری در مورد بیوفایی دنیا"
|
||||
autocomplete="off"
|
||||
style="flex:1; padding:10px 14px; border-radius:8px; border:1px solid var(--border-gold); background: var(--bg-page); color: inherit; font-size:1rem;"
|
||||
/>
|
||||
<input type="submit" value="جستوجو" />
|
||||
</form>
|
||||
|
||||
<div id="semantic-search-status" style="text-align:center; color: var(--text-muted); display:none; margin-bottom: 16px;"></div>
|
||||
|
||||
<div id="semantic-search-results"></div>
|
||||
|
||||
</div>
|
||||
|
||||
@section Scripts {
|
||||
<script>
|
||||
(function () {
|
||||
"use strict";
|
||||
|
||||
var form = document.getElementById("semantic-search-form");
|
||||
var input = document.getElementById("semantic-query");
|
||||
var status = document.getElementById("semantic-search-status");
|
||||
var results = document.getElementById("semantic-search-results");
|
||||
|
||||
form.addEventListener("submit", function (e) {
|
||||
e.preventDefault();
|
||||
var query = input.value.trim();
|
||||
if (!query) return;
|
||||
runSearch(query);
|
||||
});
|
||||
|
||||
function runSearch(query) {
|
||||
results.innerHTML = "";
|
||||
status.style.display = "block";
|
||||
status.textContent = "در حال جستوجو...";
|
||||
|
||||
$.ajax({
|
||||
type: "POST",
|
||||
url: "?handler=Search",
|
||||
data: { query: query },
|
||||
success: function (response) {
|
||||
status.style.display = "none";
|
||||
renderResults(response);
|
||||
},
|
||||
error: function (xhr) {
|
||||
status.style.display = "block";
|
||||
status.textContent = xhr.status === 503
|
||||
? "در حال حاضر جستوجوی معنایی در دسترس نیست. لطفاً کمی بعد دوباره امتحان کنید."
|
||||
: "خطایی رخ داد. لطفاً دوباره تلاش کنید.";
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
function renderResults(response) {
|
||||
if (!response || !response.results || response.results.length === 0) {
|
||||
status.style.display = "block";
|
||||
status.textContent = "نتیجهای پیدا نشد.";
|
||||
return;
|
||||
}
|
||||
|
||||
var scopeNote = "";
|
||||
if (response.detectedPoetName || response.detectedCategoryName) {
|
||||
var scopeText = [response.detectedPoetName, response.detectedCategoryName].filter(Boolean).join(" » ");
|
||||
scopeNote = '<p style="text-align:center; color: var(--gold, var(--text-muted)); font-size:0.88rem; margin-bottom:16px;">نتایج محدود به: ' + escapeHtml(scopeText) + '</p>';
|
||||
}
|
||||
|
||||
var html = scopeNote;
|
||||
response.results.forEach(function (item) {
|
||||
html += '<div class="sitem poem-excerpt" style="margin-bottom:16px; padding:14px 16px; border:1px solid var(--border-gold); border-radius:10px;">';
|
||||
html += '<a href="' + escapeAttr(item.fullUrl) + '" style="font-weight:600; font-size:1.05rem;">' + escapeHtml(item.fullTitle || item.title) + '</a>';
|
||||
html += renderVerses(item.verses);
|
||||
html += '</div>';
|
||||
});
|
||||
results.innerHTML = html;
|
||||
}
|
||||
|
||||
// Same hemistich-pairing logic as mini-ganjoor's renderVerses (see
|
||||
// https://ganjoor.github.io/mini/) - right/left couplet pairs render side by side,
|
||||
// everything else (Comment, Paragraph, etc.) renders as a plain line. Reused as-is
|
||||
// rather than reinvented, since it's already proven correct there.
|
||||
function renderVerses(verses) {
|
||||
if (!verses || verses.length === 0) return "";
|
||||
|
||||
var out = [];
|
||||
var i = 0;
|
||||
while (i < verses.length) {
|
||||
var v = verses[i];
|
||||
var pos = v.position;
|
||||
if (pos === "Right" && verses[i + 1] && verses[i + 1].position === "Left") {
|
||||
out.push(
|
||||
'<div style="display:flex; justify-content:space-between; gap:16px; padding:4px 0;">' +
|
||||
'<div style="flex:1;">' + escapeHtml(v.text) + '</div>' +
|
||||
'<div style="flex:1;">' + escapeHtml(verses[i + 1].text) + '</div>' +
|
||||
'</div>'
|
||||
);
|
||||
i += 2;
|
||||
} else {
|
||||
out.push('<div style="padding:4px 0;">' + escapeHtml(v.text) + '</div>');
|
||||
i += 1;
|
||||
}
|
||||
}
|
||||
return '<div style="margin-top:10px; color: var(--text-muted); font-size:0.95rem; line-height:2;">' + out.join("") + '</div>';
|
||||
}
|
||||
|
||||
function escapeHtml(s) {
|
||||
if (s == null) return "";
|
||||
return String(s).replace(/[&<>"']/g, function (c) {
|
||||
return { "&": "&", "<": "<", ">": ">", '"': """, "'": "'" }[c];
|
||||
});
|
||||
}
|
||||
|
||||
function escapeAttr(s) {
|
||||
return escapeHtml(s);
|
||||
}
|
||||
})();
|
||||
</script>
|
||||
}
|
||||
69
GanjooRazor/Pages/SemanticSearch.cshtml.cs
Normal file
69
GanjooRazor/Pages/SemanticSearch.cshtml.cs
Normal file
@ -0,0 +1,69 @@
|
||||
using Microsoft.AspNetCore.Mvc;
|
||||
using Microsoft.AspNetCore.Mvc.RazorPages;
|
||||
using System;
|
||||
using System.Net;
|
||||
using System.Net.Http;
|
||||
using System.Text;
|
||||
using System.Threading.Tasks;
|
||||
|
||||
namespace GanjooRazor.Pages
|
||||
{
|
||||
/// <summary>
|
||||
/// "find a poem about..." chatbox. Deliberately calls APIRoot.SemanticSearchUrl, NOT
|
||||
/// APIRoot.Url/InternetUrl — see the comment on that property in APIRoot.cs. Currently a
|
||||
/// physically separate domain/app pool (ganjgah.ir) from api.ganjoor.net, so a problem with
|
||||
/// this one feature can't affect the main site or its API, a precaution proven necessary by
|
||||
/// an actual production incident during development, not a hypothetical one.
|
||||
/// </summary>
|
||||
[IgnoreAntiforgeryToken(Order = 1001)]
|
||||
public class SemanticSearchModel : PageModel
|
||||
{
|
||||
public void OnGet()
|
||||
{
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// proxies the query to the semantic search API and passes its JSON straight back to the
|
||||
/// page's own JS — no local re-modeling of the response, so there's no risk of a shape
|
||||
/// mismatch between the two sides silently dropping a field
|
||||
/// </summary>
|
||||
public async Task<IActionResult> OnPostSearchAsync(string query)
|
||||
{
|
||||
if (string.IsNullOrWhiteSpace(query))
|
||||
{
|
||||
return BadRequest("لطفاً عبارتی برای جستوجو وارد کنید.");
|
||||
}
|
||||
|
||||
using (HttpClient client = new HttpClient())
|
||||
{
|
||||
client.Timeout = TimeSpan.FromSeconds(30); // query-time ONNX inference is not instant - give it real room, but not unbounded
|
||||
|
||||
var requestBody = "{\"query\":" + System.Text.Json.JsonSerializer.Serialize(query) + ",\"topK\":10}";
|
||||
var content = new StringContent(requestBody, Encoding.UTF8, "application/json");
|
||||
|
||||
HttpResponseMessage response;
|
||||
try
|
||||
{
|
||||
response = await client.PostAsync($"{APIRoot.SemanticSearchUrl}/api/ganjoor/search/semantic", content);
|
||||
}
|
||||
catch (Exception)
|
||||
{
|
||||
// a network-level failure reaching the isolated semantic-search domain -
|
||||
// exactly the kind of failure this domain-level isolation is meant to
|
||||
// contain to just this feature, so report it plainly rather than let an
|
||||
// unhandled exception propagate
|
||||
return StatusCode((int)HttpStatusCode.ServiceUnavailable,
|
||||
"در حال حاضر جستوجوی معنایی در دسترس نیست. لطفاً کمی بعد دوباره امتحان کنید.");
|
||||
}
|
||||
|
||||
string responseBody = await response.Content.ReadAsStringAsync();
|
||||
if (!response.IsSuccessStatusCode)
|
||||
{
|
||||
return StatusCode((int)response.StatusCode, responseBody);
|
||||
}
|
||||
|
||||
return Content(responseBody, "application/json");
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
@ -10,6 +10,7 @@
|
||||
"TrackingScript": "<script src='https://cdn.kntr.ir/tracker.js'></script><script>kntrTracking.startTracking(\"2\", \"loggedon\");</script>",
|
||||
"APIRoot": "https://api.ganjoor.net",
|
||||
"GlobalAPIRoot": "https://api.ganjoor.net",
|
||||
"SemanticSearchAPIRoot": "https://ganjgah.ir",
|
||||
"SiteUrl": "http://localhost:33081",
|
||||
"MockSpotify": "False",
|
||||
"Spotify": {
|
||||
|
||||
@ -42,7 +42,7 @@ namespace RMuseum.Controllers
|
||||
{
|
||||
try
|
||||
{
|
||||
var result = await _semanticSearchService.SearchAsync(request.Query, request.TopK);
|
||||
var result = await _semanticSearchService.SearchAsync(request);
|
||||
return Ok(result);
|
||||
}
|
||||
catch (SemanticSearchUnavailableException exp)
|
||||
|
||||
@ -13,6 +13,28 @@ namespace RMuseum.Models.Ganjoor.SemanticSearch
|
||||
/// how many results to return; a sane default is applied server-side if omitted/invalid
|
||||
/// </summary>
|
||||
public int? TopK { get; set; }
|
||||
|
||||
/// <summary>
|
||||
/// Optional explicit scope, for a future UI (e.g. a poet picker) that wants to restrict
|
||||
/// search without relying on auto-detection from the query text. Auto-detection (see
|
||||
/// SemanticSearchService.DetectQueryScope) still runs even when these are set — an
|
||||
/// explicit PoetId/CatId narrows further, it doesn't replace detection.
|
||||
/// </summary>
|
||||
public int? PoetId { get; set; }
|
||||
public int? CatId { get; set; }
|
||||
}
|
||||
|
||||
public class SemanticSearchVerseDto
|
||||
{
|
||||
public int VOrder { get; set; }
|
||||
|
||||
/// <summary>
|
||||
/// matches GanjoorVerse.VersePosition.ToString() (e.g. "Right"/"Left") - same convention
|
||||
/// already used by the main ganjoor-data export, so any existing hemistich-pairing
|
||||
/// frontend code (see mini-ganjoor's renderVerses) can be reused as-is.
|
||||
/// </summary>
|
||||
public string Position { get; set; }
|
||||
public string Text { get; set; }
|
||||
}
|
||||
|
||||
public class SemanticSearchResultDto
|
||||
@ -23,6 +45,12 @@ namespace RMuseum.Models.Ganjoor.SemanticSearch
|
||||
public string FullUrl { get; set; }
|
||||
public string PoemSummary { get; set; }
|
||||
|
||||
/// <summary>
|
||||
/// A short preview from the poem's actual text - the first couple of couplets, in
|
||||
/// original verse order. Not the whole poem; just enough for a result card.
|
||||
/// </summary>
|
||||
public List<SemanticSearchVerseDto> Verses { get; set; } = new List<SemanticSearchVerseDto>();
|
||||
|
||||
/// <summary>
|
||||
/// cosine similarity, 0..1 for these normalized vectors (in practice results cluster in
|
||||
/// a narrower band - this is a relative ranking signal, not a calibrated probability)
|
||||
@ -34,5 +62,17 @@ namespace RMuseum.Models.Ganjoor.SemanticSearch
|
||||
{
|
||||
public string Query { get; set; }
|
||||
public List<SemanticSearchResultDto> Results { get; set; } = new List<SemanticSearchResultDto>();
|
||||
|
||||
/// <summary>
|
||||
/// If the query text was recognized as referring to a specific poet and/or
|
||||
/// category/book (e.g. "در کدام شعر حافظ" -> حافظ, "در کدام بخش شاهنامه" -> شاهنامه),
|
||||
/// search was restricted to that scope and these are populated so the UI can show the
|
||||
/// user what was detected ("نتایج محدود به: حافظ") rather than silently filtering.
|
||||
/// Null/null if nothing was detected (or an explicit PoetId/CatId narrowed things
|
||||
/// without any name being recognized in the free text).
|
||||
/// </summary>
|
||||
public string DetectedPoetName { get; set; }
|
||||
public string DetectedCategoryName { get; set; }
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@ -11,7 +11,7 @@ namespace RMuseum.Services.Implementation
|
||||
{
|
||||
public interface ISemanticSearchService
|
||||
{
|
||||
Task<SemanticSearchResponseDto> SearchAsync(string query, int? topK);
|
||||
Task<SemanticSearchResponseDto> SearchAsync(SemanticSearchRequestDto request);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
@ -29,23 +29,31 @@ namespace RMuseum.Services.Implementation
|
||||
/// being constructed, taking down every endpoint under /api/ganjoor with a 503, not just
|
||||
/// semantic search. This class must never let a resource-loading failure become an unhandled
|
||||
/// exception that propagates past SearchAsync — see the catch below.
|
||||
///
|
||||
/// LazyQueryScopeIndex is a SEPARATE lazy singleton from LazySemanticSearchResources on
|
||||
/// purpose — poet/category name detection ("در کدام شعر حافظ") is a genuinely independent
|
||||
/// concern from the embedding/model loading, with its own independent failure mode; a bug in
|
||||
/// one must not be able to disable the other.
|
||||
/// </summary>
|
||||
public class SemanticSearchService : ISemanticSearchService
|
||||
{
|
||||
private const int DefaultTopK = 10;
|
||||
private const int MaxTopK = 50;
|
||||
private const int PreviewVerseCount = 4; // ~2 couplets - enough for a result-card preview, not the whole poem
|
||||
|
||||
private readonly LazySemanticSearchResources _resources;
|
||||
private readonly LazyQueryScopeIndex _queryScopeIndex;
|
||||
|
||||
public SemanticSearchService(LazySemanticSearchResources resources)
|
||||
public SemanticSearchService(LazySemanticSearchResources resources, LazyQueryScopeIndex queryScopeIndex)
|
||||
{
|
||||
_resources = resources;
|
||||
_queryScopeIndex = queryScopeIndex;
|
||||
}
|
||||
|
||||
public async Task<SemanticSearchResponseDto> SearchAsync(string query, int? topK)
|
||||
public async Task<SemanticSearchResponseDto> SearchAsync(SemanticSearchRequestDto request)
|
||||
{
|
||||
if (string.IsNullOrWhiteSpace(query))
|
||||
throw new ArgumentException("query must not be empty", nameof(query));
|
||||
if (request == null || string.IsNullOrWhiteSpace(request.Query))
|
||||
throw new ArgumentException("query must not be empty", nameof(request));
|
||||
|
||||
if (!_resources.TryGetResources(out var embeddingIndex, out var queryEmbedder, out var error))
|
||||
{
|
||||
@ -53,20 +61,66 @@ namespace RMuseum.Services.Implementation
|
||||
"Semantic search is not available right now" + (error != null ? $": {error}" : "."));
|
||||
}
|
||||
|
||||
int k = topK.GetValueOrDefault(DefaultTopK);
|
||||
int k = request.TopK.GetValueOrDefault(DefaultTopK);
|
||||
if (k <= 0 || k > MaxTopK)
|
||||
k = DefaultTopK;
|
||||
|
||||
float[] queryVector = queryEmbedder.EmbedQuery(query);
|
||||
List<(int PoemId, float Score)> topMatches = embeddingIndex.FindTopSimilar(queryVector, k);
|
||||
|
||||
var response = new SemanticSearchResponseDto { Query = query };
|
||||
var response = new SemanticSearchResponseDto { Query = request.Query };
|
||||
|
||||
// a fresh, short-lived DbContext per call - same pattern GanjoorService's background
|
||||
// jobs already use throughout this codebase, since a scoped/request DbContext can't
|
||||
// be injected into this singleton service
|
||||
using (RMuseumDbContext context = new RMuseumDbContext(new DbContextOptions<RMuseumDbContext>()))
|
||||
{
|
||||
// An explicit request.PoetId/CatId (from a future UI, e.g. a poet picker) always
|
||||
// wins; auto-detection only fills in whatever wasn't already specified. A
|
||||
// detection failure here is swallowed on top of LazyQueryScopeIndex's own
|
||||
// try/catch, as a second safety net - this is a nice-to-have, never worth
|
||||
// failing the whole search over.
|
||||
int? scopePoetId = request.PoetId;
|
||||
int? scopeCatId = request.CatId;
|
||||
try
|
||||
{
|
||||
var scopeIndex = await _queryScopeIndex.TryGetIndexAsync(context);
|
||||
if (scopeIndex != null)
|
||||
{
|
||||
var detected = scopeIndex.DetectScope(request.Query);
|
||||
if (detected.HasAny)
|
||||
{
|
||||
scopePoetId = scopePoetId ?? detected.PoetId;
|
||||
scopeCatId = scopeCatId ?? detected.CatId;
|
||||
response.DetectedPoetName = detected.PoetName;
|
||||
response.DetectedCategoryName = detected.CategoryName;
|
||||
}
|
||||
}
|
||||
}
|
||||
catch (Exception)
|
||||
{
|
||||
// scope detection is best-effort only - fall through with no scope rather
|
||||
// than fail the search
|
||||
}
|
||||
|
||||
HashSet<int> allowedPoemIds = null;
|
||||
if (scopeCatId.HasValue)
|
||||
{
|
||||
var ids = await context.GanjoorPoems.AsNoTracking()
|
||||
.Where(p => p.CatId == scopeCatId.Value)
|
||||
.Select(p => p.Id)
|
||||
.ToListAsync();
|
||||
allowedPoemIds = new HashSet<int>(ids);
|
||||
}
|
||||
else if (scopePoetId.HasValue)
|
||||
{
|
||||
var ids = await context.GanjoorPoems.AsNoTracking()
|
||||
.Where(p => p.Cat.PoetId == scopePoetId.Value)
|
||||
.Select(p => p.Id)
|
||||
.ToListAsync();
|
||||
allowedPoemIds = new HashSet<int>(ids);
|
||||
}
|
||||
|
||||
float[] queryVector = queryEmbedder.EmbedQuery(request.Query);
|
||||
List<(int PoemId, float Score)> topMatches = embeddingIndex.FindTopSimilar(queryVector, k, allowedPoemIds);
|
||||
|
||||
var poemIds = topMatches.Select(m => m.PoemId).ToList();
|
||||
var poemsById = await context.GanjoorPoems.AsNoTracking()
|
||||
.Where(p => poemIds.Contains(p.Id))
|
||||
@ -77,18 +131,34 @@ namespace RMuseum.Services.Implementation
|
||||
// a poem id present in the embedding index but missing from the live DB
|
||||
// (e.g. deleted/unpublished since the embeddings were generated) is silently
|
||||
// skipped rather than failing the whole search for everyone else's results
|
||||
if (poemsById.TryGetValue(match.PoemId, out var poem))
|
||||
if (!poemsById.TryGetValue(match.PoemId, out var poem))
|
||||
continue;
|
||||
|
||||
// one small, bounded query per result (topK is capped at 50) rather than
|
||||
// loading every verse of every matched poem just to keep the first few - a
|
||||
// poem can have hundreds of verses, no reason to pull all of them over a
|
||||
// preview snippet
|
||||
var verses = await context.GanjoorVerses.AsNoTracking()
|
||||
.Where(v => v.PoemId == poem.Id)
|
||||
.OrderBy(v => v.VOrder)
|
||||
.Take(PreviewVerseCount)
|
||||
.ToListAsync();
|
||||
|
||||
response.Results.Add(new SemanticSearchResultDto
|
||||
{
|
||||
response.Results.Add(new SemanticSearchResultDto
|
||||
PoemId = poem.Id,
|
||||
Title = poem.Title,
|
||||
FullTitle = poem.FullTitle,
|
||||
FullUrl = poem.FullUrl,
|
||||
PoemSummary = poem.PoemSummary,
|
||||
Score = match.Score,
|
||||
Verses = verses.Select(v => new SemanticSearchVerseDto
|
||||
{
|
||||
PoemId = poem.Id,
|
||||
Title = poem.Title,
|
||||
FullTitle = poem.FullTitle,
|
||||
FullUrl = poem.FullUrl,
|
||||
PoemSummary = poem.PoemSummary,
|
||||
Score = match.Score,
|
||||
});
|
||||
}
|
||||
VOrder = v.VOrder,
|
||||
Position = v.VersePosition.ToString(),
|
||||
Text = v.Text,
|
||||
}).ToList(),
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@ -336,6 +336,11 @@ namespace RMuseum
|
||||
// never throws; a failure there disables semantic search only.
|
||||
services.AddSingleton<LazySemanticSearchResources>();
|
||||
|
||||
// Separate lazy singleton from LazySemanticSearchResources - poet/category name
|
||||
// detection ("در کدام شعر حافظ") is an independent concern with its own independent
|
||||
// failure mode; a bug in one must not be able to disable the other.
|
||||
services.AddSingleton<LazyQueryScopeIndex>();
|
||||
|
||||
services.AddSingleton<ISemanticSearchService, SemanticSearchService>();
|
||||
|
||||
|
||||
|
||||
@ -105,8 +105,14 @@ namespace RMuseum.Utils.SemanticSearch
|
||||
/// similarity. queryVector must already be the SAME dimension as this index and, for the
|
||||
/// score to mean what it claims (a true cosine similarity), should already be
|
||||
/// L2-normalized the same way the indexed vectors are — see QueryEmbedder.
|
||||
///
|
||||
/// If allowedPoemIds is provided (non-null), only poems in that set are eligible —
|
||||
/// everything else is scored as excluded and can never appear in the results, however
|
||||
/// similar it might be. Used for scoped search ("در کدام شعر حافظ" -> restrict to
|
||||
/// Hafez's poems) — still a full scan either way, just with cheap early-outs for
|
||||
/// excluded rows, since this corpus is small enough that a full scan is fast regardless.
|
||||
/// </summary>
|
||||
public List<(int PoemId, float Score)> FindTopSimilar(ReadOnlySpan<float> queryVector, int topK)
|
||||
public List<(int PoemId, float Score)> FindTopSimilar(ReadOnlySpan<float> queryVector, int topK, ISet<int> allowedPoemIds = null)
|
||||
{
|
||||
if (queryVector.Length != Metadata.Dimension)
|
||||
throw new ArgumentException(
|
||||
@ -121,6 +127,12 @@ namespace RMuseum.Utils.SemanticSearch
|
||||
// this is genuinely computing cosine similarity, not just a raw dot product
|
||||
for (int i = 0; i < count; i++)
|
||||
{
|
||||
if (allowedPoemIds != null && !allowedPoemIds.Contains(_poemIds[i]))
|
||||
{
|
||||
scores[i] = float.NegativeInfinity; // excluded from this search - will never sort into the results
|
||||
continue;
|
||||
}
|
||||
|
||||
float dot = 0f;
|
||||
int baseIdx = i * Metadata.Dimension;
|
||||
for (int d = 0; d < Metadata.Dimension; d++)
|
||||
@ -135,10 +147,13 @@ namespace RMuseum.Utils.SemanticSearch
|
||||
// ~130k floats is already comfortably fast (low tens of ms) and simpler to trust
|
||||
Array.Sort(indices, (a, b) => scores[b].CompareTo(scores[a]));
|
||||
|
||||
var results = new List<(int, float)>(Math.Min(topK, count));
|
||||
for (int i = 0; i < Math.Min(topK, count); i++)
|
||||
var results = new List<(int, float)>();
|
||||
for (int i = 0; i < count && results.Count < topK; i++)
|
||||
{
|
||||
results.Add((_poemIds[indices[i]], scores[indices[i]]));
|
||||
int idx = indices[i];
|
||||
if (float.IsNegativeInfinity(scores[idx]))
|
||||
break; // sorted descending - everything from here on is also excluded
|
||||
results.Add((_poemIds[idx], scores[idx]));
|
||||
}
|
||||
return results;
|
||||
}
|
||||
|
||||
74
RMuseum/Utils/SemanticSearch/LazyQueryScopeIndex.cs
Normal file
74
RMuseum/Utils/SemanticSearch/LazyQueryScopeIndex.cs
Normal file
@ -0,0 +1,74 @@
|
||||
using Microsoft.Extensions.Logging;
|
||||
using RMuseum.DbContext;
|
||||
using System;
|
||||
using System.Threading;
|
||||
using System.Threading.Tasks;
|
||||
|
||||
namespace RMuseum.Utils.SemanticSearch
|
||||
{
|
||||
/// <summary>
|
||||
/// Same "load once, never throw" pattern as LazySemanticSearchResources, but deliberately a
|
||||
/// SEPARATE singleton with its own independent failure domain: if the poet/category name
|
||||
/// lookup fails to load for any reason, that must only disable scope auto-detection
|
||||
/// ("در کدام شعر حافظ" -> unscoped, searches everything) — it must never affect the embedding
|
||||
/// index/model loading or plain (unscoped) search, which are a completely different concern.
|
||||
///
|
||||
/// Uses a SemaphoreSlim rather than a plain `lock`, since the actual load is async (DB
|
||||
/// queries) and you can't `await` inside a `lock` block.
|
||||
/// </summary>
|
||||
public class LazyQueryScopeIndex
|
||||
{
|
||||
private readonly SemaphoreSlim _semaphore = new SemaphoreSlim(1, 1);
|
||||
private QueryScopeIndex _index;
|
||||
private bool _attempted;
|
||||
private readonly ILogger<LazyQueryScopeIndex> _logger;
|
||||
|
||||
public LazyQueryScopeIndex(ILogger<LazyQueryScopeIndex> logger)
|
||||
{
|
||||
_logger = logger;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Loads (once) and returns the scope index, or null if loading failed or hasn't
|
||||
/// succeeded yet — NEVER throws. The passed-in context is only actually used on the
|
||||
/// first call that does real work; a context created by the caller for its own
|
||||
/// SearchAsync call is reused here rather than this class creating its own, since it's
|
||||
/// only needed for the one-time load.
|
||||
/// </summary>
|
||||
public async Task<QueryScopeIndex> TryGetIndexAsync(RMuseumDbContext context)
|
||||
{
|
||||
if (_attempted)
|
||||
return _index;
|
||||
|
||||
await _semaphore.WaitAsync();
|
||||
try
|
||||
{
|
||||
if (_attempted)
|
||||
return _index;
|
||||
|
||||
try
|
||||
{
|
||||
_index = await QueryScopeIndex.LoadAsync(context);
|
||||
_logger.LogInformation("Query scope index loaded.");
|
||||
}
|
||||
catch (Exception exp)
|
||||
{
|
||||
_index = null;
|
||||
_logger.LogError(exp,
|
||||
"Query scope index failed to load — poet/category auto-detection will be " +
|
||||
"unavailable, but this must not affect plain search.");
|
||||
}
|
||||
finally
|
||||
{
|
||||
_attempted = true;
|
||||
}
|
||||
}
|
||||
finally
|
||||
{
|
||||
_semaphore.Release();
|
||||
}
|
||||
|
||||
return _index;
|
||||
}
|
||||
}
|
||||
}
|
||||
137
RMuseum/Utils/SemanticSearch/QueryScopeIndex.cs
Normal file
137
RMuseum/Utils/SemanticSearch/QueryScopeIndex.cs
Normal file
@ -0,0 +1,137 @@
|
||||
using Microsoft.EntityFrameworkCore;
|
||||
using RMuseum.DbContext;
|
||||
using System.Collections.Generic;
|
||||
using System.Linq;
|
||||
using System.Threading.Tasks;
|
||||
|
||||
namespace RMuseum.Utils.SemanticSearch
|
||||
{
|
||||
public class PoetScopeEntry
|
||||
{
|
||||
public int PoetId { get; set; }
|
||||
public string Nickname { get; set; }
|
||||
}
|
||||
|
||||
public class CategoryScopeEntry
|
||||
{
|
||||
public int CatId { get; set; }
|
||||
public int PoetId { get; set; }
|
||||
public string Title { get; set; }
|
||||
}
|
||||
|
||||
public class DetectedScope
|
||||
{
|
||||
public int? PoetId { get; set; }
|
||||
public string PoetName { get; set; }
|
||||
public int? CatId { get; set; }
|
||||
public string CategoryName { get; set; }
|
||||
|
||||
public bool HasAny => PoetId.HasValue || CatId.HasValue;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Detects when a free-text query names a specific poet and/or book/collection (e.g.
|
||||
/// "در کدام شعر حافظ" -> حافظ, "در کدام بخش شاهنامه" -> شاهنامه) so search can be scoped to
|
||||
/// just that poet/category instead of the whole corpus.
|
||||
///
|
||||
/// Deliberately simple substring matching, not real NLP/NER — good enough for the common,
|
||||
/// unambiguous case (a poet's distinctive nickname, or a book title effectively unique to one
|
||||
/// poet, like شاهنامه), and safely conservative for the ambiguous case: a generic category
|
||||
/// title shared by many poets (غزلیات appears for most of them) is only used as a scope if a
|
||||
/// specific poet was ALSO named in the same query, narrowing which one is meant — otherwise
|
||||
/// it's dropped rather than guessing which poet's غزلیات the person meant.
|
||||
/// </summary>
|
||||
public class QueryScopeIndex
|
||||
{
|
||||
private readonly List<PoetScopeEntry> _poets;
|
||||
private readonly List<CategoryScopeEntry> _categories;
|
||||
|
||||
private QueryScopeIndex(List<PoetScopeEntry> poets, List<CategoryScopeEntry> categories)
|
||||
{
|
||||
_poets = poets;
|
||||
_categories = categories;
|
||||
}
|
||||
|
||||
public static async Task<QueryScopeIndex> LoadAsync(RMuseumDbContext context)
|
||||
{
|
||||
var poets = await context.GanjoorPoets.AsNoTracking()
|
||||
.Where(p => p.Published)
|
||||
.Select(p => new PoetScopeEntry { PoetId = p.Id, Nickname = p.Nickname })
|
||||
.ToListAsync();
|
||||
|
||||
// "book"/"collection" level = direct children of a poet's own root category
|
||||
// (شاهنامه, غزلیات, دیوان شمس, ...) - not deeper structural subsections, which would
|
||||
// add a lot of short, generic, easily-false-positive titles to match against.
|
||||
var rootCatIds = await context.GanjoorCategories.AsNoTracking()
|
||||
.Where(c => c.ParentId == null)
|
||||
.Select(c => c.Id)
|
||||
.ToListAsync();
|
||||
var rootCatIdSet = new HashSet<int>(rootCatIds);
|
||||
|
||||
var categories = await context.GanjoorCategories.AsNoTracking()
|
||||
.Where(c => c.ParentId != null)
|
||||
.Select(c => new { c.Id, c.ParentId, c.PoetId, c.Title })
|
||||
.ToListAsync();
|
||||
|
||||
var scopedCategories = categories
|
||||
.Where(c => rootCatIdSet.Contains(c.ParentId.Value))
|
||||
.Select(c => new CategoryScopeEntry { CatId = c.Id, PoetId = c.PoetId, Title = c.Title })
|
||||
.ToList();
|
||||
|
||||
return new QueryScopeIndex(poets, scopedCategories);
|
||||
}
|
||||
|
||||
public DetectedScope DetectScope(string query)
|
||||
{
|
||||
var result = new DetectedScope();
|
||||
if (string.IsNullOrWhiteSpace(query))
|
||||
return result;
|
||||
|
||||
PoetScopeEntry bestPoet = null;
|
||||
foreach (var poet in _poets)
|
||||
{
|
||||
if (string.IsNullOrEmpty(poet.Nickname))
|
||||
continue;
|
||||
if (query.Contains(poet.Nickname) && (bestPoet == null || poet.Nickname.Length > bestPoet.Nickname.Length))
|
||||
{
|
||||
bestPoet = poet;
|
||||
}
|
||||
}
|
||||
|
||||
string bestCategoryTitle = null;
|
||||
foreach (var cat in _categories)
|
||||
{
|
||||
if (string.IsNullOrEmpty(cat.Title))
|
||||
continue;
|
||||
if (query.Contains(cat.Title) && (bestCategoryTitle == null || cat.Title.Length > bestCategoryTitle.Length))
|
||||
{
|
||||
bestCategoryTitle = cat.Title;
|
||||
}
|
||||
}
|
||||
|
||||
CategoryScopeEntry bestCategory = null;
|
||||
if (bestCategoryTitle != null)
|
||||
{
|
||||
var matches = _categories.Where(c => c.Title == bestCategoryTitle).ToList();
|
||||
if (matches.Count == 1)
|
||||
{
|
||||
bestCategory = matches[0]; // unambiguous - only one category anywhere has this exact title
|
||||
}
|
||||
else if (bestPoet != null)
|
||||
{
|
||||
// ambiguous title (shared by multiple poets), but a specific poet was also
|
||||
// named - use that poet's version of it, if they have one
|
||||
bestCategory = matches.FirstOrDefault(c => c.PoetId == bestPoet.PoetId);
|
||||
}
|
||||
// else: ambiguous with no poet to disambiguate against - deliberately left null
|
||||
// rather than guessing which poet's version was meant
|
||||
}
|
||||
|
||||
result.PoetId = bestPoet?.PoetId;
|
||||
result.PoetName = bestPoet?.Nickname;
|
||||
result.CatId = bestCategory?.CatId;
|
||||
result.CategoryName = bestCategory?.Title;
|
||||
return result;
|
||||
}
|
||||
}
|
||||
}
|
||||
Loading…
Reference in New Issue
Block a user