semantic search improvemenst

RMuseum/Services/Implementation/SemanticSearchService.cs

@@ -42,6 +42,8 @@ namespace RMuseum.Services.Implementation
         private const int MaxTopK = 50;
         private const int PreviewVerseCount = 4; // ~2 couplets - enough for a result-card preview, not the whole poem
         private const int MaxVersesToScanForRelevance = 200; // bounded prefix to search for a relevant couplet within
+        private const int CandidatePoolMultiplier = 3; // how much larger than k the pre-re-rank candidate pool is
+        private const string AiGeneratedSummaryPrefix = "هوش مصنوعی:";
 
         private readonly LazySemanticSearchResources _resources;
         private readonly LazyQueryScopeIndex _queryScopeIndex;
@@ -136,13 +138,41 @@ namespace RMuseum.Services.Implementation
                 }
 
                 float[] queryVector = queryEmbedder.EmbedQuery(request.Query);
-                List<(int PoemId, float Score)> topMatches = embeddingIndex.FindTopSimilar(queryVector, k, allowedPoemIds);
 
-                var poemIds = topMatches.Select(m => m.PoemId).ToList();
+                // Fetch a larger candidate pool than the final k, so there's room for the
+                // AI-summary re-ranking below to actually change the outcome — re-ranking a pool
+                // that's already exactly k results wide can only reorder them, never let a
+                // human-reviewed poem that was originally ranked k+1 overtake one that made the
+                // cut only because its (unreviewed) summary happened to score marginally higher.
+                int candidatePoolSize = Math.Min(k * CandidatePoolMultiplier, MaxTopK * CandidatePoolMultiplier);
+                List<(int PoemId, float Score)> candidates = embeddingIndex.FindTopSimilar(queryVector, candidatePoolSize, allowedPoemIds);
+
+                var candidateIds = candidates.Select(c => c.PoemId).ToList();
                 var poemsById = await context.GanjoorPoems.AsNoTracking()
-                                        .Where(p => poemIds.Contains(p.Id))
+                                        .Where(p => candidateIds.Contains(p.Id))
                                         .ToDictionaryAsync(p => p.Id);
 
+                // Gentle re-rank: a poem whose summary is still AI-generated and un-reviewed
+                // (still carries ganjoor-data's own "هوش مصنوعی:" prefix - removing it is part of
+                // that project's human-review/edit workflow) gets a small score penalty before
+                // final sorting. The DISPLAYED score stays the true, unpenalized cosine
+                // similarity (Score below uses c.Score, not AdjustedScore) - the penalty is
+                // purely an internal ordering nudge, not something that should make the shown
+                // similarity number stop meaning what it says.
+                float aiPenalty = _resources.AiSummaryScorePenalty;
+                List<(int PoemId, float Score)> topMatches = candidates
+                    .Where(c => poemsById.ContainsKey(c.PoemId))
+                    .Select(c => new
+                    {
+                        c.PoemId,
+                        c.Score,
+                        AdjustedScore = IsAiGeneratedSummary(poemsById[c.PoemId].PoemSummary) ? c.Score * aiPenalty : c.Score,
+                    })
+                    .OrderByDescending(x => x.AdjustedScore)
+                    .Take(k)
+                    .Select(x => (x.PoemId, x.Score))
+                    .ToList();
+
                 // computed once for the whole request, not per result - the query doesn't change
                 // between results
                 var queryKeywords = ExtractKeywords(request.Query);
@@ -191,6 +221,18 @@ namespace RMuseum.Services.Implementation
             return response;
         }
 
+        /// <summary>
+        /// ganjoor-data's own editing workflow requires this exact prefix be removed once a
+        /// human has reviewed/edited a poem's summary — its continued presence is a direct,
+        /// already-existing signal for "not yet human-reviewed," not something this project
+        /// invented or has to infer.
+        /// </summary>
+        private static bool IsAiGeneratedSummary(string poemSummary)
+        {
+            return !string.IsNullOrEmpty(poemSummary) &&
+                   poemSummary.StartsWith(AiGeneratedSummaryPrefix, StringComparison.Ordinal);
+        }
+
         /// <summary>
         /// Common Persian function words stripped out before keyword-matching a query against
         /// verse text (see SelectPreviewVerses) — the kind of words that appear in nearly every

RMuseum/Utils/SemanticSearch/LazySemanticSearchResources.cs

@@ -48,6 +48,20 @@ namespace RMuseum.Utils.SemanticSearch
         private readonly int _dimension;
         private readonly ILogger<LazySemanticSearchResources> _logger;
 
+        /// <summary>
+        /// A gentle re-ranking multiplier applied to results whose PoemSummary is still
+        /// AI-generated and un-reviewed (detected by the "هوش مصنوعی:" prefix ganjoor-data's own
+        /// editing workflow requires removing once a human has reviewed/edited a summary — see
+        /// SemanticSearchService for how this is actually applied). A soft nudge, not a filter:
+        /// ~95% of summaries currently carry this prefix, so excluding them outright would gut
+        /// coverage for most queries. Configurable (SemanticSearch:AiSummaryScorePenalty) rather
+        /// than hardcoded, since the right effect size here is a judgment call worth tuning
+        /// without a redeploy. Read here (not directly in SemanticSearchService) purely to reuse
+        /// the config-reading this class already does — this value has nothing to do with the
+        /// lazy-loaded embeddings/model themselves and is available even when Enabled is false.
+        /// </summary>
+        public float AiSummaryScorePenalty { get; }
+
         public LazySemanticSearchResources(IConfiguration configuration, ILogger<LazySemanticSearchResources> logger)
         {
             _enabled = string.Equals(configuration["SemanticSearch:Enabled"], "true", StringComparison.OrdinalIgnoreCase);
@@ -56,6 +70,7 @@ namespace RMuseum.Utils.SemanticSearch
             _vocabPath = configuration["SemanticSearch:VocabPath"];
             _mergesPath = configuration["SemanticSearch:MergesPath"];
             _dimension = int.TryParse(configuration["SemanticSearch:Dimension"], out var d) ? d : 1024;
+            AiSummaryScorePenalty = float.TryParse(configuration["SemanticSearch:AiSummaryScorePenalty"], out var p) ? p : 0.97f;
             _logger = logger;
 
             if (!_enabled)

RMuseum/appsettings.json

@@ -120,7 +120,8 @@
     "ModelPath": "C:\\ganjoor-model\\onnx\\model_quantized.onnx",
     "VocabPath": "C:\\ganjoor-model\\vocab.json",
     "MergesPath": "C:\\ganjoor-model\\merges.txt",
-    "Dimension": "1024"
+    "Dimension": "1024",
+    "AiSummaryScorePenalty": "0.97"
   },
   "WebServiceUrl": "http://localhost:3439",
   "ReadOnlyMode": "False",