diff --git a/RMuseum/Models/Ganjoor/SemanticSearch/SemanticSearchDtos.cs b/RMuseum/Models/Ganjoor/SemanticSearch/SemanticSearchDtos.cs new file mode 100644 index 00000000..927e27ff --- /dev/null +++ b/RMuseum/Models/Ganjoor/SemanticSearch/SemanticSearchDtos.cs @@ -0,0 +1,38 @@ +using System.Collections.Generic; + +namespace RMuseum.Models.Ganjoor.SemanticSearch +{ + /// + /// "find a poem about..." request + /// + public class SemanticSearchRequestDto + { + public string Query { get; set; } + + /// + /// how many results to return; a sane default is applied server-side if omitted/invalid + /// + public int? TopK { get; set; } + } + + public class SemanticSearchResultDto + { + public int PoemId { get; set; } + public string Title { get; set; } + public string FullTitle { get; set; } + public string FullUrl { get; set; } + public string PoemSummary { get; set; } + + /// + /// cosine similarity, 0..1 for these normalized vectors (in practice results cluster in + /// a narrower band - this is a relative ranking signal, not a calibrated probability) + /// + public float Score { get; set; } + } + + public class SemanticSearchResponseDto + { + public string Query { get; set; } + public List Results { get; set; } = new List(); + } +} diff --git a/RMuseum/RMuseum.csproj b/RMuseum/RMuseum.csproj index 31797486..4c6b345b 100644 --- a/RMuseum/RMuseum.csproj +++ b/RMuseum/RMuseum.csproj @@ -27,6 +27,8 @@ all runtime; build; native; contentfiles; analyzers; buildtransitive + + diff --git a/RMuseum/RMuseum.xml b/RMuseum/RMuseum.xml index 3b273e58..ebe582f9 100644 --- a/RMuseum/RMuseum.xml +++ b/RMuseum/RMuseum.xml @@ -11349,6 +11349,22 @@ stores nothing but the transliterated text and its order. + + + "find a poem about..." request + + + + + how many results to return; a sane default is applied server-side if omitted/invalid + + + + + cosine similarity, 0..1 for these normalized vectors (in practice results cluster in + a narrower band - this is a relative ranking signal, not a calibrated probability) + + Updating related sections logs @@ -23054,6 +23070,17 @@ + + + Deliberately NOT a GanjoorService partial, unlike everything else in this project. + EmbeddingIndex (~530MB in memory) and QueryEmbedder (a loaded ONNX model) both need to be + true singletons — constructed once at startup, never per-request — while GanjoorService + and RMuseumDbContext are scoped per-request throughout this codebase. Injecting a + singleton's dependencies into a per-request class (or vice versa) is a real DI lifetime + bug, not just an inconsistency, so this stays a separate service registered as a + singleton itself (see INTEGRATION.md for the exact registration). + + ganjoor.net banners service @@ -24386,6 +24413,84 @@ (404) is treated the same as "doesn't exist", not an error. + + + Loads the published embeddings.f32 (raw N x D float32, row-major, sorted by poem id + ascending per the export's own writer) + embeddings-index.json into memory once, and does + brute-force cosine similarity search against them. + + Brute force, not a vector database, is a deliberate choice at this scale — ~130k poems x + 1024 dimensions is ~530MB, comfortably held in RAM, and a full linear scan per query is + fast enough (low tens of milliseconds) that a dedicated vector index/database would be + solving a problem this corpus doesn't actually have. Revisit only if the corpus grows by + an order of magnitude or query volume becomes very high. + + Thread-safety: immutable after Load() — every field is only ever written once, during + loading, before this instance is published to any other thread (e.g. via DI singleton + registration). Safe for concurrent reads (searches) from multiple requests afterward. + + + + + Loads embeddings.f32 + embeddings-index.json from the given directory. Intended to be + called once at startup (e.g. from a DI factory) — this reads and holds ~530MB in + memory, not something to redo per-request. + + + + + Returns the topK poem ids most similar to queryVector, ranked descending by cosine + similarity. queryVector must already be the SAME dimension as this index and, for the + score to mean what it claims (a true cosine similarity), should already be + L2-normalized the same way the indexed vectors are — see QueryEmbedder. + + + + + Embeds a single user-typed search query using the SAME ONNX model + tokenizer as + scripts/generate_embeddings.py (the Python side that indexed the poem corpus) — this is + non-negotiable: a query embedded in a different space than the documents it's being + compared against produces meaningless similarity scores, silently (no error, just bad + results), not something that shows up as a crash. + + UNTESTED — flagging this more strongly than usual, because this file stacks two risks + that weren't present anywhere else in this project: + 1. This environment has no .NET SDK and no network access to onnxruntime/the model, so + none of this has actually been compiled or run. + 2. Even once it compiles, BpeTokenizer.Create(vocab, merges) builds a plain BPE + tokenizer from vocab.json + merges.txt — it may not capture every detail of Qwen's + full tokenizer.json spec (custom pre-tokenizer regex, added/special tokens). A + mismatch here wouldn't error either — it would silently tokenize text differently + than the Python `tokenizers` library did at index time, degrading search quality + without any visible failure. + See VERIFICATION.md for a paired Python/C# test to run BEFORE trusting search results — + do this before anything else in this phase. + + Query-time embedding is always exactly one query per call (a person typing into a search + box), never a batch — unlike the Python indexing script, which batched many poems + together and needed padding logic. That simplifies this class: no padding, no + batch-dimension complexity, last-token pooling reduces to simply the final sequence + position (no attention-mask lookup needed, since there's no padding to skip past). + + + + + Per Qwen3-Embedding's documented convention, QUERIES (unlike documents) benefit from + an instruction prefix. generate_embeddings.py deliberately does NOT add one to + documents (poem summaries) — this asymmetry is the model's own documented design, not + an inconsistency. If this template ever changes, or if it's ever added to/removed + from the document side, both sides must be updated together, or queries and documents + drift into subtly different embedding spaces. This exact wording hasn't been + benchmarked for Persian poetry specifically — reasonable to A/B test once the basic + pipeline is confirmed working (see VERIFICATION.md), not something to treat as final. + + + + + Returns an L2-normalized embedding for queryText, comparable via dot product against + EmbeddingIndex's vectors (which are normalized the same way). + + url diff --git a/RMuseum/Services/Implementation/SemanticSearchService.cs b/RMuseum/Services/Implementation/SemanticSearchService.cs new file mode 100644 index 00000000..346a8f02 --- /dev/null +++ b/RMuseum/Services/Implementation/SemanticSearchService.cs @@ -0,0 +1,87 @@ +using Microsoft.EntityFrameworkCore; +using RMuseum.DbContext; +using RMuseum.Models.Ganjoor.SemanticSearch; +using RMuseum.Utils.SemanticSearch; +using System; +using System.Collections.Generic; +using System.Linq; +using System.Threading.Tasks; + +namespace RMuseum.Services.Implementation +{ + public interface ISemanticSearchService + { + Task SearchAsync(string query, int? topK); + } + + /// + /// Deliberately NOT a GanjoorService partial, unlike everything else in this project. + /// EmbeddingIndex (~530MB in memory) and QueryEmbedder (a loaded ONNX model) both need to be + /// true singletons — constructed once at startup, never per-request — while GanjoorService + /// and RMuseumDbContext are scoped per-request throughout this codebase. Injecting a + /// singleton's dependencies into a per-request class (or vice versa) is a real DI lifetime + /// bug, not just an inconsistency, so this stays a separate service registered as a + /// singleton itself (see INTEGRATION.md for the exact registration). + /// + public class SemanticSearchService : ISemanticSearchService + { + private const int DefaultTopK = 10; + private const int MaxTopK = 50; + + private readonly EmbeddingIndex _embeddingIndex; + private readonly QueryEmbedder _queryEmbedder; + + public SemanticSearchService(EmbeddingIndex embeddingIndex, QueryEmbedder queryEmbedder) + { + _embeddingIndex = embeddingIndex; + _queryEmbedder = queryEmbedder; + } + + public async Task SearchAsync(string query, int? topK) + { + if (string.IsNullOrWhiteSpace(query)) + throw new ArgumentException("query must not be empty", nameof(query)); + + int k = topK.GetValueOrDefault(DefaultTopK); + if (k <= 0 || k > MaxTopK) + k = DefaultTopK; + + float[] queryVector = _queryEmbedder.EmbedQuery(query); + List<(int PoemId, float Score)> topMatches = _embeddingIndex.FindTopSimilar(queryVector, k); + + var response = new SemanticSearchResponseDto { Query = query }; + + // a fresh, short-lived DbContext per call - same pattern GanjoorService's background + // jobs already use throughout this codebase, since a scoped/request DbContext can't + // be injected into this singleton service + using (RMuseumDbContext context = new RMuseumDbContext(new DbContextOptions())) + { + var poemIds = topMatches.Select(m => m.PoemId).ToList(); + var poemsById = await context.GanjoorPoems.AsNoTracking() + .Where(p => poemIds.Contains(p.Id)) + .ToDictionaryAsync(p => p.Id); + + foreach (var match in topMatches) + { + // a poem id present in the embedding index but missing from the live DB + // (e.g. deleted/unpublished since the embeddings were generated) is silently + // skipped rather than failing the whole search for everyone else's results + if (poemsById.TryGetValue(match.PoemId, out var poem)) + { + response.Results.Add(new SemanticSearchResultDto + { + PoemId = poem.Id, + Title = poem.Title, + FullTitle = poem.FullTitle, + FullUrl = poem.FullUrl, + PoemSummary = poem.PoemSummary, + Score = match.Score, + }); + } + } + } + + return response; + } + } +} diff --git a/RMuseum/Utils/SemanticSearch/EmbeddingIndex.cs b/RMuseum/Utils/SemanticSearch/EmbeddingIndex.cs new file mode 100644 index 00000000..3ec8e792 --- /dev/null +++ b/RMuseum/Utils/SemanticSearch/EmbeddingIndex.cs @@ -0,0 +1,146 @@ +using System; +using System.Collections.Generic; +using System.IO; +using System.Linq; +using System.Text.Json; + +namespace RMuseum.Utils.SemanticSearch +{ + public class EmbeddingIndexMetadata + { + public string Model { get; set; } + public int Dimension { get; set; } + public string PoolingMethod { get; set; } + public bool Normalized { get; set; } + public string GeneratedAtUtc { get; set; } + public int Count { get; set; } + public int[] Ids { get; set; } + } + + /// + /// Loads the published embeddings.f32 (raw N x D float32, row-major, sorted by poem id + /// ascending per the export's own writer) + embeddings-index.json into memory once, and does + /// brute-force cosine similarity search against them. + /// + /// Brute force, not a vector database, is a deliberate choice at this scale — ~130k poems x + /// 1024 dimensions is ~530MB, comfortably held in RAM, and a full linear scan per query is + /// fast enough (low tens of milliseconds) that a dedicated vector index/database would be + /// solving a problem this corpus doesn't actually have. Revisit only if the corpus grows by + /// an order of magnitude or query volume becomes very high. + /// + /// Thread-safety: immutable after Load() — every field is only ever written once, during + /// loading, before this instance is published to any other thread (e.g. via DI singleton + /// registration). Safe for concurrent reads (searches) from multiple requests afterward. + /// + public class EmbeddingIndex + { + private readonly float[] _vectors; // flat, row-major: vectors[id_index * Dimension + d] + private readonly int[] _poemIds; // poemIds[id_index] is the poem id for that row + public EmbeddingIndexMetadata Metadata { get; } + + private EmbeddingIndex(float[] vectors, int[] poemIds, EmbeddingIndexMetadata metadata) + { + _vectors = vectors; + _poemIds = poemIds; + Metadata = metadata; + } + + /// + /// Loads embeddings.f32 + embeddings-index.json from the given directory. Intended to be + /// called once at startup (e.g. from a DI factory) — this reads and holds ~530MB in + /// memory, not something to redo per-request. + /// + public static EmbeddingIndex Load(string embeddingsDirectory) + { + string indexPath = Path.Combine(embeddingsDirectory, "embeddings-index.json"); + string binPath = Path.Combine(embeddingsDirectory, "embeddings.f32"); + + if (!File.Exists(indexPath)) + throw new FileNotFoundException($"embeddings-index.json not found in '{embeddingsDirectory}'", indexPath); + if (!File.Exists(binPath)) + throw new FileNotFoundException($"embeddings.f32 not found in '{embeddingsDirectory}'", binPath); + + var metadata = JsonSerializer.Deserialize( + File.ReadAllText(indexPath), + new JsonSerializerOptions { PropertyNameCaseInsensitive = true }); + + if (metadata == null || metadata.Count <= 0 || metadata.Dimension <= 0) + throw new InvalidDataException($"embeddings-index.json in '{embeddingsDirectory}' is missing count/dimension"); + if (metadata.Ids == null || metadata.Ids.Length != metadata.Count) + throw new InvalidDataException( + $"embeddings-index.json declares count={metadata.Count} but ids array has " + + $"{metadata.Ids?.Length ?? 0} entries — these must match"); + + long expectedBytes = (long)metadata.Count * metadata.Dimension * sizeof(float); + long actualBytes = new FileInfo(binPath).Length; + if (actualBytes != expectedBytes) + throw new InvalidDataException( + $"embeddings.f32 in '{embeddingsDirectory}' is {actualBytes} bytes, expected " + + $"{expectedBytes} (count={metadata.Count} x dimension={metadata.Dimension} x 4 bytes). " + + "The index and binary file don't agree — do not trust this data until re-verified " + + "(see verify_embeddings.py in the ganjoor-embeddings tooling)."); + + var vectors = new float[metadata.Count * metadata.Dimension]; + using (var stream = File.OpenRead(binPath)) + using (var reader = new BinaryReader(stream)) + { + var buffer = new byte[vectors.Length * sizeof(float)]; + int totalRead = 0; + while (totalRead < buffer.Length) + { + int read = reader.Read(buffer, totalRead, buffer.Length - totalRead); + if (read == 0) break; + totalRead += read; + } + if (totalRead != buffer.Length) + throw new InvalidDataException($"expected to read {buffer.Length} bytes from embeddings.f32, got {totalRead}"); + Buffer.BlockCopy(buffer, 0, vectors, 0, buffer.Length); + } + + return new EmbeddingIndex(vectors, metadata.Ids, metadata); + } + + /// + /// Returns the topK poem ids most similar to queryVector, ranked descending by cosine + /// similarity. queryVector must already be the SAME dimension as this index and, for the + /// score to mean what it claims (a true cosine similarity), should already be + /// L2-normalized the same way the indexed vectors are — see QueryEmbedder. + /// + public List<(int PoemId, float Score)> FindTopSimilar(ReadOnlySpan queryVector, int topK) + { + if (queryVector.Length != Metadata.Dimension) + throw new ArgumentException( + $"query vector has {queryVector.Length} dimensions, index has {Metadata.Dimension} — " + + "these must match (are the query and index using the same model?)"); + + int count = _poemIds.Length; + var scores = new float[count]; + + // dot product of two unit vectors == cosine similarity - the vectors were already + // L2-normalized at export time (see embeddings-index.json's "normalized": true), so + // this is genuinely computing cosine similarity, not just a raw dot product + for (int i = 0; i < count; i++) + { + float dot = 0f; + int baseIdx = i * Metadata.Dimension; + for (int d = 0; d < Metadata.Dimension; d++) + { + dot += _vectors[baseIdx + d] * queryVector[d]; + } + scores[i] = dot; + } + + var indices = Enumerable.Range(0, count).ToArray(); + // partial sort would be a fair perf optimization at larger scale; a full sort of + // ~130k floats is already comfortably fast (low tens of ms) and simpler to trust + Array.Sort(indices, (a, b) => scores[b].CompareTo(scores[a])); + + var results = new List<(int, float)>(Math.Min(topK, count)); + for (int i = 0; i < Math.Min(topK, count); i++) + { + results.Add((_poemIds[indices[i]], scores[indices[i]])); + } + return results; + } + } +} diff --git a/RMuseum/Utils/SemanticSearch/QueryEmbedder.cs b/RMuseum/Utils/SemanticSearch/QueryEmbedder.cs new file mode 100644 index 00000000..6a08067d --- /dev/null +++ b/RMuseum/Utils/SemanticSearch/QueryEmbedder.cs @@ -0,0 +1,159 @@ +using Microsoft.ML.OnnxRuntime; +using Microsoft.ML.OnnxRuntime.Tensors; +using Microsoft.ML.Tokenizers; +using System; +using System.Collections.Generic; +using System.IO; +using System.Linq; + +namespace RMuseum.Utils.SemanticSearch +{ + /// + /// Embeds a single user-typed search query using the SAME ONNX model + tokenizer as + /// scripts/generate_embeddings.py (the Python side that indexed the poem corpus) — this is + /// non-negotiable: a query embedded in a different space than the documents it's being + /// compared against produces meaningless similarity scores, silently (no error, just bad + /// results), not something that shows up as a crash. + /// + /// UNTESTED — flagging this more strongly than usual, because this file stacks two risks + /// that weren't present anywhere else in this project: + /// 1. This environment has no .NET SDK and no network access to onnxruntime/the model, so + /// none of this has actually been compiled or run. + /// 2. Even once it compiles, BpeTokenizer.Create(vocab, merges) builds a plain BPE + /// tokenizer from vocab.json + merges.txt — it may not capture every detail of Qwen's + /// full tokenizer.json spec (custom pre-tokenizer regex, added/special tokens). A + /// mismatch here wouldn't error either — it would silently tokenize text differently + /// than the Python `tokenizers` library did at index time, degrading search quality + /// without any visible failure. + /// See VERIFICATION.md for a paired Python/C# test to run BEFORE trusting search results — + /// do this before anything else in this phase. + /// + /// Query-time embedding is always exactly one query per call (a person typing into a search + /// box), never a batch — unlike the Python indexing script, which batched many poems + /// together and needed padding logic. That simplifies this class: no padding, no + /// batch-dimension complexity, last-token pooling reduces to simply the final sequence + /// position (no attention-mask lookup needed, since there's no padding to skip past). + /// + public class QueryEmbedder : IDisposable + { + // Confirmed via `python3 scripts/generate_embeddings.py --inspect-only` against the real + // Qwen/Qwen3-Embedding-0.6B ONNX export (onnx-community/Qwen3-Embedding-0.6B-ONNX), + // 2026-09-05. If the model is ever swapped for a different export/architecture, these + // MUST be re-confirmed the same way (re-run --inspect-only, read off the real shapes) — + // hardcoding them here (rather than trying to introspect symbolic dimension names from + // .NET's ONNX Runtime API, whose support for that is less certain than Python's) is a + // deliberate tradeoff: reliable given what we've already confirmed, but silently wrong + // if the model changes without updating these. + private const int NumLayers = 28; + private const int NumKvHeads = 8; + private const int HeadDim = 128; + + /// + /// Per Qwen3-Embedding's documented convention, QUERIES (unlike documents) benefit from + /// an instruction prefix. generate_embeddings.py deliberately does NOT add one to + /// documents (poem summaries) — this asymmetry is the model's own documented design, not + /// an inconsistency. If this template ever changes, or if it's ever added to/removed + /// from the document side, both sides must be updated together, or queries and documents + /// drift into subtly different embedding spaces. This exact wording hasn't been + /// benchmarked for Persian poetry specifically — reasonable to A/B test once the basic + /// pipeline is confirmed working (see VERIFICATION.md), not something to treat as final. + /// + private const string InstructionTemplate = + "Instruct: Given a search query, retrieve Persian poems whose meaning matches it\nQuery: {0}"; + + private readonly InferenceSession _session; + private readonly Tokenizer _tokenizer; + private readonly int _dimension; + + public QueryEmbedder(string modelPath, string vocabPath, string mergesPath, int dimension = 1024) + { + _session = new InferenceSession(modelPath); + + using (var vocabStream = File.OpenRead(vocabPath)) + using (var mergesStream = File.OpenRead(mergesPath)) + { + _tokenizer = BpeTokenizer.Create(vocabStream, mergesStream); + } + + _dimension = dimension; + } + + /// + /// Returns an L2-normalized embedding for queryText, comparable via dot product against + /// EmbeddingIndex's vectors (which are normalized the same way). + /// + public float[] EmbedQuery(string queryText) + { + string instructedText = string.Format(InstructionTemplate, queryText); + + IReadOnlyList tokenIds = _tokenizer.EncodeToIds(instructedText); + int seqLen = tokenIds.Count; + if (seqLen == 0) + throw new ArgumentException("query tokenized to zero tokens — empty or whitespace-only query?", nameof(queryText)); + + var inputIds = new DenseTensor(new[] { 1, seqLen }); + var attentionMask = new DenseTensor(new[] { 1, seqLen }); + var positionIds = new DenseTensor(new[] { 1, seqLen }); + for (int i = 0; i < seqLen; i++) + { + inputIds[0, i] = tokenIds[i]; + attentionMask[0, i] = 1; + positionIds[0, i] = i; + } + + var inputs = new List + { + NamedOnnxValue.CreateFromTensor("input_ids", inputIds), + NamedOnnxValue.CreateFromTensor("attention_mask", attentionMask), + NamedOnnxValue.CreateFromTensor("position_ids", positionIds), + }; + + // empty (zero-length) KV cache for every layer - a single uncached forward pass over + // the whole query, same reasoning as build_extra_inputs() in generate_embeddings.py + for (int layer = 0; layer < NumLayers; layer++) + { + var emptyKey = new DenseTensor(new[] { 1, NumKvHeads, 0, HeadDim }); + var emptyValue = new DenseTensor(new[] { 1, NumKvHeads, 0, HeadDim }); + inputs.Add(NamedOnnxValue.CreateFromTensor($"past_key_values.{layer}.key", emptyKey)); + inputs.Add(NamedOnnxValue.CreateFromTensor($"past_key_values.{layer}.value", emptyValue)); + } + + using (var outputs = _session.Run(inputs, new[] { "last_hidden_state" })) + { + var hiddenStates = outputs.First(o => o.Name == "last_hidden_state").AsTensor(); + // hiddenStates shape: (1, seqLen, _dimension) + + // last-token pooling: batch size 1, no padding, so the last real token is simply + // the final sequence position - see the class docstring for why this doesn't + // need the attention-mask lookup the Python (batched, padded) version does + var pooled = new float[_dimension]; + for (int d = 0; d < _dimension; d++) + { + pooled[d] = hiddenStates[0, seqLen - 1, d]; + } + + return L2Normalize(pooled); + } + } + + private static float[] L2Normalize(float[] vector) + { + double sumSquares = 0; + foreach (var v in vector) sumSquares += (double)v * v; + double norm = Math.Sqrt(sumSquares); + if (norm < 1e-12) norm = 1e-12; // guard against a degenerate all-zero vector + + var result = new float[vector.Length]; + for (int i = 0; i < vector.Length; i++) + { + result[i] = (float)(vector[i] / norm); + } + return result; + } + + public void Dispose() + { + _session?.Dispose(); + } + } +}