anas/divan
1
0
Fork 0
Code Issues 12 Pull Requests Actions Packages Projects Releases Wiki Activity

semantic search

Browse Source
This commit is contained in:
Hamid Reza Mohammadi 2026-09-09 11:07:09 +03:30
parent 6255d4d8ca
commit 0762f19fb4
6 changed files with 537 additions and 0 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

38
RMuseum/Models/Ganjoor/SemanticSearch/SemanticSearchDtos.cs Normal file
View File

@ -0,0 +1,38 @@
using System.Collections.Generic;
namespace RMuseum.Models.Ganjoor.SemanticSearch
{
/// <summary>
/// "find a poem about..." request
/// </summary>
public class SemanticSearchRequestDto
{
public string Query { get; set; }
/// <summary>
/// how many results to return; a sane default is applied server-side if omitted/invalid
/// </summary>
public int? TopK { get; set; }
}
public class SemanticSearchResultDto
{
public int PoemId { get; set; }
public string Title { get; set; }
public string FullTitle { get; set; }
public string FullUrl { get; set; }
public string PoemSummary { get; set; }
/// <summary>
/// cosine similarity, 0..1 for these normalized vectors (in practice results cluster in
/// a narrower band - this is a relative ranking signal, not a calibrated probability)
/// </summary>
public float Score { get; set; }
}
public class SemanticSearchResponseDto
{
public string Query { get; set; }
public List<SemanticSearchResultDto> Results { get; set; } = new List<SemanticSearchResultDto>();
}
}

2
RMuseum/RMuseum.csproj
View File

@ -27,6 +27,8 @@
<PrivateAssets>all</PrivateAssets>
<IncludeAssets>runtime; build; native; contentfiles; analyzers; buildtransitive</IncludeAssets>
</PackageReference>
<PackageReference Include="Microsoft.ML.OnnxRuntime" Version="1.29.0" />
<PackageReference Include="Microsoft.ML.Tokenizers" Version="2.0.0" />
<PackageReference Include="RSecurityBackend" Version="1.7.8" />
<PackageReference Include="Swashbuckle.AspNetCore" Version="10.2.3" />
<PackageReference Include="Swashbuckle.AspNetCore.Annotations" Version="10.2.3" />

105
RMuseum/RMuseum.xml
View File

@ -11349,6 +11349,22 @@
stores nothing but the transliterated text and its order.
</summary>
</member>
<member name="T:RMuseum.Models.Ganjoor.SemanticSearch.SemanticSearchRequestDto">
<summary>
"find a poem about..." request
</summary>
</member>
<member name="P:RMuseum.Models.Ganjoor.SemanticSearch.SemanticSearchRequestDto.TopK">
<summary>
how many results to return; a sane default is applied server-side if omitted/invalid
</summary>
</member>
<member name="P:RMuseum.Models.Ganjoor.SemanticSearch.SemanticSearchResultDto.Score">
<summary>
cosine similarity, 0..1 for these normalized vectors (in practice results cluster in
a narrower band - this is a relative ranking signal, not a calibrated probability)
</summary>
</member>
<member name="T:RMuseum.Models.Ganjoor.UpdatingRelSectsLog">
<summary>
Updating related sections logs
@ -23054,6 +23070,17 @@
</summary>
<param name="roleManager"></param>
</member>
<member name="T:RMuseum.Services.Implementation.SemanticSearchService">
<summary>
Deliberately NOT a GanjoorService partial, unlike everything else in this project.
EmbeddingIndex (~530MB in memory) and QueryEmbedder (a loaded ONNX model) both need to be
true singletons — constructed once at startup, never per-request — while GanjoorService
and RMuseumDbContext are scoped per-request throughout this codebase. Injecting a
singleton's dependencies into a per-request class (or vice versa) is a real DI lifetime
bug, not just an inconsistency, so this stays a separate service registered as a
singleton itself (see INTEGRATION.md for the exact registration).
</summary>
</member>
<member name="T:RMuseum.Services.Implementation.SiteBannersService">
<summary>
ganjoor.net banners service
@ -24386,6 +24413,84 @@
(404) is treated the same as "doesn't exist", not an error.
</summary>
</member>
<member name="T:RMuseum.Utils.SemanticSearch.EmbeddingIndex">
<summary>
Loads the published embeddings.f32 (raw N x D float32, row-major, sorted by poem id
ascending per the export's own writer) + embeddings-index.json into memory once, and does
brute-force cosine similarity search against them.
Brute force, not a vector database, is a deliberate choice at this scale — ~130k poems x
1024 dimensions is ~530MB, comfortably held in RAM, and a full linear scan per query is
fast enough (low tens of milliseconds) that a dedicated vector index/database would be
solving a problem this corpus doesn't actually have. Revisit only if the corpus grows by
an order of magnitude or query volume becomes very high.
Thread-safety: immutable after Load() — every field is only ever written once, during
loading, before this instance is published to any other thread (e.g. via DI singleton
registration). Safe for concurrent reads (searches) from multiple requests afterward.
</summary>
</member>
<member name="M:RMuseum.Utils.SemanticSearch.EmbeddingIndex.Load(System.String)">
<summary>
Loads embeddings.f32 + embeddings-index.json from the given directory. Intended to be
called once at startup (e.g. from a DI factory) — this reads and holds ~530MB in
memory, not something to redo per-request.
</summary>
</member>
<member name="M:RMuseum.Utils.SemanticSearch.EmbeddingIndex.FindTopSimilar(System.ReadOnlySpan{System.Single},System.Int32)">
<summary>
Returns the topK poem ids most similar to queryVector, ranked descending by cosine
similarity. queryVector must already be the SAME dimension as this index and, for the
score to mean what it claims (a true cosine similarity), should already be
L2-normalized the same way the indexed vectors are — see QueryEmbedder.
</summary>
</member>
<member name="T:RMuseum.Utils.SemanticSearch.QueryEmbedder">
<summary>
Embeds a single user-typed search query using the SAME ONNX model + tokenizer as
scripts/generate_embeddings.py (the Python side that indexed the poem corpus) — this is
non-negotiable: a query embedded in a different space than the documents it's being
compared against produces meaningless similarity scores, silently (no error, just bad
results), not something that shows up as a crash.
UNTESTED — flagging this more strongly than usual, because this file stacks two risks
that weren't present anywhere else in this project:
1. This environment has no .NET SDK and no network access to onnxruntime/the model, so
none of this has actually been compiled or run.
2. Even once it compiles, BpeTokenizer.Create(vocab, merges) builds a plain BPE
tokenizer from vocab.json + merges.txt — it may not capture every detail of Qwen's
full tokenizer.json spec (custom pre-tokenizer regex, added/special tokens). A
mismatch here wouldn't error either — it would silently tokenize text differently
than the Python `tokenizers` library did at index time, degrading search quality
without any visible failure.
See VERIFICATION.md for a paired Python/C# test to run BEFORE trusting search results —
do this before anything else in this phase.
Query-time embedding is always exactly one query per call (a person typing into a search
box), never a batch — unlike the Python indexing script, which batched many poems
together and needed padding logic. That simplifies this class: no padding, no
batch-dimension complexity, last-token pooling reduces to simply the final sequence
position (no attention-mask lookup needed, since there's no padding to skip past).
</summary>
</member>
<member name="F:RMuseum.Utils.SemanticSearch.QueryEmbedder.InstructionTemplate">
<summary>
Per Qwen3-Embedding's documented convention, QUERIES (unlike documents) benefit from
an instruction prefix. generate_embeddings.py deliberately does NOT add one to
documents (poem summaries) — this asymmetry is the model's own documented design, not
an inconsistency. If this template ever changes, or if it's ever added to/removed
from the document side, both sides must be updated together, or queries and documents
drift into subtly different embedding spaces. This exact wording hasn't been
benchmarked for Persian poetry specifically — reasonable to A/B test once the basic
pipeline is confirmed working (see VERIFICATION.md), not something to treat as final.
</summary>
</member>
<member name="M:RMuseum.Utils.SemanticSearch.QueryEmbedder.EmbedQuery(System.String)">
<summary>
Returns an L2-normalized embedding for queryText, comparable via dot product against
EmbeddingIndex's vectors (which are normalized the same way).
</summary>
</member>
<member name="P:RMuseum.WebServiceUrl.Url">
<summary>
url

87
RMuseum/Services/Implementation/SemanticSearchService.cs Normal file
View File

@ -0,0 +1,87 @@
using Microsoft.EntityFrameworkCore;
using RMuseum.DbContext;
using RMuseum.Models.Ganjoor.SemanticSearch;
using RMuseum.Utils.SemanticSearch;
using System;
using System.Collections.Generic;
using System.Linq;
using System.Threading.Tasks;
namespace RMuseum.Services.Implementation
{
public interface ISemanticSearchService
{
Task<SemanticSearchResponseDto> SearchAsync(string query, int? topK);
}
/// <summary>
/// Deliberately NOT a GanjoorService partial, unlike everything else in this project.
/// EmbeddingIndex (~530MB in memory) and QueryEmbedder (a loaded ONNX model) both need to be
/// true singletons — constructed once at startup, never per-request — while GanjoorService
/// and RMuseumDbContext are scoped per-request throughout this codebase. Injecting a
/// singleton's dependencies into a per-request class (or vice versa) is a real DI lifetime
/// bug, not just an inconsistency, so this stays a separate service registered as a
/// singleton itself (see INTEGRATION.md for the exact registration).
/// </summary>
public class SemanticSearchService : ISemanticSearchService
{
private const int DefaultTopK = 10;
private const int MaxTopK = 50;
private readonly EmbeddingIndex _embeddingIndex;
private readonly QueryEmbedder _queryEmbedder;
public SemanticSearchService(EmbeddingIndex embeddingIndex, QueryEmbedder queryEmbedder)
{
_embeddingIndex = embeddingIndex;
_queryEmbedder = queryEmbedder;
}
public async Task<SemanticSearchResponseDto> SearchAsync(string query, int? topK)
{
if (string.IsNullOrWhiteSpace(query))
throw new ArgumentException("query must not be empty", nameof(query));
int k = topK.GetValueOrDefault(DefaultTopK);
if (k <= 0 || k > MaxTopK)
k = DefaultTopK;
float[] queryVector = _queryEmbedder.EmbedQuery(query);
List<(int PoemId, float Score)> topMatches = _embeddingIndex.FindTopSimilar(queryVector, k);
var response = new SemanticSearchResponseDto { Query = query };
// a fresh, short-lived DbContext per call - same pattern GanjoorService's background
// jobs already use throughout this codebase, since a scoped/request DbContext can't
// be injected into this singleton service
using (RMuseumDbContext context = new RMuseumDbContext(new DbContextOptions<RMuseumDbContext>()))
{
var poemIds = topMatches.Select(m => m.PoemId).ToList();
var poemsById = await context.GanjoorPoems.AsNoTracking()
.Where(p => poemIds.Contains(p.Id))
.ToDictionaryAsync(p => p.Id);
foreach (var match in topMatches)
{
// a poem id present in the embedding index but missing from the live DB
// (e.g. deleted/unpublished since the embeddings were generated) is silently
// skipped rather than failing the whole search for everyone else's results
if (poemsById.TryGetValue(match.PoemId, out var poem))
{
response.Results.Add(new SemanticSearchResultDto
{
PoemId = poem.Id,
Title = poem.Title,
FullTitle = poem.FullTitle,
FullUrl = poem.FullUrl,
PoemSummary = poem.PoemSummary,
Score = match.Score,
});
}
}
}
return response;
}
}
}

146
RMuseum/Utils/SemanticSearch/EmbeddingIndex.cs Normal file
View File

@ -0,0 +1,146 @@
using System;
using System.Collections.Generic;
using System.IO;
using System.Linq;
using System.Text.Json;
namespace RMuseum.Utils.SemanticSearch
{
public class EmbeddingIndexMetadata
{
public string Model { get; set; }
public int Dimension { get; set; }
public string PoolingMethod { get; set; }
public bool Normalized { get; set; }
public string GeneratedAtUtc { get; set; }
public int Count { get; set; }
public int[] Ids { get; set; }
}
/// <summary>
/// Loads the published embeddings.f32 (raw N x D float32, row-major, sorted by poem id
/// ascending per the export's own writer) + embeddings-index.json into memory once, and does
/// brute-force cosine similarity search against them.
///
/// Brute force, not a vector database, is a deliberate choice at this scale — ~130k poems x
/// 1024 dimensions is ~530MB, comfortably held in RAM, and a full linear scan per query is
/// fast enough (low tens of milliseconds) that a dedicated vector index/database would be
/// solving a problem this corpus doesn't actually have. Revisit only if the corpus grows by
/// an order of magnitude or query volume becomes very high.
///
/// Thread-safety: immutable after Load() — every field is only ever written once, during
/// loading, before this instance is published to any other thread (e.g. via DI singleton
/// registration). Safe for concurrent reads (searches) from multiple requests afterward.
/// </summary>
public class EmbeddingIndex
{
private readonly float[] _vectors; // flat, row-major: vectors[id_index * Dimension + d]
private readonly int[] _poemIds; // poemIds[id_index] is the poem id for that row
public EmbeddingIndexMetadata Metadata { get; }
private EmbeddingIndex(float[] vectors, int[] poemIds, EmbeddingIndexMetadata metadata)
{
_vectors = vectors;
_poemIds = poemIds;
Metadata = metadata;
}
/// <summary>
/// Loads embeddings.f32 + embeddings-index.json from the given directory. Intended to be
/// called once at startup (e.g. from a DI factory) — this reads and holds ~530MB in
/// memory, not something to redo per-request.
/// </summary>
public static EmbeddingIndex Load(string embeddingsDirectory)
{
string indexPath = Path.Combine(embeddingsDirectory, "embeddings-index.json");
string binPath = Path.Combine(embeddingsDirectory, "embeddings.f32");
if (!File.Exists(indexPath))
throw new FileNotFoundException($"embeddings-index.json not found in '{embeddingsDirectory}'", indexPath);
if (!File.Exists(binPath))
throw new FileNotFoundException($"embeddings.f32 not found in '{embeddingsDirectory}'", binPath);
var metadata = JsonSerializer.Deserialize<EmbeddingIndexMetadata>(
File.ReadAllText(indexPath),
new JsonSerializerOptions { PropertyNameCaseInsensitive = true });
if (metadata == null || metadata.Count <= 0 || metadata.Dimension <= 0)
throw new InvalidDataException($"embeddings-index.json in '{embeddingsDirectory}' is missing count/dimension");
if (metadata.Ids == null || metadata.Ids.Length != metadata.Count)
throw new InvalidDataException(
$"embeddings-index.json declares count={metadata.Count} but ids array has " +
$"{metadata.Ids?.Length ?? 0} entries — these must match");
long expectedBytes = (long)metadata.Count * metadata.Dimension * sizeof(float);
long actualBytes = new FileInfo(binPath).Length;
if (actualBytes != expectedBytes)
throw new InvalidDataException(
$"embeddings.f32 in '{embeddingsDirectory}' is {actualBytes} bytes, expected " +
$"{expectedBytes} (count={metadata.Count} x dimension={metadata.Dimension} x 4 bytes). " +
"The index and binary file don't agree — do not trust this data until re-verified " +
"(see verify_embeddings.py in the ganjoor-embeddings tooling).");
var vectors = new float[metadata.Count * metadata.Dimension];
using (var stream = File.OpenRead(binPath))
using (var reader = new BinaryReader(stream))
{
var buffer = new byte[vectors.Length * sizeof(float)];
int totalRead = 0;
while (totalRead < buffer.Length)
{
int read = reader.Read(buffer, totalRead, buffer.Length - totalRead);
if (read == 0) break;
totalRead += read;
}
if (totalRead != buffer.Length)
throw new InvalidDataException($"expected to read {buffer.Length} bytes from embeddings.f32, got {totalRead}");
Buffer.BlockCopy(buffer, 0, vectors, 0, buffer.Length);
}
return new EmbeddingIndex(vectors, metadata.Ids, metadata);
}
/// <summary>
/// Returns the topK poem ids most similar to queryVector, ranked descending by cosine
/// similarity. queryVector must already be the SAME dimension as this index and, for the
/// score to mean what it claims (a true cosine similarity), should already be
/// L2-normalized the same way the indexed vectors are — see QueryEmbedder.
/// </summary>
public List<(int PoemId, float Score)> FindTopSimilar(ReadOnlySpan<float> queryVector, int topK)
{
if (queryVector.Length != Metadata.Dimension)
throw new ArgumentException(
$"query vector has {queryVector.Length} dimensions, index has {Metadata.Dimension} — " +
"these must match (are the query and index using the same model?)");
int count = _poemIds.Length;
var scores = new float[count];
// dot product of two unit vectors == cosine similarity - the vectors were already
// L2-normalized at export time (see embeddings-index.json's "normalized": true), so
// this is genuinely computing cosine similarity, not just a raw dot product
for (int i = 0; i < count; i++)
{
float dot = 0f;
int baseIdx = i * Metadata.Dimension;
for (int d = 0; d < Metadata.Dimension; d++)
{
dot += _vectors[baseIdx + d] * queryVector[d];
}
scores[i] = dot;
}
var indices = Enumerable.Range(0, count).ToArray();
// partial sort would be a fair perf optimization at larger scale; a full sort of
// ~130k floats is already comfortably fast (low tens of ms) and simpler to trust
Array.Sort(indices, (a, b) => scores[b].CompareTo(scores[a]));
var results = new List<(int, float)>(Math.Min(topK, count));
for (int i = 0; i < Math.Min(topK, count); i++)
{
results.Add((_poemIds[indices[i]], scores[indices[i]]));
}
return results;
}
}
}

159
RMuseum/Utils/SemanticSearch/QueryEmbedder.cs Normal file
View File

@ -0,0 +1,159 @@
using Microsoft.ML.OnnxRuntime;
using Microsoft.ML.OnnxRuntime.Tensors;
using Microsoft.ML.Tokenizers;
using System;
using System.Collections.Generic;
using System.IO;
using System.Linq;
namespace RMuseum.Utils.SemanticSearch
{
/// <summary>
/// Embeds a single user-typed search query using the SAME ONNX model + tokenizer as
/// scripts/generate_embeddings.py (the Python side that indexed the poem corpus) — this is
/// non-negotiable: a query embedded in a different space than the documents it's being
/// compared against produces meaningless similarity scores, silently (no error, just bad
/// results), not something that shows up as a crash.
///
/// UNTESTED — flagging this more strongly than usual, because this file stacks two risks
/// that weren't present anywhere else in this project:
/// 1. This environment has no .NET SDK and no network access to onnxruntime/the model, so
/// none of this has actually been compiled or run.
/// 2. Even once it compiles, BpeTokenizer.Create(vocab, merges) builds a plain BPE
/// tokenizer from vocab.json + merges.txt — it may not capture every detail of Qwen's
/// full tokenizer.json spec (custom pre-tokenizer regex, added/special tokens). A
/// mismatch here wouldn't error either — it would silently tokenize text differently
/// than the Python `tokenizers` library did at index time, degrading search quality
/// without any visible failure.
/// See VERIFICATION.md for a paired Python/C# test to run BEFORE trusting search results —
/// do this before anything else in this phase.
///
/// Query-time embedding is always exactly one query per call (a person typing into a search
/// box), never a batch — unlike the Python indexing script, which batched many poems
/// together and needed padding logic. That simplifies this class: no padding, no
/// batch-dimension complexity, last-token pooling reduces to simply the final sequence
/// position (no attention-mask lookup needed, since there's no padding to skip past).
/// </summary>
public class QueryEmbedder : IDisposable
{
// Confirmed via `python3 scripts/generate_embeddings.py --inspect-only` against the real
// Qwen/Qwen3-Embedding-0.6B ONNX export (onnx-community/Qwen3-Embedding-0.6B-ONNX),
// 2026-09-05. If the model is ever swapped for a different export/architecture, these
// MUST be re-confirmed the same way (re-run --inspect-only, read off the real shapes) —
// hardcoding them here (rather than trying to introspect symbolic dimension names from
// .NET's ONNX Runtime API, whose support for that is less certain than Python's) is a
// deliberate tradeoff: reliable given what we've already confirmed, but silently wrong
// if the model changes without updating these.
private const int NumLayers = 28;
private const int NumKvHeads = 8;
private const int HeadDim = 128;
/// <summary>
/// Per Qwen3-Embedding's documented convention, QUERIES (unlike documents) benefit from
/// an instruction prefix. generate_embeddings.py deliberately does NOT add one to
/// documents (poem summaries) — this asymmetry is the model's own documented design, not
/// an inconsistency. If this template ever changes, or if it's ever added to/removed
/// from the document side, both sides must be updated together, or queries and documents
/// drift into subtly different embedding spaces. This exact wording hasn't been
/// benchmarked for Persian poetry specifically — reasonable to A/B test once the basic
/// pipeline is confirmed working (see VERIFICATION.md), not something to treat as final.
/// </summary>
private const string InstructionTemplate =
"Instruct: Given a search query, retrieve Persian poems whose meaning matches it\nQuery: {0}";
private readonly InferenceSession _session;
private readonly Tokenizer _tokenizer;
private readonly int _dimension;
public QueryEmbedder(string modelPath, string vocabPath, string mergesPath, int dimension = 1024)
{
_session = new InferenceSession(modelPath);
using (var vocabStream = File.OpenRead(vocabPath))
using (var mergesStream = File.OpenRead(mergesPath))
{
_tokenizer = BpeTokenizer.Create(vocabStream, mergesStream);
}
_dimension = dimension;
}
/// <summary>
/// Returns an L2-normalized embedding for queryText, comparable via dot product against
/// EmbeddingIndex's vectors (which are normalized the same way).
/// </summary>
public float[] EmbedQuery(string queryText)
{
string instructedText = string.Format(InstructionTemplate, queryText);
IReadOnlyList<int> tokenIds = _tokenizer.EncodeToIds(instructedText);
int seqLen = tokenIds.Count;
if (seqLen == 0)
throw new ArgumentException("query tokenized to zero tokens — empty or whitespace-only query?", nameof(queryText));
var inputIds = new DenseTensor<long>(new[] { 1, seqLen });
var attentionMask = new DenseTensor<long>(new[] { 1, seqLen });
var positionIds = new DenseTensor<long>(new[] { 1, seqLen });
for (int i = 0; i < seqLen; i++)
{
inputIds[0, i] = tokenIds[i];
attentionMask[0, i] = 1;
positionIds[0, i] = i;
}
var inputs = new List<NamedOnnxValue>
{
NamedOnnxValue.CreateFromTensor("input_ids", inputIds),
NamedOnnxValue.CreateFromTensor("attention_mask", attentionMask),
NamedOnnxValue.CreateFromTensor("position_ids", positionIds),
};
// empty (zero-length) KV cache for every layer - a single uncached forward pass over
// the whole query, same reasoning as build_extra_inputs() in generate_embeddings.py
for (int layer = 0; layer < NumLayers; layer++)
{
var emptyKey = new DenseTensor<float>(new[] { 1, NumKvHeads, 0, HeadDim });
var emptyValue = new DenseTensor<float>(new[] { 1, NumKvHeads, 0, HeadDim });
inputs.Add(NamedOnnxValue.CreateFromTensor($"past_key_values.{layer}.key", emptyKey));
inputs.Add(NamedOnnxValue.CreateFromTensor($"past_key_values.{layer}.value", emptyValue));
}
using (var outputs = _session.Run(inputs, new[] { "last_hidden_state" }))
{
var hiddenStates = outputs.First(o => o.Name == "last_hidden_state").AsTensor<float>();
// hiddenStates shape: (1, seqLen, _dimension)
// last-token pooling: batch size 1, no padding, so the last real token is simply
// the final sequence position - see the class docstring for why this doesn't
// need the attention-mask lookup the Python (batched, padded) version does
var pooled = new float[_dimension];
for (int d = 0; d < _dimension; d++)
{
pooled[d] = hiddenStates[0, seqLen - 1, d];
}
return L2Normalize(pooled);
}
}
private static float[] L2Normalize(float[] vector)
{
double sumSquares = 0;
foreach (var v in vector) sumSquares += (double)v * v;
double norm = Math.Sqrt(sumSquares);
if (norm < 1e-12) norm = 1e-12; // guard against a degenerate all-zero vector
var result = new float[vector.Length];
for (int i = 0; i < vector.Length; i++)
{
result[i] = (float)(vector[i] / norm);
}
return result;
}
public void Dispose()
{
_session?.Dispose();
}
}
}