diff --git a/RMuseum/Services/Implementation/GanjoorService-Partials/GanjoorService-Centuries.cs b/RMuseum/Services/Implementation/GanjoorService-Partials/GanjoorService-Centuries.cs index 59d180d9..e4a21d1c 100644 --- a/RMuseum/Services/Implementation/GanjoorService-Partials/GanjoorService-Centuries.cs +++ b/RMuseum/Services/Implementation/GanjoorService-Partials/GanjoorService-Centuries.cs @@ -1,4 +1,5 @@ using Microsoft.EntityFrameworkCore; +using RMuseum.DbContext; using RMuseum.Models.Ganjoor; using RSecurityBackend.Models.Generic; using System; @@ -8,7 +9,6 @@ using System.Threading.Tasks; using System.Collections.Generic; using RMuseum.Models.Ganjoor.ViewModels; using System.Globalization; -using RMuseum.DbContext; namespace RMuseum.Services.Implementation { diff --git a/RMuseum/Services/Implementation/GanjoorService-Partials/GanjoorService-PublicDataExport.cs b/RMuseum/Services/Implementation/GanjoorService-Partials/GanjoorService-PublicDataExport.cs index ca7e3d22..abc254b0 100644 --- a/RMuseum/Services/Implementation/GanjoorService-Partials/GanjoorService-PublicDataExport.cs +++ b/RMuseum/Services/Implementation/GanjoorService-Partials/GanjoorService-PublicDataExport.cs @@ -60,6 +60,13 @@ namespace RMuseum.Services.Implementation await jobProgressServiceEF.UpdateJob(job.Id, 1, "Writing shared lookup tables"); await ExportSharedLookupTables(context, repoRoot); + // loaded once for the whole run instead of once per poem (was the + // single biggest cost in this job: ~3 round-trips per poem, tens of + // thousands of poems, on every run regardless of what changed) — see + // ExportPoetContent for how sections/verses are batched per-poet. + var metresById = (await context.GanjoorMetres.AsNoTracking().ToListAsync()) + .ToDictionary(m => m.Id); + var poets = await context.GanjoorPoets.AsNoTracking() .Include(p => p.BirthLocation) .Include(p => p.DeathLocation) @@ -92,7 +99,7 @@ namespace RMuseum.Services.Implementation await ExportPoetToJson(context, repoRoot, poet, catPoet); poetIdIndex[poet.Id] = catPoet.FullUrl; - int poemCount = await ExportCatTreeToJson(context, repoRoot, catPoet, catIdIndex, poemIdIndex); + int poemCount = await ExportPoetContent(context, repoRoot, poet, catPoet, metresById, catIdIndex, poemIdIndex); manifest.PoemsCount += poemCount; manifest.Poets.Add(new PublicExportManifestPoetEntryDto @@ -202,6 +209,35 @@ namespace RMuseum.Services.Implementation await DeterministicJsonWriter.WriteIfChangedAsync(path, dto); } + /// + /// Batch-loads this poet's entire sections/verses in two queries (instead of the + /// two-per-poem queries the old per-poem approach ran), then walks the poet's category + /// tree writing everything from memory. This is the fix for the export consistently + /// taking about as long on every run regardless of how much content changed — the + /// "skip unchanged files" logic in DeterministicJsonWriter only ever saved disk writes, + /// never the DB round-trips, which were the actual dominant cost (roughly 3 sequential + /// queries per poem — tens of thousands of round-trips for the full corpus, every run). + /// + private async Task ExportPoetContent(RMuseumDbContext context, string repoRoot, GanjoorPoet poet, GanjoorCat catPoet, + Dictionary metresById, Dictionary catIdIndex, Dictionary poemIdIndex) + { + var sectionsByPoem = (await context.GanjoorPoemSections.AsNoTracking() + .Where(s => s.Poem.Cat.PoetId == poet.Id) + .OrderBy(s => s.Index) + .ToListAsync()) + .GroupBy(s => s.PoemId) + .ToDictionary(g => g.Key, g => g.ToList()); + + var versesByPoem = (await context.GanjoorVerses.AsNoTracking() + .Where(v => v.Poem.Cat.PoetId == poet.Id) + .OrderBy(v => v.VOrder) + .ToListAsync()) + .GroupBy(v => v.PoemId) + .ToDictionary(g => g.Key, g => g.ToList()); + + return await ExportCatTreeToJson(context, repoRoot, catPoet, metresById, sectionsByPoem, versesByPoem, catIdIndex, poemIdIndex); + } + /// /// recursively writes _cat.json for and every poem directly /// under it, then recurses into child categories. Returns the number of poems written @@ -212,6 +248,8 @@ namespace RMuseum.Services.Implementation /// either; every category/poem under a published poet is exported. /// private async Task ExportCatTreeToJson(RMuseumDbContext context, string repoRoot, GanjoorCat cat, + Dictionary metresById, + Dictionary> sectionsByPoem, Dictionary> versesByPoem, Dictionary catIdIndex, Dictionary poemIdIndex) { var childCats = await context.GanjoorCategories.AsNoTracking() @@ -247,34 +285,29 @@ namespace RMuseum.Services.Implementation foreach (var poem in poems) { - await ExportPoemToJson(context, repoRoot, poem); + sectionsByPoem.TryGetValue(poem.Id, out var sections); + versesByPoem.TryGetValue(poem.Id, out var verses); + GanjoorMetre metre = poem.GanjoorMetreId != null && metresById.TryGetValue(poem.GanjoorMetreId.Value, out var m) ? m : null; + + await ExportPoemToJson(repoRoot, poem, metre, sections ?? new List(), verses ?? new List()); poemIdIndex[poem.Id] = poem.FullUrl; } foreach (var childCat in childCats) { - poemCount += await ExportCatTreeToJson(context, repoRoot, childCat, catIdIndex, poemIdIndex); + poemCount += await ExportCatTreeToJson(context, repoRoot, childCat, metresById, sectionsByPoem, versesByPoem, catIdIndex, poemIdIndex); } return poemCount; } - private async Task ExportPoemToJson(RMuseumDbContext context, string repoRoot, GanjoorPoem poem) + /// + /// Pure in-memory write — no DB access. Sections/verses/metre are pre-loaded by the caller + /// (see ) instead of queried per poem. + /// + private async Task ExportPoemToJson(string repoRoot, GanjoorPoem poem, GanjoorMetre metre, + List sections, List verses) { - var metre = poem.GanjoorMetreId == null - ? null - : await context.GanjoorMetres.AsNoTracking().Where(m => m.Id == poem.GanjoorMetreId).SingleOrDefaultAsync(); - - var sections = await context.GanjoorPoemSections.AsNoTracking() - .Where(s => s.PoemId == poem.Id) - .OrderBy(s => s.Index) - .ToListAsync(); - - var verses = await context.GanjoorVerses.AsNoTracking() - .Where(v => v.PoemId == poem.Id) - .OrderBy(v => v.VOrder) - .ToListAsync(); - var dto = new PoemPublicDto { Id = poem.Id,