#498 export tajik data

This commit is contained in:
Hamid Reza Mohammadi 2026-08-14 21:44:19 +03:30
parent 6e44e4dec1
commit c030a192d2
8 changed files with 518 additions and 0 deletions

View File

@ -232,6 +232,7 @@
<a role="button" onclick="rebuildRelatedPages()" class="up-btn up-btn--ghost up-btn--sm" title="بازسازی صفحات استقبالها">بازسازی صفحات استقبالها</a>
<a role="button" onclick="rebuildDigitalSourcesStats()" class="up-btn up-btn--ghost up-btn--sm" title="بازسازی آمار منابع دیجیتال">بازسازی آمار منابع دیجیتال</a>
<a role="button" onclick="regeneratePublicData()" class="up-btn up-btn--ghost up-btn--sm" title="بازسازی و ارسال دادهٔ عمومی گنجور">بازسازی دادهٔ عمومی</a>
<a role="button" onclick="regenerateTajikPublicData()" class="up-btn up-btn--ghost up-btn--sm" title="بازسازی و ارسال دادهٔ عمومی تاجیکی">بازسازی دادهٔ عمومی تاجیکی</a>
<a role="button" onclick="transilerate()" class="up-btn up-btn--ghost up-btn--sm" title="برگردان تاجیکی">برگردان تاجیکی</a>
</div>

View File

@ -361,6 +361,27 @@ namespace GanjooRazor.Areas.Admin.Pages
return new OkObjectResult(false);
}
/// <summary>
/// (re)generate the separate Tajik (Cyrillic) public data export — same background-job
/// pattern as the main one, different repo
/// </summary>
public async Task<IActionResult> OnPostRegenerateTajikPublicDataAsync()
{
using (HttpClient secureClient = new HttpClient(new GanjoorReloginHandler(Request, Response)))
{
if (await GanjoorSessionChecker.PrepareClient(secureClient, Request, Response))
{
HttpResponseMessage response = await secureClient.PostAsync($"{APIRoot.Url}/api/ganjoor/tajik/publicdata/batchexport", null);
if (!response.IsSuccessStatusCode)
{
return BadRequest(JsonConvert.DeserializeObject<string>(await response.Content.ReadAsStringAsync()));
}
return new OkObjectResult(true);
}
}
return new OkObjectResult(false);
}
public async Task<IActionResult> OnPostRefillSectionsCoupletCountsAsync()
{

View File

@ -205,6 +205,25 @@
});
}
async function regenerateTajikPublicData() {
var ok = await upConfirm('آیا از بازسازی و ارسال دادهٔ عمومی تاجیکی (مخزن گیت جداگانه) اطمینان دارید؟');
if (!ok) return;
var url = '?handler=RegenerateTajikPublicData';
$.ajax({
type: "POST",
url: url,
error: function () {
upToast('خطا رخ داد.', 'error');
},
success: function () {
upToast('فرایند بازسازی دادهٔ عمومی تاجیکی در پس‌زمینه شروع شد.', 'success');
},
});
}
async function refillCoupletCounts() {

View File

@ -2448,6 +2448,32 @@ namespace RMuseum.Controllers
}
}
/// <summary>
/// start exporting the Tajik (Cyrillic) overlay data to its own, separate public
/// git-tracked JSON data set — only poets/categories/poems with an actual Tajik
/// translation are included
/// </summary>
/// <returns></returns>
[HttpPost("tajik/publicdata/batchexport")]
[Authorize(Policy = RMuseumSecurableItem.GanjoorEntityShortName + ":" + RMuseumSecurableItem.ImportOperationShortName)]
[ProducesResponseType((int)HttpStatusCode.OK)]
[ProducesResponseType((int)HttpStatusCode.BadRequest, Type = typeof(string))]
[ProducesResponseType((int)HttpStatusCode.Unauthorized)]
public IActionResult StartBatchExportTajikPublicGitData()
{
try
{
var res = _ganjoorService.StartBatchExportTajikPublicGitData();
if (!string.IsNullOrEmpty(res.ExceptionString))
return BadRequest(res.ExceptionString);
return Ok();
}
catch (Exception exp)
{
return BadRequest(exp.ToString());
}
}
/// <summary>
/// (re)build local Ganjoor content from a public data export tree (local folder or HTTP) —
/// intended for local development use, not for production servers

View File

@ -0,0 +1,99 @@
using System.Collections.Generic;
namespace RMuseum.Models.Ganjoor.PublicExport
{
/// <summary>
/// Root manifest for the Tajik data export — a separate, sparser repository from the main
/// ganjoor-data one. Tajik translation only covers a subset of poets/categories/poems, so this
/// export only ever contains entries that actually have a Tajik counterpart in the database
/// (GanjoorTajikPoet/Cat/Poem/Verse) — it does not mirror the full corpus the way the main
/// export does. Ids and FullUrls match the main repo 1:1 (same underlying entity ids), so a
/// consumer resolves structure/metre/sections for a poem by fetching the same id/path from
/// ganjoor-data and merging verses by VOrder.
/// </summary>
public class TajikPublicExportManifestDto
{
public int SchemaVersion { get; set; } = 1;
public string GeneratedAtUtc { get; set; }
public int PoetsCount { get; set; }
public int PoemsCount { get; set; }
public int IdIndexShardSize { get; set; }
/// <summary>
/// where to find the structural data (metre, rhyme, sections, full verse text in Persian)
/// this repo's ids/paths correspond to
/// </summary>
public string MainDataRepo { get; set; } = "https://github.com/ganjoor/ganjoor-data";
public TajikPublicExportUrlTemplatesDto UrlTemplates { get; set; } = new TajikPublicExportUrlTemplatesDto();
public List<PublicExportManifestPoetEntryDto> Poets { get; set; } = new List<PublicExportManifestPoetEntryDto>();
}
public class TajikPublicExportUrlTemplatesDto
{
public string Poet { get; set; } = "poets/{poetSlug}/poet.json";
public string Category { get; set; } = "poets/{poetSlug}/{catPath}/_cat.json";
public string Poem { get; set; } = "poets/{poetSlug}/{catPath}/{poemSlug}.json";
public string PoetIdIndex { get; set; } = "index/poets-by-id.json";
public string CatIdIndexShard { get; set; } = "index/cats-by-id/{bucket}.json";
public string PoemIdIndexShard { get; set; } = "index/poems-by-id/{bucket}.json";
}
/// <summary>
/// poet.json — only written for a poet with an actual GanjoorTajikPoet record.
/// FullUrl matches the poet's path in the main ganjoor-data repo exactly.
/// </summary>
public class TajikPoetPublicDto
{
public int Id { get; set; }
public string TajikNickname { get; set; }
public string TajikDescription { get; set; }
public int BirthYearInLHijri { get; set; }
public string FullUrl { get; set; }
}
/// <summary>
/// _cat.json — only written for a category with an actual GanjoorTajikCat record.
/// ChildCats/Poems only list children that themselves have a Tajik translation — a consumer
/// never gets a link to a file that doesn't exist.
/// </summary>
public class TajikCatPublicDto
{
public int Id { get; set; }
public int PoetId { get; set; }
public int? ParentId { get; set; }
public string TajikTitle { get; set; }
public string TajikDescription { get; set; }
public string FullUrl { get; set; }
public List<CatChildRefDto> ChildCats { get; set; } = new List<CatChildRefDto>();
public List<PoemChildRefDto> Poems { get; set; } = new List<PoemChildRefDto>();
}
/// <summary>
/// {poem-slug}.json — only written for a poem with an actual GanjoorTajikPoem record.
/// Deliberately doesn't carry metre/rhyme/section data — GanjoorTajikPoem/Verse don't store
/// their own copy of that either; fetch it from the same id/path in the main ganjoor-data repo.
/// </summary>
public class TajikPoemPublicDto
{
public int Id { get; set; }
public int CatId { get; set; }
public string TajikTitle { get; set; }
public string FullTitle { get; set; }
public string FullUrl { get; set; }
public string TajikPlainText { get; set; }
public List<TajikVersePublicDto> Verses { get; set; } = new List<TajikVersePublicDto>();
}
/// <summary>
/// VOrder is the join key back to the corresponding verse in the main repo's poem file — no
/// position/section/couplet data is duplicated here, matching how GanjoorTajikVerse itself
/// stores nothing but the transliterated text and its order.
/// </summary>
public class TajikVersePublicDto
{
public int VOrder { get; set; }
public string TajikText { get; set; }
}
}

View File

@ -640,6 +640,13 @@ namespace RMuseum.Services
/// <returns></returns>
RServiceResult<bool> StartBatchExportPublicGitData();
/// <summary>
/// start exporting the Tajik (Cyrillic) overlay data to its own separate git-tracked JSON
/// data set — only entities that actually have a Tajik translation are included
/// </summary>
/// <returns></returns>
RServiceResult<bool> StartBatchExportTajikPublicGitData();
/// <summary>
/// (re)build local Ganjoor content (poets/categories/poems/verses/sections + their pages)
/// from a public data export tree, read locally or over HTTP. Safe to re-run — existing

View File

@ -0,0 +1,335 @@
using Microsoft.EntityFrameworkCore;
using RMuseum.DbContext;
using RMuseum.Models.Ganjoor;
using RMuseum.Models.Ganjoor.PublicExport;
using RMuseum.Utils.PublicDataExport;
using RSecurityBackend.Models.Generic;
using RSecurityBackend.Services.Implementation;
using System;
using System.Collections.Generic;
using System.IO;
using System.Linq;
using System.Threading.Tasks;
namespace RMuseum.Services.Implementation
{
/// <summary>
/// IGanjoorService implementation
/// </summary>
public partial class GanjoorService : IGanjoorService
{
/// <summary>
/// Exports the Tajik (Cyrillic transliteration) overlay data — GanjoorTajikPoet/Cat/Poem/Verse
/// — to its own git-tracked JSON repository, separate from the main ganjoor-data export.
/// Kept separate deliberately: Tajik translation coverage is sparse (most poets/categories/
/// poems have no Tajik counterpart at all), the consumer audience is different (Cyrillic
/// Tajik apps, not general Ganjoor API consumers), and update cadence is independent of the
/// main Persian corpus. This export only ever writes entries that actually have a Tajik
/// database record; everything else is skipped. Ids and FullUrls match the main
/// ganjoor-data repo exactly, so a consumer resolves structure/metre/sections for a poem by
/// fetching the same id/path from there and merging verses by VOrder.
/// </summary>
public RServiceResult<bool> StartBatchExportTajikPublicGitData()
{
try
{
// scans every DTO in RMuseum.Models.Ganjoor.PublicExport, which now includes the
// Tajik DTOs too — no separate call needed, they're covered automatically
PublicExportSafetyGuard.AssertSafe();
_backgroundTaskQueue.QueueBackgroundWorkItem
(
async token =>
{
using (RMuseumDbContext context = new RMuseumDbContext(new DbContextOptions<RMuseumDbContext>()))
{
LongRunningJobProgressServiceEF jobProgressServiceEF = new LongRunningJobProgressServiceEF(context);
var job = (await jobProgressServiceEF.NewJob("TajikPublicDataExport", "Preparing working copy")).Result;
try
{
var options = ReadTajikPublicDataExportOptions();
var publisher = new GitRepoPublisher(options);
publisher.EnsureWorkingCopyUpToDate();
string repoRoot = options.LocalWorkingCopyPath;
var manifest = new TajikPublicExportManifestDto
{
GeneratedAtUtc = DateTime.UtcNow.ToString("O"),
IdIndexShardSize = IdIndexShardSize,
};
var poetIdIndex = new Dictionary<int, string>();
var catIdIndex = new Dictionary<int, string>();
var poemIdIndex = new Dictionary<int, string>();
var tajikPoets = await context.TajikPoets.AsNoTracking().OrderBy(p => p.Id).ToListAsync();
int poetIndex = 0;
foreach (var tajikPoet in tajikPoets)
{
poetIndex++;
await jobProgressServiceEF.UpdateJob(job.Id, (int)(100.0 * poetIndex / Math.Max(1, tajikPoets.Count)), $"Exporting {tajikPoet.TajikNickname}");
// the Tajik tables have no FullUrl of their own - poet/category/
// poem paths are read from the main tables, since ids match 1:1
var catPoet = await context.GanjoorCategories.AsNoTracking()
.Where(c => c.PoetId == tajikPoet.Id && c.ParentId == null)
.SingleOrDefaultAsync();
if (catPoet == null)
continue;
await ExportTajikPoetToJson(repoRoot, tajikPoet, catPoet);
poetIdIndex[tajikPoet.Id] = catPoet.FullUrl;
int poemCount = await ExportTajikPoetContent(context, repoRoot, tajikPoet.Id, catPoet, catIdIndex, poemIdIndex);
manifest.PoemsCount += poemCount;
manifest.Poets.Add(new PublicExportManifestPoetEntryDto
{
Id = tajikPoet.Id,
Nickname = tajikPoet.TajikNickname,
FullUrl = catPoet.FullUrl,
});
}
manifest.PoetsCount = manifest.Poets.Count;
await jobProgressServiceEF.UpdateJob(job.Id, 98, "Writing id indexes");
await IdIndexWriter.WriteFlatIndexAsync(repoRoot, "index/poets-by-id.json", poetIdIndex);
await IdIndexWriter.WriteShardedIndexAsync(repoRoot, "cats", catIdIndex, IdIndexShardSize);
await IdIndexWriter.WriteShardedIndexAsync(repoRoot, "poems", poemIdIndex, IdIndexShardSize);
await DeterministicJsonWriter.WriteIfChangedAsync(Path.Combine(repoRoot, "manifest.json"), manifest);
await TextFileWriter.WriteIfChangedAsync(Path.Combine(repoRoot, "API.md"), BuildTajikApiMarkdown(manifest));
await TextFileWriter.WriteIfChangedAsync(Path.Combine(repoRoot, "README.md"), BuildTajikReadmeMarkdown(manifest));
await jobProgressServiceEF.UpdateJob(job.Id, 99, "Committing and pushing");
int changed = publisher.CommitAndPush($"data: export {manifest.PoetsCount} poets / {manifest.PoemsCount} poems — {DateTime.UtcNow:yyyy-MM-dd}");
await jobProgressServiceEF.UpdateJob(job.Id, 100, changed == 0 ? "No changes" : $"{changed} files changed", true);
}
catch (Exception exp)
{
await jobProgressServiceEF.UpdateJob(job.Id, 100, "", false, exp.ToString());
}
}
}
);
return new RServiceResult<bool>(true);
}
catch (Exception exp)
{
return new RServiceResult<bool>(false, exp.ToString());
}
}
private GitRepoPublisherOptions ReadTajikPublicDataExportOptions()
{
var section = Configuration.GetSection("TajikPublicDataExport");
return new GitRepoPublisherOptions
{
LocalWorkingCopyPath = section["LocalWorkingCopyPath"],
RemoteUrl = section["RemoteUrl"],
Branch = section["Branch"] ?? "main",
CommitAuthorName = section["CommitAuthorName"] ?? "Ganjoor Tajik Export Bot",
CommitAuthorEmail = section["CommitAuthorEmail"] ?? "bot@ganjoor.net",
PushEnabled = bool.TryParse(section["PushEnabled"], out var push) && push,
GitUserName = section["GitUserName"],
GitToken = section["GitToken"],
};
}
private async Task ExportTajikPoetToJson(string repoRoot, GanjoorTajikPoet tajikPoet, GanjoorCat catPoet)
{
var dto = new TajikPoetPublicDto
{
Id = tajikPoet.Id,
TajikNickname = tajikPoet.TajikNickname,
TajikDescription = tajikPoet.TajikDescription,
BirthYearInLHijri = tajikPoet.BirthYearInLHijri,
FullUrl = catPoet.FullUrl,
};
string path = Path.Combine(repoRoot, "poets", TrimLeadingSlash(catPoet.FullUrl), "poet.json");
await DeterministicJsonWriter.WriteIfChangedAsync(path, dto);
}
/// <summary>
/// batch-loads everything Tajik for this poet (mirrors the same "load once per poet, not
/// once per poem" fix applied to the main export) before walking the tree
/// </summary>
private async Task<int> ExportTajikPoetContent(RMuseumDbContext context, string repoRoot, int poetId, GanjoorCat catPoet,
Dictionary<int, string> catIdIndex, Dictionary<int, string> poemIdIndex)
{
var tajikCatsById = (await context.TajikCats.AsNoTracking().Where(c => c.PoetId == poetId).ToListAsync())
.ToDictionary(c => c.Id);
if (tajikCatsById.Count == 0)
return 0; // nothing translated for this poet at all
var catIds = tajikCatsById.Keys.ToList();
var tajikPoemsById = (await context.TajikPoems.AsNoTracking().Where(p => catIds.Contains(p.CatId)).ToListAsync())
.ToDictionary(p => p.Id);
var poemIds = tajikPoemsById.Keys.ToList();
var tajikVersesByPoem = (await context.TajikVerses.AsNoTracking()
.Where(v => poemIds.Contains(v.PoemId))
.OrderBy(v => v.VOrder)
.ToListAsync())
.GroupBy(v => v.PoemId)
.ToDictionary(g => g.Key, g => g.ToList());
return await ExportTajikCatTreeToJson(context, repoRoot, catPoet, tajikCatsById, tajikPoemsById, tajikVersesByPoem, catIdIndex, poemIdIndex);
}
/// <summary>
/// walks the *main* category tree (for structure/ordering — Tajik tables have no tree
/// shape of their own) but only ever writes a file where a Tajik counterpart actually
/// exists; still recurses into every child regardless, since a deeper category can have a
/// translation even when its parent doesn't.
/// </summary>
private async Task<int> ExportTajikCatTreeToJson(RMuseumDbContext context, string repoRoot, GanjoorCat cat,
Dictionary<int, GanjoorTajikCat> tajikCatsById, Dictionary<int, GanjoorTajikPoem> tajikPoemsById,
Dictionary<int, List<GanjoorTajikVerse>> tajikVersesByPoem,
Dictionary<int, string> catIdIndex, Dictionary<int, string> poemIdIndex)
{
var childCats = await context.GanjoorCategories.AsNoTracking()
.Where(c => c.ParentId == cat.Id)
.OrderBy(c => c.Id)
.ToListAsync();
int poemCount = 0;
if (tajikCatsById.TryGetValue(cat.Id, out var tajikCat))
{
var poems = await context.GanjoorPoems.AsNoTracking()
.Where(p => p.CatId == cat.Id)
.OrderBy(p => p.Id)
.ToListAsync();
var translatedChildCats = childCats.Where(c => tajikCatsById.ContainsKey(c.Id)).ToList();
var translatedPoems = poems.Where(p => tajikPoemsById.ContainsKey(p.Id)).ToList();
var catDto = new TajikCatPublicDto
{
Id = cat.Id,
PoetId = cat.PoetId,
ParentId = cat.ParentId,
TajikTitle = tajikCat.TajikTitle,
TajikDescription = tajikCat.TajikDescription,
FullUrl = cat.FullUrl,
ChildCats = translatedChildCats.Select(c => new CatChildRefDto { Id = c.Id, Title = tajikCatsById[c.Id].TajikTitle, FullUrl = c.FullUrl }).ToList(),
Poems = translatedPoems.Select(p => new PoemChildRefDto { Id = p.Id, Title = tajikPoemsById[p.Id].TajikTitle, FullUrl = p.FullUrl }).ToList(),
};
string catDir = Path.Combine(repoRoot, "poets", TrimLeadingSlash(cat.FullUrl));
await DeterministicJsonWriter.WriteIfChangedAsync(Path.Combine(catDir, "_cat.json"), catDto);
catIdIndex[cat.Id] = cat.FullUrl;
foreach (var poem in translatedPoems)
{
var tajikPoem = tajikPoemsById[poem.Id];
tajikVersesByPoem.TryGetValue(poem.Id, out var verses);
await ExportTajikPoemToJson(repoRoot, poem, tajikPoem, verses ?? new List<GanjoorTajikVerse>());
poemIdIndex[poem.Id] = poem.FullUrl;
poemCount++;
}
}
foreach (var childCat in childCats)
{
poemCount += await ExportTajikCatTreeToJson(context, repoRoot, childCat, tajikCatsById, tajikPoemsById, tajikVersesByPoem, catIdIndex, poemIdIndex);
}
return poemCount;
}
private async Task ExportTajikPoemToJson(string repoRoot, GanjoorPoem poem, GanjoorTajikPoem tajikPoem, List<GanjoorTajikVerse> verses)
{
var dto = new TajikPoemPublicDto
{
Id = poem.Id,
CatId = poem.CatId,
TajikTitle = tajikPoem.TajikTitle,
FullTitle = tajikPoem.FullTitle,
FullUrl = poem.FullUrl,
TajikPlainText = tajikPoem.TajikPlainText,
Verses = verses.OrderBy(v => v.VOrder).Select(v => new TajikVersePublicDto { VOrder = v.VOrder, TajikText = v.TajikText }).ToList(),
};
string path = Path.Combine(repoRoot, "poets", TrimLeadingSlash(poem.FullUrl) + ".json");
await DeterministicJsonWriter.WriteIfChangedAsync(path, dto);
}
private static string BuildTajikReadmeMarkdown(TajikPublicExportManifestDto manifest)
{
return
$@"# ganjoor-tajik-data
Cyrillic Tajik transliteration/translation overlay for [Ganjoor](https://ganjoor.net)'s poetry
content. This is a **separate, sparser** data set from
[ganjoor-data](https://github.com/ganjoor/ganjoor-data) — Tajik coverage only exists for some
poets/categories/poems, not the full corpus, so this repo only ever contains entries that
actually have a Tajik translation.
Ids and paths in this repo match [ganjoor-data]({manifest.MainDataRepo}) exactly (same underlying
poet/category/poem). This repo carries only the Tajik title/text; for metre, rhyme, verse
structure, and sections, fetch the same id/path from `ganjoor-data` and merge verses by `vOrder`.
Currently tracks **{manifest.PoetsCount} poets** / **{manifest.PoemsCount} poems** with Tajik
content, generated {manifest.GeneratedAtUtc}.
## Where do I start?
- **[`manifest.json`](manifest.json)** — the list of poets that have any Tajik content, plus URL
templates for every other file kind here.
- **[`API.md`](API.md)** — how to fetch any of this over plain HTTPS, and how to cross-reference
`ganjoor-data` for structural data this repo doesn't duplicate.
";
}
private static string BuildTajikApiMarkdown(TajikPublicExportManifestDto manifest)
{
var t = manifest.UrlTemplates;
return
$@"# ganjoor-tajik-data — static API
Same static-API-via-jsDelivr approach as [ganjoor-data]({manifest.MainDataRepo}) — no server,
every ""endpoint"" below is a plain HTTPS file fetch:
https://cdn.jsdelivr.net/gh/ganjoor/ganjoor-tajik-data@main/
## Content
- `GET {t.Poet}` — Tajik nickname/description for a poet
- `GET {t.Category}` — Tajik title/description for a category, plus only the children that
themselves have Tajik content
- `GET {t.Poem}` — Tajik title and per-verse text (`vOrder` + `tajikText` only)
## This repo does not stand alone
A poem file here has no metre, rhyme, or verse-position/section data — those live in
[ganjoor-data]({manifest.MainDataRepo}) under the *same* id and path. To render a Tajik poem
fully: fetch this repo's poem file for the Tajik text, fetch `ganjoor-data`'s poem file (same
`{t.Poem}` path) for structure, and merge the two verse lists by `vOrder`.
## Resolving a numeric id
Same scheme as `ganjoor-data`: `{t.PoetIdIndex}` (flat), `{t.CatIdIndexShard}` /
`{t.PoemIdIndexShard}` (bucketed, `bucket = id / {manifest.IdIndexShardSize}`). A missing id here
just means that poet/category/poem has no Tajik content yet — check `ganjoor-data`'s own id index
to confirm the id is valid at all.
## Not included
No comments, bookmarks, reading history, edit/correction history, or any other data linked to a
user account — same allowlist-only approach as `ganjoor-data`.
";
}
}
}

View File

@ -77,6 +77,16 @@
"GitUserName": "",
"GitToken": ""
},
"TajikPublicDataExport": {
"LocalWorkingCopyPath": "C:\\ganjoor-tajik-public-data",
"RemoteUrl": "https://github.com/ganjoor/ganjoor-tajik-data.git",
"Branch": "main",
"CommitAuthorName": "Ganjoor Tajik Export Bot",
"CommitAuthorEmail": "bot@ganjoor.net",
"PushEnabled": "False",
"GitUserName": "",
"GitToken": ""
},
"ExternalFTPServer": {
"Host": "localhost",
"Port": "22",