diff --git a/RMuseum/Controllers/GanjoorController.cs b/RMuseum/Controllers/GanjoorController.cs
index 57d1ff37..e9debb17 100644
--- a/RMuseum/Controllers/GanjoorController.cs
+++ b/RMuseum/Controllers/GanjoorController.cs
@@ -2423,6 +2423,30 @@ namespace RMuseum.Controllers
}
}
+ ///
+ /// start exporting all published Ganjoor data to the public git-tracked JSON data set
+ ///
+ ///
+ [HttpPost("publicdata/batchexport")]
+ [Authorize(Policy = RMuseumSecurableItem.GanjoorEntityShortName + ":" + RMuseumSecurableItem.ReviewSongs)]
+ [ProducesResponseType((int)HttpStatusCode.OK)]
+ [ProducesResponseType((int)HttpStatusCode.BadRequest, Type = typeof(string))]
+ [ProducesResponseType((int)HttpStatusCode.Unauthorized)]
+ public IActionResult StartBatchExportPublicGitData()
+ {
+ try
+ {
+ var res = _ganjoorService.StartBatchExportPublicGitData();
+ if (!string.IsNullOrEmpty(res.ExceptionString))
+ return BadRequest(res.ExceptionString);
+ return Ok();
+ }
+ catch (Exception exp)
+ {
+ return BadRequest(exp.ToString());
+ }
+ }
+
///
/// Get user public profile
///
diff --git a/RMuseum/Models/Ganjoor/PublicExport/PublicExportDtos.cs b/RMuseum/Models/Ganjoor/PublicExport/PublicExportDtos.cs
new file mode 100644
index 00000000..58ef5965
--- /dev/null
+++ b/RMuseum/Models/Ganjoor/PublicExport/PublicExportDtos.cs
@@ -0,0 +1,173 @@
+using System.Collections.Generic;
+
+namespace RMuseum.Models.Ganjoor.PublicExport
+{
+ ///
+ /// Root manifest written at the repository root (manifest.json).
+ /// Lets consuming apps detect schema changes and do cheap "did anything change" checks
+ /// without downloading the whole tree.
+ ///
+ public class PublicExportManifestDto
+ {
+ ///
+ /// bump this whenever a DTO shape changes in a way consumers should know about
+ ///
+ public int SchemaVersion { get; set; } = 1;
+
+ ///
+ /// UTC generation timestamp of this run (informational only — never embed per-file
+ /// timestamps inside poem/cat files, that would defeat deterministic diffs)
+ ///
+ public string GeneratedAtUtc { get; set; }
+
+ public int PoetsCount { get; set; }
+
+ public int PoemsCount { get; set; }
+
+ public List Poets { get; set; } = new List();
+ }
+
+ public class PublicExportManifestPoetEntryDto
+ {
+ public int Id { get; set; }
+ public string Nickname { get; set; }
+ public string FullUrl { get; set; }
+ }
+
+ ///
+ /// poet.json — biographical data only, no account/user linkage exists on GanjoorPoet at all
+ ///
+ public class PoetPublicDto
+ {
+ public int Id { get; set; }
+ public string Name { get; set; }
+ public string Nickname { get; set; }
+ public string Description { get; set; }
+ public string FullUrl { get; set; }
+ public string ImageUrl { get; set; }
+ public int BirthYearInLHijri { get; set; }
+ public bool ValidBirthDate { get; set; }
+ public int DeathYearInLHijri { get; set; }
+ public bool ValidDeathDate { get; set; }
+ public string BirthPlace { get; set; }
+ public string DeathPlace { get; set; }
+ }
+
+ ///
+ /// _cat.json — one per category/collection folder
+ ///
+ public class CatPublicDto
+ {
+ public int Id { get; set; }
+ public int PoetId { get; set; }
+ public int? ParentId { get; set; }
+ public string Title { get; set; }
+ public string FullUrl { get; set; }
+ public string Description { get; set; }
+ public string DescriptionHtml { get; set; }
+ public string BookName { get; set; }
+
+ ///
+ /// child categories, sorted by Id for deterministic output
+ ///
+ public List ChildCats { get; set; } = new List();
+
+ ///
+ /// poems directly under this category, sorted by Id for deterministic output
+ ///
+ public List Poems { get; set; } = new List();
+ }
+
+ public class CatChildRefDto
+ {
+ public int Id { get; set; }
+ public string Title { get; set; }
+ public string FullUrl { get; set; }
+ }
+
+ public class PoemChildRefDto
+ {
+ public int Id { get; set; }
+ public string Title { get; set; }
+ public string FullUrl { get; set; }
+ }
+
+ ///
+ /// {poem-slug}.json — the poem text itself. GanjoorPoem/GanjoorVerse/GanjoorPoemSection carry
+ /// no user/account reference in the source schema, so this is a straight field allowlist,
+ /// not a redaction pass.
+ ///
+ public class PoemPublicDto
+ {
+ public int Id { get; set; }
+ public int CatId { get; set; }
+ public string Title { get; set; }
+ public string FullTitle { get; set; }
+ public string FullUrl { get; set; }
+ public string RhymeLetters { get; set; }
+ public string SourceName { get; set; }
+ public string SourceUrlSlug { get; set; }
+ public string Language { get; set; }
+ public string PoemSummary { get; set; }
+ public MetreRefDto Metre { get; set; }
+ public List Sections { get; set; } = new List();
+ public List Verses { get; set; } = new List();
+ }
+
+ public class MetreRefDto
+ {
+ public int Id { get; set; }
+ public string Rhythm { get; set; }
+ public string Name { get; set; }
+ }
+
+ public class PoemSectionPublicDto
+ {
+ public int Index { get; set; }
+ public int Number { get; set; }
+ public string SectionType { get; set; }
+ public string VerseType { get; set; }
+ public string RhymeLetters { get; set; }
+ public string PlainText { get; set; }
+ public string HtmlText { get; set; }
+ public string PoemFormat { get; set; }
+ public string Language { get; set; }
+ public int CoupletsCount { get; set; }
+ }
+
+ public class VersePublicDto
+ {
+ public int VOrder { get; set; }
+ public string Position { get; set; }
+ public string Text { get; set; }
+ public int? CoupletIndex { get; set; }
+ public int? SectionIndex1 { get; set; }
+ public int? SectionIndex2 { get; set; }
+ public int? SectionIndex3 { get; set; }
+ public int? SectionIndex4 { get; set; }
+ }
+
+ ///
+ /// metres.json — shared lookup table written once at the repo root
+ ///
+ public class MetrePublicDto
+ {
+ public int Id { get; set; }
+ public string UrlSlug { get; set; }
+ public string Rhythm { get; set; }
+ public string Name { get; set; }
+ public string Description { get; set; }
+ }
+
+ ///
+ /// languages.json — shared lookup table written once at the repo root
+ ///
+ public class LanguagePublicDto
+ {
+ public int Id { get; set; }
+ public string Name { get; set; }
+ public string Code { get; set; }
+ public string NativeName { get; set; }
+ public bool RightToLeft { get; set; }
+ }
+}
diff --git a/RMuseum/RMuseum.csproj b/RMuseum/RMuseum.csproj
index b9265e84..a8f40b3a 100644
--- a/RMuseum/RMuseum.csproj
+++ b/RMuseum/RMuseum.csproj
@@ -22,6 +22,7 @@
+
all
diff --git a/RMuseum/RMuseum.xml b/RMuseum/RMuseum.xml
index 2434d438..c127d051 100644
--- a/RMuseum/RMuseum.xml
+++ b/RMuseum/RMuseum.xml
@@ -1775,6 +1775,12 @@
+
+
+ start exporting all published Ganjoor data to the public git-tracked JSON data set
+
+
+
Get user public profile
@@ -11174,6 +11180,61 @@
couplets (virtual) for Masnavi
+
+
+ Root manifest written at the repository root (manifest.json).
+ Lets consuming apps detect schema changes and do cheap "did anything change" checks
+ without downloading the whole tree.
+
+
+
+
+ bump this whenever a DTO shape changes in a way consumers should know about
+
+
+
+
+ UTC generation timestamp of this run (informational only — never embed per-file
+ timestamps inside poem/cat files, that would defeat deterministic diffs)
+
+
+
+
+ poet.json — biographical data only, no account/user linkage exists on GanjoorPoet at all
+
+
+
+
+ _cat.json — one per category/collection folder
+
+
+
+
+ child categories, sorted by Id for deterministic output
+
+
+
+
+ poems directly under this category, sorted by Id for deterministic output
+
+
+
+
+ {poem-slug}.json — the poem text itself. GanjoorPoem/GanjoorVerse/GanjoorPoemSection carry
+ no user/account reference in the source schema, so this is a straight field allowlist,
+ not a redaction pass.
+
+
+
+
+ metres.json — shared lookup table written once at the repo root
+
+
+
+
+ languages.json — shared lookup table written once at the repo root
+
+
Updating related sections logs
@@ -17383,6 +17444,13 @@
+
+
+ start exporting all published data (poets/categories/poems/verses) to the public
+ git-tracked JSON data set, committing and pushing changes since the last run
+
+
+
examine site pages for broken links
@@ -20273,6 +20341,9 @@
IGanjoorService implementation
+
+ IGanjoorService implementation
+
@@ -21006,6 +21077,22 @@
+
+
+ start exporting all published Ganjoor data (poets/categories/poems/verses) to a
+ git-tracked JSON tree and pushing it to the configured remote. User-linked tables
+ (comments, bookmarks, visits, corrections, accounting, ...) are never touched by
+ this code path — see RMuseum.Models.Ganjoor.PublicExport for the allowlisted shape
+ of what actually gets written.
+
+
+
+
+ recursively writes _cat.json for and every published poem directly
+ under it, then recurses into published child categories. Returns the number of poems written
+ in this subtree (for manifest counts).
+
+
moderate quoted poems
@@ -23787,6 +23874,97 @@
+
+
+ Writes JSON in a way that is stable across runs: fixed indentation, LF line endings,
+ UTF-8 without BOM, a single trailing newline, and Persian/Arabic text left unescaped
+ (the default encoder \u-escapes non-ASCII, which is both hard to review in a diff and
+ bloats file size for a mostly-Persian dataset).
+
+ Callers are responsible for sorting any collections before serializing — this class only
+ guarantees stable *encoding*, not stable *ordering*, since ordering is a data decision.
+
+ Files are only actually written to disk when the content differs from what's already
+ there, so an unmodified poem never shows up in `git status` even if the whole tree is
+ regenerated every run.
+
+
+
+
+ Serializes and writes it to only if the
+ resulting bytes differ from the file's current content. Creates parent directories as needed.
+ Returns true if the file was created or changed, false if it was already up to date.
+
+
+
+
+ Options for publishing the public export tree to a git remote.
+ Bind this from the "PublicDataExport" configuration section.
+
+
+
+
+ local working copy path (e.g. C:\ganjoor-public-data or /var/ganjoor/public-data)
+
+
+
+
+ remote URL, e.g. https://github.com/ganjoor/ganjoor-data.git
+
+
+
+
+ if false, everything is written and committed locally but never pushed —
+ useful for a first dry run before wiring up real credentials
+
+
+
+
+ git username for the push (for GitHub, a PAT is used as both username-independent
+ and as the token below — GitHub only checks the token)
+
+
+
+
+ personal access token / app token with push rights to RemoteUrl. Keep this in
+ user-secrets / environment variables, never committed to appsettings.json.
+
+
+
+
+ Ensures the local working copy exists and is up to date with the remote before the
+ export job starts writing files into it. Clones on first run, fetches + hard-resets
+ to the remote branch on subsequent runs (this working copy is bot-owned; nobody
+ should be hand-editing it, so a hard reset is safe and keeps the job idempotent).
+
+
+
+
+ Stages every change under the working copy and, if anything actually changed,
+ commits and (if enabled) pushes. Returns the number of files touched (0 means
+ nothing changed since last run — a normal, expected outcome on most nightly runs).
+
+
+
+
+ Belt-and-suspenders check on top of the allowlist design: even though the export DTOs
+ only ever declare the fields we explicitly want published, this scans every DTO type in
+ the RMuseum.Models.Ganjoor.PublicExport namespace by reflection and throws if a property
+ name or type looks like it could carry personal data. Call this once at application
+ startup (Development/CI) and/or from a test — see AssertSafe().
+
+
+
+
+ property-name patterns that must never appear on an export DTO
+
+
+
+
+ Scans every public class in the PublicExport DTO namespace. Throws InvalidOperationException
+ naming the offending type/property if anything trips the checks.
+
+
url
diff --git a/RMuseum/Services/IGanjoorService.cs b/RMuseum/Services/IGanjoorService.cs
index 90b011e2..140f3d97 100644
--- a/RMuseum/Services/IGanjoorService.cs
+++ b/RMuseum/Services/IGanjoorService.cs
@@ -633,6 +633,13 @@ namespace RMuseum.Services
///
RServiceResult StartBatchGenerateGDBFiles();
+ ///
+ /// start exporting all published data (poets/categories/poems/verses) to the public
+ /// git-tracked JSON data set, committing and pushing changes since the last run
+ ///
+ ///
+ RServiceResult StartBatchExportPublicGitData();
+
///
/// examine site pages for broken links
///
diff --git a/RMuseum/Services/Implementation/GanjoorService-Partials/GanjoorService-PublicDataExport.cs b/RMuseum/Services/Implementation/GanjoorService-Partials/GanjoorService-PublicDataExport.cs
new file mode 100644
index 00000000..8569ad82
--- /dev/null
+++ b/RMuseum/Services/Implementation/GanjoorService-Partials/GanjoorService-PublicDataExport.cs
@@ -0,0 +1,298 @@
+using Microsoft.EntityFrameworkCore;
+using RMuseum.DbContext;
+using RMuseum.Models.Ganjoor;
+using RMuseum.Models.Ganjoor.PublicExport;
+using RMuseum.Utils.PublicDataExport;
+using RSecurityBackend.Models.Generic;
+using RSecurityBackend.Services.Implementation;
+using System;
+using System.Collections.Generic;
+using System.IO;
+using System.Linq;
+using System.Threading.Tasks;
+
+namespace RMuseum.Services.Implementation
+{
+ ///
+ /// IGanjoorService implementation
+ ///
+ public partial class GanjoorService : IGanjoorService
+ {
+ ///
+ /// start exporting all published Ganjoor data (poets/categories/poems/verses) to a
+ /// git-tracked JSON tree and pushing it to the configured remote. User-linked tables
+ /// (comments, bookmarks, visits, corrections, accounting, ...) are never touched by
+ /// this code path — see RMuseum.Models.Ganjoor.PublicExport for the allowlisted shape
+ /// of what actually gets written.
+ ///
+ public RServiceResult StartBatchExportPublicGitData()
+ {
+ try
+ {
+ PublicExportSafetyGuard.AssertSafe();
+
+ _backgroundTaskQueue.QueueBackgroundWorkItem
+ (
+ async token =>
+ {
+ using (RMuseumDbContext context = new RMuseumDbContext(new DbContextOptions()))
+ {
+ LongRunningJobProgressServiceEF jobProgressServiceEF = new LongRunningJobProgressServiceEF(context);
+ var job = (await jobProgressServiceEF.NewJob("PublicDataExport", "Preparing working copy")).Result;
+
+ try
+ {
+ var options = ReadPublicDataExportOptions();
+ var publisher = new GitRepoPublisher(options);
+ publisher.EnsureWorkingCopyUpToDate();
+
+ string repoRoot = options.LocalWorkingCopyPath;
+
+ await jobProgressServiceEF.UpdateJob(job.Id, 1, "Writing shared lookup tables");
+ await ExportSharedLookupTables(context, repoRoot);
+
+ var poets = await context.GanjoorPoets.AsNoTracking()
+ .Include(p => p.BirthLocation)
+ .Include(p => p.DeathLocation)
+ .Where(p => p.Published)
+ .OrderBy(p => p.Id)
+ .ToListAsync();
+
+ var manifest = new PublicExportManifestDto
+ {
+ GeneratedAtUtc = DateTime.UtcNow.ToString("O"),
+ };
+
+ int poetIndex = 0;
+ foreach (var poet in poets)
+ {
+ poetIndex++;
+ await jobProgressServiceEF.UpdateJob(job.Id, (int)(100.0 * poetIndex / Math.Max(1, poets.Count)), $"Exporting {poet.Nickname}");
+
+ var catPoet = await context.GanjoorCategories.AsNoTracking()
+ .Where(c => c.PoetId == poet.Id && c.ParentId == null)
+ .SingleOrDefaultAsync();
+ if (catPoet == null || !catPoet.Published)
+ continue;
+
+ await ExportPoetToJson(context, repoRoot, poet, catPoet);
+
+ int poemCount = await ExportCatTreeToJson(context, repoRoot, catPoet);
+ manifest.PoemsCount += poemCount;
+
+ manifest.Poets.Add(new PublicExportManifestPoetEntryDto
+ {
+ Id = poet.Id,
+ Nickname = poet.Nickname,
+ FullUrl = catPoet.FullUrl,
+ });
+ }
+
+ manifest.PoetsCount = manifest.Poets.Count;
+
+ await DeterministicJsonWriter.WriteIfChangedAsync(Path.Combine(repoRoot, "manifest.json"), manifest);
+
+ await jobProgressServiceEF.UpdateJob(job.Id, 99, "Committing and pushing");
+ int changed = publisher.CommitAndPush($"data: export {manifest.PoetsCount} poets / {manifest.PoemsCount} poems — {DateTime.UtcNow:yyyy-MM-dd}");
+
+ await jobProgressServiceEF.UpdateJob(job.Id, 100, changed == 0 ? "No changes" : $"{changed} files changed", true);
+ }
+ catch (Exception exp)
+ {
+ await jobProgressServiceEF.UpdateJob(job.Id, 100, "", false, exp.ToString());
+ }
+ }
+ }
+ );
+
+ return new RServiceResult(true);
+ }
+ catch (Exception exp)
+ {
+ return new RServiceResult(false, exp.ToString());
+ }
+ }
+
+ private GitRepoPublisherOptions ReadPublicDataExportOptions()
+ {
+ var section = Configuration.GetSection("PublicDataExport");
+ return new GitRepoPublisherOptions
+ {
+ LocalWorkingCopyPath = section["LocalWorkingCopyPath"],
+ RemoteUrl = section["RemoteUrl"],
+ Branch = section["Branch"] ?? "main",
+ CommitAuthorName = section["CommitAuthorName"] ?? "Ganjoor Export Bot",
+ CommitAuthorEmail = section["CommitAuthorEmail"] ?? "bot@ganjoor.net",
+ PushEnabled = bool.TryParse(section["PushEnabled"], out var push) && push,
+ GitUserName = section["GitUserName"],
+ GitToken = section["GitToken"],
+ };
+ }
+
+ private async Task ExportSharedLookupTables(RMuseumDbContext context, string repoRoot)
+ {
+ var metres = await context.GanjoorMetres.AsNoTracking()
+ .OrderBy(m => m.Id)
+ .Select(m => new MetrePublicDto
+ {
+ Id = m.Id,
+ UrlSlug = m.UrlSlug,
+ Rhythm = m.Rhythm,
+ Name = m.Name,
+ Description = m.Description,
+ })
+ .ToListAsync();
+ await DeterministicJsonWriter.WriteIfChangedAsync(Path.Combine(repoRoot, "metres.json"), metres);
+
+ var languages = await context.GanjoorLanguages.AsNoTracking()
+ .OrderBy(l => l.Id)
+ .Select(l => new LanguagePublicDto
+ {
+ Id = l.Id,
+ Name = l.Name,
+ Code = l.Code,
+ NativeName = l.NativeName,
+ RightToLeft = l.RightToLeft,
+ })
+ .ToListAsync();
+ await DeterministicJsonWriter.WriteIfChangedAsync(Path.Combine(repoRoot, "languages.json"), languages);
+ }
+
+ private async Task ExportPoetToJson(RMuseumDbContext context, string repoRoot, GanjoorPoet poet, GanjoorCat catPoet)
+ {
+ var dto = new PoetPublicDto
+ {
+ Id = poet.Id,
+ Name = poet.Name,
+ Nickname = poet.Nickname,
+ Description = poet.Description,
+ FullUrl = catPoet.FullUrl,
+ ImageUrl = poet.RImageId == null ? null : $"https://ganjoor.net/api/ganjoor/poet/image{catPoet.FullUrl}.gif",
+ BirthYearInLHijri = poet.BirthYearInLHijri,
+ ValidBirthDate = poet.ValidBirthDate,
+ DeathYearInLHijri = poet.DeathYearInLHijri,
+ ValidDeathDate = poet.ValidDeathDate,
+ BirthPlace = poet.BirthLocation?.Name,
+ DeathPlace = poet.DeathLocation?.Name,
+ };
+
+ string path = Path.Combine(repoRoot, "poets", TrimLeadingSlash(catPoet.FullUrl), "poet.json");
+ await DeterministicJsonWriter.WriteIfChangedAsync(path, dto);
+ }
+
+ ///
+ /// recursively writes _cat.json for and every published poem directly
+ /// under it, then recurses into published child categories. Returns the number of poems written
+ /// in this subtree (for manifest counts).
+ ///
+ private async Task ExportCatTreeToJson(RMuseumDbContext context, string repoRoot, GanjoorCat cat)
+ {
+ var childCats = await context.GanjoorCategories.AsNoTracking()
+ .Where(c => c.ParentId == cat.Id && c.Published)
+ .OrderBy(c => c.Id)
+ .ToListAsync();
+
+ var poems = await context.GanjoorPoems.AsNoTracking()
+ .Where(p => p.CatId == cat.Id && p.Published)
+ .OrderBy(p => p.Id)
+ .ToListAsync();
+
+ var catDto = new CatPublicDto
+ {
+ Id = cat.Id,
+ PoetId = cat.PoetId,
+ ParentId = cat.ParentId,
+ Title = cat.Title,
+ FullUrl = cat.FullUrl,
+ Description = cat.Description,
+ DescriptionHtml = cat.DescriptionHtml,
+ BookName = cat.BookName,
+ ChildCats = childCats.Select(c => new CatChildRefDto { Id = c.Id, Title = c.Title, FullUrl = c.FullUrl }).ToList(),
+ Poems = poems.Select(p => new PoemChildRefDto { Id = p.Id, Title = p.Title, FullUrl = p.FullUrl }).ToList(),
+ };
+
+ string catDir = Path.Combine(repoRoot, "poets", TrimLeadingSlash(cat.FullUrl));
+ await DeterministicJsonWriter.WriteIfChangedAsync(Path.Combine(catDir, "_cat.json"), catDto);
+
+ int poemCount = poems.Count;
+
+ foreach (var poem in poems)
+ {
+ await ExportPoemToJson(context, repoRoot, poem);
+ }
+
+ foreach (var childCat in childCats)
+ {
+ poemCount += await ExportCatTreeToJson(context, repoRoot, childCat);
+ }
+
+ return poemCount;
+ }
+
+ private async Task ExportPoemToJson(RMuseumDbContext context, string repoRoot, GanjoorPoem poem)
+ {
+ var metre = poem.GanjoorMetreId == null
+ ? null
+ : await context.GanjoorMetres.AsNoTracking().Where(m => m.Id == poem.GanjoorMetreId).SingleOrDefaultAsync();
+
+ var sections = await context.GanjoorPoemSections.AsNoTracking()
+ .Where(s => s.PoemId == poem.Id)
+ .OrderBy(s => s.Index)
+ .ToListAsync();
+
+ var verses = await context.GanjoorVerses.AsNoTracking()
+ .Where(v => v.PoemId == poem.Id)
+ .OrderBy(v => v.VOrder)
+ .ToListAsync();
+
+ var dto = new PoemPublicDto
+ {
+ Id = poem.Id,
+ CatId = poem.CatId,
+ Title = poem.Title,
+ FullTitle = poem.FullTitle,
+ FullUrl = poem.FullUrl,
+ RhymeLetters = poem.RhymeLetters,
+ SourceName = poem.SourceName,
+ SourceUrlSlug = poem.SourceUrlSlug,
+ Language = poem.Language,
+ PoemSummary = poem.PoemSummary,
+ Metre = metre == null ? null : new MetreRefDto { Id = metre.Id, Rhythm = metre.Rhythm, Name = metre.Name },
+ Sections = sections.Select(s => new PoemSectionPublicDto
+ {
+ Index = s.Index,
+ Number = s.Number,
+ SectionType = s.SectionType.ToString(),
+ VerseType = s.VerseType.ToString(),
+ RhymeLetters = s.RhymeLetters,
+ PlainText = s.PlainText,
+ HtmlText = s.HtmlText,
+ PoemFormat = s.PoemFormat?.ToString(),
+ Language = s.Language,
+ CoupletsCount = s.CoupletsCount,
+ }).ToList(),
+ Verses = verses.Select(v => new VersePublicDto
+ {
+ VOrder = v.VOrder,
+ Position = v.VersePosition.ToString(),
+ Text = v.Text,
+ CoupletIndex = v.CoupletIndex,
+ SectionIndex1 = v.SectionIndex1,
+ SectionIndex2 = v.SectionIndex2,
+ SectionIndex3 = v.SectionIndex3,
+ SectionIndex4 = v.SectionIndex4,
+ }).ToList(),
+ };
+
+ string path = Path.Combine(repoRoot, "poets", TrimLeadingSlash(poem.FullUrl) + ".json");
+ await DeterministicJsonWriter.WriteIfChangedAsync(path, dto);
+ }
+
+ private static string TrimLeadingSlash(string url)
+ {
+ if (string.IsNullOrEmpty(url)) return url;
+ url = url.Replace('/', Path.DirectorySeparatorChar);
+ return url.TrimStart(Path.DirectorySeparatorChar);
+ }
+ }
+}
diff --git a/RMuseum/Utils/PublicDataExport/DeterministicJsonWriter.cs b/RMuseum/Utils/PublicDataExport/DeterministicJsonWriter.cs
new file mode 100644
index 00000000..4157833e
--- /dev/null
+++ b/RMuseum/Utils/PublicDataExport/DeterministicJsonWriter.cs
@@ -0,0 +1,75 @@
+using System.IO;
+using System.Text;
+using System.Text.Encodings.Web;
+using System.Text.Json;
+using System.Text.Unicode;
+using System.Threading.Tasks;
+
+namespace RMuseum.Utils.PublicDataExport
+{
+ ///
+ /// Writes JSON in a way that is stable across runs: fixed indentation, LF line endings,
+ /// UTF-8 without BOM, a single trailing newline, and Persian/Arabic text left unescaped
+ /// (the default encoder \u-escapes non-ASCII, which is both hard to review in a diff and
+ /// bloats file size for a mostly-Persian dataset).
+ ///
+ /// Callers are responsible for sorting any collections before serializing — this class only
+ /// guarantees stable *encoding*, not stable *ordering*, since ordering is a data decision.
+ ///
+ /// Files are only actually written to disk when the content differs from what's already
+ /// there, so an unmodified poem never shows up in `git status` even if the whole tree is
+ /// regenerated every run.
+ ///
+ public static class DeterministicJsonWriter
+ {
+ private static readonly JsonSerializerOptions _options = new JsonSerializerOptions
+ {
+ WriteIndented = true,
+ Encoder = JavaScriptEncoder.Create(UnicodeRanges.All),
+ DefaultIgnoreCondition = System.Text.Json.Serialization.JsonIgnoreCondition.WhenWritingNull,
+ };
+
+ private static readonly UTF8Encoding _utf8NoBom = new UTF8Encoding(encoderShouldEmitUTF8Identifier: false);
+
+ ///
+ /// Serializes and writes it to only if the
+ /// resulting bytes differ from the file's current content. Creates parent directories as needed.
+ /// Returns true if the file was created or changed, false if it was already up to date.
+ ///
+ public static async Task WriteIfChangedAsync(string path, T value)
+ {
+ string json = JsonSerializer.Serialize(value, _options);
+ // normalize to LF and ensure exactly one trailing newline, regardless of host OS
+ json = json.Replace("\r\n", "\n").TrimEnd('\n') + "\n";
+ byte[] newBytes = _utf8NoBom.GetBytes(json);
+
+ string dir = Path.GetDirectoryName(path);
+ if (!string.IsNullOrEmpty(dir) && !Directory.Exists(dir))
+ {
+ Directory.CreateDirectory(dir);
+ }
+
+ if (File.Exists(path))
+ {
+ byte[] existingBytes = await File.ReadAllBytesAsync(path);
+ if (BytesEqual(existingBytes, newBytes))
+ {
+ return false;
+ }
+ }
+
+ await File.WriteAllBytesAsync(path, newBytes);
+ return true;
+ }
+
+ private static bool BytesEqual(byte[] a, byte[] b)
+ {
+ if (a.Length != b.Length) return false;
+ for (int i = 0; i < a.Length; i++)
+ {
+ if (a[i] != b[i]) return false;
+ }
+ return true;
+ }
+ }
+}
diff --git a/RMuseum/Utils/PublicDataExport/GitRepoPublisher.cs b/RMuseum/Utils/PublicDataExport/GitRepoPublisher.cs
new file mode 100644
index 00000000..03f28c26
--- /dev/null
+++ b/RMuseum/Utils/PublicDataExport/GitRepoPublisher.cs
@@ -0,0 +1,141 @@
+using LibGit2Sharp;
+using System;
+using System.IO;
+using System.Linq;
+
+namespace RMuseum.Utils.PublicDataExport
+{
+ ///
+ /// Options for publishing the public export tree to a git remote.
+ /// Bind this from the "PublicDataExport" configuration section.
+ ///
+ public class GitRepoPublisherOptions
+ {
+ ///
+ /// local working copy path (e.g. C:\ganjoor-public-data or /var/ganjoor/public-data)
+ ///
+ public string LocalWorkingCopyPath { get; set; }
+
+ ///
+ /// remote URL, e.g. https://github.com/ganjoor/ganjoor-data.git
+ ///
+ public string RemoteUrl { get; set; }
+
+ public string Branch { get; set; } = "main";
+
+ public string CommitAuthorName { get; set; } = "Ganjoor Export Bot";
+
+ public string CommitAuthorEmail { get; set; } = "bot@ganjoor.net";
+
+ ///
+ /// if false, everything is written and committed locally but never pushed —
+ /// useful for a first dry run before wiring up real credentials
+ ///
+ public bool PushEnabled { get; set; }
+
+ ///
+ /// git username for the push (for GitHub, a PAT is used as both username-independent
+ /// and as the token below — GitHub only checks the token)
+ ///
+ public string GitUserName { get; set; }
+
+ ///
+ /// personal access token / app token with push rights to RemoteUrl. Keep this in
+ /// user-secrets / environment variables, never committed to appsettings.json.
+ ///
+ public string GitToken { get; set; }
+ }
+
+ public class GitRepoPublisher
+ {
+ private readonly GitRepoPublisherOptions _options;
+
+ public GitRepoPublisher(GitRepoPublisherOptions options)
+ {
+ _options = options;
+ }
+
+ ///
+ /// Ensures the local working copy exists and is up to date with the remote before the
+ /// export job starts writing files into it. Clones on first run, fetches + hard-resets
+ /// to the remote branch on subsequent runs (this working copy is bot-owned; nobody
+ /// should be hand-editing it, so a hard reset is safe and keeps the job idempotent).
+ ///
+ public void EnsureWorkingCopyUpToDate()
+ {
+ if (!Directory.Exists(_options.LocalWorkingCopyPath) ||
+ !Directory.Exists(Path.Combine(_options.LocalWorkingCopyPath, ".git")))
+ {
+ Directory.CreateDirectory(_options.LocalWorkingCopyPath);
+ var cloneOptions = new CloneOptions();
+ if (RemoteRequiresAuth())
+ {
+ cloneOptions.FetchOptions.CredentialsProvider = CredentialsHandler;
+ }
+ Repository.Clone(_options.RemoteUrl, _options.LocalWorkingCopyPath, cloneOptions);
+ return;
+ }
+
+ using var repo = new Repository(_options.LocalWorkingCopyPath);
+ var remote = repo.Network.Remotes["origin"];
+ var fetchOptions = new FetchOptions();
+ if (RemoteRequiresAuth())
+ {
+ fetchOptions.CredentialsProvider = CredentialsHandler;
+ }
+ Commands.Fetch(repo, remote.Name, remote.FetchRefSpecs.Select(r => r.Specification), fetchOptions, null);
+
+ var remoteBranch = repo.Branches[$"origin/{_options.Branch}"];
+ if (remoteBranch != null)
+ {
+ repo.Reset(ResetMode.Hard, remoteBranch.Tip);
+ }
+ }
+
+ ///
+ /// Stages every change under the working copy and, if anything actually changed,
+ /// commits and (if enabled) pushes. Returns the number of files touched (0 means
+ /// nothing changed since last run — a normal, expected outcome on most nightly runs).
+ ///
+ public int CommitAndPush(string commitMessage)
+ {
+ using var repo = new Repository(_options.LocalWorkingCopyPath);
+
+ Commands.Stage(repo, "*");
+
+ var status = repo.RetrieveStatus();
+ int changedCount = status.Count(s => s.State != FileStatus.Ignored && s.State != FileStatus.Unaltered);
+ if (changedCount == 0)
+ {
+ return 0;
+ }
+
+ var signature = new Signature(_options.CommitAuthorName, _options.CommitAuthorEmail, DateTimeOffset.UtcNow);
+ repo.Commit(commitMessage, signature, signature);
+
+ if (_options.PushEnabled)
+ {
+ var pushOptions = new PushOptions();
+ if (RemoteRequiresAuth())
+ {
+ pushOptions.CredentialsProvider = CredentialsHandler;
+ }
+ var branch = repo.Branches[_options.Branch] ?? repo.CreateBranch(_options.Branch);
+ repo.Network.Push(branch, pushOptions);
+ }
+
+ return changedCount;
+ }
+
+ private bool RemoteRequiresAuth() => !string.IsNullOrEmpty(_options.GitToken);
+
+ private Credentials CredentialsHandler(string url, string usernameFromUrl, SupportedCredentialTypes types)
+ {
+ return new UsernamePasswordCredentials
+ {
+ Username = string.IsNullOrEmpty(_options.GitUserName) ? _options.GitToken : _options.GitUserName,
+ Password = _options.GitToken,
+ };
+ }
+ }
+}
diff --git a/RMuseum/Utils/PublicDataExport/PublicExportSafetyGuard.cs b/RMuseum/Utils/PublicDataExport/PublicExportSafetyGuard.cs
new file mode 100644
index 00000000..804b3f35
--- /dev/null
+++ b/RMuseum/Utils/PublicDataExport/PublicExportSafetyGuard.cs
@@ -0,0 +1,77 @@
+using RMuseum.Models.Ganjoor.PublicExport;
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using System.Reflection;
+using System.Text.RegularExpressions;
+
+namespace RMuseum.Utils.PublicDataExport
+{
+ ///
+ /// Belt-and-suspenders check on top of the allowlist design: even though the export DTOs
+ /// only ever declare the fields we explicitly want published, this scans every DTO type in
+ /// the RMuseum.Models.Ganjoor.PublicExport namespace by reflection and throws if a property
+ /// name or type looks like it could carry personal data. Call this once at application
+ /// startup (Development/CI) and/or from a test — see AssertSafe().
+ ///
+ public static class PublicExportSafetyGuard
+ {
+ ///
+ /// property-name patterns that must never appear on an export DTO
+ ///
+ private static readonly Regex[] _forbiddenNamePatterns = new[]
+ {
+ new Regex("UserId", RegexOptions.IgnoreCase),
+ new Regex("OwnerId", RegexOptions.IgnoreCase),
+ new Regex("ReviewerId", RegexOptions.IgnoreCase),
+ new Regex("^Email$", RegexOptions.IgnoreCase),
+ new Regex("Email$", RegexOptions.IgnoreCase),
+ new Regex("^Ip$", RegexOptions.IgnoreCase),
+ new Regex("IpAddress", RegexOptions.IgnoreCase),
+ new Regex("Password", RegexOptions.IgnoreCase),
+ new Regex("Token", RegexOptions.IgnoreCase),
+ new Regex("PhoneNumber", RegexOptions.IgnoreCase),
+ new Regex("^AuthorName$", RegexOptions.IgnoreCase),
+ new Regex("^AuthorUrl$", RegexOptions.IgnoreCase),
+ };
+
+ ///
+ /// Scans every public class in the PublicExport DTO namespace. Throws InvalidOperationException
+ /// naming the offending type/property if anything trips the checks.
+ ///
+ public static void AssertSafe()
+ {
+ var dtoTypes = typeof(PoemPublicDto).Assembly
+ .GetTypes()
+ .Where(t => t.IsClass && t.Namespace == typeof(PoemPublicDto).Namespace)
+ .ToList();
+
+ var violations = new List();
+
+ foreach (var type in dtoTypes)
+ {
+ foreach (var prop in type.GetProperties(BindingFlags.Public | BindingFlags.Instance))
+ {
+ if (_forbiddenNamePatterns.Any(p => p.IsMatch(prop.Name)))
+ {
+ violations.Add($"{type.Name}.{prop.Name} matches a forbidden field-name pattern");
+ }
+
+ // a bare Guid property on a public export DTO is almost always an entity/user
+ // reference (our public ids are all ints); flag it for manual review
+ if (prop.PropertyType == typeof(Guid) || prop.PropertyType == typeof(Guid?))
+ {
+ violations.Add($"{type.Name}.{prop.Name} is a Guid — likely an internal entity/user reference, not public data");
+ }
+ }
+ }
+
+ if (violations.Count > 0)
+ {
+ throw new InvalidOperationException(
+ "PublicExportSafetyGuard failed — the following fields must not exist on a public export DTO:" +
+ Environment.NewLine + string.Join(Environment.NewLine, violations));
+ }
+ }
+ }
+}
diff --git a/RMuseum/appsettings.json b/RMuseum/appsettings.json
index a1ea04ae..9056f3da 100644
--- a/RMuseum/appsettings.json
+++ b/RMuseum/appsettings.json
@@ -67,6 +67,16 @@
"MaxWordLengthInComments": 200,
"TajikSitemapLocation": "C:\\inetpub\\tj\\wwwroot\\sitemap.xml"
},
+ "PublicDataExport": {
+ "LocalWorkingCopyPath": "C:\\ganjoor-public-data",
+ "RemoteUrl": "https://github.com/ganjoor/ganjoor-data.git",
+ "Branch": "main",
+ "CommitAuthorName": "Ganjoor Export Bot",
+ "CommitAuthorEmail": "bot@ganjoor.net",
+ "PushEnabled": "False",
+ "GitUserName": "",
+ "GitToken": ""
+ },
"ExternalFTPServer": {
"Host": "localhost",
"Port": "22",