anas/divan
1
0
Fork 0
Code Issues 12 Pull Requests Actions Packages Projects Releases Wiki Activity

Open source all public data

Browse Source 
#498
This commit is contained in:
Hamid Reza Mohammadi 2026-08-14 13:37:43 +03:30
parent f3db50d512
commit ae50557acc
10 changed files with 984 additions and 0 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

24
RMuseum/Controllers/GanjoorController.cs
View File

@ -2423,6 +2423,30 @@ namespace RMuseum.Controllers
}
}
/// <summary>
/// start exporting all published Ganjoor data to the public git-tracked JSON data set
/// </summary>
/// <returns></returns>
[HttpPost("publicdata/batchexport")]
[Authorize(Policy = RMuseumSecurableItem.GanjoorEntityShortName + ":" + RMuseumSecurableItem.ReviewSongs)]
[ProducesResponseType((int)HttpStatusCode.OK)]
[ProducesResponseType((int)HttpStatusCode.BadRequest, Type = typeof(string))]
[ProducesResponseType((int)HttpStatusCode.Unauthorized)]
public IActionResult StartBatchExportPublicGitData()
{
try
{
var res = _ganjoorService.StartBatchExportPublicGitData();
if (!string.IsNullOrEmpty(res.ExceptionString))
return BadRequest(res.ExceptionString);
return Ok();
}
catch (Exception exp)
{
return BadRequest(exp.ToString());
}
}
/// <summary>
/// Get user public profile
/// </summary>

173
RMuseum/Models/Ganjoor/PublicExport/PublicExportDtos.cs Normal file
View File

@ -0,0 +1,173 @@
using System.Collections.Generic;
namespace RMuseum.Models.Ganjoor.PublicExport
{
/// <summary>
/// Root manifest written at the repository root (manifest.json).
/// Lets consuming apps detect schema changes and do cheap "did anything change" checks
/// without downloading the whole tree.
/// </summary>
public class PublicExportManifestDto
{
/// <summary>
/// bump this whenever a DTO shape changes in a way consumers should know about
/// </summary>
public int SchemaVersion { get; set; } = 1;
/// <summary>
/// UTC generation timestamp of this run (informational only — never embed per-file
/// timestamps inside poem/cat files, that would defeat deterministic diffs)
/// </summary>
public string GeneratedAtUtc { get; set; }
public int PoetsCount { get; set; }
public int PoemsCount { get; set; }
public List<PublicExportManifestPoetEntryDto> Poets { get; set; } = new List<PublicExportManifestPoetEntryDto>();
}
public class PublicExportManifestPoetEntryDto
{
public int Id { get; set; }
public string Nickname { get; set; }
public string FullUrl { get; set; }
}
/// <summary>
/// poet.json — biographical data only, no account/user linkage exists on GanjoorPoet at all
/// </summary>
public class PoetPublicDto
{
public int Id { get; set; }
public string Name { get; set; }
public string Nickname { get; set; }
public string Description { get; set; }
public string FullUrl { get; set; }
public string ImageUrl { get; set; }
public int BirthYearInLHijri { get; set; }
public bool ValidBirthDate { get; set; }
public int DeathYearInLHijri { get; set; }
public bool ValidDeathDate { get; set; }
public string BirthPlace { get; set; }
public string DeathPlace { get; set; }
}
/// <summary>
/// _cat.json — one per category/collection folder
/// </summary>
public class CatPublicDto
{
public int Id { get; set; }
public int PoetId { get; set; }
public int? ParentId { get; set; }
public string Title { get; set; }
public string FullUrl { get; set; }
public string Description { get; set; }
public string DescriptionHtml { get; set; }
public string BookName { get; set; }
/// <summary>
/// child categories, sorted by Id for deterministic output
/// </summary>
public List<CatChildRefDto> ChildCats { get; set; } = new List<CatChildRefDto>();
/// <summary>
/// poems directly under this category, sorted by Id for deterministic output
/// </summary>
public List<PoemChildRefDto> Poems { get; set; } = new List<PoemChildRefDto>();
}
public class CatChildRefDto
{
public int Id { get; set; }
public string Title { get; set; }
public string FullUrl { get; set; }
}
public class PoemChildRefDto
{
public int Id { get; set; }
public string Title { get; set; }
public string FullUrl { get; set; }
}
/// <summary>
/// {poem-slug}.json — the poem text itself. GanjoorPoem/GanjoorVerse/GanjoorPoemSection carry
/// no user/account reference in the source schema, so this is a straight field allowlist,
/// not a redaction pass.
/// </summary>
public class PoemPublicDto
{
public int Id { get; set; }
public int CatId { get; set; }
public string Title { get; set; }
public string FullTitle { get; set; }
public string FullUrl { get; set; }
public string RhymeLetters { get; set; }
public string SourceName { get; set; }
public string SourceUrlSlug { get; set; }
public string Language { get; set; }
public string PoemSummary { get; set; }
public MetreRefDto Metre { get; set; }
public List<PoemSectionPublicDto> Sections { get; set; } = new List<PoemSectionPublicDto>();
public List<VersePublicDto> Verses { get; set; } = new List<VersePublicDto>();
}
public class MetreRefDto
{
public int Id { get; set; }
public string Rhythm { get; set; }
public string Name { get; set; }
}
public class PoemSectionPublicDto
{
public int Index { get; set; }
public int Number { get; set; }
public string SectionType { get; set; }
public string VerseType { get; set; }
public string RhymeLetters { get; set; }
public string PlainText { get; set; }
public string HtmlText { get; set; }
public string PoemFormat { get; set; }
public string Language { get; set; }
public int CoupletsCount { get; set; }
}
public class VersePublicDto
{
public int VOrder { get; set; }
public string Position { get; set; }
public string Text { get; set; }
public int? CoupletIndex { get; set; }
public int? SectionIndex1 { get; set; }
public int? SectionIndex2 { get; set; }
public int? SectionIndex3 { get; set; }
public int? SectionIndex4 { get; set; }
}
/// <summary>
/// metres.json — shared lookup table written once at the repo root
/// </summary>
public class MetrePublicDto
{
public int Id { get; set; }
public string UrlSlug { get; set; }
public string Rhythm { get; set; }
public string Name { get; set; }
public string Description { get; set; }
}
/// <summary>
/// languages.json — shared lookup table written once at the repo root
/// </summary>
public class LanguagePublicDto
{
public int Id { get; set; }
public string Name { get; set; }
public string Code { get; set; }
public string NativeName { get; set; }
public bool RightToLeft { get; set; }
}
}

1
RMuseum/RMuseum.csproj
View File

@ -22,6 +22,7 @@
<PackageReference Include="DNTPersianUtils.Core" Version="7.0.0" />
<PackageReference Include="FluentFTP" Version="54.2.0" />
<PackageReference Include="HtmlSanitizer" Version="9.1.982" />
<PackageReference Include="LibGit2Sharp" Version="0.31.0" />
<PackageReference Include="Microsoft.Data.Sqlite" Version="10.0.10" />
<PackageReference Include="Microsoft.EntityFrameworkCore.Design" Version="10.0.10">
<PrivateAssets>all</PrivateAssets>

178
RMuseum/RMuseum.xml
View File

@ -1775,6 +1775,12 @@
</summary>
<returns></returns>
</member>
<member name="M:RMuseum.Controllers.GanjoorController.StartBatchExportPublicGitData">
<summary>
start exporting all published Ganjoor data to the public git-tracked JSON data set
</summary>
<returns></returns>
</member>
<member name="M:RMuseum.Controllers.GanjoorController.GetUserPublicProfile(System.Guid)">
<summary>
Get user public profile
@ -11174,6 +11180,61 @@
couplets (virtual) for Masnavi
</summary>
</member>
<member name="T:RMuseum.Models.Ganjoor.PublicExport.PublicExportManifestDto">
<summary>
Root manifest written at the repository root (manifest.json).
Lets consuming apps detect schema changes and do cheap "did anything change" checks
without downloading the whole tree.
</summary>
</member>
<member name="P:RMuseum.Models.Ganjoor.PublicExport.PublicExportManifestDto.SchemaVersion">
<summary>
bump this whenever a DTO shape changes in a way consumers should know about
</summary>
</member>
<member name="P:RMuseum.Models.Ganjoor.PublicExport.PublicExportManifestDto.GeneratedAtUtc">
<summary>
UTC generation timestamp of this run (informational only — never embed per-file
timestamps inside poem/cat files, that would defeat deterministic diffs)
</summary>
</member>
<member name="T:RMuseum.Models.Ganjoor.PublicExport.PoetPublicDto">
<summary>
poet.json — biographical data only, no account/user linkage exists on GanjoorPoet at all
</summary>
</member>
<member name="T:RMuseum.Models.Ganjoor.PublicExport.CatPublicDto">
<summary>
_cat.json — one per category/collection folder
</summary>
</member>
<member name="P:RMuseum.Models.Ganjoor.PublicExport.CatPublicDto.ChildCats">
<summary>
child categories, sorted by Id for deterministic output
</summary>
</member>
<member name="P:RMuseum.Models.Ganjoor.PublicExport.CatPublicDto.Poems">
<summary>
poems directly under this category, sorted by Id for deterministic output
</summary>
</member>
<member name="T:RMuseum.Models.Ganjoor.PublicExport.PoemPublicDto">
<summary>
{poem-slug}.json — the poem text itself. GanjoorPoem/GanjoorVerse/GanjoorPoemSection carry
no user/account reference in the source schema, so this is a straight field allowlist,
not a redaction pass.
</summary>
</member>
<member name="T:RMuseum.Models.Ganjoor.PublicExport.MetrePublicDto">
<summary>
metres.json — shared lookup table written once at the repo root
</summary>
</member>
<member name="T:RMuseum.Models.Ganjoor.PublicExport.LanguagePublicDto">
<summary>
languages.json — shared lookup table written once at the repo root
</summary>
</member>
<member name="T:RMuseum.Models.Ganjoor.UpdatingRelSectsLog">
<summary>
Updating related sections logs
@ -17383,6 +17444,13 @@
</summary>
<returns></returns>
</member>
<member name="M:RMuseum.Services.IGanjoorService.StartBatchExportPublicGitData">
<summary>
start exporting all published data (poets/categories/poems/verses) to the public
git-tracked JSON data set, committing and pushing changes since the last run
</summary>
<returns></returns>
</member>
<member name="M:RMuseum.Services.IGanjoorService.HealthCheckContents">
<summary>
examine site pages for broken links
@ -20273,6 +20341,9 @@
<summary>
IGanjoorService implementation
</summary>
<summary>
IGanjoorService implementation
</summary>
</member>
<member name="M:RMuseum.Services.Implementation.GanjoorService.SwitchCoupletBookmark(System.Guid,System.Int32,System.Int32)">
<summary>
@ -21006,6 +21077,22 @@
<param name="metre"></param>
<returns></returns>
</member>
<member name="M:RMuseum.Services.Implementation.GanjoorService.StartBatchExportPublicGitData">
<summary>
start exporting all published Ganjoor data (poets/categories/poems/verses) to a
git-tracked JSON tree and pushing it to the configured remote. User-linked tables
(comments, bookmarks, visits, corrections, accounting, ...) are never touched by
this code path — see RMuseum.Models.Ganjoor.PublicExport for the allowlisted shape
of what actually gets written.
</summary>
</member>
<member name="M:RMuseum.Services.Implementation.GanjoorService.ExportCatTreeToJson(RMuseum.DbContext.RMuseumDbContext,System.String,RMuseum.Models.Ganjoor.GanjoorCat)">
<summary>
recursively writes _cat.json for <paramref name="cat"/> and every published poem directly
under it, then recurses into published child categories. Returns the number of poems written
in this subtree (for manifest counts).
</summary>
</member>
<member name="M:RMuseum.Services.Implementation.GanjoorService.ModerateGanjoorQuotedPoemAsync(RMuseum.Models.Ganjoor.ViewModels.GanjoorQuotedPoemModerationViewModel,System.Guid)">
<summary>
moderate quoted poems
@ -23787,6 +23874,97 @@
<param name="poemHtml"></param>
<returns></returns>
</member>
<member name="T:RMuseum.Utils.PublicDataExport.DeterministicJsonWriter">
<summary>
Writes JSON in a way that is stable across runs: fixed indentation, LF line endings,
UTF-8 without BOM, a single trailing newline, and Persian/Arabic text left unescaped
(the default encoder \u-escapes non-ASCII, which is both hard to review in a diff and
bloats file size for a mostly-Persian dataset).
Callers are responsible for sorting any collections before serializing — this class only
guarantees stable *encoding*, not stable *ordering*, since ordering is a data decision.
Files are only actually written to disk when the content differs from what's already
there, so an unmodified poem never shows up in `git status` even if the whole tree is
regenerated every run.
</summary>
</member>
<member name="M:RMuseum.Utils.PublicDataExport.DeterministicJsonWriter.WriteIfChangedAsync``1(System.String,``0)">
<summary>
Serializes <paramref name="value"/> and writes it to <paramref name="path"/> only if the
resulting bytes differ from the file's current content. Creates parent directories as needed.
Returns true if the file was created or changed, false if it was already up to date.
</summary>
</member>
<member name="T:RMuseum.Utils.PublicDataExport.GitRepoPublisherOptions">
<summary>
Options for publishing the public export tree to a git remote.
Bind this from the "PublicDataExport" configuration section.
</summary>
</member>
<member name="P:RMuseum.Utils.PublicDataExport.GitRepoPublisherOptions.LocalWorkingCopyPath">
<summary>
local working copy path (e.g. C:\ganjoor-public-data or /var/ganjoor/public-data)
</summary>
</member>
<member name="P:RMuseum.Utils.PublicDataExport.GitRepoPublisherOptions.RemoteUrl">
<summary>
remote URL, e.g. https://github.com/ganjoor/ganjoor-data.git
</summary>
</member>
<member name="P:RMuseum.Utils.PublicDataExport.GitRepoPublisherOptions.PushEnabled">
<summary>
if false, everything is written and committed locally but never pushed —
useful for a first dry run before wiring up real credentials
</summary>
</member>
<member name="P:RMuseum.Utils.PublicDataExport.GitRepoPublisherOptions.GitUserName">
<summary>
git username for the push (for GitHub, a PAT is used as both username-independent
and as the token below — GitHub only checks the token)
</summary>
</member>
<member name="P:RMuseum.Utils.PublicDataExport.GitRepoPublisherOptions.GitToken">
<summary>
personal access token / app token with push rights to RemoteUrl. Keep this in
user-secrets / environment variables, never committed to appsettings.json.
</summary>
</member>
<member name="M:RMuseum.Utils.PublicDataExport.GitRepoPublisher.EnsureWorkingCopyUpToDate">
<summary>
Ensures the local working copy exists and is up to date with the remote before the
export job starts writing files into it. Clones on first run, fetches + hard-resets
to the remote branch on subsequent runs (this working copy is bot-owned; nobody
should be hand-editing it, so a hard reset is safe and keeps the job idempotent).
</summary>
</member>
<member name="M:RMuseum.Utils.PublicDataExport.GitRepoPublisher.CommitAndPush(System.String)">
<summary>
Stages every change under the working copy and, if anything actually changed,
commits and (if enabled) pushes. Returns the number of files touched (0 means
nothing changed since last run — a normal, expected outcome on most nightly runs).
</summary>
</member>
<member name="T:RMuseum.Utils.PublicDataExport.PublicExportSafetyGuard">
<summary>
Belt-and-suspenders check on top of the allowlist design: even though the export DTOs
only ever declare the fields we explicitly want published, this scans every DTO type in
the RMuseum.Models.Ganjoor.PublicExport namespace by reflection and throws if a property
name or type looks like it could carry personal data. Call this once at application
startup (Development/CI) and/or from a test — see AssertSafe().
</summary>
</member>
<member name="F:RMuseum.Utils.PublicDataExport.PublicExportSafetyGuard._forbiddenNamePatterns">
<summary>
property-name patterns that must never appear on an export DTO
</summary>
</member>
<member name="M:RMuseum.Utils.PublicDataExport.PublicExportSafetyGuard.AssertSafe">
<summary>
Scans every public class in the PublicExport DTO namespace. Throws InvalidOperationException
naming the offending type/property if anything trips the checks.
</summary>
</member>
<member name="P:RMuseum.WebServiceUrl.Url">
<summary>
url

7
RMuseum/Services/IGanjoorService.cs
View File

@ -633,6 +633,13 @@ namespace RMuseum.Services
/// <returns></returns>
RServiceResult<bool> StartBatchGenerateGDBFiles();
/// <summary>
/// start exporting all published data (poets/categories/poems/verses) to the public
/// git-tracked JSON data set, committing and pushing changes since the last run
/// </summary>
/// <returns></returns>
RServiceResult<bool> StartBatchExportPublicGitData();
/// <summary>
/// examine site pages for broken links
/// </summary>

298
RMuseum/Services/Implementation/GanjoorService-Partials/GanjoorService-PublicDataExport.cs Normal file
View File

@ -0,0 +1,298 @@
using Microsoft.EntityFrameworkCore;
using RMuseum.DbContext;
using RMuseum.Models.Ganjoor;
using RMuseum.Models.Ganjoor.PublicExport;
using RMuseum.Utils.PublicDataExport;
using RSecurityBackend.Models.Generic;
using RSecurityBackend.Services.Implementation;
using System;
using System.Collections.Generic;
using System.IO;
using System.Linq;
using System.Threading.Tasks;
namespace RMuseum.Services.Implementation
{
/// <summary>
/// IGanjoorService implementation
/// </summary>
public partial class GanjoorService : IGanjoorService
{
/// <summary>
/// start exporting all published Ganjoor data (poets/categories/poems/verses) to a
/// git-tracked JSON tree and pushing it to the configured remote. User-linked tables
/// (comments, bookmarks, visits, corrections, accounting, ...) are never touched by
/// this code path — see RMuseum.Models.Ganjoor.PublicExport for the allowlisted shape
/// of what actually gets written.
/// </summary>
public RServiceResult<bool> StartBatchExportPublicGitData()
{
try
{
PublicExportSafetyGuard.AssertSafe();
_backgroundTaskQueue.QueueBackgroundWorkItem
(
async token =>
{
using (RMuseumDbContext context = new RMuseumDbContext(new DbContextOptions<RMuseumDbContext>()))
{
LongRunningJobProgressServiceEF jobProgressServiceEF = new LongRunningJobProgressServiceEF(context);
var job = (await jobProgressServiceEF.NewJob("PublicDataExport", "Preparing working copy")).Result;
try
{
var options = ReadPublicDataExportOptions();
var publisher = new GitRepoPublisher(options);
publisher.EnsureWorkingCopyUpToDate();
string repoRoot = options.LocalWorkingCopyPath;
await jobProgressServiceEF.UpdateJob(job.Id, 1, "Writing shared lookup tables");
await ExportSharedLookupTables(context, repoRoot);
var poets = await context.GanjoorPoets.AsNoTracking()
.Include(p => p.BirthLocation)
.Include(p => p.DeathLocation)
.Where(p => p.Published)
.OrderBy(p => p.Id)
.ToListAsync();
var manifest = new PublicExportManifestDto
{
GeneratedAtUtc = DateTime.UtcNow.ToString("O"),
};
int poetIndex = 0;
foreach (var poet in poets)
{
poetIndex++;
await jobProgressServiceEF.UpdateJob(job.Id, (int)(100.0 * poetIndex / Math.Max(1, poets.Count)), $"Exporting {poet.Nickname}");
var catPoet = await context.GanjoorCategories.AsNoTracking()
.Where(c => c.PoetId == poet.Id && c.ParentId == null)
.SingleOrDefaultAsync();
if (catPoet == null || !catPoet.Published)
continue;
await ExportPoetToJson(context, repoRoot, poet, catPoet);
int poemCount = await ExportCatTreeToJson(context, repoRoot, catPoet);
manifest.PoemsCount += poemCount;
manifest.Poets.Add(new PublicExportManifestPoetEntryDto
{
Id = poet.Id,
Nickname = poet.Nickname,
FullUrl = catPoet.FullUrl,
});
}
manifest.PoetsCount = manifest.Poets.Count;
await DeterministicJsonWriter.WriteIfChangedAsync(Path.Combine(repoRoot, "manifest.json"), manifest);
await jobProgressServiceEF.UpdateJob(job.Id, 99, "Committing and pushing");
int changed = publisher.CommitAndPush($"data: export {manifest.PoetsCount} poets / {manifest.PoemsCount} poems — {DateTime.UtcNow:yyyy-MM-dd}");
await jobProgressServiceEF.UpdateJob(job.Id, 100, changed == 0 ? "No changes" : $"{changed} files changed", true);
}
catch (Exception exp)
{
await jobProgressServiceEF.UpdateJob(job.Id, 100, "", false, exp.ToString());
}
}
}
);
return new RServiceResult<bool>(true);
}
catch (Exception exp)
{
return new RServiceResult<bool>(false, exp.ToString());
}
}
private GitRepoPublisherOptions ReadPublicDataExportOptions()
{
var section = Configuration.GetSection("PublicDataExport");
return new GitRepoPublisherOptions
{
LocalWorkingCopyPath = section["LocalWorkingCopyPath"],
RemoteUrl = section["RemoteUrl"],
Branch = section["Branch"] ?? "main",
CommitAuthorName = section["CommitAuthorName"] ?? "Ganjoor Export Bot",
CommitAuthorEmail = section["CommitAuthorEmail"] ?? "bot@ganjoor.net",
PushEnabled = bool.TryParse(section["PushEnabled"], out var push) && push,
GitUserName = section["GitUserName"],
GitToken = section["GitToken"],
};
}
private async Task ExportSharedLookupTables(RMuseumDbContext context, string repoRoot)
{
var metres = await context.GanjoorMetres.AsNoTracking()
.OrderBy(m => m.Id)
.Select(m => new MetrePublicDto
{
Id = m.Id,
UrlSlug = m.UrlSlug,
Rhythm = m.Rhythm,
Name = m.Name,
Description = m.Description,
})
.ToListAsync();
await DeterministicJsonWriter.WriteIfChangedAsync(Path.Combine(repoRoot, "metres.json"), metres);
var languages = await context.GanjoorLanguages.AsNoTracking()
.OrderBy(l => l.Id)
.Select(l => new LanguagePublicDto
{
Id = l.Id,
Name = l.Name,
Code = l.Code,
NativeName = l.NativeName,
RightToLeft = l.RightToLeft,
})
.ToListAsync();
await DeterministicJsonWriter.WriteIfChangedAsync(Path.Combine(repoRoot, "languages.json"), languages);
}
private async Task ExportPoetToJson(RMuseumDbContext context, string repoRoot, GanjoorPoet poet, GanjoorCat catPoet)
{
var dto = new PoetPublicDto
{
Id = poet.Id,
Name = poet.Name,
Nickname = poet.Nickname,
Description = poet.Description,
FullUrl = catPoet.FullUrl,
ImageUrl = poet.RImageId == null ? null : $"https://ganjoor.net/api/ganjoor/poet/image{catPoet.FullUrl}.gif",
BirthYearInLHijri = poet.BirthYearInLHijri,
ValidBirthDate = poet.ValidBirthDate,
DeathYearInLHijri = poet.DeathYearInLHijri,
ValidDeathDate = poet.ValidDeathDate,
BirthPlace = poet.BirthLocation?.Name,
DeathPlace = poet.DeathLocation?.Name,
};
string path = Path.Combine(repoRoot, "poets", TrimLeadingSlash(catPoet.FullUrl), "poet.json");
await DeterministicJsonWriter.WriteIfChangedAsync(path, dto);
}
/// <summary>
/// recursively writes _cat.json for <paramref name="cat"/> and every published poem directly
/// under it, then recurses into published child categories. Returns the number of poems written
/// in this subtree (for manifest counts).
/// </summary>
private async Task<int> ExportCatTreeToJson(RMuseumDbContext context, string repoRoot, GanjoorCat cat)
{
var childCats = await context.GanjoorCategories.AsNoTracking()
.Where(c => c.ParentId == cat.Id && c.Published)
.OrderBy(c => c.Id)
.ToListAsync();
var poems = await context.GanjoorPoems.AsNoTracking()
.Where(p => p.CatId == cat.Id && p.Published)
.OrderBy(p => p.Id)
.ToListAsync();
var catDto = new CatPublicDto
{
Id = cat.Id,
PoetId = cat.PoetId,
ParentId = cat.ParentId,
Title = cat.Title,
FullUrl = cat.FullUrl,
Description = cat.Description,
DescriptionHtml = cat.DescriptionHtml,
BookName = cat.BookName,
ChildCats = childCats.Select(c => new CatChildRefDto { Id = c.Id, Title = c.Title, FullUrl = c.FullUrl }).ToList(),
Poems = poems.Select(p => new PoemChildRefDto { Id = p.Id, Title = p.Title, FullUrl = p.FullUrl }).ToList(),
};
string catDir = Path.Combine(repoRoot, "poets", TrimLeadingSlash(cat.FullUrl));
await DeterministicJsonWriter.WriteIfChangedAsync(Path.Combine(catDir, "_cat.json"), catDto);
int poemCount = poems.Count;
foreach (var poem in poems)
{
await ExportPoemToJson(context, repoRoot, poem);
}
foreach (var childCat in childCats)
{
poemCount += await ExportCatTreeToJson(context, repoRoot, childCat);
}
return poemCount;
}
private async Task ExportPoemToJson(RMuseumDbContext context, string repoRoot, GanjoorPoem poem)
{
var metre = poem.GanjoorMetreId == null
? null
: await context.GanjoorMetres.AsNoTracking().Where(m => m.Id == poem.GanjoorMetreId).SingleOrDefaultAsync();
var sections = await context.GanjoorPoemSections.AsNoTracking()
.Where(s => s.PoemId == poem.Id)
.OrderBy(s => s.Index)
.ToListAsync();
var verses = await context.GanjoorVerses.AsNoTracking()
.Where(v => v.PoemId == poem.Id)
.OrderBy(v => v.VOrder)
.ToListAsync();
var dto = new PoemPublicDto
{
Id = poem.Id,
CatId = poem.CatId,
Title = poem.Title,
FullTitle = poem.FullTitle,
FullUrl = poem.FullUrl,
RhymeLetters = poem.RhymeLetters,
SourceName = poem.SourceName,
SourceUrlSlug = poem.SourceUrlSlug,
Language = poem.Language,
PoemSummary = poem.PoemSummary,
Metre = metre == null ? null : new MetreRefDto { Id = metre.Id, Rhythm = metre.Rhythm, Name = metre.Name },
Sections = sections.Select(s => new PoemSectionPublicDto
{
Index = s.Index,
Number = s.Number,
SectionType = s.SectionType.ToString(),
VerseType = s.VerseType.ToString(),
RhymeLetters = s.RhymeLetters,
PlainText = s.PlainText,
HtmlText = s.HtmlText,
PoemFormat = s.PoemFormat?.ToString(),
Language = s.Language,
CoupletsCount = s.CoupletsCount,
}).ToList(),
Verses = verses.Select(v => new VersePublicDto
{
VOrder = v.VOrder,
Position = v.VersePosition.ToString(),
Text = v.Text,
CoupletIndex = v.CoupletIndex,
SectionIndex1 = v.SectionIndex1,
SectionIndex2 = v.SectionIndex2,
SectionIndex3 = v.SectionIndex3,
SectionIndex4 = v.SectionIndex4,
}).ToList(),
};
string path = Path.Combine(repoRoot, "poets", TrimLeadingSlash(poem.FullUrl) + ".json");
await DeterministicJsonWriter.WriteIfChangedAsync(path, dto);
}
private static string TrimLeadingSlash(string url)
{
if (string.IsNullOrEmpty(url)) return url;
url = url.Replace('/', Path.DirectorySeparatorChar);
return url.TrimStart(Path.DirectorySeparatorChar);
}
}
}

75
RMuseum/Utils/PublicDataExport/DeterministicJsonWriter.cs Normal file
View File

@ -0,0 +1,75 @@
using System.IO;
using System.Text;
using System.Text.Encodings.Web;
using System.Text.Json;
using System.Text.Unicode;
using System.Threading.Tasks;
namespace RMuseum.Utils.PublicDataExport
{
/// <summary>
/// Writes JSON in a way that is stable across runs: fixed indentation, LF line endings,
/// UTF-8 without BOM, a single trailing newline, and Persian/Arabic text left unescaped
/// (the default encoder \u-escapes non-ASCII, which is both hard to review in a diff and
/// bloats file size for a mostly-Persian dataset).
///
/// Callers are responsible for sorting any collections before serializing — this class only
/// guarantees stable *encoding*, not stable *ordering*, since ordering is a data decision.
///
/// Files are only actually written to disk when the content differs from what's already
/// there, so an unmodified poem never shows up in `git status` even if the whole tree is
/// regenerated every run.
/// </summary>
public static class DeterministicJsonWriter
{
private static readonly JsonSerializerOptions _options = new JsonSerializerOptions
{
WriteIndented = true,
Encoder = JavaScriptEncoder.Create(UnicodeRanges.All),
DefaultIgnoreCondition = System.Text.Json.Serialization.JsonIgnoreCondition.WhenWritingNull,
};
private static readonly UTF8Encoding _utf8NoBom = new UTF8Encoding(encoderShouldEmitUTF8Identifier: false);
/// <summary>
/// Serializes <paramref name="value"/> and writes it to <paramref name="path"/> only if the
/// resulting bytes differ from the file's current content. Creates parent directories as needed.
/// Returns true if the file was created or changed, false if it was already up to date.
/// </summary>
public static async Task<bool> WriteIfChangedAsync<T>(string path, T value)
{
string json = JsonSerializer.Serialize(value, _options);
// normalize to LF and ensure exactly one trailing newline, regardless of host OS
json = json.Replace("\r\n", "\n").TrimEnd('\n') + "\n";
byte[] newBytes = _utf8NoBom.GetBytes(json);
string dir = Path.GetDirectoryName(path);
if (!string.IsNullOrEmpty(dir) && !Directory.Exists(dir))
{
Directory.CreateDirectory(dir);
}
if (File.Exists(path))
{
byte[] existingBytes = await File.ReadAllBytesAsync(path);
if (BytesEqual(existingBytes, newBytes))
{
return false;
}
}
await File.WriteAllBytesAsync(path, newBytes);
return true;
}
private static bool BytesEqual(byte[] a, byte[] b)
{
if (a.Length != b.Length) return false;
for (int i = 0; i < a.Length; i++)
{
if (a[i] != b[i]) return false;
}
return true;
}
}
}

141
RMuseum/Utils/PublicDataExport/GitRepoPublisher.cs Normal file
View File

@ -0,0 +1,141 @@
using LibGit2Sharp;
using System;
using System.IO;
using System.Linq;
namespace RMuseum.Utils.PublicDataExport
{
/// <summary>
/// Options for publishing the public export tree to a git remote.
/// Bind this from the "PublicDataExport" configuration section.
/// </summary>
public class GitRepoPublisherOptions
{
/// <summary>
/// local working copy path (e.g. C:\ganjoor-public-data or /var/ganjoor/public-data)
/// </summary>
public string LocalWorkingCopyPath { get; set; }
/// <summary>
/// remote URL, e.g. https://github.com/ganjoor/ganjoor-data.git
/// </summary>
public string RemoteUrl { get; set; }
public string Branch { get; set; } = "main";
public string CommitAuthorName { get; set; } = "Ganjoor Export Bot";
public string CommitAuthorEmail { get; set; } = "bot@ganjoor.net";
/// <summary>
/// if false, everything is written and committed locally but never pushed —
/// useful for a first dry run before wiring up real credentials
/// </summary>
public bool PushEnabled { get; set; }
/// <summary>
/// git username for the push (for GitHub, a PAT is used as both username-independent
/// and as the token below — GitHub only checks the token)
/// </summary>
public string GitUserName { get; set; }
/// <summary>
/// personal access token / app token with push rights to RemoteUrl. Keep this in
/// user-secrets / environment variables, never committed to appsettings.json.
/// </summary>
public string GitToken { get; set; }
}
public class GitRepoPublisher
{
private readonly GitRepoPublisherOptions _options;
public GitRepoPublisher(GitRepoPublisherOptions options)
{
_options = options;
}
/// <summary>
/// Ensures the local working copy exists and is up to date with the remote before the
/// export job starts writing files into it. Clones on first run, fetches + hard-resets
/// to the remote branch on subsequent runs (this working copy is bot-owned; nobody
/// should be hand-editing it, so a hard reset is safe and keeps the job idempotent).
/// </summary>
public void EnsureWorkingCopyUpToDate()
{
if (!Directory.Exists(_options.LocalWorkingCopyPath) ||
!Directory.Exists(Path.Combine(_options.LocalWorkingCopyPath, ".git")))
{
Directory.CreateDirectory(_options.LocalWorkingCopyPath);
var cloneOptions = new CloneOptions();
if (RemoteRequiresAuth())
{
cloneOptions.FetchOptions.CredentialsProvider = CredentialsHandler;
}
Repository.Clone(_options.RemoteUrl, _options.LocalWorkingCopyPath, cloneOptions);
return;
}
using var repo = new Repository(_options.LocalWorkingCopyPath);
var remote = repo.Network.Remotes["origin"];
var fetchOptions = new FetchOptions();
if (RemoteRequiresAuth())
{
fetchOptions.CredentialsProvider = CredentialsHandler;
}
Commands.Fetch(repo, remote.Name, remote.FetchRefSpecs.Select(r => r.Specification), fetchOptions, null);
var remoteBranch = repo.Branches[$"origin/{_options.Branch}"];
if (remoteBranch != null)
{
repo.Reset(ResetMode.Hard, remoteBranch.Tip);
}
}
/// <summary>
/// Stages every change under the working copy and, if anything actually changed,
/// commits and (if enabled) pushes. Returns the number of files touched (0 means
/// nothing changed since last run — a normal, expected outcome on most nightly runs).
/// </summary>
public int CommitAndPush(string commitMessage)
{
using var repo = new Repository(_options.LocalWorkingCopyPath);
Commands.Stage(repo, "*");
var status = repo.RetrieveStatus();
int changedCount = status.Count(s => s.State != FileStatus.Ignored && s.State != FileStatus.Unaltered);
if (changedCount == 0)
{
return 0;
}
var signature = new Signature(_options.CommitAuthorName, _options.CommitAuthorEmail, DateTimeOffset.UtcNow);
repo.Commit(commitMessage, signature, signature);
if (_options.PushEnabled)
{
var pushOptions = new PushOptions();
if (RemoteRequiresAuth())
{
pushOptions.CredentialsProvider = CredentialsHandler;
}
var branch = repo.Branches[_options.Branch] ?? repo.CreateBranch(_options.Branch);
repo.Network.Push(branch, pushOptions);
}
return changedCount;
}
private bool RemoteRequiresAuth() => !string.IsNullOrEmpty(_options.GitToken);
private Credentials CredentialsHandler(string url, string usernameFromUrl, SupportedCredentialTypes types)
{
return new UsernamePasswordCredentials
{
Username = string.IsNullOrEmpty(_options.GitUserName) ? _options.GitToken : _options.GitUserName,
Password = _options.GitToken,
};
}
}
}

77
RMuseum/Utils/PublicDataExport/PublicExportSafetyGuard.cs Normal file
View File

@ -0,0 +1,77 @@
using RMuseum.Models.Ganjoor.PublicExport;
using System;
using System.Collections.Generic;
using System.Linq;
using System.Reflection;
using System.Text.RegularExpressions;
namespace RMuseum.Utils.PublicDataExport
{
/// <summary>
/// Belt-and-suspenders check on top of the allowlist design: even though the export DTOs
/// only ever declare the fields we explicitly want published, this scans every DTO type in
/// the RMuseum.Models.Ganjoor.PublicExport namespace by reflection and throws if a property
/// name or type looks like it could carry personal data. Call this once at application
/// startup (Development/CI) and/or from a test — see AssertSafe().
/// </summary>
public static class PublicExportSafetyGuard
{
/// <summary>
/// property-name patterns that must never appear on an export DTO
/// </summary>
private static readonly Regex[] _forbiddenNamePatterns = new[]
{
new Regex("UserId", RegexOptions.IgnoreCase),
new Regex("OwnerId", RegexOptions.IgnoreCase),
new Regex("ReviewerId", RegexOptions.IgnoreCase),
new Regex("^Email$", RegexOptions.IgnoreCase),
new Regex("Email$", RegexOptions.IgnoreCase),
new Regex("^Ip$", RegexOptions.IgnoreCase),
new Regex("IpAddress", RegexOptions.IgnoreCase),
new Regex("Password", RegexOptions.IgnoreCase),
new Regex("Token", RegexOptions.IgnoreCase),
new Regex("PhoneNumber", RegexOptions.IgnoreCase),
new Regex("^AuthorName$", RegexOptions.IgnoreCase),
new Regex("^AuthorUrl$", RegexOptions.IgnoreCase),
};
/// <summary>
/// Scans every public class in the PublicExport DTO namespace. Throws InvalidOperationException
/// naming the offending type/property if anything trips the checks.
/// </summary>
public static void AssertSafe()
{
var dtoTypes = typeof(PoemPublicDto).Assembly
.GetTypes()
.Where(t => t.IsClass && t.Namespace == typeof(PoemPublicDto).Namespace)
.ToList();
var violations = new List<string>();
foreach (var type in dtoTypes)
{
foreach (var prop in type.GetProperties(BindingFlags.Public | BindingFlags.Instance))
{
if (_forbiddenNamePatterns.Any(p => p.IsMatch(prop.Name)))
{
violations.Add($"{type.Name}.{prop.Name} matches a forbidden field-name pattern");
}
// a bare Guid property on a public export DTO is almost always an entity/user
// reference (our public ids are all ints); flag it for manual review
if (prop.PropertyType == typeof(Guid) || prop.PropertyType == typeof(Guid?))
{
violations.Add($"{type.Name}.{prop.Name} is a Guid — likely an internal entity/user reference, not public data");
}
}
}
if (violations.Count > 0)
{
throw new InvalidOperationException(
"PublicExportSafetyGuard failed — the following fields must not exist on a public export DTO:" +
Environment.NewLine + string.Join(Environment.NewLine, violations));
}
}
}
}

10
RMuseum/appsettings.json
View File

@ -67,6 +67,16 @@
"MaxWordLengthInComments": 200,
"TajikSitemapLocation": "C:\\inetpub\\tj\\wwwroot\\sitemap.xml"
},
"PublicDataExport": {
"LocalWorkingCopyPath": "C:\\ganjoor-public-data",
"RemoteUrl": "https://github.com/ganjoor/ganjoor-data.git",
"Branch": "main",
"CommitAuthorName": "Ganjoor Export Bot",
"CommitAuthorEmail": "bot@ganjoor.net",
"PushEnabled": "False",
"GitUserName": "",
"GitToken": ""
},
"ExternalFTPServer": {
"Host": "localhost",
"Port": "22",