532d005cdb
Projects (GanjooRazor -> DivanRazor, GanjoorService.sln -> DivanService.sln), files, folders, classes, namespaces, settings, API routes (/api/divan) and DB tables (Divan*). External addresses (ganjoor.net, github.com/ganjoor) unchanged. Migrations renamed consistently (no pending model changes); requires a fresh database. Upstream RUNNING_LOCALLY/SEMANTIC_SEARCH docs archived unmodified under docs/. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
487 lines
33 KiB
C#
487 lines
33 KiB
C#
using DNTPersianUtils.Core;
|
|
using Microsoft.EntityFrameworkCore;
|
|
using RMuseum.DbContext;
|
|
using RMuseum.Models.Divan;
|
|
using RMuseum.Models.Divan.ViewModels;
|
|
using RSecurityBackend.Models.Generic;
|
|
using RSecurityBackend.Services.Implementation;
|
|
using System;
|
|
using System.Collections.Generic;
|
|
using System.Data;
|
|
using System.Globalization;
|
|
using System.Linq;
|
|
using System.Threading.Tasks;
|
|
|
|
namespace RMuseum.Services.Implementation
|
|
{
|
|
/// <summary>
|
|
/// IDivanService implementation
|
|
/// </summary>
|
|
public partial class DivanService : IDivanService
|
|
{
|
|
/// <summary>
|
|
/// get digital source from tag
|
|
/// </summary>
|
|
/// <param name="sourceUrlSlug"></param>
|
|
/// <returns></returns>
|
|
public async Task<RServiceResult<DigitalSource>> GetDigitalSourceFromTagAsync(string sourceUrlSlug)
|
|
{
|
|
try
|
|
{
|
|
return new RServiceResult<DigitalSource>(await _context.DigitalSources.AsNoTracking().Where(d => d.UrlSlug == sourceUrlSlug).SingleOrDefaultAsync());
|
|
}
|
|
catch (Exception exp)
|
|
{
|
|
return new RServiceResult<DigitalSource>(null, exp.ToString());
|
|
}
|
|
}
|
|
/// <summary>
|
|
/// tag with sources
|
|
/// </summary>
|
|
/// <param name="catId"></param>
|
|
/// <param name="source"></param>
|
|
public void TagCategoryWithSource(int catId, DigitalSource source)
|
|
{
|
|
_backgroundTaskQueue.QueueBackgroundWorkItem
|
|
(
|
|
async token =>
|
|
{
|
|
using (RMuseumDbContext context = new RMuseumDbContext(new DbContextOptions<RMuseumDbContext>())) //this is long running job, so _context might be already been freed/collected by GC
|
|
{
|
|
LongRunningJobProgressServiceEF jobProgressServiceEF = new LongRunningJobProgressServiceEF(context);
|
|
var job = (await jobProgressServiceEF.NewJob($"TagCategoryWithSource(catId: {catId} - source:{source.UrlSlug})", "Query data")).Result;
|
|
try
|
|
{
|
|
var pages = await context.DivanPages.AsNoTracking().Where(p => p.FullUrl.StartsWith("/sources/")).ToListAsync();
|
|
foreach (var page in pages)
|
|
{
|
|
if (false == await context.DigitalSources.Where(d => d.UrlSlug == page.UrlSlug).AnyAsync())
|
|
{
|
|
string shortName = page.Title;
|
|
switch (page.UrlSlug)
|
|
{
|
|
case "wikidorj":
|
|
shortName = "ویکیدرج";
|
|
break;
|
|
case "frankfurt":
|
|
shortName = "دانشگاه فرانکفورت";
|
|
break;
|
|
case "tariqmo":
|
|
shortName = "طریق التحقیق دکتر مؤذنی";
|
|
break;
|
|
case "tebyan":
|
|
shortName = "تبیان";
|
|
break;
|
|
}
|
|
context.DigitalSources.Add
|
|
(
|
|
new DigitalSource()
|
|
{
|
|
UrlSlug = page.UrlSlug,
|
|
ShortName = shortName,
|
|
FullName = page.Title,
|
|
SourceType = "",
|
|
CoupletsCount = 0,
|
|
}
|
|
);
|
|
await context.SaveChangesAsync();
|
|
}
|
|
}
|
|
|
|
var digitalSource = await context.DigitalSources.Where(p => p.UrlSlug == source.UrlSlug).SingleOrDefaultAsync();
|
|
if (digitalSource == null)
|
|
{
|
|
context.DigitalSources.Add
|
|
(
|
|
new DigitalSource()
|
|
{
|
|
UrlSlug = source.UrlSlug,
|
|
ShortName = source.ShortName,
|
|
FullName = source.FullName,
|
|
SourceType = source.SourceType,
|
|
CoupletsCount = 0,
|
|
}
|
|
);
|
|
|
|
var parentPage = await context.DivanPages.AsNoTracking().Where(p => p.UrlSlug == "sources").SingleAsync();
|
|
var newPageId = 1 + await context.DivanPages.MaxAsync(p => p.Id);
|
|
while (await context.DivanPoems.Where(p => p.Id == newPageId).AnyAsync())
|
|
newPageId++;
|
|
context.DivanPages.Add
|
|
(
|
|
new DivanPage()
|
|
{
|
|
Id = newPageId,
|
|
DivanPageType = DivanPageType.None,
|
|
Published = true,
|
|
PageOrder = -1,
|
|
Title = source.FullName,
|
|
FullTitle = $"{parentPage.FullTitle} » {source.FullName}",
|
|
FullUrl = $"{parentPage.FullUrl}/{source.UrlSlug}",
|
|
UrlSlug = source.UrlSlug,
|
|
HtmlText = $"<p>{source.FullName}</p>",
|
|
PostDate = DateTime.Now,
|
|
ParentId = parentPage.Id,
|
|
}
|
|
);
|
|
|
|
await context.SaveChangesAsync();
|
|
digitalSource = await context.DigitalSources.Where(p => p.UrlSlug == source.UrlSlug).SingleAsync();
|
|
}
|
|
|
|
List<int> catIdList = new List<int>
|
|
{
|
|
catId
|
|
};
|
|
await _populateCategoryChildren(context, catId, catIdList);
|
|
int poemCount = 0;
|
|
int progress = 0;
|
|
foreach (int catId in catIdList)
|
|
{
|
|
var poems = await context.DivanPoems.AsNoTracking().Where(p => p.CatId == catId).ToListAsync();
|
|
poemCount += poems.Count;
|
|
foreach (var poem in poems)
|
|
{
|
|
poem.SourceName = digitalSource.ShortName;
|
|
poem.SourceUrlSlug = source.UrlSlug;
|
|
context.Update(poem);
|
|
|
|
int coupletCount = await context.DivanVerses.AsNoTracking()
|
|
.Where(v =>
|
|
v.PoemId == poem.Id
|
|
&&
|
|
(v.VersePosition == VersePosition.Right || v.VersePosition == VersePosition.CenteredVerse1)
|
|
).CountAsync();
|
|
digitalSource.CoupletsCount += coupletCount;
|
|
await jobProgressServiceEF.UpdateJob(job.Id, progress, $"{progress} از {poemCount}");
|
|
}
|
|
}
|
|
|
|
context.Update(digitalSource);
|
|
await jobProgressServiceEF.UpdateJob(job.Id, 100, "", true);
|
|
}
|
|
catch (Exception exp)
|
|
{
|
|
await jobProgressServiceEF.UpdateJob(job.Id, 100, "", false, exp.ToString());
|
|
}
|
|
|
|
}
|
|
});
|
|
}
|
|
|
|
/// <summary>
|
|
/// update digital sources stats
|
|
/// </summary>
|
|
/// <param name="editingUserId"></param>
|
|
public void UpdateDigitalSourcesStats(Guid editingUserId)
|
|
{
|
|
_backgroundTaskQueue.QueueBackgroundWorkItem
|
|
(
|
|
async token =>
|
|
{
|
|
using (RMuseumDbContext context = new RMuseumDbContext(new DbContextOptions<RMuseumDbContext>())) //this is long running job, so _context might be already been freed/collected by GC
|
|
{
|
|
LongRunningJobProgressServiceEF jobProgressServiceEF = new LongRunningJobProgressServiceEF(context);
|
|
var job = (await jobProgressServiceEF.NewJob($"UpdateDigitalSourcesStats", "Query data")).Result;
|
|
try
|
|
{
|
|
var pages = await context.DivanPages.AsNoTracking().Where(p => p.FullUrl.StartsWith("/sources/")).ToListAsync();
|
|
foreach (var page in pages)
|
|
{
|
|
if (false == await context.DigitalSources.Where(d => d.UrlSlug == page.UrlSlug).AnyAsync())
|
|
{
|
|
string shortName = page.Title;
|
|
switch (page.UrlSlug)
|
|
{
|
|
case "wikidorj":
|
|
shortName = "ویکیدرج";
|
|
break;
|
|
case "frankfurt":
|
|
shortName = "دانشگاه فرانکفورت";
|
|
break;
|
|
case "tariqmo":
|
|
shortName = "طریق التحقیق دکتر مؤذنی";
|
|
break;
|
|
case "tebyan":
|
|
shortName = "تبیان";
|
|
break;
|
|
}
|
|
context.DigitalSources.Add
|
|
(
|
|
new DigitalSource()
|
|
{
|
|
UrlSlug = page.UrlSlug,
|
|
ShortName = shortName,
|
|
FullName = page.Title,
|
|
SourceType = "",
|
|
CoupletsCount = 0,
|
|
}
|
|
);
|
|
await context.SaveChangesAsync();
|
|
}
|
|
}
|
|
|
|
var digitalSources = await context.DigitalSources.ToArrayAsync();
|
|
Dictionary<int, List<int>> digitalSourcesCats = new Dictionary<int, List<int>>();
|
|
int totalCoupletsCount = 0;
|
|
foreach (var digitalSource in digitalSources)
|
|
{
|
|
digitalSource.CoupletsCount = 0;
|
|
|
|
List<int> cats = new List<int>();
|
|
|
|
var poems = await context.DivanPoems.AsNoTracking().Where(p => p.SourceUrlSlug == digitalSource.UrlSlug).ToArrayAsync();
|
|
foreach (var poem in poems)
|
|
{
|
|
int coupletCount = await context.DivanVerses.AsNoTracking()
|
|
.Where(v =>
|
|
v.PoemId == poem.Id
|
|
&&
|
|
(v.VersePosition == VersePosition.Right || v.VersePosition == VersePosition.CenteredVerse1)
|
|
).CountAsync();
|
|
digitalSource.CoupletsCount += coupletCount;
|
|
if(!cats.Contains(poem.CatId))
|
|
{
|
|
cats.Add(poem.CatId);
|
|
}
|
|
}
|
|
totalCoupletsCount += digitalSource.CoupletsCount;
|
|
digitalSourcesCats.Add(digitalSource.Id, cats);
|
|
context.Update(digitalSource);
|
|
await jobProgressServiceEF.UpdateJob(job.Id, 50, digitalSource.FullName);
|
|
}
|
|
|
|
var noSourceUrlPoems = await context.DivanPoems.AsNoTracking().Where(p => string.IsNullOrEmpty(p.SourceUrlSlug)).ToArrayAsync();
|
|
int untaggaed = 0;
|
|
foreach (var noSourceUrlPoem in noSourceUrlPoems)
|
|
{
|
|
int coupletCount = await context.DivanVerses.AsNoTracking()
|
|
.Where(v =>
|
|
v.PoemId == noSourceUrlPoem.Id
|
|
&&
|
|
(v.VersePosition == VersePosition.Right || v.VersePosition == VersePosition.CenteredVerse1)
|
|
).CountAsync();
|
|
totalCoupletsCount += coupletCount;
|
|
untaggaed += coupletCount;
|
|
}
|
|
|
|
//now rebuild /sources page contents
|
|
await jobProgressServiceEF.UpdateJob(job.Id, 75, "Rebuild Sources Page");
|
|
|
|
var sources = await context.DigitalSources.AsNoTracking().Where(d => d.CoupletsCount > 0).OrderByDescending(d => d.CoupletsCount).ToListAsync();
|
|
|
|
int totalCount = sources.Sum(s => s.CoupletsCount);
|
|
string htmlText = $"<p>بخش عمدهٔ محتوای گنجور که در حال حاضر شامل {LanguageUtils.FormatMoney(totalCount)} بیت شعر است از نرمافزارها و وبگاههای دیگر وارد شده است و در این زمینه علاقمندان ادبیات مدیون بزرگوارانی هستند که پیش و بیش از گنجور روی دیجیتالی کردن میراث ادب ایرانزمین سرمایهگذاری کردهاند. آمار زیر نشان دهندهٔ ترکیب محتوای گنجور بر اساس نوع منبع دیجیتالی شدن آنهاست.</p>{Environment.NewLine}";
|
|
htmlText += $"<table>{Environment.NewLine}" +
|
|
$"<tr class=\"h\">{Environment.NewLine}" +
|
|
$"<td class=\"c1\">ردیف</td>{Environment.NewLine}" +
|
|
$"<td class=\"c2\">نوع منبع</td>{Environment.NewLine}" +
|
|
$"<td class=\"c3\">تعداد ابیات</td>{Environment.NewLine}" +
|
|
$"<td class=\"c4\">درصد از کل</td>{Environment.NewLine}" +
|
|
$"</tr>{Environment.NewLine}";
|
|
var sourcesByType = await context.DigitalSources.AsNoTracking().Where(d => d.CoupletsCount > 0).GroupBy(d => d.SourceType).Select(d => new { SourceType = d.Key, CoupletsCount = d.Sum(s => s.CoupletsCount) }).OrderByDescending(d => d.CoupletsCount).ToListAsync();
|
|
for (int i = 0; i < sourcesByType.Count; i++)
|
|
{
|
|
if (i % 2 == 0)
|
|
htmlText += $"<tr class=\"e\">{Environment.NewLine}";
|
|
else
|
|
htmlText += $"<tr>{Environment.NewLine}";
|
|
|
|
htmlText += $"<td class=\"c1\">{(i + 1).ToPersianNumbers()}</td>{Environment.NewLine}";
|
|
htmlText += $"<td class=\"c2\">{sourcesByType[i].SourceType}</td>{Environment.NewLine}";
|
|
htmlText += $"<td class=\"c3\">{LanguageUtils.FormatMoney(sourcesByType[i].CoupletsCount)}</td>{Environment.NewLine}";
|
|
htmlText += $"<td class=\"c4\">{(sourcesByType[i].CoupletsCount * 100.0 / totalCount).ToString("N2", new CultureInfo("ur-PK")).ToPersianNumbers()}</td>{Environment.NewLine}";
|
|
|
|
htmlText += $"</tr>{Environment.NewLine}";
|
|
}
|
|
htmlText += $"</table>{Environment.NewLine}";
|
|
|
|
htmlText += $"<p>جدول زیر نشاندهندهٔ آمار ریز منابع دیجیتال گنجور است:</p>{Environment.NewLine}";
|
|
htmlText += $"<table>{Environment.NewLine}" +
|
|
$"<tr class=\"h\">{Environment.NewLine}" +
|
|
$"<td class=\"c1\">ردیف</td>{Environment.NewLine}" +
|
|
$"<td class=\"c2\">نام منبع</td>{Environment.NewLine}" +
|
|
$"<td class=\"c3\">تعداد ابیات</td>{Environment.NewLine}" +
|
|
$"<td class=\"c4\">درصد از کل</td>{Environment.NewLine}" +
|
|
$"</tr>{Environment.NewLine}";
|
|
|
|
for (int i = 0; i < sources.Count; i++)
|
|
{
|
|
if (i % 2 == 0)
|
|
htmlText += $"<tr class=\"e\">{Environment.NewLine}";
|
|
else
|
|
htmlText += $"<tr>{Environment.NewLine}";
|
|
|
|
htmlText += $"<td class=\"c1\">{(i + 1).ToPersianNumbers()}</td>{Environment.NewLine}";
|
|
htmlText += $"<td class=\"c2\"><a href=\"/sources/{sources[i].UrlSlug}\">{sources[i].FullName}</a></td>{Environment.NewLine}";
|
|
htmlText += $"<td class=\"c3\">{LanguageUtils.FormatMoney(sources[i].CoupletsCount)}</td>{Environment.NewLine}";
|
|
htmlText += $"<td class=\"c4\">{(sources[i].CoupletsCount * 100.0 / totalCount).ToString("N2", new CultureInfo("ur-PK")).ToPersianNumbers()}</td>{Environment.NewLine}";
|
|
|
|
htmlText += $"</tr>{Environment.NewLine}";
|
|
}
|
|
htmlText += $"</table>{Environment.NewLine}";
|
|
|
|
var dbPage = await context.DivanPages.Where(p => p.FullUrl == "/sources").SingleAsync();
|
|
await _UpdatePageHtmlText(context, editingUserId, dbPage, "به روزرسانی خودکار صفحهٔ آمار منابع", htmlText);
|
|
|
|
await jobProgressServiceEF.UpdateJob(job.Id, 80, "Rebuild Sources SubPages");
|
|
foreach (var digitalSource in digitalSources)
|
|
{
|
|
List<DivanCatViewModel> includedCats = new List<DivanCatViewModel>();
|
|
foreach (var catId in digitalSourcesCats[digitalSource.Id])
|
|
{
|
|
var resCat = await _GetCatById(context, catId, false, false, false);
|
|
var cat = resCat.Result.Cat;
|
|
if(cat.Ancestors.Any())
|
|
{
|
|
var lastParent = cat.Ancestors.Last();
|
|
if (includedCats.Any(f => f.Id == lastParent.Id))
|
|
continue;
|
|
var resParent = await _GetCatById(context, lastParent.Id, false, false, false);
|
|
var parent = resParent.Result.Cat;
|
|
bool includesAllChildren = true;
|
|
foreach (var child in parent.Children)
|
|
{
|
|
if(!digitalSourcesCats[digitalSource.Id].Contains(child.Id))
|
|
{
|
|
includesAllChildren = false;
|
|
break;
|
|
}
|
|
}
|
|
if(includesAllChildren)
|
|
{
|
|
includedCats.Add(parent);
|
|
}
|
|
else
|
|
{
|
|
includedCats.Add(cat);
|
|
}
|
|
}
|
|
else
|
|
{
|
|
includedCats.Add(cat);
|
|
}
|
|
}
|
|
|
|
List<DivanCatViewModel> finalCats = new List<DivanCatViewModel>();
|
|
foreach (var cat in includedCats)
|
|
{
|
|
if(!cat.Ancestors.Any())
|
|
{
|
|
if (!finalCats.Where(c => c.Id == cat.Id).Any())
|
|
{
|
|
finalCats.Add(cat);
|
|
}
|
|
}
|
|
else
|
|
{
|
|
var thisCat = cat;
|
|
var lastParent = cat.Ancestors.Last();
|
|
if(lastParent != null)
|
|
{
|
|
var resParent = await _GetCatById(context, lastParent.Id, false, false, false);
|
|
var parent = resParent.Result.Cat;
|
|
while (parent != null)
|
|
{
|
|
bool includesAllChildren = false;
|
|
foreach (var child in parent.Children)
|
|
{
|
|
if (!includedCats.Any(c => c.Id == child.Id))
|
|
{
|
|
includesAllChildren = true;
|
|
break;
|
|
}
|
|
}
|
|
if (includesAllChildren)
|
|
{
|
|
thisCat = parent;
|
|
lastParent = thisCat.Ancestors.LastOrDefault();
|
|
if(lastParent == null)
|
|
{
|
|
break;
|
|
}
|
|
resParent = await _GetCatById(context, lastParent.Id, false, false, false);
|
|
parent = resParent.Result.Cat;
|
|
}
|
|
else
|
|
{
|
|
break;
|
|
}
|
|
}
|
|
}
|
|
if(!finalCats.Where(c => c.Id == thisCat.Id).Any())
|
|
{
|
|
finalCats.Add(thisCat);
|
|
}
|
|
}
|
|
}
|
|
|
|
var dbPageDigitalPage = await context.DivanPages.Where(p => p.FullUrl == $"/sources/{digitalSource.UrlSlug}").FirstOrDefaultAsync();
|
|
if(dbPageDigitalPage != null)
|
|
{
|
|
string commentTag = $"{Environment.NewLine}<!-- STATIC -->{Environment.NewLine}";
|
|
|
|
dbPageDigitalPage.HtmlText =
|
|
dbPageDigitalPage.HtmlText.Replace(
|
|
$"{Environment.NewLine}<!- STATIC -->{Environment.NewLine}",
|
|
commentTag
|
|
);
|
|
|
|
|
|
if (!dbPageDigitalPage.HtmlText.Contains(commentTag))
|
|
{
|
|
|
|
}
|
|
if (dbPageDigitalPage.HtmlText.Contains(commentTag))
|
|
{
|
|
htmlText = dbPageDigitalPage.HtmlText.Substring(0, dbPageDigitalPage.HtmlText.IndexOf(commentTag));
|
|
}
|
|
else
|
|
{
|
|
htmlText = dbPageDigitalPage.HtmlText;
|
|
}
|
|
htmlText += commentTag;
|
|
htmlText += $"<p>{digitalSource.FullName} منبع دیجیتال {LanguageUtils.FormatMoney(digitalSource.CoupletsCount)} بیت شعر از بخشهای زیر است:</p>{Environment.NewLine}";
|
|
htmlText += $"<table>{Environment.NewLine}" +
|
|
$"<tr class=\"h\">{Environment.NewLine}" +
|
|
$"<td class=\"c1\">ردیف</td>{Environment.NewLine}" +
|
|
$"<td class=\"c2\">نام بخش</td>{Environment.NewLine}" +
|
|
$"</tr>{Environment.NewLine}";
|
|
int rowIndex = 0;
|
|
foreach (var cat in finalCats)
|
|
{
|
|
if (rowIndex % 2 == 0)
|
|
htmlText += $"<tr class=\"e\">{Environment.NewLine}";
|
|
else
|
|
htmlText += $"<tr>{Environment.NewLine}";
|
|
|
|
htmlText += $"<td class=\"c1\">{(rowIndex + 1).ToPersianNumbers()}</td>{Environment.NewLine}";
|
|
string catTitle = cat.Title;
|
|
for (int a = cat.Ancestors.Count - 1; a >= 0; a--)
|
|
{
|
|
catTitle = cat.Ancestors.ElementAt(a).Title + " » " + catTitle;
|
|
}
|
|
htmlText += $"<td class=\"c2\"><a href=\"{cat.FullUrl}\">{catTitle}</a></td>{Environment.NewLine}";
|
|
htmlText += $"</tr>{Environment.NewLine}";
|
|
rowIndex++;
|
|
}
|
|
htmlText += $"</table>{Environment.NewLine}";
|
|
|
|
await _UpdatePageHtmlText(context, editingUserId, dbPageDigitalPage, "به روزرسانی خودکار صفحهٔ آمار منابع", htmlText);
|
|
}
|
|
}
|
|
|
|
await jobProgressServiceEF.UpdateJob(job.Id, 100, $"total: {totalCoupletsCount} - untagged: {untaggaed}", true);
|
|
}
|
|
catch (Exception exp)
|
|
{
|
|
await jobProgressServiceEF.UpdateJob(job.Id, 100, "", false, exp.ToString());
|
|
}
|
|
|
|
}
|
|
});
|
|
}
|
|
}
|
|
|
|
|
|
} |