divan/RMuseum/Services/Implementation/DivanService-Partials/DivanService-SQLiteExport.cs
Anas Rashid 532d005cdb Rename Ganjoor -> Divan throughout the code
Projects (GanjooRazor -> DivanRazor, GanjoorService.sln -> DivanService.sln), files,
folders, classes, namespaces, settings, API routes (/api/divan) and DB tables (Divan*).
External addresses (ganjoor.net, github.com/ganjoor) unchanged. Migrations renamed
consistently (no pending model changes); requires a fresh database.
Upstream RUNNING_LOCALLY/SEMANTIC_SEARCH docs archived unmodified under docs/.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-10-04 23:49:01 +02:00

607 lines
36 KiB
C#

using Dapper;
using DNTPersianUtils.Core;
using FluentFTP;
using divan;
using Microsoft.AspNetCore.Http;
using Microsoft.Data.Sqlite;
using Microsoft.EntityFrameworkCore;
using RMuseum.DbContext;
using RMuseum.Models.Artifact;
using RMuseum.Models.Divan;
using RSecurityBackend.Models.Generic;
using RSecurityBackend.Services.Implementation;
using System;
using System.Collections.Generic;
using System.Data;
using System.IO;
using System.IO.Compression;
using System.Linq;
using System.Threading.Tasks;
namespace RMuseum.Services.Implementation
{
/// <summary>
/// IDivanService implementation
/// </summary>
public partial class DivanService : IDivanService
{
/// <summary>
/// start generating gdb files
/// </summary>
/// <returns></returns>
public RServiceResult<bool> StartBatchGenerateGDBFiles()
{
try
{
_backgroundTaskQueue.QueueBackgroundWorkItem
(
async token =>
{
using (RMuseumDbContext context = new RMuseumDbContext(new DbContextOptions<RMuseumDbContext>())) //this is long running job, so _context might be already been freed/collected by GC
{
LongRunningJobProgressServiceEF jobProgressServiceEF = new LongRunningJobProgressServiceEF(context);
var job = (await jobProgressServiceEF.NewJob("BatchGenerateGDBFiles", "Query Data")).Result;
try
{
string outDir = Configuration.GetSection("Divan")["GDBStorage"];
string imgDir = Configuration.GetSection("Divan")["GDBStorageImageSource"];
string xmlFile = Configuration.GetSection("Divan")["GDBListXMLFile"];
string preExisitingListXMLFile = Configuration.GetSection("Divan")["GDBPreExisitingListXMLFile"];
List<GDBInfo> programList = new List<GDBInfo>();
if (!string.IsNullOrEmpty(preExisitingListXMLFile))
{
if (File.Exists(preExisitingListXMLFile))
{
programList = GDBListProcessor.RetrieveListFromFile(preExisitingListXMLFile, out string exception);
if (programList == null)
{
await jobProgressServiceEF.UpdateJob(job.Id, 100, "GDBListProcessor.RetrieveListFromFile", false, exception);
return;
}
}
}
var poets = await context.DivanPoets.AsNoTracking().ToListAsync();
List<GDBInfo> lstFiles = new List<GDBInfo>();
foreach (var poet in poets)
{
if (!await context.DivanPoems
.Include(p => p.Cat)
.Where(p => p.Cat.PoetId == poet.Id).AnyAsync())
continue;
await jobProgressServiceEF.UpdateJob(job.Id, poet.Id);
var gdbGeneration = await _ExportToSqlite(context, poet.Id, outDir, null, false);
if(!string.IsNullOrEmpty(gdbGeneration.ExceptionString))
{
await jobProgressServiceEF.UpdateJob(job.Id, 100, "", false, gdbGeneration.ExceptionString);
return;
}
string gdbFile = gdbGeneration.Result;
string pngFile = Path.Combine(imgDir, $"{poet.Id}.png");
bool hasImage = File.Exists(pngFile);
using (var archiveStream = new MemoryStream())
{
using (var archive = new ZipArchive(archiveStream, ZipArchiveMode.Create, true))
{
var zipGDBFileEntry = archive.CreateEntry(Path.GetFileName(gdbFile), CompressionLevel.Optimal);
using (var zipStream = zipGDBFileEntry.Open())
{
var gdbBytes = File.ReadAllBytes(gdbFile);
zipStream.Write(gdbBytes, 0, gdbBytes.Length);
}
if (hasImage)
{
var zipImgFileEntry = archive.CreateEntry(Path.GetFileName(pngFile), CompressionLevel.Optimal);
using (var zipStream = zipImgFileEntry.Open())
{
var pngBytes = File.ReadAllBytes(pngFile);
zipStream.Write(pngBytes, 0, pngBytes.Length);
}
}
}
string zipFile = Path.Combine(outDir, Path.GetFileNameWithoutExtension(gdbFile) + ".zip");
if (File.Exists(zipFile))
File.Delete(zipFile);
byte[] zipArray = archiveStream.ToArray();
File.WriteAllBytes(zipFile, zipArray);
var catPoet = await context.DivanCategories.AsNoTracking().Where(c => c.PoetId == poet.Id && c.ParentId == null).SingleAsync();
var lowestPoemID = await context.DivanPoems
.Include(p => p.Cat)
.Where(p => p.Cat.PoetId == poet.Id)
.MinAsync(p => p.Id);
lstFiles.Add
(
new GDBInfo()
{
CatName = poet.Nickname,
CatID = catPoet.Id,
PoetID = poet.Id,
DownloadUrl = $"https://i.ganjoor.net/android/gdb/{Path.GetFileName(zipFile)}",
BlogUrl = "",
FileExt = ".zip",
ImageUrl = (hasImage ? $"https://i.ganjoor.net/android/img/{poet.Id}.png" : ""),
FileSizeInByte = zipArray.Length,
LowestPoemID = lowestPoemID,
PubDate = DateTime.Now
}
);
}
File.Delete(gdbFile);
}
if (File.Exists(xmlFile))
File.Delete(xmlFile);
if(programList.Count > 0)
{
lstFiles.AddRange(programList);
}
lstFiles.Sort((a, b) => a.CatName.CompareTo(b.CatName));
GDBListProcessor.Save(xmlFile, "مجموعه‌های قابل دریافت از گنجور", "", "", lstFiles);
if (bool.Parse(Configuration.GetSection("ExternalFTPServer")["UploadEnabled"]))
{
var ftpClient = new AsyncFtpClient
(
Configuration.GetSection("ExternalFTPServer")["Host"],
Configuration.GetSection("ExternalFTPServer")["Username"],
Configuration.GetSection("ExternalFTPServer")["Password"]
);
ftpClient.ValidateCertificate += FtpClient_ValidateCertificate;
await ftpClient.AutoConnect();
ftpClient.Config.RetryAttempts = 3;
string dir = Path.GetDirectoryName(xmlFile);
string rootDirName = Path.GetFileName(dir);
string gdbDirName = Path.GetFileName(Path.GetDirectoryName(outDir));
foreach (var localFilePath in Directory.GetFiles(Path.Combine(dir, gdbDirName), "*.zip"))
{
var remoteFilePath = $"{Configuration.GetSection("ExternalFTPServer")["RootPath"]}/{rootDirName}/{gdbDirName}/{Path.GetFileName(localFilePath)}";
await ftpClient.UploadFile(localFilePath, remoteFilePath, createRemoteDir: true);
}
string imgDirName = Path.GetFileName(Path.GetDirectoryName(imgDir));
foreach (var localFilePath in Directory.GetFiles(Path.Combine(dir, imgDirName), "*.png"))
{
var remoteFilePath = $"{Configuration.GetSection("ExternalFTPServer")["RootPath"]}/{rootDirName}/{imgDirName}/{Path.GetFileName(localFilePath)}";
await ftpClient.UploadFile(localFilePath, remoteFilePath, createRemoteDir: true);
}
await ftpClient.UploadFile(xmlFile, $"{Configuration.GetSection("ExternalFTPServer")["RootPath"]}/{rootDirName}/{Path.GetFileName(xmlFile)}", createRemoteDir: true);
await ftpClient.Disconnect();
}
if (bool.Parse(Configuration.GetSection("BackupFTPServer")["UploadEnabled"]))
{
var ftpClient = new AsyncFtpClient
(
Configuration.GetSection("BackupFTPServer")["Host"],
Configuration.GetSection("BackupFTPServer")["Username"],
Configuration.GetSection("BackupFTPServer")["Password"]
);
ftpClient.ValidateCertificate += FtpClient_ValidateCertificate;
await ftpClient.AutoConnect();
ftpClient.Config.RetryAttempts = 3;
string dir = Path.GetDirectoryName(xmlFile);
string rootDirName = Path.GetFileName(dir);
string gdbDirName = Path.GetFileName(Path.GetDirectoryName(outDir));
foreach (var localFilePath in Directory.GetFiles(Path.Combine(dir, gdbDirName), "*.zip"))
{
var remoteFilePath = $"{Configuration.GetSection("BackupFTPServer")["RootPath"]}/{rootDirName}/{gdbDirName}/{Path.GetFileName(localFilePath)}";
await ftpClient.UploadFile(localFilePath, remoteFilePath, createRemoteDir: true);
}
string imgDirName = Path.GetFileName(Path.GetDirectoryName(imgDir));
foreach (var localFilePath in Directory.GetFiles(Path.Combine(dir, imgDirName), "*.png"))
{
var remoteFilePath = $"{Configuration.GetSection("BackupFTPServer")["RootPath"]}/{rootDirName}/{imgDirName}/{Path.GetFileName(localFilePath)}";
await ftpClient.UploadFile(localFilePath, remoteFilePath, createRemoteDir: true);
}
await ftpClient.UploadFile(xmlFile, $"{Configuration.GetSection("BackupFTPServer")["RootPath"]}/{rootDirName}/{Path.GetFileName(xmlFile)}", createRemoteDir: true);
await ftpClient.Disconnect();
}
await jobProgressServiceEF.UpdateJob(job.Id, 100, "", true);
}
catch (Exception exp)
{
await jobProgressServiceEF.UpdateJob(job.Id, 100, "", false, exp.ToString());
}
}
}
);
return new RServiceResult<bool>(true);
}
catch (Exception exp)
{
return new RServiceResult<bool>(false, exp.ToString());
}
}
/// <summary>
/// export to sqlite
/// </summary>
/// <param name="poetId"></param>
/// <returns></returns>
public async Task<RServiceResult<string>> ExportToSqlite(int poetId)
{
return await _ExportToSqlite(_context, poetId, Path.Combine($"{Configuration.GetSection("PictureFileService")["StoragePath"]}", "SQLiteExports"));
}
private async Task<RServiceResult<string>> _ExportToSqlite(RMuseumDbContext context, int poetId, string dir, string fileName = null, bool ignoreBio = false)
{
try
{
var poet = await context.DivanPoets.AsNoTracking().Where(p => p.Id == poetId).SingleAsync();
var catPoet = await context.DivanCategories.AsNoTracking().Where(c => c.PoetId == poetId && c.ParentId == null).SingleAsync();
if (!Directory.Exists(dir))
Directory.CreateDirectory(dir);
if(fileName == null)
{
fileName = catPoet.UrlSlug;
}
string filePath = Path.Combine(dir, $"{fileName}.gdb");
if (File.Exists(filePath))
File.Delete(filePath);
SqliteConnectionStringBuilder connectionStringBuilder = new SqliteConnectionStringBuilder();
connectionStringBuilder.DataSource = filePath;
connectionStringBuilder.Mode = SqliteOpenMode.ReadWriteCreate;
using (SqliteConnection sqliteConnection = new SqliteConnection(connectionStringBuilder.ToString()))
{
await sqliteConnection.OpenAsync();
string q = "BEGIN TRANSACTION;" +
"CREATE TABLE [cat] ([id] INTEGER PRIMARY KEY NOT NULL,[poet_id] INTEGER NULL,[text] NVARCHAR(100) NULL,[parent_id] INTEGER NULL,[url] NVARCHAR(255) NULL);"
+
"CREATE TABLE [poem] (id INTEGER PRIMARY KEY, cat_id INTEGER, title NVARCHAR(255), url NVARCHAR(255));"
+
"CREATE TABLE [poet] ([id] INTEGER PRIMARY KEY NOT NULL,[name] NVARCHAR(20) NULL,[cat_id] INTEGER NULL NULL, [description] TEXT);"
+
"CREATE TABLE [verse] ([poem_id] INTEGER NULL,[vorder] INTEGER NULL,[position] INTEGER NULL,[text] TEXT NULL);"
+
"COMMIT;";
await sqliteConnection.ExecuteAsync(q);
await sqliteConnection.ExecuteAsync("BEGIN;");
// Parameterized instead of string-interpolated: poet.Description (bio) is
// free-form user-editable text, so splicing it into the SQL text directly let a
// bio containing a "'" escape the string literal. The old code worked around that
// by stripping quote characters out of the bio before export (silently corrupting
// any bio that legitimately used one) rather than fixing the actual cause; Dapper's
// parameter binding here makes that stripping unnecessary.
string bio = poet.Description ?? "";
await sqliteConnection.ExecuteAsync(
"INSERT INTO poet (id, name, cat_id, description) VALUES (@Id, @Name, @CatId, @Description);",
new { Id = poet.Id, Name = poet.Nickname, CatId = catPoet.Id, Description = ignoreBio ? "" : bio }
);
await ExportCatToSqlite(context, sqliteConnection, catPoet);
await sqliteConnection.ExecuteAsync("COMMIT;");
}
SqliteConnection.ClearAllPools();
return new RServiceResult<string>(filePath);
}
catch(Exception exp)
{
return new RServiceResult<string>(null, exp.ToString());
}
}
private async Task ExportCatToSqlite(RMuseumDbContext context, SqliteConnection sqliteConnection, DivanCat cat)
{
int parentId = cat.ParentId == null ? 0 : (int)cat.ParentId;
// Parameterized instead of string-interpolated (see the matching note in
// _ExportToSqlite): cat.Title, poem.Title and verse.Text are all free-form user-editable
// text, so splicing them into the SQL text directly let a "'" in any of them escape the
// string literal - unlike the bio field above, nothing here even worked around that.
await sqliteConnection.ExecuteAsync(
"INSERT INTO cat (id, poet_id, text, parent_id, url) VALUES (@Id, @PoetId, @Text, @ParentId, @Url);",
new { Id = cat.Id, PoetId = cat.PoetId, Text = cat.Title, ParentId = parentId, Url = $"https://ganjoor.net{cat.FullUrl}" }
);
var poems = await context.DivanPoems.AsNoTracking().Where(p => p.CatId == cat.Id).ToListAsync();
foreach(var poem in poems)
{
await sqliteConnection.ExecuteAsync(
"INSERT INTO poem (id, cat_id, title, url) VALUES (@Id, @CatId, @Title, @Url);",
new { Id = poem.Id, CatId = poem.CatId, Title = poem.Title, Url = $"https://ganjoor.net{poem.FullUrl}" }
);
foreach (var verse in await context.DivanVerses.AsNoTracking().Where(v => v.PoemId == poem.Id).OrderBy(v => v.VOrder).ToListAsync())
await sqliteConnection.ExecuteAsync(
"INSERT INTO verse (poem_id, vorder, position, text) VALUES (@PoemId, @VOrder, @Position, @Text);",
new { PoemId = poem.Id, VOrder = verse.VOrder, Position = (int)verse.VersePosition, Text = verse.Text }
);
}
foreach (var child in await context.DivanCategories.AsNoTracking().Where(c => c.ParentId == cat.Id).ToListAsync())
await ExportCatToSqlite(context, sqliteConnection, child);
}
/// <summary>
/// Apply corrections from sqlite
/// </summary>
/// <param name="poetId"></param>
/// <param name="file"></param>
/// <param name="note"></param>
/// <returns></returns>
public async Task<RServiceResult<bool>> ApplyCorrectionsFromSqlite(int poetId, IFormFile file, string note)
{
try
{
string dir = Path.Combine($"{Configuration.GetSection("PictureFileService")["StoragePath"]}", "SQLiteImports");
if (!Directory.Exists(dir))
Directory.CreateDirectory(dir);
string filePath = Path.Combine(dir, file.FileName);
if (File.Exists(filePath))
File.Delete(filePath);
using (FileStream fsMain = new FileStream(filePath, FileMode.Create))
{
await file.CopyToAsync(fsMain);
}
string email = $"{Configuration.GetSection("Divan")["SystemEmail"]}";
var userId = (await _appUserService.FindUserByEmail(email)).Result.Id;
_backgroundTaskQueue.QueueBackgroundWorkItem
(
async token =>
{
using (RMuseumDbContext context = new RMuseumDbContext(new DbContextOptions<RMuseumDbContext>())) //this is long running job, so _context might be already been freed/collected by GC
{
LongRunningJobProgressServiceEF jobProgressServiceEF = new LongRunningJobProgressServiceEF(context);
var job = (await jobProgressServiceEF.NewJob("ApplyCorrectionsFromSqlite", "Query data")).Result;
try
{
SqliteConnectionStringBuilder connectionStringBuilder = new SqliteConnectionStringBuilder();
connectionStringBuilder.DataSource = filePath;
using (SqliteConnection sqliteConnection = new SqliteConnection(connectionStringBuilder.ToString()))
{
await sqliteConnection.OpenAsync();
IDbConnection sqlite = sqliteConnection;
var poets = (await sqlite.QueryAsync("SELECT * FROM poet")).ToList();
if (poets.Count != 1)
{
await jobProgressServiceEF.UpdateJob(job.Id, 100, "", false, "poets count in sqlite db is not equal to 1");
}
int poemNumber = 0;
foreach (var poem in await sqlite.QueryAsync($"SELECT * FROM poem ORDER BY id"))
{
poemNumber++;
await jobProgressServiceEF.UpdateJob(job.Id, poemNumber, "", false);
int poemId = (int)poem.id;
DivanPoem dbPoem = await context.DivanPoems.Include(p => p.Cat).Where(p => p.Id == poemId).SingleOrDefaultAsync();
if (dbPoem == null)
continue;
if (dbPoem.Cat.PoetId != poetId)
continue;
string comment = $"<p>تغییرات حاصل از پردازش {note}</p>{Environment.NewLine}";
bool anyChanges = false;
var dbPage = await context.DivanPages.Where(p => p.Id == poemId).SingleOrDefaultAsync();
DivanPageSnapshot snapshot = new DivanPageSnapshot()
{
DivanPageId = poemId,
MadeObsoleteByUserId = (Guid)userId,
RecordDate = DateTime.Now,
Note = note,
Title = dbPage.Title,
UrlSlug = dbPage.UrlSlug,
HtmlText = dbPage.HtmlText,
};
string poemTitle = poem.title;
if(poemTitle != dbPoem.Title)
{
anyChanges = true;
comment += $"<p>تغییر عنوان از «{dbPoem.Title}» به «{poemTitle}»</p>{Environment.NewLine}";
dbPoem.Title = poemTitle;
dbPoem.FullTitle = $"{dbPoem.Cat.FullUrl} » {dbPoem.Title}";
context.DivanPoems.Update(dbPoem);
}
var sqliteVerses = new List<dynamic>(await sqlite.QueryAsync($"SELECT * FROM verse WHERE poem_id = {poem.id} ORDER BY vorder"));
var dbVerses = await context.DivanVerses.Where(v => v.PoemId == poemId).OrderBy(v => v.VOrder).ToListAsync();
int vIndex = 0;
while(vIndex < sqliteVerses.Count && vIndex < dbVerses.Count)
{
if (sqliteVerses[vIndex].vorder != dbVerses[vIndex].VOrder)
{
vIndex = -1;
break;
}
string text = sqliteVerses[vIndex].text;
text = text.Replace("ـ", "").Replace(" ", " ").ApplyCorrectYeKe().Trim();
if (text == dbVerses[vIndex].Text)
{
vIndex++;
continue;
}
comment += $"<p>تغییر مصرع {vIndex + 1} از «{dbVerses[vIndex].Text}» به «{text}»</p>{Environment.NewLine}".ToPersianNumbers();
dbVerses[vIndex].Text = text;
context.DivanVerses.Update(dbVerses[vIndex]);
anyChanges = true;
vIndex++;
}
if(vIndex != -1)
{
while (vIndex < dbVerses.Count)
{
comment += $"<p>حذف مصرع {vIndex + 1} با متن «{dbVerses[vIndex].Text}»</p>{Environment.NewLine}".ToPersianNumbers();
context.DivanVerses.Remove(dbVerses[vIndex]);
vIndex++;
anyChanges = true;
}
while (vIndex < sqliteVerses.Count)
{
string text = sqliteVerses[vIndex].text;
text = text.Replace("ـ", "").Replace(" ", " ").ApplyCorrectYeKe().Trim();
int vOrder = int.Parse(sqliteVerses[vIndex].vorder.ToString());
int position = int.Parse(sqliteVerses[vIndex].position.ToString());
comment += $"<p>اضافه شدن مصرع {vIndex + 1} با متن «{text}»</p>{Environment.NewLine}".ToPersianNumbers();
context.DivanVerses.Add
(
new DivanVerse()
{
PoemId = poemId,
VOrder = vOrder,
VersePosition = (VersePosition)position,
Text = text
}
);
vIndex++;
anyChanges = true;
}
if (anyChanges)
{
await _FillPoemCoupletIndices(context, poemId);
DivanComment sysComment = new DivanComment()
{
UserId = userId,
AuthorIpAddress = "127.0.0.1",
CommentDate = DateTime.Now,
HtmlComment = comment,
PoemId = poemId,
Status = PublishStatus.Published,
};
context.DivanComments.Add(sysComment);
context.DivanPageSnapshots.Add(snapshot);
await context.SaveChangesAsync();
var poemVerses = await context.DivanVerses.Where(v => v.PoemId == poemId).OrderBy(v => v.VOrder).ToListAsync();
bool needsNewVOrder = false;
for (int i = 0; i < poemVerses.Count; i++)
{
if(poemVerses[i].VOrder != (i + 1))
{
poemVerses[i].VOrder = i + 1;
needsNewVOrder = true;
}
}
if(needsNewVOrder)
{
context.DivanVerses.UpdateRange(poemVerses);
}
dbPoem.PlainText = PreparePlainText(poemVerses, dbPoem.Title);
dbPoem.HtmlText = PrepareHtmlText(poemVerses);
dbPage.HtmlText = dbPoem.HtmlText;
dbPage.Title = dbPoem.Title;
dbPage.FullTitle = dbPoem.FullTitle;
try
{
var poemRhymeLettersRes = LanguageUtils.FindRhyme(poemVerses);
if (!string.IsNullOrEmpty(poemRhymeLettersRes.Rhyme))
{
dbPoem.RhymeLetters = poemRhymeLettersRes.Rhyme;
}
}
catch
{
}
context.DivanPoems.Update(dbPoem);
context.DivanPages.Update(dbPage);
await context.SaveChangesAsync();
}
}
}
await jobProgressServiceEF.UpdateJob(job.Id, 100, "", true);
}
}
catch (Exception exp)
{
await jobProgressServiceEF.UpdateJob(job.Id, 100, "", false, exp.ToString());
}
}
File.Delete(filePath);
}
);
}
catch (Exception exp)
{
return new RServiceResult<bool>(false, exp.ToString());
}
return new RServiceResult<bool>(true);
}
}
}