using Dapper; using DNTPersianUtils.Core; using FluentFTP; using diwan; using Microsoft.AspNetCore.Http; using Microsoft.Data.Sqlite; using Microsoft.EntityFrameworkCore; using RMuseum.DbContext; using RMuseum.Models.Artifact; using RMuseum.Models.Diwan; using RSecurityBackend.Models.Generic; using RSecurityBackend.Services.Implementation; using System; using System.Collections.Generic; using System.Data; using System.IO; using System.IO.Compression; using System.Linq; using System.Threading.Tasks; namespace RMuseum.Services.Implementation { /// /// IDiwanService implementation /// public partial class DiwanService : IDiwanService { /// /// start generating gdb files /// /// public RServiceResult StartBatchGenerateGDBFiles() { try { _backgroundTaskQueue.QueueBackgroundWorkItem ( async token => { using (RMuseumDbContext context = new RMuseumDbContext(new DbContextOptions())) //this is long running job, so _context might be already been freed/collected by GC { LongRunningJobProgressServiceEF jobProgressServiceEF = new LongRunningJobProgressServiceEF(context); var job = (await jobProgressServiceEF.NewJob("BatchGenerateGDBFiles", "Query Data")).Result; try { string outDir = Configuration.GetSection("Diwan")["GDBStorage"]; string imgDir = Configuration.GetSection("Diwan")["GDBStorageImageSource"]; string xmlFile = Configuration.GetSection("Diwan")["GDBListXMLFile"]; string preExisitingListXMLFile = Configuration.GetSection("Diwan")["GDBPreExisitingListXMLFile"]; List programList = new List(); if (!string.IsNullOrEmpty(preExisitingListXMLFile)) { if (File.Exists(preExisitingListXMLFile)) { programList = GDBListProcessor.RetrieveListFromFile(preExisitingListXMLFile, out string exception); if (programList == null) { await jobProgressServiceEF.UpdateJob(job.Id, 100, "GDBListProcessor.RetrieveListFromFile", false, exception); return; } } } var poets = await context.DiwanPoets.AsNoTracking().ToListAsync(); List lstFiles = new List(); foreach (var poet in poets) { if (!await context.DiwanPoems .Include(p => p.Cat) .Where(p => p.Cat.PoetId == poet.Id).AnyAsync()) continue; await jobProgressServiceEF.UpdateJob(job.Id, poet.Id); var gdbGeneration = await _ExportToSqlite(context, poet.Id, outDir, null, false); if(!string.IsNullOrEmpty(gdbGeneration.ExceptionString)) { await jobProgressServiceEF.UpdateJob(job.Id, 100, "", false, gdbGeneration.ExceptionString); return; } string gdbFile = gdbGeneration.Result; string pngFile = Path.Combine(imgDir, $"{poet.Id}.png"); bool hasImage = File.Exists(pngFile); using (var archiveStream = new MemoryStream()) { using (var archive = new ZipArchive(archiveStream, ZipArchiveMode.Create, true)) { var zipGDBFileEntry = archive.CreateEntry(Path.GetFileName(gdbFile), CompressionLevel.Optimal); using (var zipStream = zipGDBFileEntry.Open()) { var gdbBytes = File.ReadAllBytes(gdbFile); zipStream.Write(gdbBytes, 0, gdbBytes.Length); } if (hasImage) { var zipImgFileEntry = archive.CreateEntry(Path.GetFileName(pngFile), CompressionLevel.Optimal); using (var zipStream = zipImgFileEntry.Open()) { var pngBytes = File.ReadAllBytes(pngFile); zipStream.Write(pngBytes, 0, pngBytes.Length); } } } string zipFile = Path.Combine(outDir, Path.GetFileNameWithoutExtension(gdbFile) + ".zip"); if (File.Exists(zipFile)) File.Delete(zipFile); byte[] zipArray = archiveStream.ToArray(); File.WriteAllBytes(zipFile, zipArray); var catPoet = await context.DiwanCategories.AsNoTracking().Where(c => c.PoetId == poet.Id && c.ParentId == null).SingleAsync(); var lowestPoemID = await context.DiwanPoems .Include(p => p.Cat) .Where(p => p.Cat.PoetId == poet.Id) .MinAsync(p => p.Id); lstFiles.Add ( new GDBInfo() { CatName = poet.Nickname, CatID = catPoet.Id, PoetID = poet.Id, DownloadUrl = $"https://i.ganjoor.net/android/gdb/{Path.GetFileName(zipFile)}", BlogUrl = "", FileExt = ".zip", ImageUrl = (hasImage ? $"https://i.ganjoor.net/android/img/{poet.Id}.png" : ""), FileSizeInByte = zipArray.Length, LowestPoemID = lowestPoemID, PubDate = DateTime.Now } ); } File.Delete(gdbFile); } if (File.Exists(xmlFile)) File.Delete(xmlFile); if(programList.Count > 0) { lstFiles.AddRange(programList); } lstFiles.Sort((a, b) => a.CatName.CompareTo(b.CatName)); GDBListProcessor.Save(xmlFile, "مجموعه‌های قابل دریافت از گنجور", "", "", lstFiles); if (bool.Parse(Configuration.GetSection("ExternalFTPServer")["UploadEnabled"])) { var ftpClient = new AsyncFtpClient ( Configuration.GetSection("ExternalFTPServer")["Host"], Configuration.GetSection("ExternalFTPServer")["Username"], Configuration.GetSection("ExternalFTPServer")["Password"] ); ftpClient.ValidateCertificate += FtpClient_ValidateCertificate; await ftpClient.AutoConnect(); ftpClient.Config.RetryAttempts = 3; string dir = Path.GetDirectoryName(xmlFile); string rootDirName = Path.GetFileName(dir); string gdbDirName = Path.GetFileName(Path.GetDirectoryName(outDir)); foreach (var localFilePath in Directory.GetFiles(Path.Combine(dir, gdbDirName), "*.zip")) { var remoteFilePath = $"{Configuration.GetSection("ExternalFTPServer")["RootPath"]}/{rootDirName}/{gdbDirName}/{Path.GetFileName(localFilePath)}"; await ftpClient.UploadFile(localFilePath, remoteFilePath, createRemoteDir: true); } string imgDirName = Path.GetFileName(Path.GetDirectoryName(imgDir)); foreach (var localFilePath in Directory.GetFiles(Path.Combine(dir, imgDirName), "*.png")) { var remoteFilePath = $"{Configuration.GetSection("ExternalFTPServer")["RootPath"]}/{rootDirName}/{imgDirName}/{Path.GetFileName(localFilePath)}"; await ftpClient.UploadFile(localFilePath, remoteFilePath, createRemoteDir: true); } await ftpClient.UploadFile(xmlFile, $"{Configuration.GetSection("ExternalFTPServer")["RootPath"]}/{rootDirName}/{Path.GetFileName(xmlFile)}", createRemoteDir: true); await ftpClient.Disconnect(); } if (bool.Parse(Configuration.GetSection("BackupFTPServer")["UploadEnabled"])) { var ftpClient = new AsyncFtpClient ( Configuration.GetSection("BackupFTPServer")["Host"], Configuration.GetSection("BackupFTPServer")["Username"], Configuration.GetSection("BackupFTPServer")["Password"] ); ftpClient.ValidateCertificate += FtpClient_ValidateCertificate; await ftpClient.AutoConnect(); ftpClient.Config.RetryAttempts = 3; string dir = Path.GetDirectoryName(xmlFile); string rootDirName = Path.GetFileName(dir); string gdbDirName = Path.GetFileName(Path.GetDirectoryName(outDir)); foreach (var localFilePath in Directory.GetFiles(Path.Combine(dir, gdbDirName), "*.zip")) { var remoteFilePath = $"{Configuration.GetSection("BackupFTPServer")["RootPath"]}/{rootDirName}/{gdbDirName}/{Path.GetFileName(localFilePath)}"; await ftpClient.UploadFile(localFilePath, remoteFilePath, createRemoteDir: true); } string imgDirName = Path.GetFileName(Path.GetDirectoryName(imgDir)); foreach (var localFilePath in Directory.GetFiles(Path.Combine(dir, imgDirName), "*.png")) { var remoteFilePath = $"{Configuration.GetSection("BackupFTPServer")["RootPath"]}/{rootDirName}/{imgDirName}/{Path.GetFileName(localFilePath)}"; await ftpClient.UploadFile(localFilePath, remoteFilePath, createRemoteDir: true); } await ftpClient.UploadFile(xmlFile, $"{Configuration.GetSection("BackupFTPServer")["RootPath"]}/{rootDirName}/{Path.GetFileName(xmlFile)}", createRemoteDir: true); await ftpClient.Disconnect(); } await jobProgressServiceEF.UpdateJob(job.Id, 100, "", true); } catch (Exception exp) { await jobProgressServiceEF.UpdateJob(job.Id, 100, "", false, exp.ToString()); } } } ); return new RServiceResult(true); } catch (Exception exp) { return new RServiceResult(false, exp.ToString()); } } /// /// export to sqlite /// /// /// public async Task> ExportToSqlite(int poetId) { return await _ExportToSqlite(_context, poetId, Path.Combine($"{Configuration.GetSection("PictureFileService")["StoragePath"]}", "SQLiteExports")); } private async Task> _ExportToSqlite(RMuseumDbContext context, int poetId, string dir, string fileName = null, bool ignoreBio = false) { try { var poet = await context.DiwanPoets.AsNoTracking().Where(p => p.Id == poetId).SingleAsync(); var catPoet = await context.DiwanCategories.AsNoTracking().Where(c => c.PoetId == poetId && c.ParentId == null).SingleAsync(); if (!Directory.Exists(dir)) Directory.CreateDirectory(dir); if(fileName == null) { fileName = catPoet.UrlSlug; } string filePath = Path.Combine(dir, $"{fileName}.gdb"); if (File.Exists(filePath)) File.Delete(filePath); SqliteConnectionStringBuilder connectionStringBuilder = new SqliteConnectionStringBuilder(); connectionStringBuilder.DataSource = filePath; connectionStringBuilder.Mode = SqliteOpenMode.ReadWriteCreate; using (SqliteConnection sqliteConnection = new SqliteConnection(connectionStringBuilder.ToString())) { await sqliteConnection.OpenAsync(); string q = "BEGIN TRANSACTION;" + "CREATE TABLE [cat] ([id] INTEGER PRIMARY KEY NOT NULL,[poet_id] INTEGER NULL,[text] NVARCHAR(100) NULL,[parent_id] INTEGER NULL,[url] NVARCHAR(255) NULL);" + "CREATE TABLE [poem] (id INTEGER PRIMARY KEY, cat_id INTEGER, title NVARCHAR(255), url NVARCHAR(255));" + "CREATE TABLE [poet] ([id] INTEGER PRIMARY KEY NOT NULL,[name] NVARCHAR(20) NULL,[cat_id] INTEGER NULL NULL, [description] TEXT);" + "CREATE TABLE [verse] ([poem_id] INTEGER NULL,[vorder] INTEGER NULL,[position] INTEGER NULL,[text] TEXT NULL);" + "COMMIT;"; await sqliteConnection.ExecuteAsync(q); await sqliteConnection.ExecuteAsync("BEGIN;"); // Parameterized instead of string-interpolated: poet.Description (bio) is // free-form user-editable text, so splicing it into the SQL text directly let a // bio containing a "'" escape the string literal. The old code worked around that // by stripping quote characters out of the bio before export (silently corrupting // any bio that legitimately used one) rather than fixing the actual cause; Dapper's // parameter binding here makes that stripping unnecessary. string bio = poet.Description ?? ""; await sqliteConnection.ExecuteAsync( "INSERT INTO poet (id, name, cat_id, description) VALUES (@Id, @Name, @CatId, @Description);", new { Id = poet.Id, Name = poet.Nickname, CatId = catPoet.Id, Description = ignoreBio ? "" : bio } ); await ExportCatToSqlite(context, sqliteConnection, catPoet); await sqliteConnection.ExecuteAsync("COMMIT;"); } SqliteConnection.ClearAllPools(); return new RServiceResult(filePath); } catch(Exception exp) { return new RServiceResult(null, exp.ToString()); } } private async Task ExportCatToSqlite(RMuseumDbContext context, SqliteConnection sqliteConnection, DiwanCat cat) { int parentId = cat.ParentId == null ? 0 : (int)cat.ParentId; // Parameterized instead of string-interpolated (see the matching note in // _ExportToSqlite): cat.Title, poem.Title and verse.Text are all free-form user-editable // text, so splicing them into the SQL text directly let a "'" in any of them escape the // string literal - unlike the bio field above, nothing here even worked around that. await sqliteConnection.ExecuteAsync( "INSERT INTO cat (id, poet_id, text, parent_id, url) VALUES (@Id, @PoetId, @Text, @ParentId, @Url);", new { Id = cat.Id, PoetId = cat.PoetId, Text = cat.Title, ParentId = parentId, Url = $"https://ganjoor.net{cat.FullUrl}" } ); var poems = await context.DiwanPoems.AsNoTracking().Where(p => p.CatId == cat.Id).ToListAsync(); foreach(var poem in poems) { await sqliteConnection.ExecuteAsync( "INSERT INTO poem (id, cat_id, title, url) VALUES (@Id, @CatId, @Title, @Url);", new { Id = poem.Id, CatId = poem.CatId, Title = poem.Title, Url = $"https://ganjoor.net{poem.FullUrl}" } ); foreach (var verse in await context.DiwanVerses.AsNoTracking().Where(v => v.PoemId == poem.Id).OrderBy(v => v.VOrder).ToListAsync()) await sqliteConnection.ExecuteAsync( "INSERT INTO verse (poem_id, vorder, position, text) VALUES (@PoemId, @VOrder, @Position, @Text);", new { PoemId = poem.Id, VOrder = verse.VOrder, Position = (int)verse.VersePosition, Text = verse.Text } ); } foreach (var child in await context.DiwanCategories.AsNoTracking().Where(c => c.ParentId == cat.Id).ToListAsync()) await ExportCatToSqlite(context, sqliteConnection, child); } /// /// Apply corrections from sqlite /// /// /// /// /// public async Task> ApplyCorrectionsFromSqlite(int poetId, IFormFile file, string note) { try { string dir = Path.Combine($"{Configuration.GetSection("PictureFileService")["StoragePath"]}", "SQLiteImports"); if (!Directory.Exists(dir)) Directory.CreateDirectory(dir); string filePath = Path.Combine(dir, file.FileName); if (File.Exists(filePath)) File.Delete(filePath); using (FileStream fsMain = new FileStream(filePath, FileMode.Create)) { await file.CopyToAsync(fsMain); } string email = $"{Configuration.GetSection("Diwan")["SystemEmail"]}"; var userId = (await _appUserService.FindUserByEmail(email)).Result.Id; _backgroundTaskQueue.QueueBackgroundWorkItem ( async token => { using (RMuseumDbContext context = new RMuseumDbContext(new DbContextOptions())) //this is long running job, so _context might be already been freed/collected by GC { LongRunningJobProgressServiceEF jobProgressServiceEF = new LongRunningJobProgressServiceEF(context); var job = (await jobProgressServiceEF.NewJob("ApplyCorrectionsFromSqlite", "Query data")).Result; try { SqliteConnectionStringBuilder connectionStringBuilder = new SqliteConnectionStringBuilder(); connectionStringBuilder.DataSource = filePath; using (SqliteConnection sqliteConnection = new SqliteConnection(connectionStringBuilder.ToString())) { await sqliteConnection.OpenAsync(); IDbConnection sqlite = sqliteConnection; var poets = (await sqlite.QueryAsync("SELECT * FROM poet")).ToList(); if (poets.Count != 1) { await jobProgressServiceEF.UpdateJob(job.Id, 100, "", false, "poets count in sqlite db is not equal to 1"); } int poemNumber = 0; foreach (var poem in await sqlite.QueryAsync($"SELECT * FROM poem ORDER BY id")) { poemNumber++; await jobProgressServiceEF.UpdateJob(job.Id, poemNumber, "", false); int poemId = (int)poem.id; DiwanPoem dbPoem = await context.DiwanPoems.Include(p => p.Cat).Where(p => p.Id == poemId).SingleOrDefaultAsync(); if (dbPoem == null) continue; if (dbPoem.Cat.PoetId != poetId) continue; string comment = $"

تغییرات حاصل از پردازش {note}

{Environment.NewLine}"; bool anyChanges = false; var dbPage = await context.DiwanPages.Where(p => p.Id == poemId).SingleOrDefaultAsync(); DiwanPageSnapshot snapshot = new DiwanPageSnapshot() { DiwanPageId = poemId, MadeObsoleteByUserId = (Guid)userId, RecordDate = DateTime.Now, Note = note, Title = dbPage.Title, UrlSlug = dbPage.UrlSlug, HtmlText = dbPage.HtmlText, }; string poemTitle = poem.title; if(poemTitle != dbPoem.Title) { anyChanges = true; comment += $"

تغییر عنوان از «{dbPoem.Title}» به «{poemTitle}»

{Environment.NewLine}"; dbPoem.Title = poemTitle; dbPoem.FullTitle = $"{dbPoem.Cat.FullUrl} » {dbPoem.Title}"; context.DiwanPoems.Update(dbPoem); } var sqliteVerses = new List(await sqlite.QueryAsync($"SELECT * FROM verse WHERE poem_id = {poem.id} ORDER BY vorder")); var dbVerses = await context.DiwanVerses.Where(v => v.PoemId == poemId).OrderBy(v => v.VOrder).ToListAsync(); int vIndex = 0; while(vIndex < sqliteVerses.Count && vIndex < dbVerses.Count) { if (sqliteVerses[vIndex].vorder != dbVerses[vIndex].VOrder) { vIndex = -1; break; } string text = sqliteVerses[vIndex].text; text = text.Replace("ـ", "").Replace(" ", " ").ApplyCorrectYeKe().Trim(); if (text == dbVerses[vIndex].Text) { vIndex++; continue; } comment += $"

تغییر مصرع {vIndex + 1} از «{dbVerses[vIndex].Text}» به «{text}»

{Environment.NewLine}".ToPersianNumbers(); dbVerses[vIndex].Text = text; context.DiwanVerses.Update(dbVerses[vIndex]); anyChanges = true; vIndex++; } if(vIndex != -1) { while (vIndex < dbVerses.Count) { comment += $"

حذف مصرع {vIndex + 1} با متن «{dbVerses[vIndex].Text}»

{Environment.NewLine}".ToPersianNumbers(); context.DiwanVerses.Remove(dbVerses[vIndex]); vIndex++; anyChanges = true; } while (vIndex < sqliteVerses.Count) { string text = sqliteVerses[vIndex].text; text = text.Replace("ـ", "").Replace(" ", " ").ApplyCorrectYeKe().Trim(); int vOrder = int.Parse(sqliteVerses[vIndex].vorder.ToString()); int position = int.Parse(sqliteVerses[vIndex].position.ToString()); comment += $"

اضافه شدن مصرع {vIndex + 1} با متن «{text}»

{Environment.NewLine}".ToPersianNumbers(); context.DiwanVerses.Add ( new DiwanVerse() { PoemId = poemId, VOrder = vOrder, VersePosition = (VersePosition)position, Text = text } ); vIndex++; anyChanges = true; } if (anyChanges) { await _FillPoemCoupletIndices(context, poemId); DiwanComment sysComment = new DiwanComment() { UserId = userId, AuthorIpAddress = "127.0.0.1", CommentDate = DateTime.Now, HtmlComment = comment, PoemId = poemId, Status = PublishStatus.Published, }; context.DiwanComments.Add(sysComment); context.DiwanPageSnapshots.Add(snapshot); await context.SaveChangesAsync(); var poemVerses = await context.DiwanVerses.Where(v => v.PoemId == poemId).OrderBy(v => v.VOrder).ToListAsync(); bool needsNewVOrder = false; for (int i = 0; i < poemVerses.Count; i++) { if(poemVerses[i].VOrder != (i + 1)) { poemVerses[i].VOrder = i + 1; needsNewVOrder = true; } } if(needsNewVOrder) { context.DiwanVerses.UpdateRange(poemVerses); } dbPoem.PlainText = PreparePlainText(poemVerses, dbPoem.Title); dbPoem.HtmlText = PrepareHtmlText(poemVerses); dbPage.HtmlText = dbPoem.HtmlText; dbPage.Title = dbPoem.Title; dbPage.FullTitle = dbPoem.FullTitle; try { var poemRhymeLettersRes = LanguageUtils.FindRhyme(poemVerses); if (!string.IsNullOrEmpty(poemRhymeLettersRes.Rhyme)) { dbPoem.RhymeLetters = poemRhymeLettersRes.Rhyme; } } catch { } context.DiwanPoems.Update(dbPoem); context.DiwanPages.Update(dbPage); await context.SaveChangesAsync(); } } } await jobProgressServiceEF.UpdateJob(job.Id, 100, "", true); } } catch (Exception exp) { await jobProgressServiceEF.UpdateJob(job.Id, 100, "", false, exp.ToString()); } } File.Delete(filePath); } ); } catch (Exception exp) { return new RServiceResult(false, exp.ToString()); } return new RServiceResult(true); } } }