using Dapper; using Microsoft.Data.Sqlite; using Microsoft.EntityFrameworkCore; using Microsoft.Extensions.Configuration; using MySql.Data.MySqlClient; using RMuseum.DbContext; using RMuseum.Models.Ganjoor; using RMuseum.Models.Ganjoor.ViewModels; using RMuseum.Models.GanjoorAudio; using RMuseum.Models.GanjoorAudio.ViewModels; using RMuseum.Models.GanjoorIntegration.ViewModels; using RMuseum.Models.MusicCatalogue; using RSecurityBackend.Models.Generic; using RSecurityBackend.Models.Generic.Db; using RSecurityBackend.Services; using RSecurityBackend.Services.Implementation; using System; using System.Collections.Generic; using System.Data; using System.Data.Common; using System.Globalization; using System.Linq; using System.Threading.Tasks; namespace RMuseum.Services.Implementation { /// /// IGanjoorService implementation /// public class GanjoorService : IGanjoorService { /// /// Get List of poets /// /// /// /// public async Task> GetPoets(bool websitePoets, bool includeBio = true) { try { var res = await (from poet in _context.GanjoorPoets join cat in _context.GanjoorCategories.Where(c => c.ParentId == null) on poet.Id equals cat.PoetId where !websitePoets || poet.Id < 200 orderby poet.Name descending select new GanjoorPoetViewModel() { Id = poet.Id, Name = poet.Name, Description = includeBio ? poet.Description : null, FullUrl = cat.FullUrl, RootCatId = cat.Id } ) .ToListAsync(); StringComparer fa = StringComparer.Create(new CultureInfo("fa-IR"), true); res.Sort((a, b) => fa.Compare(a.Name, b.Name)); return new RServiceResult ( res.ToArray() ); ; } catch (Exception exp) { return new RServiceResult(null, exp.ToString()); } } /// /// get poet by id /// /// /// public async Task> GetPoetById(int id) { try { var poet = await _context.GanjoorPoets.Where(p => p.Id == id).FirstOrDefaultAsync(); if (poet == null) return new RServiceResult(null); var cat = await _context.GanjoorCategories.Where(c => c.ParentId == null && c.PoetId == id).FirstOrDefaultAsync(); return await GetCatById(cat.Id); } catch(Exception exp) { return new RServiceResult(null, exp.ToString()); } } /// /// get poet by url /// /// /// public async Task> GetPoetByUrl(string url) { try { // /hafez/ => /hafez : if(url.LastIndexOf('/') == url.Length - 1) { url = url.Substring(0, url.Length - 1); } var cat = await _context.GanjoorCategories.Where(c => c.FullUrl == url && c.ParentId == null).SingleOrDefaultAsync(); if (cat == null) return new RServiceResult(null); return await GetCatById(cat.Id); } catch (Exception exp) { return new RServiceResult(null, exp.ToString()); } } /// /// get cat by url /// /// /// /// public async Task> GetCatByUrl(string url, bool poems = true) { try { // /hafez/ => /hafez : if (url.LastIndexOf('/') == url.Length - 1) { url = url.Substring(0, url.Length - 1); } var cat = await _context.GanjoorCategories.Where(c => c.FullUrl == url).SingleOrDefaultAsync(); if (cat == null) return new RServiceResult(null); return await GetCatById(cat.Id); } catch (Exception exp) { return new RServiceResult(null, exp.ToString()); } } /// /// get cat by id /// /// /// /// public async Task> GetCatById(int id, bool poems = true) { try { var cat = await _context.GanjoorCategories.Include(c => c.Poet).Include(c => c.Parent).Where(c => c.Id == id).FirstOrDefaultAsync(); if (cat == null) return new RServiceResult(null); List ancetors = new List(); var parent = cat.Parent; while (parent != null) { ancetors.Insert(0, new GanjoorCatViewModel() { Id = parent.Id, Title = parent.Title, UrlSlug = parent.UrlSlug, FullUrl = parent.FullUrl }); parent = await _context.GanjoorCategories.Where(c => c.Id == parent.ParentId).FirstOrDefaultAsync(); } int nextCatId = await _context.GanjoorCategories.Where(c => c.PoetId == cat.PoetId && c.ParentId == cat.ParentId && c.Id > id).AnyAsync() ? await _context.GanjoorCategories.Where(c => c.PoetId == cat.PoetId && c.ParentId == cat.ParentId && c.Id > id).MinAsync(c => c.Id) : 0; var nextCat = nextCatId == 0 ? null : await _context .GanjoorCategories .Where(c => c.Id == nextCatId) .Select ( c => new GanjoorCatViewModel() { Id = c.Id, Title = c.Title, UrlSlug = c.UrlSlug, FullUrl = c.FullUrl //other fields null } ).SingleOrDefaultAsync(); int preCatId = await _context.GanjoorCategories.Where(c => c.PoetId == cat.PoetId && c.ParentId == cat.ParentId && c.Id < id).AnyAsync() ? await _context.GanjoorCategories.Where(c => c.PoetId == cat.PoetId && c.ParentId == cat.ParentId && c.Id < id).MaxAsync(c => c.Id) : 0; var preCat = preCatId == 0 ? null : await _context .GanjoorCategories .Where(c => c.Id == preCatId) .Select ( c => new GanjoorCatViewModel() { Id = c.Id, Title = c.Title, UrlSlug = c.UrlSlug, FullUrl = c.FullUrl //other fields null } ).SingleOrDefaultAsync(); GanjoorCatViewModel catViewModel = new GanjoorCatViewModel() { Id = cat.Id, Title = cat.Title, UrlSlug = cat.UrlSlug, FullUrl = cat.FullUrl, Next = nextCat, Previous = preCat, Ancestors = ancetors, Children = await _context.GanjoorCategories.Where(c => c.ParentId == cat.Id).OrderBy(cat => cat.Id).Select ( c => new GanjoorCatViewModel() { Id = c.Id, Title = c.Title, UrlSlug = c.UrlSlug, FullUrl = c.FullUrl } ).ToListAsync(), Poems = poems ? await _context.GanjoorPoems.Where(p => p.CatId == cat.Id).OrderBy(p => p.Id).Select ( p => new GanjoorPoemSummaryViewModel() { Id = p.Id, Title = p.Title, UrlSlug = p.UrlSlug, Excerpt = _context.GanjoorVerses.Where(v => v.PoemId == p.Id && v.VOrder == 1).FirstOrDefault().Text } ).ToListAsync() : null }; return new RServiceResult ( new GanjoorPoetCompleteViewModel() { Poet = await _context.GanjoorPoets.Where(p => p.Id == cat.PoetId) .Select(p => new GanjoorPoetViewModel() { Id = p.Id, Name = p.Name, Description = p.Description, FullUrl = _context.GanjoorCategories.Where(c => c.PoetId == p.Id && c.ParentId == null).Single().FullUrl, RootCatId = _context.GanjoorCategories.Where(c => c.PoetId == p.Id && c.ParentId == null).Single().Id }) .FirstOrDefaultAsync(), Cat = catViewModel } ); } catch (Exception exp) { return new RServiceResult(null, exp.ToString()); } } /// /// get page by url /// /// /// /// public async Task> GetPageByUrl(string url, bool catPoems = true) { try { if(url.IndexOf('?') != -1) { url = url.Substring(0, url.IndexOf('?')); } // /hafez/ => /hafez : if (url.LastIndexOf('/') == url.Length - 1) { url = url.Substring(0, url.Length - 1); } var dbPage = await _context.GanjoorPages.Where(p => p.FullUrl == url).SingleOrDefaultAsync(); if (dbPage == null) return new RServiceResult(null); //not found var secondPoet = dbPage.SecondPoetId == null ? null : await (from poet in _context.GanjoorPoets join cat in _context.GanjoorCategories.Where(c => c.ParentId == null) on poet.Id equals cat.PoetId where poet.Id == (int)dbPage.SecondPoetId orderby poet.Name descending select new GanjoorPoetViewModel() { Id = poet.Id, Name = poet.Name, FullUrl = cat.FullUrl, RootCatId = cat.Id } ) .SingleAsync(); GanjoorPageCompleteViewModel page = new GanjoorPageCompleteViewModel() { Id = dbPage.Id, GanjoorPageType = dbPage.GanjoorPageType, Title = dbPage.Title, FullTitle = dbPage.FullTitle, UrlSlug = dbPage.UrlSlug, FullUrl = dbPage.FullUrl, HtmlText = dbPage.HtmlText, SecondPoet = secondPoet }; switch (page.GanjoorPageType) { case GanjoorPageType.PoemPage: { var poemRes = await GetPoemById((int)dbPage.PoemId); if(!string.IsNullOrEmpty(poemRes.ExceptionString)) { return new RServiceResult(null, poemRes.ExceptionString); } page.Poem = poemRes.Result; } break; case GanjoorPageType.CatPage: { var catRes = await GetCatById((int)dbPage.CatId); if(!string.IsNullOrEmpty(catRes.ExceptionString)) { return new RServiceResult(null, catRes.ExceptionString); } page.PoetOrCat = catRes.Result; } break; default: { if (dbPage.PoetId != null) { var poetRes = await GetPoetById((int)dbPage.PoetId); if (!string.IsNullOrEmpty(poetRes.ExceptionString)) { return new RServiceResult(null, poetRes.ExceptionString); } page.PoetOrCat = poetRes.Result; } } break; } return new RServiceResult(page); } catch(Exception exp) { return new RServiceResult(null, exp.ToString()); } } /// /// get poem recitations (PlainText/HtmlText are intentionally empty) /// /// /// public async Task> GetPoemRecitations(int id) { try { var source = from audio in _context.Recitations join poem in _context.GanjoorPoems on audio.GanjoorPostId equals poem.Id where audio.ReviewStatus == AudioReviewStatus.Approved && poem.Id == id orderby audio.AudioOrder select new PublicRecitationViewModel() { Id = audio.Id, PoemId = audio.GanjoorPostId, PoemFullTitle = poem.FullTitle, PoemFullUrl = poem.FullUrl, AudioTitle = audio.AudioTitle, AudioArtist = audio.AudioArtist, AudioArtistUrl = audio.AudioArtistUrl, AudioSrc = audio.AudioSrc, AudioSrcUrl = audio.AudioSrcUrl, LegacyAudioGuid = audio.LegacyAudioGuid, Mp3FileCheckSum = audio.Mp3FileCheckSum, Mp3SizeInBytes = audio.Mp3SizeInBytes, PublishDate = audio.ReviewDate, FileLastUpdated = audio.FileLastUpdated, Mp3Url = $"https://ganjgah.ir/api/audio/file/{audio.Id}.mp3", XmlText = $"https://ganjgah.ir/api/audio/xml/{audio.Id}", PlainText = "", //poem.PlainText HtmlText = "",//poem.HtmlText }; return new RServiceResult(await source.ToArrayAsync()); } catch (Exception exp) { return new RServiceResult(null, exp.ToString()); } } /// /// get poem images by id (some fields are intentionally field with blank or null), /// EntityImageId : the most important data field, image url is https://ganjgah.ir/api/images/thumb/{EntityImageId}.jpg or https://ganjgah.ir/api/images/norm/{EntityImageId}.jpg /// /// /// public async Task> GetPoemImages(int id) { try { var museumSrc = from link in _context.GanjoorLinks.Include(l => l.Artifact).Include(l => l.Item).ThenInclude(i => i.Images) join poem in _context.GanjoorPoems on link.GanjoorPostId equals poem.Id where link.ReviewResult == Models.GanjoorIntegration.ReviewResult.Approved && poem.Id == id orderby link.ReviewDate select new PoemRelatedImage() { PoemRelatedImageType = PoemRelatedImageType.MuseumLink, ThumbnailImageUrl = $"https://ganjgah.ir/api/images/thumb/{link.Item.Images.First().Id}.jpg", TargetPageUrl = $"https://museum.ganjoor.net/items/{link.Artifact.FriendlyUrl}/{link.Item.FriendlyUrl}", AltText = $"{link.Artifact.Name} » {link.Item.Name}", }; List museumImages = await museumSrc.ToListAsync(); var externalSrc = from link in _context.PinterestLinks join poem in _context.GanjoorPoems on link.GanjoorPostId equals poem.Id where link.ReviewResult == Models.GanjoorIntegration.ReviewResult.Approved && poem.Id == id orderby link.ReviewDate select new PoemRelatedImage() { PoemRelatedImageType = PoemRelatedImageType.ExternalLink, ThumbnailImageUrl = $"https://ganjgah.ir/api/images/thumb/{link.Item.Images.First().Id}.jpg", TargetPageUrl = link.PinterestUrl, AltText = link.AltText, }; museumImages.AddRange(await externalSrc.ToListAsync()); for(int i=0; i(museumImages.ToArray()); } catch (Exception exp) { return new RServiceResult(null, exp.ToString()); } } private int _GetRandomPoemId() { //this is magic number based method! int startPoemId = 2130; int endPoemId = 2624 + 1; //one is added for مژده ای دل که مسیحا نفسی می‌آید Random r = new Random(DateTime.Now.Millisecond); int poemId = r.Next(startPoemId, endPoemId); if (poemId == endPoemId) { poemId = 33179;//مژده ای دل که مسیحا نفسی می‌آید } return poemId; } /// /// Get Poem By Url /// /// /// /// /// /// /// /// /// /// /// /// public async Task> GetPoemByUrl(string url, bool catInfo = true, bool catPoems = false, bool rhymes = true, bool recitations = true, bool images = true, bool songs = true, bool comments = true, bool verseDetails = true, bool navigation = true) { try { // /hafez/ => /hafez : if (url.LastIndexOf('/') == url.Length - 1) { url = url.Substring(0, url.Length - 1); } var poem = await _context.GanjoorPoems.Where(p => p.FullUrl == url).SingleOrDefaultAsync(); if (poem == null) { return new RServiceResult(null); //not found } return await GetPoemById(poem.Id, catInfo, catPoems, rhymes, recitations, images, songs, comments, verseDetails, navigation); } catch(Exception exp) { return new RServiceResult(null, exp.ToString()); } } /// /// Get Poem By Id /// /// /// /// /// /// /// /// /// /// /// /// public async Task> GetPoemById(int id, bool catInfo = true, bool catPoems = false , bool rhymes = true, bool recitations = true, bool images = true, bool songs = true, bool comments = true, bool verseDetails = true, bool navigation = true) { try { var poem = await _context.GanjoorPoems.Include(p => p.GanjoorMetre).Where(p => p.Id == id).SingleOrDefaultAsync(); if(poem == null) { return new RServiceResult(null); //not found } GanjoorPoetCompleteViewModel cat = null; if(catInfo) { var catRes = await GetCatById(poem.CatId, catPoems); if(!string.IsNullOrEmpty(catRes.ExceptionString)) { return new RServiceResult(null, catRes.ExceptionString); } cat = catRes.Result; } GanjoorPoemSummaryViewModel next = null; if(navigation) { int nextId = await _context.GanjoorPoems .Where(p => p.CatId == poem.CatId && p.Id > poem.Id) .AnyAsync() ? await _context.GanjoorPoems .Where(p => p.CatId == poem.CatId && p.Id > poem.Id) .MinAsync(p => p.Id) : 0; if(nextId != 0) { next = await _context.GanjoorPoems.Where(p => p.Id == nextId).Select ( p => new GanjoorPoemSummaryViewModel() { Id = p.Id, Title = p.Title, UrlSlug = p.UrlSlug, Excerpt = _context.GanjoorVerses.Where(v => v.PoemId == p.Id && v.VOrder == 1).FirstOrDefault().Text } ).SingleAsync(); } } GanjoorPoemSummaryViewModel previous = null; if (navigation) { int preId = await _context.GanjoorPoems .Where(p => p.CatId == poem.CatId && p.Id < poem.Id) .AnyAsync() ? await _context.GanjoorPoems .Where(p => p.CatId == poem.CatId && p.Id < poem.Id) .MaxAsync(p => p.Id) : 0; if (preId != 0) { previous = await _context.GanjoorPoems.Where(p => p.Id == preId).Select ( p => new GanjoorPoemSummaryViewModel() { Id = p.Id, Title = p.Title, UrlSlug = p.UrlSlug, Excerpt = _context.GanjoorVerses.Where(v => v.PoemId == p.Id && v.VOrder == 1).FirstOrDefault().Text } ).SingleAsync(); } } PublicRecitationViewModel[] rc = null; if(recitations) { var rcRes = await GetPoemRecitations(id); if (!string.IsNullOrEmpty(rcRes.ExceptionString)) return new RServiceResult(null, rcRes.ExceptionString); rc = rcRes.Result; } PoemRelatedImage[] imgs = null; if(images) { var imgsRes = await GetPoemImages(id); if (!string.IsNullOrEmpty(imgsRes.ExceptionString)) return new RServiceResult(null, imgsRes.ExceptionString); imgs = imgsRes.Result; } GanjoorVerseViewModel[] verses = null; if(verseDetails) { verses = await _context.GanjoorVerses .Where(v => v.PoemId == id) .OrderBy(v => v.VOrder) .Select ( v => new GanjoorVerseViewModel() { Id = v.Id, VOrder = v.VOrder, VersePosition = v.VersePosition, Text = v.Text } ).ToArrayAsync(); }; PoemMusicTrackViewModel[] tracks = null; if(songs) { tracks = await _context.GanjoorPoemMusicTracks .Where(t => t.PoemId == id && t.Approved) .OrderBy(t => t.Id) .Select ( t => new PoemMusicTrackViewModel() { Id = t.Id, TrackType = t.TrackType, ArtistName = t.ArtistName, ArtistUrl = t.ArtistUrl, AlbumName = t.AlbumName, AlbumUrl = t.AlbumUrl, TrackName = t.TrackName, TrackUrl = t.TrackUrl, Description = t.Description, BrokenLink = t.BrokenLink } ).ToArrayAsync(); } return new RServiceResult ( new GanjoorPoemCompleteViewModel() { Id = poem.Id, Title = poem.Title, FullTitle = poem.FullTitle, FullUrl = poem.FullUrl, UrlSlug = poem.UrlSlug, HtmlText = poem.HtmlText, PlainText = poem.PlainText, GanjoorMetre = poem.GanjoorMetre, RhymeLetters = poem.RhymeLetters, SourceName = poem.SourceName, SourceUrlSlug = poem.SourceUrlSlug, Category = cat, Next = next, Previous = previous, Recitations = rc, Images = imgs, Verses = verses, Songs = tracks } ); } catch (Exception exp) { return new RServiceResult(null, exp.ToString()); } } /// /// suggest song /// /// /// /// public async Task> SuggestSong(Guid userId, PoemMusicTrackViewModel song) { try { var alreadySuggestedSong = await _context.GanjoorPoemMusicTracks.Where(t => t.PoemId == song.PoemId && t.TrackType == song.TrackType && t.TrackUrl == song.TrackUrl).FirstOrDefaultAsync(); if(alreadySuggestedSong != null) { return new RServiceResult(null, "این آهنگ پیشتر برای این شعر پیشنهاد داده شده است."); } var sug = new PoemMusicTrack() { TrackType = song.TrackType, PoemId = song.PoemId, ArtistName = song.ArtistName, ArtistUrl = song.ArtistUrl, AlbumName = song.AlbumName, AlbumUrl = song.AlbumUrl, TrackName = song.TrackName, TrackUrl = song.TrackUrl, SuggestedById = userId }; GanjoorSinger singer = await _context.GanjoorSingers.Where(s => s.Url == song.ArtistUrl).FirstOrDefaultAsync(); if (singer != null) { sug.SingerId = singer.Id; } _context.GanjoorPoemMusicTracks.Add ( sug ); await _context.SaveChangesAsync(); song.Id = sug.Id; return new RServiceResult(song); } catch(Exception exp) { return new RServiceResult(null, exp.ToString()); } } /// /// get a random poem from hafez /// /// public async Task> Faal() { try { int poemId = _GetRandomPoemId(); var poem = await _context.GanjoorPoems.Where(p => p.Id == poemId).SingleOrDefaultAsync(); PublicRecitationViewModel[] recitations = poem == null ? new PublicRecitationViewModel[] { } : (await GetPoemRecitations(poemId)).Result; int loopPreventer = 0; while (poem == null || recitations.Length == 0) { poem = await _context.GanjoorPoems.Where(p => p.Id == poemId).SingleOrDefaultAsync(); recitations = poem == null ? new PublicRecitationViewModel[] { } : (await GetPoemRecitations(poemId)).Result; loopPreventer++; if (loopPreventer > 5) break; } return await GetPoemById(poemId, false, false, false, true, false, false, false, false, false); } catch (Exception exp) { return new RServiceResult(null, exp.ToString()); } } #region Date import / modifications /// /// imports unimported poem data from a locally accessible ganjoor SqlLite database /// /// public async Task> ImportLocalSQLiteDb() { try { SqliteConnectionStringBuilder connectionStringBuilder = new SqliteConnectionStringBuilder(); connectionStringBuilder.DataSource = Configuration.GetSection("LocalSqliteImport")["FilePath"]; using (SqliteConnection sqliteConnection = new SqliteConnection(connectionStringBuilder.ToString())) { await sqliteConnection.OpenAsync(); IDbConnection dapper = sqliteConnection; using (var sqlConnection = _context.Database.GetDbConnection()) { await sqlConnection.OpenAsync(); foreach (var poet in await dapper.QueryAsync("SELECT * FROM poet ORDER BY id")) { int poetId = (int)poet.id; if ((await _context.GanjoorPoets.Where(p => p.Id == poetId).FirstOrDefaultAsync()) != null) { continue; } DbTransaction transaction = await sqlConnection.BeginTransactionAsync(); try { using (var command = sqlConnection.CreateCommand()) { command.Transaction = transaction; command.CommandText = $"INSERT INTO GanjoorPoets (Id, Name, Description) VALUES (${poet.id}, N'{poet.name}', N'{poet.description}')"; await command.ExecuteNonQueryAsync(); await _ImportSQLiteCatChildren(command, dapper, poetId, 0, "", ""); await transaction.CommitAsync(); } } catch (Exception exp2) { await transaction.RollbackAsync(); return new RServiceResult(false, exp2.ToString()); } } } } } catch(Exception exp) { return new RServiceResult(false, exp.ToString()); } return new RServiceResult(true); } private async Task _ImportSQLiteCatChildren(DbCommand command, IDbConnection dapper, int poetId, int catId, string itemFullTitle, string itemFullSlug) { foreach(var cat in await dapper.QueryAsync($"SELECT * FROM cat WHERE poet_id = {poetId} AND parent_id = {catId} ORDER BY id")) { if (catId == 0) { command.CommandText = $"INSERT INTO GanjoorCategories (Id, PoetId, Title, UrlSlug, FullUrl) VALUES (${cat.id}, {poetId}, N'{cat.text}', '{ _ExtractUrlSlug(cat.url)}', '{itemFullSlug}/{ _ExtractUrlSlug(cat.url)}')"; } else { command.CommandText = $"INSERT INTO GanjoorCategories (Id, PoetId, Title, ParentId, UrlSlug, FullUrl) VALUES (${cat.id}, {poetId}, N'{cat.text}', {catId}, '{ _ExtractUrlSlug(cat.url)}', '{itemFullSlug}/{ _ExtractUrlSlug(cat.url)}')"; } await command.ExecuteNonQueryAsync(); foreach (var poem in await dapper.QueryAsync($"SELECT * FROM poem WHERE cat_id = {cat.id} ORDER BY id")) { command.CommandText = $"INSERT INTO GanjoorPoems (Id, CatId, Title, UrlSlug, FullTitle, FullUrl) VALUES (${poem.id}, {cat.id}, N'{poem.title}', '{ _ExtractUrlSlug(poem.url)}', N'{$"{itemFullTitle}{cat.text} » {poem.title}"}', '{itemFullSlug}/{ _ExtractUrlSlug(cat.url)}/{_ExtractUrlSlug(poem.url)}')"; await command.ExecuteNonQueryAsync(); foreach (var verse in await dapper.QueryAsync($"SELECT * FROM verse WHERE poem_id = {poem.id} ORDER BY vorder")) { command.CommandText = $"INSERT INTO GanjoorVerses (PoemId, VOrder, VersePosition, Text) VALUES (${poem.id}, {verse.vorder}, {verse.position}, N'{verse.text}')"; await command.ExecuteNonQueryAsync(); } } await _ImportSQLiteCatChildren(command, dapper, poetId, (int)cat.id, $"{itemFullTitle}{cat.text} » ", $"{itemFullSlug}/{ _ExtractUrlSlug(cat.url)}"); } } private static string _ExtractUrlSlug(string slug) { if (!string.IsNullOrEmpty(slug)) { //sample: https://ganjoor.net/hafez if (slug.LastIndexOf('/') != -1) { // => result = hafez slug = slug.Substring(slug.LastIndexOf('/') + 1); } } return slug; } /// /// updates poems text /// /// public RServiceResult UpdatePoemsText() { try { _backgroundTaskQueue.QueueBackgroundWorkItem ( async token => { using (RMuseumDbContext context = new RMuseumDbContext(Configuration)) //this is long running job, so _context might be already been freed/collected by GC { var poems = await context.GanjoorPoems.Where(p => p.PlainText == null).ToListAsync(); int n = 0; foreach (GanjoorPoem poem in poems) { var verses = await context.GanjoorVerses.Where(v => v.PoemId == poem.Id).OrderBy(v => v.VOrder).ToListAsync(); string plainText = ""; string htmlText = ""; for (int vIndex = 0; vIndex < verses.Count; vIndex++) { GanjoorVerse v = verses[vIndex]; if (string.IsNullOrEmpty(plainText)) plainText = v.Text; else { plainText += $" {v.Text}"; } switch (v.VersePosition) { case VersePosition.CenteredVerse1: if (((vIndex + 1) < verses.Count) && (verses[vIndex + 1].VersePosition == VersePosition.CenteredVerse2)) { htmlText += $"

{v.Text.Replace("ـ", "").Trim()}

{Environment.NewLine}"; //div is not closed } else { htmlText += $"

{v.Text.Replace("ـ", "").Trim()}

{Environment.NewLine}"; } break; case VersePosition.CenteredVerse2: htmlText += $"

{v.Text.Replace("ـ", "").Trim()}

{Environment.NewLine}"; break; case VersePosition.Right: htmlText += $"

{v.Text.Replace("ـ", "").Trim()}

{Environment.NewLine}"; break; case VersePosition.Left: htmlText += $"

{v.Text.Replace("ـ", "").Trim()}

{Environment.NewLine}"; break; case VersePosition.Paragraph: case VersePosition.Single: { string[] lines = v.Text.Replace("ـ", "").Split(new char[] { '\r', '\n' }, StringSplitOptions.RemoveEmptyEntries); if (lines.Length != 0) { if (v.Text.Replace("ـ", "").Length / lines.Length < 150) { htmlText += $"

{v.Text.Replace("ـ", "").Replace("\r\n", " ")}

{Environment.NewLine}"; } else { foreach (string line in lines) htmlText += $"

{line}

{Environment.NewLine}"; } } } break; } } poem.PlainText = plainText; poem.HtmlText = htmlText; context.Update(poem); n++; if (n >= 1000) { await context.SaveChangesAsync(); n = 0; } } if(n > 0) await context.SaveChangesAsync(); } }); return new RServiceResult(true); } catch(Exception exp) { return new RServiceResult(false, exp.ToString()); } } /// /// import GanjoorPage entity data from MySql /// /// public RServiceResult ImportFromMySql() { try { _backgroundTaskQueue.QueueBackgroundWorkItem ( async token => { using (RMuseumDbContext context = new RMuseumDbContext(Configuration)) //this is long running job, so _context might be already been freed/collected by GC using (RMuseumDbContext contextReport = new RMuseumDbContext(Configuration)) //this is long running job, so _context might be already been freed/collected by GC { LongRunningJobProgressServiceEF jobProgressServiceEF = new LongRunningJobProgressServiceEF(contextReport); var job = (await jobProgressServiceEF.NewJob("GanjoorService:ImportFromMySql", "pre open connection")).Result; MusicCatalogueService catalogueService = new MusicCatalogueService(Configuration); RServiceResult musicCatalogueRes = await catalogueService.ImportFromMySql(jobProgressServiceEF, job); if (!musicCatalogueRes.Result) return; try { using (MySqlConnection connection = new MySqlConnection ( $"server={Configuration.GetSection("AudioMySqlServer")["Server"]};uid={Configuration.GetSection("AudioMySqlServer")["Username"]};pwd={Configuration.GetSection("AudioMySqlServer")["Password"]};database={Configuration.GetSection("AudioMySqlServer")["Database"]};charset=utf8;convert zero datetime=True" )) { connection.Open(); using (MySqlDataAdapter src = new MySqlDataAdapter( "SELECT ID, post_author, post_date, post_date_gmt, post_content, post_title, post_category, post_excerpt, post_status, comment_status, ping_status, post_password, post_name, to_ping, pinged, post_modified, post_modified_gmt, post_content_filtered, post_parent, guid, menu_order, post_type, post_mime_type, comment_count, " + "COALESCE((SELECT meta_value FROM ganja_postmeta WHERE post_id = ID AND meta_key='_wp_page_template'), '') AS template," + "(SELECT meta_value FROM ganja_postmeta WHERE post_id = ID AND meta_key='otherpoetid') AS other_poet_id " + "FROM ganja_posts", connection)) { using (DataTable srcData = new DataTable()) { job = (await jobProgressServiceEF.UpdateJob(job.Id, 0, "phase 1 - mysql 1")).Result; await src.FillAsync(srcData); job = (await jobProgressServiceEF.UpdateJob(job.Id, 0, "phase 1 - processing mysql data")).Result; foreach (DataRow row in srcData.Rows) { GanjoorPageType pageType = row["post_type"].ToString() == "post" && row["comment_status"].ToString() != "closed" ? GanjoorPageType.PoemPage : row["template"].ToString() == "comspage.php" ? GanjoorPageType.AllComments : row["template"].ToString() == "relations.php" ? GanjoorPageType.ProsodySimilars : row["template"].ToString() == "vazn.php" ? GanjoorPageType.ProsodyAndStats : GanjoorPageType.None; int? poetId = row["post_author"].ToString() == "1" ? (int?)null : int.Parse(row["post_author"].ToString()); if (poetId == 36)//رشحه { continue; } if (poetId != null) { if (!await context.GanjoorPoets.Where(poet => poet.Id == poetId).AnyAsync()) { continue; } } GanjoorPage page = new GanjoorPage() { Id = int.Parse(row["ID"].ToString()), GanjoorPageType = pageType, Published = true, PageOrder = -1, Title = row["post_title"].ToString(), UrlSlug = row["post_name"].ToString(), HtmlText = row["post_content"].ToString(), ParentId = row["post_parent"].ToString() == "0" ? (int?)null : int.Parse(row["post_parent"].ToString()), PoetId = poetId, SecondPoetId = row["other_poet_id"] == DBNull.Value ? (int?)null : int.Parse(row["other_poet_id"].ToString()), PostDate = (DateTime)row["post_date"] }; if (pageType == GanjoorPageType.PoemPage) { var poem = await context.GanjoorPoems.Where(p => p.Id == page.Id).FirstOrDefaultAsync(); if (poem == null) continue; page.PoemId = poem.Id; } if (poetId != null && pageType == GanjoorPageType.None) { GanjoorCat cat = await context.GanjoorCategories.Where(c => c.PoetId == poetId && c.ParentId == null && c.UrlSlug == page.UrlSlug).SingleOrDefaultAsync(); if (cat != null) { page.GanjoorPageType = GanjoorPageType.PoetPage; page.CatId = cat.Id; } else { cat = await context.GanjoorCategories.Where(c => c.PoetId == poetId && c.ParentId != null && c.UrlSlug == page.UrlSlug).SingleOrDefaultAsync(); if (cat != null) { page.GanjoorPageType = GanjoorPageType.CatPage; page.CatId = cat.Id; } } } context.GanjoorPages.Add(page); } } } } job = (await jobProgressServiceEF.UpdateJob(job.Id, 0, "phase 1 - finalizing")).Result; await context.SaveChangesAsync(); job = (await jobProgressServiceEF.UpdateJob(job.Id, 0, "phase 2 - pre fetch data")).Result; var orphanPages = await context.GanjoorPages.Include(p => p.Poem).Where(p => p.FullUrl == null).ToListAsync(); job = (await jobProgressServiceEF.UpdateJob(job.Id, 0, "phase 2 - post fetch data")).Result; double count = orphanPages.Count; int i = 0; foreach (var page in orphanPages) { job = (await jobProgressServiceEF.UpdateJob(job.Id, i++, "phase 2")).Result; string fullUrl = page.UrlSlug; string fullTitle = page.Title; if (page.GanjoorPageType == GanjoorPageType.PoemPage) { fullTitle = page.Poem.FullTitle; fullUrl = page.Poem.FullUrl; } else { if (page.ParentId != null) { GanjoorPage parent = await context.GanjoorPages.Where(p => p.Id == page.ParentId).SingleAsync(); while (parent != null) { fullUrl = parent.UrlSlug + "/" + fullUrl; fullTitle = parent.Title + " » " + fullTitle; parent = parent.ParentId == null ? null : await context.GanjoorPages.Where(p => p.Id == parent.ParentId).SingleAsync(); } } else { GanjoorCat cat = await context.GanjoorCategories.Where(c => c.PoetId == page.PoetId && c.UrlSlug == page.UrlSlug).SingleOrDefaultAsync(); if (cat != null) { fullUrl = cat.FullUrl; while (cat.ParentId != null) { cat = await context.GanjoorCategories.Where(c => c.Id == cat.ParentId).SingleOrDefaultAsync(); if (cat != null) { fullTitle = cat.Title + " » " + fullTitle; } } } else { cat = await context.GanjoorCategories.Where(c => c.PoetId == page.PoetId && c.ParentId == null).SingleOrDefaultAsync(); if(cat != null) { fullUrl = $"{cat.UrlSlug}/{page.UrlSlug}"; } } } } if (!string.IsNullOrEmpty(fullUrl) && fullUrl.IndexOf('/') != 0) fullUrl = $"/{fullUrl}"; page.FullUrl = fullUrl; page.FullTitle = fullTitle; context.Update(page); } job = (await jobProgressServiceEF.UpdateJob(job.Id, job.Progress, "phase 2 - finalizing")).Result; await context.SaveChangesAsync(); job = (await jobProgressServiceEF.UpdateJob(job.Id, 0, "phase 3 - pre mysql data fetch")).Result; using (MySqlConnection connection = new MySqlConnection ( $"server={Configuration.GetSection("AudioMySqlServer")["Server"]};uid={Configuration.GetSection("AudioMySqlServer")["Username"]};pwd={Configuration.GetSection("AudioMySqlServer")["Password"]};database={Configuration.GetSection("AudioMySqlServer")["Database"]};charset=utf8;convert zero datetime=True" )) { connection.Open(); using (MySqlDataAdapter src = new MySqlDataAdapter( "SELECT meta_key, post_id, meta_value FROM ganja_postmeta WHERE meta_key IN ( 'vazn', 'ravi', 'src', 'srcslug' )", connection)) { job = (await jobProgressServiceEF.UpdateJob(job.Id, 0, "phase 3 - mysql 2")).Result; using (DataTable srcData = new DataTable()) { await src.FillAsync(srcData); job = (await jobProgressServiceEF.UpdateJob(job.Id, 0, "phase 3 - processing meta data")).Result; int r = 0; foreach(DataRow row in srcData.Rows) { job = (await jobProgressServiceEF.UpdateJob(job.Id, r++, "phase 3 - processing meta data")).Result; int poemId = int.Parse(row["post_id"].ToString()); var poem = await context.GanjoorPoems.Where(p => p.Id == poemId).FirstOrDefaultAsync(); if (poem == null) continue; string metaKey = row["meta_key"].ToString(); string metaValue = row["meta_value"].ToString(); switch (metaKey) { case "vazn": { GanjoorMetre metre = await context.GanjoorMetres.Where(m => m.Rhythm == metaValue).SingleOrDefaultAsync(); if (metre == null) { metre = new GanjoorMetre() { Rhythm = metaValue, VerseCount = 0 }; context.GanjoorMetres.Add(metre); await context.SaveChangesAsync(); } poem.GanjoorMetreId = metre.Id; } break; case "ravi": poem.RhymeLetters = metaValue; break; case "src": poem.SourceName = metaValue; break; case "srcslug": poem.SourceUrlSlug = metaValue; break; } context.GanjoorPoems.Update(poem); } } } } job = (await jobProgressServiceEF.UpdateJob(job.Id, 0, "phase 3 - finalizing meta data")).Result; await context.SaveChangesAsync(); var resApprovedPoemSongs = await _ImportPoemSongsDataFromMySql(context, jobProgressServiceEF, job, true); if(!resApprovedPoemSongs.Result) { await jobProgressServiceEF.UpdateJob(job.Id, job.Progress, "", false, resApprovedPoemSongs.ExceptionString); } var resPendingPoemSongs = await _ImportPoemSongsDataFromMySql(context, jobProgressServiceEF, job, false); if (!resPendingPoemSongs.Result) { await jobProgressServiceEF.UpdateJob(job.Id, job.Progress, "", false, resPendingPoemSongs.ExceptionString); } await jobProgressServiceEF.UpdateJob(job.Id, 100, "Finished", true); } catch(Exception jobExp) { await jobProgressServiceEF.UpdateJob(job.Id, job.Progress, "", false, jobExp.ToString()); } } }); return new RServiceResult(true); } catch(Exception exp) { return new RServiceResult(false, exp.ToString()); } } private async Task> _ImportPoemSongsDataFromMySql(RMuseumDbContext context, LongRunningJobProgressServiceEF jobProgressServiceEF, RLongRunningJobStatus job, bool approved) { try { job = (await jobProgressServiceEF.UpdateJob(job.Id, 0, "phase 5 - pre mysql data fetch")).Result; string connectionString = approved ? $"server={Configuration.GetSection("AudioMySqlServer")["Server"]};uid={Configuration.GetSection("AudioMySqlServer")["Username"]};pwd={Configuration.GetSection("AudioMySqlServer")["Password"]};database={Configuration.GetSection("AudioMySqlServer")["Database"]};charset=utf8;convert zero datetime=True" : $"server={Configuration.GetSection("AudioMySqlServer")["Server"]};uid={Configuration.GetSection("AudioMySqlServer")["SongsUsername"]};pwd={Configuration.GetSection("AudioMySqlServer")["SongsPassword"]};database={Configuration.GetSection("AudioMySqlServer")["SongsDatabase"]};charset=utf8;convert zero datetime=True"; using (MySqlConnection connection = new MySqlConnection ( connectionString )) { connection.Open(); using (MySqlDataAdapter src = new MySqlDataAdapter( "SELECT poem_id, artist_name, artist_beeptunesurl, album_name, album_beeptunesurl, track_name, track_beeptunesurl, ptrack_typeid FROM ganja_ptracks ORDER BY id", connection)) { job = (await jobProgressServiceEF.UpdateJob(job.Id, 0, "phase 5 - mysql")).Result; using (DataTable data = new DataTable()) { await src.FillAsync(data); job = (await jobProgressServiceEF.UpdateJob(job.Id, 0, "phase 5 - processing approved poem songs")).Result; foreach (DataRow row in data.Rows) { PoemMusicTrack track = new PoemMusicTrack() { TrackType = (PoemMusicTrackType)int.Parse(row["ptrack_typeid"].ToString()), PoemId = int.Parse(row["poem_id"].ToString()), ArtistName = row["artist_name"].ToString(), ArtistUrl = row["artist_beeptunesurl"].ToString(), AlbumName = row["album_name"].ToString(), AlbumUrl = row["album_beeptunesurl"].ToString(), TrackName = row["track_name"].ToString(), TrackUrl = row["track_beeptunesurl"].ToString(), ApprovalDate = DateTime.Now, Description = "", Approved = approved }; var poem = await context.GanjoorPoems.Where(p => p.Id == track.PoemId).SingleOrDefaultAsync(); if (poem == null) continue; switch(track.TrackType) { case PoemMusicTrackType.BeepTunesOrKhosousi: case PoemMusicTrackType.iTunes: { GanjoorTrack catalogueTrack = await context.GanjoorMusicCatalogueTracks.Where(m => m.Url == track.TrackUrl).FirstOrDefaultAsync(); if(catalogueTrack != null) { track.GanjoorTrackId = catalogueTrack.Id; } GanjoorSinger singer = await context.GanjoorSingers.Where(s => s.Url == track.ArtistUrl).FirstOrDefaultAsync(); if(singer != null) { track.SingerId = singer.Id; } } break; case PoemMusicTrackType.Golha: { track.AlbumName = $"{track.ArtistName} » {track.AlbumName}"; track.ArtistName = ""; track.GolhaTrackId = int.Parse(track.ArtistUrl); track.ArtistUrl = ""; } break; } context.GanjoorPoemMusicTracks.Add(track); } job = (await jobProgressServiceEF.UpdateJob(job.Id, 0, "phase 5 - finalizing approved poem songs data")).Result; await context.SaveChangesAsync(); } } } return new RServiceResult(true); } catch(Exception exp) { return new RServiceResult(false, exp.ToString()); } } private List _extractVersesFromPoemHtmlText(string poemtext) { List verses = new List(); //this spagetti code has been imported from my internal utilities: while (poemtext.IndexOf("", ahrefStart) + 1, poemtext.IndexOf("") - (poemtext.IndexOf(">", ahrefStart) + 1)); poemtext = part1 + part2 + poemtext.Substring(poemtext.IndexOf("") + 4, poemtext.Length - (poemtext.IndexOf("") + 4)); } while (poemtext.IndexOf("", acroStart) + 1, poemtext.IndexOf("") - (poemtext.IndexOf(">", acroStart) + 1)); poemtext = part1 + part2 + poemtext.Substring(poemtext.IndexOf("") + 10, poemtext.Length - (poemtext.IndexOf("") + 10)); } catch { part2 = poemtext.Substring(poemtext.IndexOf(">", acroStart) + 1, poemtext.IndexOf("") - (poemtext.IndexOf(">", acroStart) + 1)); poemtext = part1 + part2 + poemtext.Substring(poemtext.IndexOf("") + 10, poemtext.Length - (poemtext.IndexOf("") + 10)); } } while (poemtext.IndexOf("") != -1) { string part1 = poemtext.Substring(0, poemtext.IndexOf("")); try { poemtext = part1 + poemtext.Substring(poemtext.IndexOf("") + 6, poemtext.Length - (poemtext.IndexOf("") + 6)); poemtext = poemtext.Replace(" ", " "); } catch { throw new Exception($"poemtext.IndexOf(\"\": {poemtext}"); } } poemtext = poemtext.Replace("Adaptation du milieu", "یییییییییییییییییییی"); poemtext = poemtext.Replace("Empirique", "ببببببببب"); poemtext = poemtext.Replace("
", "
").Replace("
", "
").Replace("
", "
").Replace("
", "
").Replace("
", "
").Replace("

", "

").Replace("

", "

").Replace("

", "").Replace(" ", "").Replace("
", "").Replace("
", "
"); poemtext = poemtext.Replace("", "").Replace("", ""); poemtext = poemtext.Replace("", "").Replace("", "").Replace("", "").Replace("", ""); poemtext = poemtext.Replace("", "").Replace("", "").Replace("", "").Replace("", ""); poemtext = poemtext.Replace("


", "").Replace("
", ""); if (poemtext.IndexOf("\r\n") == 0) poemtext = poemtext.Substring(2); poemtext = poemtext.Replace("\r", "").Replace("\n", ""); poemtext = poemtext.Replace("
", "").Replace("

", ""); poemtext = poemtext.Replace("
", "a"); poemtext = poemtext.Replace("
", "b"); poemtext = poemtext.Replace("
", "m"); poemtext = poemtext.Replace("
", "n"); poemtext = poemtext.Replace("
", ""); poemtext = poemtext.Replace("

", "p"); poemtext = poemtext.Replace("bmp", "b"); poemtext = poemtext.Replace("np", "n"); poemtext = poemtext.Replace("ap", "a"); poemtext = poemtext.Replace("\"", "").Replace("'", ""); if (poemtext.IndexOfAny(new char[] { '<', '>' }) != -1) throw new Exception($"Invalid Characteres: {poemtext}"); if (poemtext.IndexOf("mp") != -1) throw new Exception($"مصرع اول بدون مصرع دوم: {poemtext}"); if (poemtext.Length > 0) { int idx = poemtext.IndexOfAny(new char[] { 'a', 'b', 'm', 'n', 'p' }); bool preWasBand = false; while (idx != -1) { GanjoorVerse verse = new GanjoorVerse(); verse.VOrder = verses.Count + 1; switch (poemtext[idx]) { case 'p': if (preWasBand) verse.VersePosition = VersePosition.CenteredVerse2; else verse.VersePosition = VersePosition.Paragraph; preWasBand = false; break; case 'b': verse.VersePosition = VersePosition.Right; preWasBand = false; break; case 'n': verse.VersePosition = VersePosition.Left; preWasBand = false; break; case 'a': verse.VersePosition = VersePosition.CenteredVerse1; preWasBand = true; break; } int nextIdx = poemtext.IndexOfAny(new char[] { 'a', 'b', 'm', 'n', 'p' }, idx + 1); if (nextIdx == -1) { verse.Text = poemtext.Substring(idx + 1).Replace("یییییییییییییییییییی", "Adaptation du milieu").Replace("ببببببببب", "Empirique"); } else { verse.Text = poemtext.Substring(idx + 1, nextIdx - idx - 1).Replace("یییییییییییییییییییی", "Adaptation du milieu").Replace("ببببببببب", "Empirique"); } verses.Add(verse); idx = nextIdx; } } return verses; } #endregion ///

/// Database Contetxt /// protected readonly RMuseumDbContext _context; /// /// Configuration /// protected IConfiguration Configuration { get; } /// /// Background Task Queue Instance /// protected readonly IBackgroundTaskQueue _backgroundTaskQueue; /// /// constructor /// /// /// /// public GanjoorService(RMuseumDbContext context, IConfiguration configuration, IBackgroundTaskQueue backgroundTaskQueue) { _context = context; _backgroundTaskQueue = backgroundTaskQueue; Configuration = configuration; } } }