diff --git a/RMuseum/RMuseum.xml b/RMuseum/RMuseum.xml index ad9af9e2..cd6f1309 100644 --- a/RMuseum/RMuseum.xml +++ b/RMuseum/RMuseum.xml @@ -6625,6 +6625,15 @@ IGanjoorService implementation + + IGanjoorService implementation + + + + + import GanjoorPage entity data from MySql + + @@ -6638,12 +6647,6 @@ - - - import GanjoorPage entity data from MySql - - - Get List of poets diff --git a/RMuseum/Services/Implementation/GanjoorService-Imports/GanjoorService-MySqlImports.cs b/RMuseum/Services/Implementation/GanjoorService-Imports/GanjoorService-MySqlImports.cs index b954329c..da5feebd 100644 --- a/RMuseum/Services/Implementation/GanjoorService-Imports/GanjoorService-MySqlImports.cs +++ b/RMuseum/Services/Implementation/GanjoorService-Imports/GanjoorService-MySqlImports.cs @@ -1,6 +1,4 @@ -using Dapper; -using Microsoft.Data.Sqlite; -using Microsoft.EntityFrameworkCore; +using Microsoft.EntityFrameworkCore; using MySql.Data.MySqlClient; using RMuseum.DbContext; using RMuseum.Models.Ganjoor; @@ -11,7 +9,6 @@ using RSecurityBackend.Services.Implementation; using System; using System.Collections.Generic; using System.Data; -using System.Data.Common; using System.Linq; using System.Threading.Tasks; @@ -22,227 +19,6 @@ namespace RMuseum.Services.Implementation /// public partial class GanjoorService : IGanjoorService { - #region Date import / modifications - /// - /// imports unimported poem data from a locally accessible ganjoor SqlLite database - /// - /// - public async Task> ImportLocalSQLiteDb() - { - try - { - SqliteConnectionStringBuilder connectionStringBuilder = new SqliteConnectionStringBuilder(); - connectionStringBuilder.DataSource = Configuration.GetSection("LocalSqliteImport")["FilePath"]; - using (SqliteConnection sqliteConnection = new SqliteConnection(connectionStringBuilder.ToString())) - { - await sqliteConnection.OpenAsync(); - IDbConnection dapper = sqliteConnection; - using (var sqlConnection = _context.Database.GetDbConnection()) - { - await sqlConnection.OpenAsync(); - - foreach (var poet in await dapper.QueryAsync("SELECT * FROM poet ORDER BY id")) - { - int poetId = (int)poet.id; - if ((await _context.GanjoorPoets.Where(p => p.Id == poetId).FirstOrDefaultAsync()) != null) - { - continue; - } - - DbTransaction transaction = await sqlConnection.BeginTransactionAsync(); - try - { - using (var command = sqlConnection.CreateCommand()) - { - command.Transaction = transaction; - command.CommandText = - $"INSERT INTO GanjoorPoets (Id, Name, Description) VALUES (${poet.id}, N'{poet.name}', N'{poet.description}')"; - await command.ExecuteNonQueryAsync(); - await _ImportSQLiteCatChildren(command, dapper, poetId, 0, "", ""); - await transaction.CommitAsync(); - } - } - catch (Exception exp2) - { - await transaction.RollbackAsync(); - return new RServiceResult(false, exp2.ToString()); - } - - - } - } - - } - } - catch (Exception exp) - { - return new RServiceResult(false, exp.ToString()); - } - return new RServiceResult(true); - } - private async Task _ImportSQLiteCatChildren(DbCommand command, IDbConnection dapper, int poetId, int catId, string itemFullTitle, string itemFullSlug) - { - - foreach (var cat in await dapper.QueryAsync($"SELECT * FROM cat WHERE poet_id = {poetId} AND parent_id = {catId} ORDER BY id")) - { - - if (catId == 0) - { - command.CommandText = - $"INSERT INTO GanjoorCategories (Id, PoetId, Title, UrlSlug, FullUrl) VALUES (${cat.id}, {poetId}, N'{cat.text}', '{ _ExtractUrlSlug(cat.url)}', '{itemFullSlug}/{ _ExtractUrlSlug(cat.url)}')"; - } - else - { - command.CommandText = - $"INSERT INTO GanjoorCategories (Id, PoetId, Title, ParentId, UrlSlug, FullUrl) VALUES (${cat.id}, {poetId}, N'{cat.text}', {catId}, '{ _ExtractUrlSlug(cat.url)}', '{itemFullSlug}/{ _ExtractUrlSlug(cat.url)}')"; - } - - await command.ExecuteNonQueryAsync(); - - - foreach (var poem in await dapper.QueryAsync($"SELECT * FROM poem WHERE cat_id = {cat.id} ORDER BY id")) - { - command.CommandText = - $"INSERT INTO GanjoorPoems (Id, CatId, Title, UrlSlug, FullTitle, FullUrl) VALUES (${poem.id}, {cat.id}, N'{poem.title}', '{ _ExtractUrlSlug(poem.url)}', N'{$"{itemFullTitle}{cat.text} » {poem.title}"}', '{itemFullSlug}/{ _ExtractUrlSlug(cat.url)}/{_ExtractUrlSlug(poem.url)}')"; - await command.ExecuteNonQueryAsync(); - - foreach (var verse in await dapper.QueryAsync($"SELECT * FROM verse WHERE poem_id = {poem.id} ORDER BY vorder")) - { - command.CommandText = - $"INSERT INTO GanjoorVerses (PoemId, VOrder, VersePosition, Text) VALUES (${poem.id}, {verse.vorder}, {verse.position}, N'{verse.text}')"; - await command.ExecuteNonQueryAsync(); - } - - } - - - await _ImportSQLiteCatChildren(command, dapper, poetId, (int)cat.id, $"{itemFullTitle}{cat.text} » ", $"{itemFullSlug}/{ _ExtractUrlSlug(cat.url)}"); - - - } - } - private static string _ExtractUrlSlug(string slug) - { - if (!string.IsNullOrEmpty(slug)) - { - //sample: https://ganjoor.net/hafez - if (slug.LastIndexOf('/') != -1) - { - // => result = hafez - slug = slug.Substring(slug.LastIndexOf('/') + 1); - } - } - return slug; - } - - /// - /// updates poems text - /// - /// - public RServiceResult UpdatePoemsText() - { - try - { - _backgroundTaskQueue.QueueBackgroundWorkItem - ( - async token => - { - using (RMuseumDbContext context = new RMuseumDbContext(Configuration)) //this is long running job, so _context might be already been freed/collected by GC - { - var poems = await context.GanjoorPoems.Where(p => p.PlainText == null).ToListAsync(); - int n = 0; - foreach (GanjoorPoem poem in poems) - { - var verses = await context.GanjoorVerses.Where(v => v.PoemId == poem.Id).OrderBy(v => v.VOrder).ToListAsync(); - string plainText = ""; - string htmlText = ""; - for (int vIndex = 0; vIndex < verses.Count; vIndex++) - { - GanjoorVerse v = verses[vIndex]; - - if (string.IsNullOrEmpty(plainText)) - plainText = v.Text; - else - { - plainText += $" {v.Text}"; - } - - switch (v.VersePosition) - { - case VersePosition.CenteredVerse1: - if (((vIndex + 1) < verses.Count) && (verses[vIndex + 1].VersePosition == VersePosition.CenteredVerse2)) - { - htmlText += $"

{v.Text.Replace("ـ", "").Trim()}

{Environment.NewLine}"; //div is not closed - } - else - { - htmlText += $"

{v.Text.Replace("ـ", "").Trim()}

{Environment.NewLine}"; - } - break; - case VersePosition.CenteredVerse2: - htmlText += $"

{v.Text.Replace("ـ", "").Trim()}

{Environment.NewLine}"; - break; - case VersePosition.Right: - htmlText += $"

{v.Text.Replace("ـ", "").Trim()}

{Environment.NewLine}"; - break; - case VersePosition.Left: - htmlText += $"

{v.Text.Replace("ـ", "").Trim()}

{Environment.NewLine}"; - break; - case VersePosition.Paragraph: - case VersePosition.Single: - { - string[] lines = v.Text.Replace("ـ", "").Split(new char[] { '\r', '\n' }, StringSplitOptions.RemoveEmptyEntries); - - if (lines.Length != 0) - { - if (v.Text.Replace("ـ", "").Length / lines.Length < 150) - { - htmlText += $"

{v.Text.Replace("ـ", "").Replace("\r\n", " ")}

{Environment.NewLine}"; - } - else - { - foreach (string line in lines) - htmlText += $"

{line}

{Environment.NewLine}"; - } - } - } - break; - } - } - - poem.PlainText = plainText; - poem.HtmlText = htmlText; - - context.Update(poem); - - n++; - - if (n >= 1000) - { - await context.SaveChangesAsync(); - n = 0; - } - } - - if (n > 0) - await context.SaveChangesAsync(); - - - } - - - }); - - - return new RServiceResult(true); - } - catch (Exception exp) - { - return new RServiceResult(false, exp.ToString()); - } - - } - /// /// import GanjoorPage entity data from MySql @@ -815,6 +591,5 @@ namespace RMuseum.Services.Implementation return verses; } - #endregion } } diff --git a/RMuseum/Services/Implementation/GanjoorService-Imports/GanjoorService-SQLiteImport.cs b/RMuseum/Services/Implementation/GanjoorService-Imports/GanjoorService-SQLiteImport.cs new file mode 100644 index 00000000..2270c57b --- /dev/null +++ b/RMuseum/Services/Implementation/GanjoorService-Imports/GanjoorService-SQLiteImport.cs @@ -0,0 +1,241 @@ +using Dapper; +using Microsoft.Data.Sqlite; +using Microsoft.EntityFrameworkCore; +using RMuseum.DbContext; +using RMuseum.Models.Ganjoor; +using RSecurityBackend.Models.Generic; +using System; +using System.Data; +using System.Data.Common; +using System.Linq; +using System.Threading.Tasks; + +namespace RMuseum.Services.Implementation +{ + /// + /// IGanjoorService implementation + /// + public partial class GanjoorService : IGanjoorService + { + /// + /// imports unimported poem data from a locally accessible ganjoor SqlLite database + /// + /// + public async Task> ImportLocalSQLiteDb() + { + try + { + SqliteConnectionStringBuilder connectionStringBuilder = new SqliteConnectionStringBuilder(); + connectionStringBuilder.DataSource = Configuration.GetSection("LocalSqliteImport")["FilePath"]; + using (SqliteConnection sqliteConnection = new SqliteConnection(connectionStringBuilder.ToString())) + { + await sqliteConnection.OpenAsync(); + IDbConnection dapper = sqliteConnection; + using (var sqlConnection = _context.Database.GetDbConnection()) + { + await sqlConnection.OpenAsync(); + + foreach (var poet in await dapper.QueryAsync("SELECT * FROM poet ORDER BY id")) + { + int poetId = (int)poet.id; + if ((await _context.GanjoorPoets.Where(p => p.Id == poetId).FirstOrDefaultAsync()) != null) + { + continue; + } + + DbTransaction transaction = await sqlConnection.BeginTransactionAsync(); + try + { + using (var command = sqlConnection.CreateCommand()) + { + command.Transaction = transaction; + command.CommandText = + $"INSERT INTO GanjoorPoets (Id, Name, Description) VALUES (${poet.id}, N'{poet.name}', N'{poet.description}')"; + await command.ExecuteNonQueryAsync(); + await _ImportSQLiteCatChildren(command, dapper, poetId, 0, "", ""); + await transaction.CommitAsync(); + } + } + catch (Exception exp2) + { + await transaction.RollbackAsync(); + return new RServiceResult(false, exp2.ToString()); + } + + + } + } + + } + } + catch (Exception exp) + { + return new RServiceResult(false, exp.ToString()); + } + return new RServiceResult(true); + } + private async Task _ImportSQLiteCatChildren(DbCommand command, IDbConnection dapper, int poetId, int catId, string itemFullTitle, string itemFullSlug) + { + + foreach (var cat in await dapper.QueryAsync($"SELECT * FROM cat WHERE poet_id = {poetId} AND parent_id = {catId} ORDER BY id")) + { + + if (catId == 0) + { + command.CommandText = + $"INSERT INTO GanjoorCategories (Id, PoetId, Title, UrlSlug, FullUrl) VALUES (${cat.id}, {poetId}, N'{cat.text}', '{ _ExtractUrlSlug(cat.url)}', '{itemFullSlug}/{ _ExtractUrlSlug(cat.url)}')"; + } + else + { + command.CommandText = + $"INSERT INTO GanjoorCategories (Id, PoetId, Title, ParentId, UrlSlug, FullUrl) VALUES (${cat.id}, {poetId}, N'{cat.text}', {catId}, '{ _ExtractUrlSlug(cat.url)}', '{itemFullSlug}/{ _ExtractUrlSlug(cat.url)}')"; + } + + await command.ExecuteNonQueryAsync(); + + + foreach (var poem in await dapper.QueryAsync($"SELECT * FROM poem WHERE cat_id = {cat.id} ORDER BY id")) + { + command.CommandText = + $"INSERT INTO GanjoorPoems (Id, CatId, Title, UrlSlug, FullTitle, FullUrl) VALUES (${poem.id}, {cat.id}, N'{poem.title}', '{ _ExtractUrlSlug(poem.url)}', N'{$"{itemFullTitle}{cat.text} » {poem.title}"}', '{itemFullSlug}/{ _ExtractUrlSlug(cat.url)}/{_ExtractUrlSlug(poem.url)}')"; + await command.ExecuteNonQueryAsync(); + + foreach (var verse in await dapper.QueryAsync($"SELECT * FROM verse WHERE poem_id = {poem.id} ORDER BY vorder")) + { + command.CommandText = + $"INSERT INTO GanjoorVerses (PoemId, VOrder, VersePosition, Text) VALUES (${poem.id}, {verse.vorder}, {verse.position}, N'{verse.text}')"; + await command.ExecuteNonQueryAsync(); + } + + } + + + await _ImportSQLiteCatChildren(command, dapper, poetId, (int)cat.id, $"{itemFullTitle}{cat.text} » ", $"{itemFullSlug}/{ _ExtractUrlSlug(cat.url)}"); + + + } + } + private static string _ExtractUrlSlug(string slug) + { + if (!string.IsNullOrEmpty(slug)) + { + //sample: https://ganjoor.net/hafez + if (slug.LastIndexOf('/') != -1) + { + // => result = hafez + slug = slug.Substring(slug.LastIndexOf('/') + 1); + } + } + return slug; + } + + /// + /// updates poems text + /// + /// + public RServiceResult UpdatePoemsText() + { + try + { + _backgroundTaskQueue.QueueBackgroundWorkItem + ( + async token => + { + using (RMuseumDbContext context = new RMuseumDbContext(Configuration)) //this is long running job, so _context might be already been freed/collected by GC + { + var poems = await context.GanjoorPoems.Where(p => p.PlainText == null).ToListAsync(); + int n = 0; + foreach (GanjoorPoem poem in poems) + { + var verses = await context.GanjoorVerses.Where(v => v.PoemId == poem.Id).OrderBy(v => v.VOrder).ToListAsync(); + string plainText = ""; + string htmlText = ""; + for (int vIndex = 0; vIndex < verses.Count; vIndex++) + { + GanjoorVerse v = verses[vIndex]; + + if (string.IsNullOrEmpty(plainText)) + plainText = v.Text; + else + { + plainText += $" {v.Text}"; + } + + switch (v.VersePosition) + { + case VersePosition.CenteredVerse1: + if (((vIndex + 1) < verses.Count) && (verses[vIndex + 1].VersePosition == VersePosition.CenteredVerse2)) + { + htmlText += $"

{v.Text.Replace("ـ", "").Trim()}

{Environment.NewLine}"; //div is not closed + } + else + { + htmlText += $"

{v.Text.Replace("ـ", "").Trim()}

{Environment.NewLine}"; + } + break; + case VersePosition.CenteredVerse2: + htmlText += $"

{v.Text.Replace("ـ", "").Trim()}

{Environment.NewLine}"; + break; + case VersePosition.Right: + htmlText += $"

{v.Text.Replace("ـ", "").Trim()}

{Environment.NewLine}"; + break; + case VersePosition.Left: + htmlText += $"

{v.Text.Replace("ـ", "").Trim()}

{Environment.NewLine}"; + break; + case VersePosition.Paragraph: + case VersePosition.Single: + { + string[] lines = v.Text.Replace("ـ", "").Split(new char[] { '\r', '\n' }, StringSplitOptions.RemoveEmptyEntries); + + if (lines.Length != 0) + { + if (v.Text.Replace("ـ", "").Length / lines.Length < 150) + { + htmlText += $"

{v.Text.Replace("ـ", "").Replace("\r\n", " ")}

{Environment.NewLine}"; + } + else + { + foreach (string line in lines) + htmlText += $"

{line}

{Environment.NewLine}"; + } + } + } + break; + } + } + + poem.PlainText = plainText; + poem.HtmlText = htmlText; + + context.Update(poem); + + n++; + + if (n >= 1000) + { + await context.SaveChangesAsync(); + n = 0; + } + } + + if (n > 0) + await context.SaveChangesAsync(); + + + } + + + }); + + + return new RServiceResult(true); + } + catch (Exception exp) + { + return new RServiceResult(false, exp.ToString()); + } + + } + + } +}