From e95052666e9f800c678458010b67fab197563e7d Mon Sep 17 00:00:00 2001 From: Hamid Reza Mohammadi Date: Wed, 6 Jan 2021 20:35:01 +0330 Subject: [PATCH] #59 changing mysqlimport method to a long running job (incomplete) --- RMuseum/Controllers/GanjoorController.cs | 4 +- RMuseum/Services/IGanjoorService.cs | 2 +- .../Services/Implementation/GanjoorService.cs | 259 +++++++++--------- .../Generic/Db/RLongRunningJobStatus.cs | 49 ++++ RSecurityBackend/RSecurityBackend.xml | 45 +++ 5 files changed, 231 insertions(+), 128 deletions(-) create mode 100644 RSecurityBackend/Models/Generic/Db/RLongRunningJobStatus.cs diff --git a/RMuseum/Controllers/GanjoorController.cs b/RMuseum/Controllers/GanjoorController.cs index 6acee3b7..f1cb798c 100644 --- a/RMuseum/Controllers/GanjoorController.cs +++ b/RMuseum/Controllers/GanjoorController.cs @@ -312,10 +312,10 @@ namespace RMuseum.Controllers [Authorize(Policy = RMuseumSecurableItem.GanjoorEntityShortName + ":" + RMuseumSecurableItem.ImportOperationShortName)] [ProducesResponseType((int)HttpStatusCode.OK, Type = typeof(bool))] [ProducesResponseType((int)HttpStatusCode.BadRequest, Type = typeof(string))] - public async Task ImportFromMySql() + public IActionResult ImportFromMySql() { RServiceResult res = - await _ganjoorService.ImportFromMySql(); + _ganjoorService.ImportFromMySql(); if (res.Result) return Ok(); return BadRequest(res.ExceptionString); diff --git a/RMuseum/Services/IGanjoorService.cs b/RMuseum/Services/IGanjoorService.cs index 061358be..a347fa27 100644 --- a/RMuseum/Services/IGanjoorService.cs +++ b/RMuseum/Services/IGanjoorService.cs @@ -127,6 +127,6 @@ namespace RMuseum.Services /// import GanjoorPage entity data from MySql /// /// - Task> ImportFromMySql(); + RServiceResult ImportFromMySql(); } } diff --git a/RMuseum/Services/Implementation/GanjoorService.cs b/RMuseum/Services/Implementation/GanjoorService.cs index ce7c55c8..6a208227 100644 --- a/RMuseum/Services/Implementation/GanjoorService.cs +++ b/RMuseum/Services/Implementation/GanjoorService.cs @@ -935,156 +935,165 @@ namespace RMuseum.Services.Implementation /// import GanjoorPage entity data from MySql /// /// - public async Task> ImportFromMySql() + public RServiceResult ImportFromMySql() { try { - using (MySqlConnection connection = new MySqlConnection - ( - $"server={Configuration.GetSection("AudioMySqlServer")["Server"]};uid={Configuration.GetSection("AudioMySqlServer")["Username"]};pwd={Configuration.GetSection("AudioMySqlServer")["Password"]};database={Configuration.GetSection("AudioMySqlServer")["Database"]};charset=utf8;convert zero datetime=True" - )) + _backgroundTaskQueue.QueueBackgroundWorkItem + ( + async token => { - connection.Open(); - using(MySqlDataAdapter src = new MySqlDataAdapter( - "SELECT ID, post_author, post_date, post_date_gmt, post_content, post_title, post_category, post_excerpt, post_status, comment_status, ping_status, post_password, post_name, to_ping, pinged, post_modified, post_modified_gmt, post_content_filtered, post_parent, guid, menu_order, post_type, post_mime_type, comment_count, " + - "COALESCE((SELECT meta_value FROM ganja_postmeta WHERE post_id = ID AND meta_key='_wp_page_template'), '') AS template," + - "(SELECT meta_value FROM ganja_postmeta WHERE post_id = ID AND meta_key='otherpoetid') AS other_poet_id " + - "FROM ganja_posts", - connection)) + using (RMuseumDbContext context = new RMuseumDbContext(Configuration)) //this is long running job, so _context might be already been freed/collected by GC { - using (DataTable srcData = new DataTable()) + using (MySqlConnection connection = new MySqlConnection + ( + $"server={Configuration.GetSection("AudioMySqlServer")["Server"]};uid={Configuration.GetSection("AudioMySqlServer")["Username"]};pwd={Configuration.GetSection("AudioMySqlServer")["Password"]};database={Configuration.GetSection("AudioMySqlServer")["Database"]};charset=utf8;convert zero datetime=True" + )) { - await src.FillAsync(srcData); - foreach (DataRow row in srcData.Rows) + connection.Open(); + using (MySqlDataAdapter src = new MySqlDataAdapter( + "SELECT ID, post_author, post_date, post_date_gmt, post_content, post_title, post_category, post_excerpt, post_status, comment_status, ping_status, post_password, post_name, to_ping, pinged, post_modified, post_modified_gmt, post_content_filtered, post_parent, guid, menu_order, post_type, post_mime_type, comment_count, " + + "COALESCE((SELECT meta_value FROM ganja_postmeta WHERE post_id = ID AND meta_key='_wp_page_template'), '') AS template," + + "(SELECT meta_value FROM ganja_postmeta WHERE post_id = ID AND meta_key='otherpoetid') AS other_poet_id " + + "FROM ganja_posts", + connection)) { - GanjoorPageType pageType = - row["post_type"].ToString() == "post" && row["comment_status"].ToString() != "closed" ? - GanjoorPageType.PoemPage - : - row["template"].ToString() == "comspage.php" ? - GanjoorPageType.AllComments - : - row["template"].ToString() == "relations.php" ? - GanjoorPageType.ProsodySimilars - : - row["template"].ToString() == "vazn.php" ? - GanjoorPageType.ProsodyAndStats - : - GanjoorPageType.None; - - int? poetId = row["post_author"].ToString() == "0" ? (int?)null : int.Parse(row["post_author"].ToString()); - if(poetId == 36)//رشحه + using (DataTable srcData = new DataTable()) { - continue; - } - - if(poetId != null) - { - if(!await _context.GanjoorPoets.Where(poet => poet.Id == poetId).AnyAsync()) + await src.FillAsync(srcData); + foreach (DataRow row in srcData.Rows) { - continue; + GanjoorPageType pageType = + row["post_type"].ToString() == "post" && row["comment_status"].ToString() != "closed" ? + GanjoorPageType.PoemPage + : + row["template"].ToString() == "comspage.php" ? + GanjoorPageType.AllComments + : + row["template"].ToString() == "relations.php" ? + GanjoorPageType.ProsodySimilars + : + row["template"].ToString() == "vazn.php" ? + GanjoorPageType.ProsodyAndStats + : + GanjoorPageType.None; + + int? poetId = row["post_author"].ToString() == "0" ? (int?)null : int.Parse(row["post_author"].ToString()); + if (poetId == 36)//رشحه + { + continue; + } + + if (poetId != null) + { + if (!await context.GanjoorPoets.Where(poet => poet.Id == poetId).AnyAsync()) + { + continue; + } + } + + GanjoorPage page = new GanjoorPage() + { + Id = int.Parse(row["ID"].ToString()), + GanjoorPageType = pageType, + Published = true, + PageOrder = -1, + Title = row["post_title"].ToString(), + UrlSlug = row["post_name"].ToString(), + HtmlText = row["post_content"].ToString(), + ParentId = row["post_parent"].ToString() == "0" ? (int?)null : int.Parse(row["post_parent"].ToString()), + PoetId = row["post_author"].ToString() == "1" ? (int?)null : int.Parse(row["post_author"].ToString()), + SecondPoetId = row["other_poet_id"] == DBNull.Value ? (int?)null : int.Parse(row["other_poet_id"].ToString()), + PostDate = (DateTime)row["post_date"] + }; + + + + if (pageType == GanjoorPageType.PoemPage) + { + var poem = await context.GanjoorPoems.Where(p => p.Id == page.Id).FirstOrDefaultAsync(); + if (poem == null) + continue; + page.PoemId = poem.Id; + } + if (poetId != null && pageType == GanjoorPageType.None) + { + GanjoorCat cat = await context.GanjoorCategories.Where(c => c.PoetId == poetId && c.ParentId == null && c.UrlSlug == page.UrlSlug).SingleOrDefaultAsync(); + if (cat != null) + { + page.GanjoorPageType = GanjoorPageType.PoetPage; + page.CatId = cat.Id; + } + else + { + cat = await context.GanjoorCategories.Where(c => c.PoetId == poetId && c.ParentId != null && c.UrlSlug == page.UrlSlug).SingleOrDefaultAsync(); + if (cat != null) + { + page.GanjoorPageType = GanjoorPageType.CatPage; + page.CatId = cat.Id; + } + } + } + + context.GanjoorPages.Add(page); } } + } + } + await context.SaveChangesAsync(); - GanjoorPage page = new GanjoorPage() + + var orphanPages = await context.GanjoorPages.Include(p => p.Poem).Where(p => p.FullUrl == null).ToListAsync(); + foreach (var page in orphanPages) + { + string fullUrl = page.UrlSlug; + string fullTitle = page.Title; + + if (page.GanjoorPageType == GanjoorPageType.PoemPage) + { + fullTitle = page.Poem.FullTitle; + fullUrl = page.Poem.FullUrl; + } + else + { + if (page.ParentId != null) { - Id = int.Parse(row["ID"].ToString()), - GanjoorPageType = pageType, - Published = true, - PageOrder = -1, - Title = row["post_title"].ToString(), - UrlSlug = row["post_name"].ToString(), - HtmlText = row["post_content"].ToString(), - ParentId = row["post_parent"].ToString() == "0" ? (int?)null : int.Parse(row["post_parent"].ToString()), - PoetId = row["post_author"].ToString() == "1" ? (int?)null : int.Parse(row["post_author"].ToString()), - SecondPoetId = row["other_poet_id"] == DBNull.Value ? (int?)null : int.Parse(row["other_poet_id"].ToString()), - PostDate = (DateTime)row["post_date"] - }; - - - - if(pageType == GanjoorPageType.PoemPage) - { - var poem = await _context.GanjoorPoems.Where(p => p.Id == page.Id).FirstOrDefaultAsync(); - if (poem == null) - continue; - page.PoemId = poem.Id; + GanjoorPage parent = await context.GanjoorPages.Where(p => p.Id == page.ParentId).SingleAsync(); + while (parent != null) + { + fullUrl = parent.UrlSlug + "/" + fullUrl; + fullTitle = parent.Title + " » " + fullTitle; + parent = parent.ParentId == null ? null : await context.GanjoorPages.Where(p => p.Id == parent.ParentId).SingleAsync(); + } } - if(poetId != null && pageType == GanjoorPageType.None) + else { - GanjoorCat cat = await _context.GanjoorCategories.Where(c => c.PoetId == poetId && c.ParentId == null && c.UrlSlug == page.UrlSlug).SingleOrDefaultAsync(); + GanjoorCat cat = await context.GanjoorCategories.Where(c => c.PoetId == page.PoetId && c.UrlSlug == page.UrlSlug).SingleOrDefaultAsync(); if (cat != null) { - page.GanjoorPageType = GanjoorPageType.PoetPage; - page.CatId = cat.Id; - } - else - { - cat = await _context.GanjoorCategories.Where(c => c.PoetId == poetId && c.ParentId != null && c.UrlSlug == page.UrlSlug).SingleOrDefaultAsync(); - if (cat != null) + fullUrl = cat.FullUrl; + while (cat.ParentId != null) { - page.GanjoorPageType = GanjoorPageType.CatPage; - page.CatId = cat.Id; + cat = await context.GanjoorCategories.Where(c => c.Id == cat.Id).SingleOrDefaultAsync(); + if (cat != null) + { + fullTitle = cat.Title + " » " + fullTitle; + } } } } - _context.GanjoorPages.Add(page); } + page.FullUrl = fullUrl; + page.FullTitle = fullTitle; + + context.Update(page); } + + await context.SaveChangesAsync(); } - } - await _context.SaveChangesAsync(); - - - var orphanPages = await _context.GanjoorPages.Include(p => p.Poem).Where(p => p.FullUrl == null).ToListAsync(); - foreach(var page in orphanPages) - { - string fullUrl = page.UrlSlug; - string fullTitle = page.Title; - - if(page.GanjoorPageType == GanjoorPageType.PoemPage) - { - fullTitle = page.Poem.FullTitle; - fullUrl = page.Poem.FullUrl; - } - else - { - if(page.ParentId != null) - { - GanjoorPage parent = await _context.GanjoorPages.Where(p => p.Id == page.ParentId).SingleAsync(); - while (parent != null) - { - fullUrl = parent.UrlSlug + "/" + fullUrl; - fullTitle = parent.Title + " » " + fullTitle; - parent = parent.ParentId == null ? null : await _context.GanjoorPages.Where(p => p.Id == parent.ParentId).SingleAsync(); - } - } - else - { - GanjoorCat cat = await _context.GanjoorCategories.Where(c => c.PoetId == page.PoetId && c.UrlSlug == page.UrlSlug).SingleOrDefaultAsync(); - if(cat != null) - { - fullUrl = cat.FullUrl; - while(cat.ParentId != null) - { - cat = await _context.GanjoorCategories.Where(c => c.Id == cat.Id).SingleOrDefaultAsync(); - if(cat != null) - { - fullTitle = cat.Title + " » " + fullTitle; - } - } - } - } - - } - page.FullUrl = fullUrl; - page.FullTitle = fullTitle; - - _context.Update(page); - } - - await _context.SaveChangesAsync(); + }); + return new RServiceResult(true); } diff --git a/RSecurityBackend/Models/Generic/Db/RLongRunningJobStatus.cs b/RSecurityBackend/Models/Generic/Db/RLongRunningJobStatus.cs new file mode 100644 index 00000000..5cfbda72 --- /dev/null +++ b/RSecurityBackend/Models/Generic/Db/RLongRunningJobStatus.cs @@ -0,0 +1,49 @@ +using System; + +namespace RSecurityBackend.Models.Generic.Db +{ + /// + /// Long Running Job Status + /// + public class RLongRunningJobStatus + { + /// + /// Id + /// + public Guid Id { get; set; } + /// + /// Name + /// + public string Name { get; set; } + + /// + /// progress value (percent of custom value) + /// + public double Progress { get; set; } + + /// + /// Start Time + /// + public DateTime StartTime { get; set; } + + /// + /// End Time + /// + public DateTime EndTime { get; set; } + + /// + /// Current Step + /// + public string Step { get; set; } + + /// + /// Exception + /// + public string Exception { get; set; } + + /// + /// finished + /// + public bool Finished { get; set; } + } +} diff --git a/RSecurityBackend/RSecurityBackend.xml b/RSecurityBackend/RSecurityBackend.xml index 7f2aaf52..84d287b6 100644 --- a/RSecurityBackend/RSecurityBackend.xml +++ b/RSecurityBackend/RSecurityBackend.xml @@ -1629,6 +1629,51 @@ Amount + + + Long Running Job Status + + + + + Id + + + + + Name + + + + + progress value (percent of custom value) + + + + + Start Time + + + + + End Time + + + + + Current Step + + + + + Exception + + + + + finished + + pagination metadat