From 305bcad7cdec111c8dd5b87f943c5ddb05d12d94 Mon Sep 17 00:00:00 2001 From: Hamid Reza Mohammadi Date: Sun, 27 Mar 2022 18:07:01 +0430 Subject: [PATCH] docs: #302 added an API for finding duplicates --- RMuseum/Controllers/GanjoorController.cs | 18 ++++ RMuseum/DbContext/RMuseumDbContext.cs | 5 + RMuseum/RMuseum.xml | 29 +++++ RMuseum/Services/IGanjoorService.cs | 8 ++ .../GanjoorService-DuplicateFinder.cs | 100 ++++++++++++++++-- 5 files changed, 149 insertions(+), 11 deletions(-) diff --git a/RMuseum/Controllers/GanjoorController.cs b/RMuseum/Controllers/GanjoorController.cs index 864f1d61..f7040552 100644 --- a/RMuseum/Controllers/GanjoorController.cs +++ b/RMuseum/Controllers/GanjoorController.cs @@ -2635,6 +2635,24 @@ namespace RMuseum.Controllers return Ok(); } + /// + /// Finding Category Poems Duplicates + /// + /// + /// + /// + [HttpPost("duplicates/{srcCatId}/{destCatId}")] + [Authorize(Policy = RMuseumSecurableItem.GanjoorEntityShortName + ":" + SecurableItem.ModifyOperationShortName)] + [ProducesResponseType((int)HttpStatusCode.OK)] + [ProducesResponseType((int)HttpStatusCode.BadRequest, Type = typeof(string))] + public IActionResult StartFindingCategoryPoemsDuplicates(int srcCatId, int destCatId) + { + var res = _ganjoorService.StartFindingCategoryPoemsDuplicates(srcCatId, destCatId); + if (!string.IsNullOrEmpty(res.ExceptionString)) + return BadRequest(res.ExceptionString); + return Ok(); + } + /// /// readonly mode /// diff --git a/RMuseum/DbContext/RMuseumDbContext.cs b/RMuseum/DbContext/RMuseumDbContext.cs index 02da06a5..6b3d308e 100644 --- a/RMuseum/DbContext/RMuseumDbContext.cs +++ b/RMuseum/DbContext/RMuseumDbContext.cs @@ -452,5 +452,10 @@ namespace RMuseum.DbContext /// public DbSet FAQItems { get; set; } + /// + /// Temporary Model contianing duplicated poems information + /// + public DbSet GanjoorDuplicates { get; set; } + } } diff --git a/RMuseum/RMuseum.xml b/RMuseum/RMuseum.xml index 76e08a59..c18db9d3 100644 --- a/RMuseum/RMuseum.xml +++ b/RMuseum/RMuseum.xml @@ -1633,6 +1633,14 @@ + + + Finding Category Poems Duplicates + + + + + readonly mode @@ -2799,6 +2807,11 @@ faq items + + + Temporary Model contianing duplicated poems information + + donation expenditures @@ -11331,6 +11344,14 @@ + + + Finding Category Poems Duplicates + + + + + translation service implementation @@ -12772,6 +12793,14 @@ + + + Finding Category Poems Duplicates + + + + + find category poem rhymes diff --git a/RMuseum/Services/IGanjoorService.cs b/RMuseum/Services/IGanjoorService.cs index ae5f59cd..f805562d 100644 --- a/RMuseum/Services/IGanjoorService.cs +++ b/RMuseum/Services/IGanjoorService.cs @@ -854,5 +854,13 @@ namespace RMuseum.Services /// /// RServiceResult StartRegeneratingTOCs(Guid userId); + + /// + /// Finding Category Poems Duplicates + /// + /// + /// + /// + RServiceResult StartFindingCategoryPoemsDuplicates(int srcCatId, int destCatId); } } diff --git a/RMuseum/Services/Implementation/GanjoorService-Partials/GanjoorService-DuplicateFinder.cs b/RMuseum/Services/Implementation/GanjoorService-Partials/GanjoorService-DuplicateFinder.cs index 66e5cf2c..d2ee7fef 100644 --- a/RMuseum/Services/Implementation/GanjoorService-Partials/GanjoorService-DuplicateFinder.cs +++ b/RMuseum/Services/Implementation/GanjoorService-Partials/GanjoorService-DuplicateFinder.cs @@ -1,15 +1,10 @@ using Microsoft.EntityFrameworkCore; using RMuseum.DbContext; using RMuseum.Models.Ganjoor; -using RMuseum.Models.Ganjoor.ViewModels; -using RMuseum.Services.Implementation.ImportedFromDesktopGanjoor; using RSecurityBackend.Models.Generic; -using RSecurityBackend.Models.Generic.Db; using RSecurityBackend.Services.Implementation; using System; -using System.Collections.Generic; using System.Data; -using System.Globalization; using System.Linq; using System.Threading.Tasks; @@ -20,12 +15,99 @@ namespace RMuseum.Services.Implementation /// public partial class GanjoorService : IGanjoorService { - private async Task> _FindCateryPoemsDuplicates(RMuseumDbContext context, int srcCatId, int destCatId) + /// + /// Finding Category Poems Duplicates + /// + /// + /// + /// + public RServiceResult StartFindingCategoryPoemsDuplicates(int srcCatId, int destCatId) { try { - var poems = await context.GanjoorPoems.AsNoTracking().Where(p => p.CatId == srcCatId).ToListAsync(); + _backgroundTaskQueue.QueueBackgroundWorkItem + ( + async token => + { + using (RMuseumDbContext context = new RMuseumDbContext(new DbContextOptions())) //this is long running job, so _context might be already been freed/collected by GC + { + LongRunningJobProgressServiceEF jobProgressServiceEF = new LongRunningJobProgressServiceEF(context); + var job = (await jobProgressServiceEF.NewJob("StartFindingCategoryPoemsDuplicates", "Query data")).Result; + try + { + await _FindCategoryPoemsDuplicates(context, srcCatId, destCatId); + await jobProgressServiceEF.UpdateJob(job.Id, 100, "", true); + } + catch (Exception exp) + { + await jobProgressServiceEF.UpdateJob(job.Id, 100, "", false, exp.ToString()); + } + + } + }); + return new RServiceResult(true); + } + catch (Exception exp) + { + return new RServiceResult(false, exp.ToString()); + } + } + private async Task> _FindCategoryPoemsDuplicates(RMuseumDbContext context, int srcCatId, int destCatId) + { + try + { + if (srcCatId == destCatId) + return new RServiceResult(false, "srcCatId == destCatId"); + var poems = await context.GanjoorPoems.AsNoTracking().Where(p => p.CatId == srcCatId).ToListAsync(); + var targetPoems = await context.GanjoorPoems.AsNoTracking().Where(p => p.CatId == destCatId).ToListAsync(); + var alreayFoundOnes = await context.GanjoorDuplicates.AsNoTracking().Where(p => p.SrcCatId == srcCatId).ToListAsync(); + foreach (var poem in poems) + { + var firstVerse = await context.GanjoorVerses.AsNoTracking().Where(p => p.PoemId == poem.Id).OrderBy(v => v.VOrder).FirstAsync(); + if (alreayFoundOnes.Any(p => p.SrcPoemId == poem.Id)) + continue; + var probablyTheOnes = targetPoems.Where(p => p.GanjoorMetreId == poem.GanjoorMetreId && p.RhymeLetters == poem.RhymeLetters).ToList(); + bool found = false; + foreach (var probable in probablyTheOnes) + { + var meVerse = await context.GanjoorVerses.AsNoTracking().Where(p => p.PoemId == probable.Id).OrderBy(v => v.VOrder).FirstAsync(); + if(_AreSimilar(firstVerse.Text, meVerse.Text, false)) + { + context.GanjoorDuplicates.Add + ( + new GanjoorDuplicate() + { + SrcCatId = srcCatId, + SrcPoemId = poem.Id, + DestPoemId = probable.Id + } + ); + await context.SaveChangesAsync(); + found = true; + break; + } + } + if (found) continue; + foreach (var probable in targetPoems) + { + var meVerse = await context.GanjoorVerses.AsNoTracking().Where(p => p.PoemId == probable.Id).OrderBy(v => v.VOrder).FirstAsync(); + if (_AreSimilar(firstVerse.Text, meVerse.Text, false)) + { + context.GanjoorDuplicates.Add + ( + new GanjoorDuplicate() + { + SrcCatId = srcCatId, + SrcPoemId = poem.Id, + DestPoemId = probable.Id + } + ); + await context.SaveChangesAsync(); + break; + } + } + } return new RServiceResult(true); } catch (Exception exp) @@ -38,9 +120,7 @@ namespace RMuseum.Services.Implementation { if (string.IsNullOrEmpty(str1) || string.IsNullOrEmpty(str2)) return false; - string[] words2 = str2.Split(new char[] { ' ' }, StringSplitOptions.RemoveEmptyEntries); - int total = words2.Length; int found = 0; for (int i = 0; i < total; i++) @@ -48,10 +128,8 @@ namespace RMuseum.Services.Implementation if (str1.IndexOf(words2[i]) != -1) found++; } - if (!reverse) return (float)found / total > 0.7f; - return (float)found / total > 0.7f && _AreSimilar(str2, str1, false); } }