anas/divan
1
0
Fork 0
Code Issues 12 Pull Requests Actions Packages Projects Releases Wiki Activity

docs: #302 added an API for finding duplicates

Browse Source
This commit is contained in:
Hamid Reza Mohammadi 2022-03-27 18:07:01 +04:30
parent 28be6aa9c8
commit 305bcad7cd
5 changed files with 149 additions and 11 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

18
RMuseum/Controllers/GanjoorController.cs
View File

@ -2635,6 +2635,24 @@ namespace RMuseum.Controllers
return Ok();
}
/// <summary>
/// Finding Category Poems Duplicates
/// </summary>
/// <param name="srcCatId"></param>
/// <param name="destCatId"></param>
/// <returns></returns>
[HttpPost("duplicates/{srcCatId}/{destCatId}")]
[Authorize(Policy = RMuseumSecurableItem.GanjoorEntityShortName + ":" + SecurableItem.ModifyOperationShortName)]
[ProducesResponseType((int)HttpStatusCode.OK)]
[ProducesResponseType((int)HttpStatusCode.BadRequest, Type = typeof(string))]
public IActionResult StartFindingCategoryPoemsDuplicates(int srcCatId, int destCatId)
{
var res = _ganjoorService.StartFindingCategoryPoemsDuplicates(srcCatId, destCatId);
if (!string.IsNullOrEmpty(res.ExceptionString))
return BadRequest(res.ExceptionString);
return Ok();
}
/// <summary>
/// readonly mode
/// </summary>

5
RMuseum/DbContext/RMuseumDbContext.cs
View File

@ -452,5 +452,10 @@ namespace RMuseum.DbContext
/// </summary>
public DbSet<FAQItem> FAQItems { get; set; }
/// <summary>
/// Temporary Model contianing duplicated poems information
/// </summary>
public DbSet<GanjoorDuplicate> GanjoorDuplicates { get; set; }
}
}

29
RMuseum/RMuseum.xml
View File

@ -1633,6 +1633,14 @@
<param name="id"></param>
<returns></returns>
</member>
<member name="M:RMuseum.Controllers.GanjoorController.StartFindingCategoryPoemsDuplicates(System.Int32,System.Int32)">
<summary>
Finding Category Poems Duplicates
</summary>
<param name="srcCatId"></param>
<param name="destCatId"></param>
<returns></returns>
</member>
<member name="P:RMuseum.Controllers.GanjoorController.ReadOnlyMode">
<summary>
readonly mode
@ -2799,6 +2807,11 @@
faq items
</summary>
</member>
<member name="P:RMuseum.DbContext.RMuseumDbContext.GanjoorDuplicates">
<summary>
Temporary Model contianing duplicated poems information
</summary>
</member>
<member name="T:RMuseum.Models.Accounting.DonationExpenditure">
<summary>
donation expenditures
@ -11331,6 +11344,14 @@
<param name="userId"></param>
<returns></returns>
</member>
<member name="M:RMuseum.Services.IGanjoorService.StartFindingCategoryPoemsDuplicates(System.Int32,System.Int32)">
<summary>
Finding Category Poems Duplicates
</summary>
<param name="srcCatId"></param>
<param name="destCatId"></param>
<returns></returns>
</member>
<member name="T:RMuseum.Services.IGanjoorTranslationService">
<summary>
translation service implementation
@ -12772,6 +12793,14 @@
</summary>
<returns></returns>
</member>
<member name="M:RMuseum.Services.Implementation.GanjoorService.StartFindingCategoryPoemsDuplicates(System.Int32,System.Int32)">
<summary>
Finding Category Poems Duplicates
</summary>
<param name="srcCatId"></param>
<param name="destCatId"></param>
<returns></returns>
</member>
<member name="M:RMuseum.Services.Implementation.GanjoorService.FindCategoryPoemsRhythms(System.Int32,System.Boolean,System.String)">
<summary>
find category poem rhymes

8
RMuseum/Services/IGanjoorService.cs
View File

@ -854,5 +854,13 @@ namespace RMuseum.Services
/// <param name="userId"></param>
/// <returns></returns>
RServiceResult<bool> StartRegeneratingTOCs(Guid userId);
/// <summary>
/// Finding Category Poems Duplicates
/// </summary>
/// <param name="srcCatId"></param>
/// <param name="destCatId"></param>
/// <returns></returns>
RServiceResult<bool> StartFindingCategoryPoemsDuplicates(int srcCatId, int destCatId);
}
}

100
RMuseum/Services/Implementation/GanjoorService-Partials/GanjoorService-DuplicateFinder.cs
View File

@ -1,15 +1,10 @@
using Microsoft.EntityFrameworkCore;
using RMuseum.DbContext;
using RMuseum.Models.Ganjoor;
using RMuseum.Models.Ganjoor.ViewModels;
using RMuseum.Services.Implementation.ImportedFromDesktopGanjoor;
using RSecurityBackend.Models.Generic;
using RSecurityBackend.Models.Generic.Db;
using RSecurityBackend.Services.Implementation;
using System;
using System.Collections.Generic;
using System.Data;
using System.Globalization;
using System.Linq;
using System.Threading.Tasks;
@ -20,12 +15,99 @@ namespace RMuseum.Services.Implementation
/// </summary>
public partial class GanjoorService : IGanjoorService
{
private async Task<RServiceResult<bool>> _FindCateryPoemsDuplicates(RMuseumDbContext context, int srcCatId, int destCatId)
/// <summary>
/// Finding Category Poems Duplicates
/// </summary>
/// <param name="srcCatId"></param>
/// <param name="destCatId"></param>
/// <returns></returns>
public RServiceResult<bool> StartFindingCategoryPoemsDuplicates(int srcCatId, int destCatId)
{
try
{
var poems = await context.GanjoorPoems.AsNoTracking().Where(p => p.CatId == srcCatId).ToListAsync();
_backgroundTaskQueue.QueueBackgroundWorkItem
(
async token =>
{
using (RMuseumDbContext context = new RMuseumDbContext(new DbContextOptions<RMuseumDbContext>())) //this is long running job, so _context might be already been freed/collected by GC
{
LongRunningJobProgressServiceEF jobProgressServiceEF = new LongRunningJobProgressServiceEF(context);
var job = (await jobProgressServiceEF.NewJob("StartFindingCategoryPoemsDuplicates", "Query data")).Result;
try
{
await _FindCategoryPoemsDuplicates(context, srcCatId, destCatId);
await jobProgressServiceEF.UpdateJob(job.Id, 100, "", true);
}
catch (Exception exp)
{
await jobProgressServiceEF.UpdateJob(job.Id, 100, "", false, exp.ToString());
}
}
});
return new RServiceResult<bool>(true);
}
catch (Exception exp)
{
return new RServiceResult<bool>(false, exp.ToString());
}
}
private async Task<RServiceResult<bool>> _FindCategoryPoemsDuplicates(RMuseumDbContext context, int srcCatId, int destCatId)
{
try
{
if (srcCatId == destCatId)
return new RServiceResult<bool>(false, "srcCatId == destCatId");
var poems = await context.GanjoorPoems.AsNoTracking().Where(p => p.CatId == srcCatId).ToListAsync();
var targetPoems = await context.GanjoorPoems.AsNoTracking().Where(p => p.CatId == destCatId).ToListAsync();
var alreayFoundOnes = await context.GanjoorDuplicates.AsNoTracking().Where(p => p.SrcCatId == srcCatId).ToListAsync();
foreach (var poem in poems)
{
var firstVerse = await context.GanjoorVerses.AsNoTracking().Where(p => p.PoemId == poem.Id).OrderBy(v => v.VOrder).FirstAsync();
if (alreayFoundOnes.Any(p => p.SrcPoemId == poem.Id))
continue;
var probablyTheOnes = targetPoems.Where(p => p.GanjoorMetreId == poem.GanjoorMetreId && p.RhymeLetters == poem.RhymeLetters).ToList();
bool found = false;
foreach (var probable in probablyTheOnes)
{
var meVerse = await context.GanjoorVerses.AsNoTracking().Where(p => p.PoemId == probable.Id).OrderBy(v => v.VOrder).FirstAsync();
if(_AreSimilar(firstVerse.Text, meVerse.Text, false))
{
context.GanjoorDuplicates.Add
(
new GanjoorDuplicate()
{
SrcCatId = srcCatId,
SrcPoemId = poem.Id,
DestPoemId = probable.Id
}
);
await context.SaveChangesAsync();
found = true;
break;
}
}
if (found) continue;
foreach (var probable in targetPoems)
{
var meVerse = await context.GanjoorVerses.AsNoTracking().Where(p => p.PoemId == probable.Id).OrderBy(v => v.VOrder).FirstAsync();
if (_AreSimilar(firstVerse.Text, meVerse.Text, false))
{
context.GanjoorDuplicates.Add
(
new GanjoorDuplicate()
{
SrcCatId = srcCatId,
SrcPoemId = poem.Id,
DestPoemId = probable.Id
}
);
await context.SaveChangesAsync();
break;
}
}
}
return new RServiceResult<bool>(true);
}
catch (Exception exp)
@ -38,9 +120,7 @@ namespace RMuseum.Services.Implementation
{
if (string.IsNullOrEmpty(str1) || string.IsNullOrEmpty(str2))
return false;
string[] words2 = str2.Split(new char[] { ' ' }, StringSplitOptions.RemoveEmptyEntries);
int total = words2.Length;
int found = 0;
for (int i = 0; i < total; i++)
@ -48,10 +128,8 @@ namespace RMuseum.Services.Implementation
if (str1.IndexOf(words2[i]) != -1)
found++;
}
if (!reverse)
return (float)found / total > 0.7f;
return (float)found / total > 0.7f && _AreSimilar(str2, str1, false);
}
}