anas/divan
1
0
Fork 0
Code Issues 12 Pull Requests Actions Packages Projects Releases Wiki Activity

DiscoverPoetNaskbanPaperSources

Browse Source
This commit is contained in:
Hamid Reza Mohammadi 2024-07-26 11:38:45 +03:30
parent feb150920a
commit 28c2d36c7c
4 changed files with 184 additions and 7 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

22
RMuseum/Controllers/GanjoorController.cs
View File

@ -3743,6 +3743,28 @@ namespace RMuseum.Controllers
return Ok();
}
/// <summary>
/// discover poet naskban paper sources
/// </summary>
/// <param name="poetId"></param>
/// <param name="loginViewModel"></param>
/// <returns></returns>
[HttpPut("naskban/import/poetbooks/{poetId}")]
[Authorize(Policy = RMuseumSecurableItem.GanjoorEntityShortName + ":" + RMuseumSecurableItem.ModerateOperationShortName)]
[ProducesResponseType((int)HttpStatusCode.OK)]
[ProducesResponseType((int)HttpStatusCode.BadRequest, Type = typeof(string))]
public IActionResult DiscoverPoetNaskbanPaperSources(
int poetId,
[AuditIgnore]
[FromBody]
LoginViewModel loginViewModel
)
{
_ganjoorService.DiscoverPoetNaskbanPaperSources(poetId, loginViewModel.Username, loginViewModel.Password);
return Ok();
}
/// <summary>
/// delete poem related naskban images by url
/// </summary>

24
RMuseum/RMuseum.xml
View File

@ -2046,6 +2046,14 @@
<param name="loginViewModel"></param>
<returns></returns>
</member>
<member name="M:RMuseum.Controllers.GanjoorController.DiscoverPoetNaskbanPaperSources(System.Int32,RSecurityBackend.Models.Auth.ViewModels.LoginViewModel)">
<summary>
discover poet naskban paper sources
</summary>
<param name="poetId"></param>
<param name="loginViewModel"></param>
<returns></returns>
</member>
<member name="M:RMuseum.Controllers.GanjoorController.DeletePoemRelatedNaskbanImagesByNaskbanUrlAsync(System.String)">
<summary>
delete poem related naskban images by url
@ -16616,6 +16624,14 @@
import paper sources from museum
</summary>
</member>
<member name="M:RMuseum.Services.IGanjoorService.DiscoverPoetNaskbanPaperSources(System.Int32,System.String,System.String)">
<summary>
discover poet naskban paper sources
</summary>
<param name="poetid"></param>
<param name="naskbanUserName"></param>
<param name="naskbanPassword"></param>
</member>
<member name="M:RMuseum.Services.IGanjoorService.GetCategoryPaperSourcesAsync(System.Int32)">
<summary>
category paper sources
@ -18790,6 +18806,14 @@
<param name="categoryId"></param>
<returns></returns>
</member>
<member name="M:RMuseum.Services.Implementation.GanjoorService.DiscoverPoetNaskbanPaperSources(System.Int32,System.String,System.String)">
<summary>
discover poet naskban paper sources
</summary>
<param name="poetid"></param>
<param name="naskbanUserName"></param>
<param name="naskbanPassword"></param>
</member>
<member name="M:RMuseum.Services.Implementation.GanjoorService.ImportPaperSourcesFromMuseum">
<summary>
import paper sources from museum

9
RMuseum/Services/IGanjoorService.cs
View File

@ -1241,6 +1241,14 @@ namespace RMuseum.Services
/// </summary>
void ImportPaperSourcesFromMuseum();
/// <summary>
/// discover poet naskban paper sources
/// </summary>
/// <param name="poetid"></param>
/// <param name="naskbanUserName"></param>
/// <param name="naskbanPassword"></param>
void DiscoverPoetNaskbanPaperSources(int poetid, string naskbanUserName, string naskbanPassword);
/// <summary>
/// category paper sources
/// </summary>
@ -1248,6 +1256,7 @@ namespace RMuseum.Services
/// <returns></returns>
Task<RServiceResult<GanjoorPaperSource[]>> GetCategoryPaperSourcesAsync(int categoryId);
/// <summary>
/// mark naskban links as human reviewed
/// </summary>

136
RMuseum/Services/Implementation/GanjoorService-Partials/GanjoorService-PaperSources.cs
View File

@ -9,6 +9,15 @@ using RMuseum.DbContext;
using RSecurityBackend.Services.Implementation;
using RSecurityBackend.Models.Generic.Db;
using System.Collections.Generic;
using Newtonsoft.Json;
using RMuseum.Models.PDFLibrary;
using System.Net.Http;
using System.Net;
using System.Reflection.PortableExecutable;
using RMuseum.Models.Auth.ViewModel;
using RSecurityBackend.Models.Auth.ViewModels;
using System.Text;
using System.Net.Http.Headers;
namespace RMuseum.Services.Implementation
@ -38,6 +47,119 @@ namespace RMuseum.Services.Implementation
}
}
/// <summary>
/// discover poet naskban paper sources
/// </summary>
/// <param name="poetid"></param>
/// <param name="naskbanUserName"></param>
/// <param name="naskbanPassword"></param>
public void DiscoverPoetNaskbanPaperSources(int poetid, string naskbanUserName, string naskbanPassword)
{
_backgroundTaskQueue.QueueBackgroundWorkItem
(
async token =>
{
using (RMuseumDbContext context = new RMuseumDbContext(new DbContextOptions<RMuseumDbContext>())) //this is long running job, so _context might be already been freed/collected by GC
{
LongRunningJobProgressServiceEF jobProgressServiceEF = new LongRunningJobProgressServiceEF(context);
var job = (await jobProgressServiceEF.NewJob("DiscoverCategoryPaperSources", "Query data")).Result;
try
{
var cat = await context.GanjoorCategories.AsNoTracking().Where(c => c.PoetId == poetid && c.ParentId == null).SingleAsync();
List<int> catIdList = new List<int>
{
cat.Id
};
await _populateCategoryChildren(context, cat.Id, catIdList);
LoggedOnUserModelEx loggedOnUser;
using (HttpClient client = new HttpClient())
{
LoginViewModel loginViewModel = new LoginViewModel()
{
Username = naskbanUserName,
Password = naskbanPassword,
ClientAppName = "Ganjoor API",
Language = "fa-IR"
};
var loginResponse = await client.PostAsync("https://api.naskban.ir/api/users/login", new StringContent(JsonConvert.SerializeObject(loginViewModel), Encoding.UTF8, "application/json"));
if (loginResponse.StatusCode != HttpStatusCode.OK)
{
await jobProgressServiceEF.UpdateJob(job.Id, 100, "", false, "login error: " + JsonConvert.DeserializeObject<string>(await loginResponse.Content.ReadAsStringAsync()));
return;
}
loggedOnUser = JsonConvert.DeserializeObject<LoggedOnUserModelEx>(await loginResponse.Content.ReadAsStringAsync());
}
using (HttpClient secureClient = new HttpClient())
{
secureClient.DefaultRequestHeaders.Authorization = new AuthenticationHeaderValue("Bearer", loggedOnUser.Token);
var unsyncedResponse = await secureClient.GetAsync("https://api.naskban.ir/api/pdf/ganjoor/matching?notStarted=false&notFinished=false");
if (!unsyncedResponse.IsSuccessStatusCode)
{
await jobProgressServiceEF.UpdateJob(job.Id, 100, "", false, "unsync error: " + JsonConvert.DeserializeObject<string>(await unsyncedResponse.Content.ReadAsStringAsync()));
return;
}
foreach (int subCatId in catIdList)
{
await jobProgressServiceEF.UpdateJob(job.Id, subCatId);
var poems = await context.GanjoorPoems.AsNoTracking().Where(p => p.CatId == subCatId).ToListAsync();
foreach (var poem in poems)
{
var sources = await context.PinterestLinks.Where(l => l.GanjoorPostId == poem.Id && l.NaskbanLinkId != null).ToListAsync();
foreach (var matching in sources)
{
if (false == await context.GanjoorPaperSources.Where(p => p.NaskbanBookId == matching.PDFBookId && p.GanjoorCatId == cat.Id).AnyAsync())
{
HttpResponseMessage responseBook = await secureClient.GetAsync($"https://api.naskban.ir/api/pdf/{matching.PDFBookId}?includePages=false&includeBookText=false&includePageText=false");
if (responseBook.StatusCode != HttpStatusCode.OK)
{
await jobProgressServiceEF.UpdateJob(job.Id, 1000, "", false, $"book fetch error bookid = {matching.PDFBookId} matching id = {matching.Id} - " + JsonConvert.DeserializeObject<string>(await responseBook.Content.ReadAsStringAsync()));
return;
}
responseBook.EnsureSuccessStatusCode();
var book = JsonConvert.DeserializeObject<PDFBook>(await responseBook.Content.ReadAsStringAsync());
GanjoorPaperSource paperSource = new GanjoorPaperSource()
{
GanjoorPoetId = cat.PoetId,
GanjoorCatId = cat.Id,
GanjoorCatFullTitle = cat.Title,
GanjoorCatFullUrl = cat.FullUrl,
BookType = LinkType.Naskban,
BookFullUrl = $"https://naskban.ir/{matching.PDFBookId}",
NaskbanBookId = matching.PDFBookId,
BookFullTitle = book.Title,
CoverThumbnailImageUrl = book.ExtenalCoverImageUrl,
Description = "",
IsTextOriginalSource = matching.IsTextOriginalSource,
MatchPercent = matching.MatchPercent,
HumanReviewed = matching.HumanReviewed,
OrderIndicator = 0,
};
context.GanjoorPaperSources.Add(paperSource);
await context.SaveChangesAsync();
}
}
}
}
}
await jobProgressServiceEF.UpdateJob(job.Id, 100, succeeded: true);
}
catch (Exception exp)
{
await jobProgressServiceEF.UpdateJob(job.Id, 100, "", false, exp.ToString());
}
}
}
);
}
/// <summary>
/// import paper sources from museum
/// </summary>
@ -52,7 +174,7 @@ namespace RMuseum.Services.Implementation
LongRunningJobProgressServiceEF jobProgressServiceEF = new LongRunningJobProgressServiceEF(context);
var job = (await jobProgressServiceEF.NewJob("ImportPaperSourcesFromMuseum", "Query data")).Result;
await _ImportPaperSourcesFromMuseumAsync(context, jobProgressServiceEF, job);
}
});
}
@ -72,10 +194,10 @@ namespace RMuseum.Services.Implementation
foreach (var link in links)
{
var poem = await context.GanjoorPoems.AsNoTracking().Where(p => p.Id == link.GanjoorPostId).SingleOrDefaultAsync();
if(poem == null) continue;
if (poem == null) continue;
var cat = await context.GanjoorCategories.AsNoTracking().Where(c => c.Id == poem.CatId).SingleOrDefaultAsync();
if(cat == null) continue;
if(poets.ContainsKey(cat.PoetId))
if (cat == null) continue;
if (poets.ContainsKey(cat.PoetId))
{
poets[cat.PoetId]++;
}
@ -85,18 +207,18 @@ namespace RMuseum.Services.Implementation
}
}
if(poets.Count > 0)
if (poets.Count > 0)
{
var mainPoet = poets.First();
foreach (var poet in poets)
{
if(poet.Value > mainPoet.Value)
if (poet.Value > mainPoet.Value)
{
mainPoet = poet;
}
}
if(mainPoet.Value > 3)
if (mainPoet.Value > 3)
{
var poet = await context.GanjoorPoets.AsNoTracking().Where(p => p.Id == mainPoet.Key).SingleAsync();
var cat = await context.GanjoorCategories.AsNoTracking().Where(c => c.PoetId == poet.Id & c.ParentId == null).SingleAsync();