using Microsoft.EntityFrameworkCore; using RSecurityBackend.Models.Generic; using System; using System.Data; using System.Linq; using System.Threading.Tasks; using RMuseum.Models.DivanIntegration; using RMuseum.DbContext; using RSecurityBackend.Services.Implementation; using RSecurityBackend.Models.Generic.Db; using System.Collections.Generic; using Newtonsoft.Json; using RMuseum.Models.PDFLibrary; using System.Net.Http; using System.Net; using System.Reflection.PortableExecutable; using RMuseum.Models.Auth.ViewModel; using RSecurityBackend.Models.Auth.ViewModels; using System.Text; using System.Net.Http.Headers; namespace RMuseum.Services.Implementation { /// /// IDivanService implementation /// public partial class DivanService : IDivanService { /// /// category paper sources /// /// /// public async Task> GetCategoryPaperSourcesAsync(int categoryId) { try { return new RServiceResult ( await _context.DivanPaperSources.AsNoTracking().Where(p => p.DivanCatId == categoryId).OrderByDescending(c => c.IsTextOriginalSource).OrderBy(c => c.OrderIndicator).ThenBy(c => c.Id).ToArrayAsync() ); } catch (Exception exp) { return new RServiceResult(null, exp.ToString()); } } /// /// discover poet naskban paper sources /// /// /// /// public void DiscoverPoetNaskbanPaperSources(int poetid, string naskbanUserName, string naskbanPassword) { _backgroundTaskQueue.QueueBackgroundWorkItem ( async token => { using (RMuseumDbContext context = new RMuseumDbContext(new DbContextOptions())) //this is long running job, so _context might be already been freed/collected by GC { LongRunningJobProgressServiceEF jobProgressServiceEF = new LongRunningJobProgressServiceEF(context); var job = (await jobProgressServiceEF.NewJob("DiscoverCategoryPaperSources", "Query data")).Result; try { var cat = await context.DivanCategories.AsNoTracking().Where(c => c.PoetId == poetid && c.ParentId == null).SingleAsync(); List catIdList = new List { cat.Id }; await _populateCategoryChildren(context, cat.Id, catIdList); LoggedOnUserModelEx loggedOnUser; using (HttpClient client = new HttpClient()) { LoginViewModel loginViewModel = new LoginViewModel() { Username = naskbanUserName, Password = naskbanPassword, ClientAppName = "Divan API", Language = "ur-PK" }; var loginResponse = await client.PostAsync("https://api.naskban.ir/api/users/login", new StringContent(JsonConvert.SerializeObject(loginViewModel), Encoding.UTF8, "application/json")); if (loginResponse.StatusCode != HttpStatusCode.OK) { await jobProgressServiceEF.UpdateJob(job.Id, 100, "", false, "login error: " + JsonConvert.DeserializeObject(await loginResponse.Content.ReadAsStringAsync())); return; } loggedOnUser = JsonConvert.DeserializeObject(await loginResponse.Content.ReadAsStringAsync()); } using (HttpClient secureClient = new HttpClient()) { secureClient.DefaultRequestHeaders.Authorization = new AuthenticationHeaderValue("Bearer", loggedOnUser.Token); var unsyncedResponse = await secureClient.GetAsync("https://api.naskban.ir/api/pdf/divan/matching?notStarted=false¬Finished=false"); if (!unsyncedResponse.IsSuccessStatusCode) { await jobProgressServiceEF.UpdateJob(job.Id, 100, "", false, "unsync error: " + JsonConvert.DeserializeObject(await unsyncedResponse.Content.ReadAsStringAsync())); return; } foreach (int subCatId in catIdList) { await jobProgressServiceEF.UpdateJob(job.Id, subCatId); var poems = await context.DivanPoems.AsNoTracking().Where(p => p.CatId == subCatId).ToListAsync(); foreach (var poem in poems) { var sources = await context.PinterestLinks.Where(l => l.DivanPostId == poem.Id && l.NaskbanLinkId != null).ToListAsync(); foreach (var matching in sources) { if (false == await context.DivanPaperSources.Where(p => p.NaskbanBookId == matching.PDFBookId && p.DivanCatId == cat.Id).AnyAsync()) { HttpResponseMessage responseBook = await secureClient.GetAsync($"https://api.naskban.ir/api/pdf/{matching.PDFBookId}?includePages=false&includeBookText=false&includePageText=false"); if (responseBook.StatusCode != HttpStatusCode.OK) { await jobProgressServiceEF.UpdateJob(job.Id, 1000, "", false, $"book fetch error bookid = {matching.PDFBookId} matching id = {matching.Id} - " + JsonConvert.DeserializeObject(await responseBook.Content.ReadAsStringAsync())); return; } responseBook.EnsureSuccessStatusCode(); var book = JsonConvert.DeserializeObject(await responseBook.Content.ReadAsStringAsync()); DivanPaperSource paperSource = new DivanPaperSource() { DivanPoetId = cat.PoetId, DivanCatId = cat.Id, DivanCatFullTitle = cat.Title, DivanCatFullUrl = cat.FullUrl, BookType = LinkType.Naskban, BookFullUrl = $"https://naskban.ir/{matching.PDFBookId}", NaskbanBookId = matching.PDFBookId, BookFullTitle = book.Title, CoverThumbnailImageUrl = book.ExtenalCoverImageUrl, Description = "", IsTextOriginalSource = matching.IsTextOriginalSource, MatchPercent = matching.MatchPercent, HumanReviewed = matching.HumanReviewed, OrderIndicator = 0, }; context.DivanPaperSources.Add(paperSource); await context.SaveChangesAsync(); } } } } } await jobProgressServiceEF.UpdateJob(job.Id, 100, succeeded: true); } catch (Exception exp) { await jobProgressServiceEF.UpdateJob(job.Id, 100, "", false, exp.ToString()); } } } ); } /// /// import paper sources from museum /// /// public void ImportPaperSourcesFromMuseum(int poetid) { _backgroundTaskQueue.QueueBackgroundWorkItem ( async token => { using (RMuseumDbContext context = new RMuseumDbContext(new DbContextOptions())) //this is long running job, so _context might be already been freed/collected by GC { LongRunningJobProgressServiceEF jobProgressServiceEF = new LongRunningJobProgressServiceEF(context); var job = (await jobProgressServiceEF.NewJob("ImportPaperSourcesFromMuseum", "Query data")).Result; await _ImportPaperSourcesFromMuseumAsync(context, jobProgressServiceEF, job, poetid); } }); } private async Task _ImportPaperSourcesFromMuseumAsync(RMuseumDbContext context, LongRunningJobProgressServiceEF jobProgressServiceEF, RLongRunningJobStatus job, int poetid) { try { var artifacts = await context.Artifacts.AsNoTracking().Include(c => c.CoverImage).ToListAsync(); int index = 0; foreach (var artifact in artifacts) { await jobProgressServiceEF.UpdateJob(job.Id, index++, $"{index} از {artifacts.Count} - {artifact.Name}"); var links = await context.DivanLinks.AsNoTracking().Where(l => l.ArtifactId == artifact.Id).ToListAsync(); Dictionary poets = new Dictionary(); foreach (var link in links) { var poem = await context.DivanPoems.AsNoTracking().Where(p => p.Id == link.DivanPostId).SingleOrDefaultAsync(); if (poem == null) continue; var cat = await context.DivanCategories.AsNoTracking().Where(c => c.Id == poem.CatId).SingleOrDefaultAsync(); if (cat == null) continue; if(poetid !=0) { if(cat.PoetId != poetid) continue; } if (poets.ContainsKey(cat.PoetId)) { poets[cat.PoetId]++; } else { poets[cat.PoetId] = 1; } } if (poets.Count > 0) { var mainPoet = poets.First(); foreach (var poet in poets) { if (poet.Value > mainPoet.Value) { mainPoet = poet; } } if (mainPoet.Value > 3) { var poet = await context.DivanPoets.AsNoTracking().Where(p => p.Id == mainPoet.Key).SingleAsync(); var cat = await context.DivanCategories.AsNoTracking().Where(c => c.PoetId == poet.Id & c.ParentId == null).SingleAsync(); if (false == await context.DivanPaperSources.Where(p => p.BookFullUrl == $"https://museum.ganjoor.net/items/{artifact.FriendlyUrl}" && p.DivanCatId == cat.Id).AnyAsync()) { DivanPaperSource paperSource = new DivanPaperSource() { DivanPoetId = poet.Id, DivanCatId = cat.Id, DivanCatFullTitle = poet.Nickname, DivanCatFullUrl = cat.FullUrl, BookType = LinkType.Museum, BookFullUrl = $"https://museum.ganjoor.net/items/{artifact.FriendlyUrl}", NaskbanBookId = 0, BookFullTitle = artifact.Name, CoverThumbnailImageUrl = artifact.CoverImage.ExternalNormalSizeImageUrl.Replace("/norm/", "/thumb/").Replace("/orig/", "/thumb/"), Description = "", IsTextOriginalSource = await context.DivanLinks.AsNoTracking().Where(l => l.ArtifactId == artifact.Id && l.IsTextOriginalSource == true).AnyAsync(), MatchPercent = 100, HumanReviewed = true, OrderIndicator = 1, }; context.DivanPaperSources.Add(paperSource); } } } } await jobProgressServiceEF.UpdateJob(job.Id, 100, "", true); } catch (Exception exp) { await jobProgressServiceEF.UpdateJob(job.Id, 100, "", false, exp.ToString()); } } } }