diff --git a/RMuseum/RMuseum.xml b/RMuseum/RMuseum.xml index e4c9cb5e..f44a4d22 100644 --- a/RMuseum/RMuseum.xml +++ b/RMuseum/RMuseum.xml @@ -2780,6 +2780,141 @@ IArtifactService implementation + + IArtifactService implementation + + + IArtifactService implementation + + + IArtifactService implementation + + + IArtifactService implementation + + + IArtifactService implementation + + + IArtifactService implementation + + + IArtifactService implementation + + + IArtifactService implementation + + + IArtifactService implementation + + + IArtifactService implementation + + + IArtifactService implementation + + + + + import from http://www.qajarwomen.org + + 43117279 + atame + http://www.qajarwomen.org/fa/items/1018A10.html + + + + + from http://www.bl.uk + + grenville_xli_f001r + + + + + + from http://cudl.lib.cam.ac.uk + + MS-RAS-00258 + + + + + + import from https://viewer.cbl.ie + + 119 + golestan-baysonghori + + + + + from https://curiosity.lib.harvard.edu + + example: https://curiosity.lib.harvard.edu/islamic-heritage-project/catalog/40-990114893240203941 + + + + + + from https://catalog.hathitrust.org + + 006814127 + + + + + + from https://www.loc.gov + + + + m084 + + + + + boostan1207 + + + + + plmp + + + + + + from http://www.library.upenn.edu/ + + MEDREN_9949222153503681 + + + + + + import from server folder + + C:\Tools\batches\florence + shahname-florence + https://t.me/dr_khatibi_abolfazl/888 + + + + + from http://www.thedigitalwalters.org/01_ACCESS_WALTERS_MANUSCRIPTS.html + + W619 + golestan-walters-01 + + + + + from http://pudl.princeton.edu/ + + dj52w476m + + @@ -3017,108 +3152,6 @@ - - - from https://www.loc.gov - - - - m084 - - - - - boostan1207 - - - - - plmp - - - - - - from http://pudl.princeton.edu/ - - dj52w476m - - - - - - from https://curiosity.lib.harvard.edu - - example: https://curiosity.lib.harvard.edu/islamic-heritage-project/catalog/40-990114893240203941 - - - - - - import from http://www.qajarwomen.org - - 43117279 - atame - http://www.qajarwomen.org/fa/items/1018A10.html - - - - - from https://catalog.hathitrust.org - - 006814127 - - - - - - from http://www.library.upenn.edu/ - - MEDREN_9949222153503681 - - - - - - from http://cudl.lib.cam.ac.uk - - MS-RAS-00258 - - - - - - from http://www.bl.uk - - grenville_xli_f001r - - - - - - import from server folder - - C:\Tools\batches\florence - shahname-florence - https://t.me/dr_khatibi_abolfazl/888 - - - - - from http://www.thedigitalwalters.org/01_ACCESS_WALTERS_MANUSCRIPTS.html - - W619 - golestan-walters-01 - - - - - import from https://viewer.cbl.ie - - 119 - golestan-baysonghori - - reschedule jobs diff --git a/RMuseum/Services/Implementation/ArtifactService-Imports/ArtifactService-HarvardDirectly.cs b/RMuseum/Services/Implementation/ArtifactService-Imports/ArtifactService-HarvardDirectly.cs new file mode 100644 index 00000000..b32e3022 --- /dev/null +++ b/RMuseum/Services/Implementation/ArtifactService-Imports/ArtifactService-HarvardDirectly.cs @@ -0,0 +1,212 @@ +using Microsoft.AspNetCore.Http; +using Microsoft.EntityFrameworkCore; +using Microsoft.Extensions.Configuration; +using Newtonsoft.Json.Linq; +using RMuseum.DbContext; +using RMuseum.Models.Artifact; +using RMuseum.Models.Artifact.ViewModels; +using RMuseum.Models.Bookmark; +using RMuseum.Models.Bookmark.ViewModels; +using RMuseum.Models.GanjoorIntegration; +using RMuseum.Models.GanjoorIntegration.ViewModels; +using RMuseum.Models.ImportJob; +using RMuseum.Models.Note; +using RMuseum.Models.Note.ViewModels; +using RMuseum.Models.Notification; +using RSecurityBackend.Models.Auth.ViewModels; +using RSecurityBackend.Models.Generic; +using RSecurityBackend.Services; +using RSecurityBackend.Services.Implementation; +using System; +using System.Collections.Generic; +using System.IO; +using System.Linq; +using System.Net; +using System.Net.Http; +using System.Security.Cryptography; +using System.Text.RegularExpressions; +using System.Threading; +using System.Threading.Tasks; +using System.Xml.Linq; +using System.Drawing; +using System.Drawing.Imaging; + +namespace RMuseum.Services.Implementation +{ + + /// + /// IArtifactService implementation + /// + public partial class ArtifactService : IArtifactService + { + /// + /// import from http://www.qajarwomen.org + /// + /// 43117279 + /// atame + /// http://www.qajarwomen.org/fa/items/1018A10.html + /// + private async Task> StartImportingFromHarvardDirectly(string hardvardResourceNumber, string friendlyUrl, string srcUrl) + { + try + { + if ( + ( + await _context.ImportJobs + .Where(j => j.JobType == JobType.HarvardDirect && j.ResourceNumber == hardvardResourceNumber && !(j.Status == ImportJobStatus.Failed || j.Status == ImportJobStatus.Aborted)) + .SingleOrDefaultAsync() + ) + != + null + ) + { + return new RServiceResult(false, $"Job is already scheduled or running for importing harvard direct resource number {hardvardResourceNumber}"); + } + + if (string.IsNullOrEmpty(friendlyUrl)) + { + return new RServiceResult(false, $"Friendly url is empty, harvard direct resource number {hardvardResourceNumber}"); + } + + if ( + (await _context.Artifacts.Where(a => a.FriendlyUrl == friendlyUrl).SingleOrDefaultAsync()) + != + null + ) + { + return new RServiceResult(false, $"duplicated friendly url '{friendlyUrl}'"); + } + + ImportJob job = new ImportJob() + { + JobType = JobType.HarvardDirect, + ResourceNumber = hardvardResourceNumber, + FriendlyUrl = friendlyUrl, + SrcUrl = srcUrl, + QueueTime = DateTime.Now, + ProgressPercent = 0, + Status = ImportJobStatus.NotStarted + }; + + + await _context.ImportJobs.AddAsync + ( + job + ); + + await _context.SaveChangesAsync(); + + _backgroundTaskQueue.QueueBackgroundWorkItem + ( + async token => + { + try + { + using (RMuseumDbContext context = new RMuseumDbContext(Configuration)) + { + RArtifactMasterRecord book = new RArtifactMasterRecord($"extracted from harvard resource number {job.ResourceNumber}", $"extracted from harvard resource number {job.ResourceNumber}") + { + Status = PublishStatus.Draft, + DateTime = DateTime.Now, + LastModified = DateTime.Now, + CoverItemIndex = 0, + FriendlyUrl = friendlyUrl + }; + + + List meta = new List(); + RTagValue tag; + + + + + + tag = await TagHandler.PrepareAttribute(context, "Notes", "وارد شده از سایت دنیای زنان در عصر قاجار", 1); + meta.Add(tag); + + + + tag = await TagHandler.PrepareAttribute(context, "Type", "Book", 1); + meta.Add(tag); + + tag = await TagHandler.PrepareAttribute(context, "Type", "Manuscript", 1); + meta.Add(tag); + + + tag = await TagHandler.PrepareAttribute(context, "Source", "دنیای زنان در عصر قاجار", 1); + tag.ValueSupplement = $"{job.SrcUrl}"; + + meta.Add(tag); + + using (RMuseumDbContext importJobUpdaterDb = new RMuseumDbContext(Configuration)) + { + job.StartTime = DateTime.Now; + job.Status = ImportJobStatus.Running; + job.SrcContent = $"https://iiif.lib.harvard.edu/manifests/drs:{hardvardResourceNumber}"; + importJobUpdaterDb.Update(job); + await importJobUpdaterDb.SaveChangesAsync(); + } + + List pages = (await _InternalHarvardJsonImport(hardvardResourceNumber, job, friendlyUrl, context, book, meta)).Result; + if (pages == null) + { + return; + } + + + book.Tags = meta.ToArray(); + + book.Items = pages.ToArray(); + book.ItemCount = pages.Count; + + if (pages.Count == 0) + { + using (RMuseumDbContext importJobUpdaterDb = new RMuseumDbContext(Configuration)) + { + job.EndTime = DateTime.Now; + job.Status = ImportJobStatus.Failed; + job.Exception = "Pages.Count == 0"; + importJobUpdaterDb.Update(job); + await importJobUpdaterDb.SaveChangesAsync(); + } + return; + } + + await context.Artifacts.AddAsync(book); + await context.SaveChangesAsync(); + + job.ProgressPercent = 100; + job.Status = ImportJobStatus.Succeeded; + job.ArtifactId = book.Id; + job.EndTime = DateTime.Now; + context.Update(job); + await context.SaveChangesAsync(); + + + + } + } + catch (Exception exp) + { + using (RMuseumDbContext importJobUpdaterDb = new RMuseumDbContext(Configuration)) + { + job.EndTime = DateTime.Now; + job.Status = ImportJobStatus.Failed; + job.Exception = exp.ToString(); + importJobUpdaterDb.Update(job); + await importJobUpdaterDb.SaveChangesAsync(); + } + + } + } + ); + + return new RServiceResult(true); + } + catch (Exception exp) + { + return new RServiceResult(false, exp.ToString()); + } + } + } +} diff --git a/RMuseum/Services/Implementation/ArtifactService-Imports/ArtifactService-ImportBritishLibrary.cs b/RMuseum/Services/Implementation/ArtifactService-Imports/ArtifactService-ImportBritishLibrary.cs new file mode 100644 index 00000000..c4e1c47b --- /dev/null +++ b/RMuseum/Services/Implementation/ArtifactService-Imports/ArtifactService-ImportBritishLibrary.cs @@ -0,0 +1,721 @@ +using Microsoft.AspNetCore.Http; +using Microsoft.EntityFrameworkCore; +using Microsoft.Extensions.Configuration; +using Newtonsoft.Json.Linq; +using RMuseum.DbContext; +using RMuseum.Models.Artifact; +using RMuseum.Models.Artifact.ViewModels; +using RMuseum.Models.Bookmark; +using RMuseum.Models.Bookmark.ViewModels; +using RMuseum.Models.GanjoorIntegration; +using RMuseum.Models.GanjoorIntegration.ViewModels; +using RMuseum.Models.ImportJob; +using RMuseum.Models.Note; +using RMuseum.Models.Note.ViewModels; +using RMuseum.Models.Notification; +using RSecurityBackend.Models.Auth.ViewModels; +using RSecurityBackend.Models.Generic; +using RSecurityBackend.Services; +using RSecurityBackend.Services.Implementation; +using System; +using System.Collections.Generic; +using System.IO; +using System.Linq; +using System.Net; +using System.Net.Http; +using System.Security.Cryptography; +using System.Text.RegularExpressions; +using System.Threading; +using System.Threading.Tasks; +using System.Xml.Linq; +using System.Drawing; +using System.Drawing.Imaging; + +namespace RMuseum.Services.Implementation +{ + + /// + /// IArtifactService implementation + /// + public partial class ArtifactService : IArtifactService + { + /// + /// from http://www.bl.uk + /// + /// grenville_xli_f001r + /// + /// + private async Task> StartImportingFromBritishLibrary(string resourceNumber, string friendlyUrl) + { + string url = $"http://www.bl.uk/manuscripts/Viewer.aspx?ref={resourceNumber}"; + if ( + ( + await _context.ImportJobs + .Where(j => j.JobType == JobType.BritishLibrary && j.ResourceNumber == resourceNumber && !(j.Status == ImportJobStatus.Failed || j.Status == ImportJobStatus.Aborted)) + .SingleOrDefaultAsync() + ) + != + null + ) + { + return new RServiceResult(false, $"Job is already scheduled or running for importing {url}"); + } + + if (string.IsNullOrEmpty(friendlyUrl)) + { + friendlyUrl = resourceNumber; + } + + if ( + (await _context.Artifacts.Where(a => a.FriendlyUrl == friendlyUrl).SingleOrDefaultAsync()) + != + null + ) + { + return new RServiceResult(false, $"duplicated artifact friendly url '{friendlyUrl}'"); + } + + + + ImportJob job = new ImportJob() + { + JobType = JobType.BritishLibrary, + ResourceNumber = resourceNumber, + FriendlyUrl = friendlyUrl, + SrcUrl = url, + QueueTime = DateTime.Now, + ProgressPercent = 0, + Status = ImportJobStatus.NotStarted + }; + + + await _context.ImportJobs.AddAsync + ( + job + ); + + await _context.SaveChangesAsync(); + + + try + { + + _backgroundTaskQueue.QueueBackgroundWorkItem + ( + async token => + { + try + { + + + using (var client = new HttpClient()) + { + client.Timeout = TimeSpan.FromMinutes(5); + using (var result = await client.GetAsync(url)) + { + if (result.IsSuccessStatusCode) + { + using (RMuseumDbContext context = new RMuseumDbContext(Configuration)) + { + if ( + (await context.Artifacts.Where(a => a.FriendlyUrl == job.FriendlyUrl).SingleOrDefaultAsync()) + != + null + ) + { + using (RMuseumDbContext importJobUpdaterDb = new RMuseumDbContext(Configuration)) + { + job.EndTime = DateTime.Now; + job.Status = ImportJobStatus.Failed; + job.Exception = "aborted because of duplicated friendly url"; + importJobUpdaterDb.Update(job); + await importJobUpdaterDb.SaveChangesAsync(); + } + return; + } + RArtifactMasterRecord book = new RArtifactMasterRecord($"extracted from {url}", $"extracted from {url}") + { + Status = PublishStatus.Draft, + DateTime = DateTime.Now, + LastModified = DateTime.Now, + CoverItemIndex = 0, + FriendlyUrl = friendlyUrl + }; + + + List meta = new List(); + RTagValue tag; + + string html = await result.Content.ReadAsStringAsync(); + + + using (RMuseumDbContext importJobUpdaterDb = new RMuseumDbContext(Configuration)) + { + job.StartTime = DateTime.Now; + job.Status = ImportJobStatus.Running; + job.SrcContent = html; + importJobUpdaterDb.Update(job); + await importJobUpdaterDb.SaveChangesAsync(); + } + + + int nIdxStart = html.IndexOf("PageList"); + if (nIdxStart == -1) + { + using (RMuseumDbContext importJobUpdaterDb = new RMuseumDbContext(Configuration)) + { + job.EndTime = DateTime.Now; + job.Status = ImportJobStatus.Failed; + job.Exception = "PageList not found"; + importJobUpdaterDb.Update(job); + await importJobUpdaterDb.SaveChangesAsync(); + } + return; + } + + nIdxStart = html.IndexOf("value=\"", nIdxStart); + if (nIdxStart == -1) + { + using (RMuseumDbContext importJobUpdaterDb = new RMuseumDbContext(Configuration)) + { + job.EndTime = DateTime.Now; + job.Status = ImportJobStatus.Failed; + job.Exception = "value after PageList not found"; + importJobUpdaterDb.Update(job); + await importJobUpdaterDb.SaveChangesAsync(); + } + return; + } + + nIdxStart += "value=\"".Length; + + string strPageList = html.Substring(nIdxStart, html.IndexOf('"', nIdxStart) - nIdxStart); + + nIdxStart = html.IndexOf("TextList"); + if (nIdxStart == -1) + { + using (RMuseumDbContext importJobUpdaterDb = new RMuseumDbContext(Configuration)) + { + job.EndTime = DateTime.Now; + job.Status = ImportJobStatus.Failed; + job.Exception = "TextList not found"; + importJobUpdaterDb.Update(job); + await importJobUpdaterDb.SaveChangesAsync(); + } + return; + } + + nIdxStart = html.IndexOf("value=\"", nIdxStart); + if (nIdxStart == -1) + { + using (RMuseumDbContext importJobUpdaterDb = new RMuseumDbContext(Configuration)) + { + job.EndTime = DateTime.Now; + job.Status = ImportJobStatus.Failed; + job.Exception = "value after TextList not found"; + importJobUpdaterDb.Update(job); + await importJobUpdaterDb.SaveChangesAsync(); + } + return; + } + + nIdxStart += "value=\"".Length; + + string strTextList = html.Substring(nIdxStart, html.IndexOf('"', nIdxStart) - nIdxStart); + + nIdxStart = html.IndexOf("TitleList"); + if (nIdxStart == -1) + { + using (RMuseumDbContext importJobUpdaterDb = new RMuseumDbContext(Configuration)) + { + job.EndTime = DateTime.Now; + job.Status = ImportJobStatus.Failed; + job.Exception = "TitleList not found"; + importJobUpdaterDb.Update(job); + await importJobUpdaterDb.SaveChangesAsync(); + } + return; + } + + nIdxStart = html.IndexOf("value=\"", nIdxStart); + if (nIdxStart == -1) + { + using (RMuseumDbContext importJobUpdaterDb = new RMuseumDbContext(Configuration)) + { + job.EndTime = DateTime.Now; + job.Status = ImportJobStatus.Failed; + job.Exception = "value after TitleList not found"; + importJobUpdaterDb.Update(job); + await importJobUpdaterDb.SaveChangesAsync(); + } + return; + } + + nIdxStart += "value=\"".Length; + + string strTitleList = html.Substring(nIdxStart, html.IndexOf('"', nIdxStart) - nIdxStart); + + string[] PageUrls = strPageList.Split("||", StringSplitOptions.None); + string[] PageTexts = strTextList.Split("||", StringSplitOptions.None); + string[] PageTitles = strTitleList.Split("||", StringSplitOptions.None); + + if (PageUrls.Length != PageTexts.Length || PageTexts.Length != PageTitles.Length) + { + using (RMuseumDbContext importJobUpdaterDb = new RMuseumDbContext(Configuration)) + { + job.EndTime = DateTime.Now; + job.Status = ImportJobStatus.Failed; + job.Exception = "PageUrls.Length != PageTexts.Length || PageTexts.Length != PageTitles.Length"; + importJobUpdaterDb.Update(job); + await importJobUpdaterDb.SaveChangesAsync(); + } + return; + } + + tag = await TagHandler.PrepareAttribute(context, "Title", "Untitled", 1); + meta.Add(tag); + + tag = await TagHandler.PrepareAttribute(context, "Contributor Names", "Unknown", 1); + meta.Add(tag); + + tag = await TagHandler.PrepareAttribute(context, "Type", "Book", 1); + meta.Add(tag); + + tag = await TagHandler.PrepareAttribute(context, "Type", "Manuscript", 1); + meta.Add(tag); + + book.Tags = meta.ToArray(); + + + tag = await TagHandler.PrepareAttribute(context, "Source", "British Library", 1); + string viewerUrl = $"http://www.bl.uk/manuscripts/FullDisplay.aspx?ref={resourceNumber.Substring(0, resourceNumber.LastIndexOf('_'))}"; + tag.ValueSupplement = viewerUrl; + + meta.Add(tag); + + List pages = new List(); + + int order = 0; + for (int i = 0; i < PageUrls.Length; i++) + { + if (PageUrls[i] == "##") + continue; + order++; + + using (RMuseumDbContext importJobUpdaterDb = new RMuseumDbContext(Configuration)) + { + job.ProgressPercent = order; + importJobUpdaterDb.Update(job); + await importJobUpdaterDb.SaveChangesAsync(); + } + + + RArtifactItemRecord page = new RArtifactItemRecord() + { + Name = $"تصویر {order}", + NameInEnglish = $"Image {order}", + Description = "", + DescriptionInEnglish = "", + Order = order, + FriendlyUrl = $"p{$"{order}".PadLeft(4, '0')}", + LastModified = DateTime.Now + }; + + + List pageTags = new List(); + tag = await TagHandler.PrepareAttribute(context, "Source", "British Library", 1); + tag.ValueSupplement = $"http://www.bl.uk/manuscripts/Viewer.aspx?ref={PageUrls[i]}"; + pageTags.Add(tag); + + + if (!string.IsNullOrEmpty(PageTitles[i])) + { + RTagValue toc = await TagHandler.PrepareAttribute(context, "Title in TOC", PageTitles[i], 1); + toc.ValueSupplement = "1";//font size + pageTags.Add(toc); + } + + if (!string.IsNullOrEmpty(PageTexts[i])) + { + tag = await TagHandler.PrepareAttribute(context, "Label", PageTexts[i], 1); + pageTags.Add(tag); + + } + + page.Tags = pageTags.ToArray(); + + + bool recovered = false; + if ( + File.Exists + ( + Path.Combine(Path.Combine(Path.Combine(_pictureFileService.ImageStoragePath, friendlyUrl), "orig"), $"{order}".PadLeft(4, '0') + ".jpg") + ) + && + File.Exists + ( + Path.Combine(Path.Combine(Path.Combine(_pictureFileService.ImageStoragePath, friendlyUrl), "norm"), $"{order}".PadLeft(4, '0') + ".jpg") + ) + && + File.Exists + ( + Path.Combine(Path.Combine(Path.Combine(_pictureFileService.ImageStoragePath, friendlyUrl), "thumb"), $"{order}".PadLeft(4, '0') + ".jpg") + ) + ) + { + RServiceResult picture = await _pictureFileService.RecoverFromeFiles(page.Name, page.Description, 1, + viewerUrl, + Path.Combine(Path.Combine(Path.Combine(_pictureFileService.ImageStoragePath, friendlyUrl), "orig"), $"{order}".PadLeft(4, '0') + ".jpg"), + Path.Combine(Path.Combine(Path.Combine(_pictureFileService.ImageStoragePath, friendlyUrl), "norm"), $"{order}".PadLeft(4, '0') + ".jpg"), + Path.Combine(Path.Combine(Path.Combine(_pictureFileService.ImageStoragePath, friendlyUrl), "thumb"), $"{order}".PadLeft(4, '0') + ".jpg"), + $"{order}".PadLeft(4, '0') + ".jpg", friendlyUrl); + if (picture.Result != null) + { + recovered = true; + page.Images = new RPictureFile[] { picture.Result }; + page.CoverImageIndex = 0; + + if (book.CoverItemIndex == (order - 1)) + { + book.CoverImage = RPictureFile.Duplicate(picture.Result); + } + + pages.Add(page); + } + + } + if (!recovered) + { + if ( + File.Exists + ( + Path.Combine(Path.Combine(Path.Combine(_pictureFileService.ImageStoragePath, friendlyUrl), "orig"), $"{order}".PadLeft(4, '0') + ".jpg") + ) + + ) + { + File.Delete + ( + Path.Combine(Path.Combine(Path.Combine(_pictureFileService.ImageStoragePath, friendlyUrl), "orig"), $"{order}".PadLeft(4, '0') + ".jpg") + ); + } + if ( + + File.Exists + ( + Path.Combine(Path.Combine(Path.Combine(_pictureFileService.ImageStoragePath, friendlyUrl), "norm"), $"{order}".PadLeft(4, '0') + ".jpg") + ) + + ) + { + File.Delete + ( + Path.Combine(Path.Combine(Path.Combine(_pictureFileService.ImageStoragePath, friendlyUrl), "norm"), $"{order}".PadLeft(4, '0') + ".jpg") + ); + } + if ( + + File.Exists + ( + Path.Combine(Path.Combine(Path.Combine(_pictureFileService.ImageStoragePath, friendlyUrl), "thumb"), $"{order}".PadLeft(4, '0') + ".jpg") + ) + ) + { + File.Delete + ( + Path.Combine(Path.Combine(Path.Combine(_pictureFileService.ImageStoragePath, friendlyUrl), "thumb"), $"{order}".PadLeft(4, '0') + ".jpg") + ); + } + + /* + failed multithread attempt: + + BLTileMixer mixer = new BLTileMixer(); + RServiceResult blResult = await mixer.DownloadMix(PageUrls[i], order); + */ + + + Dictionary<(int x, int y), Image> tiles = new Dictionary<(int x, int y), Image>(); + int max_x = -1; + for (int x = 0; ; x++) + { + string imageUrl = $"http://www.bl.uk/manuscripts/Proxy.ashx?view={PageUrls[i]}_files/13/{x}_0.jpg"; + var imageResult = await client.GetAsync(imageUrl); + + int _ImportRetryCount = 5; + int _ImportRetryInitialSleep = 500; + int retryCount = 0; + while (retryCount < _ImportRetryCount && !imageResult.IsSuccessStatusCode && imageResult.StatusCode == HttpStatusCode.ServiceUnavailable) + { + imageResult.Dispose(); + Thread.Sleep(_ImportRetryInitialSleep * (retryCount + 1)); + imageResult = await client.GetAsync(imageUrl); + retryCount++; + } + + if (imageResult.IsSuccessStatusCode) + { + using (Stream imageStream = await imageResult.Content.ReadAsStreamAsync()) + { + + imageStream.Position = 0; + try + { + Image tile = Image.FromStream(imageStream); + tiles.Add((x, 0), tile); + max_x = x; + } + catch (Exception aexp) + { + if (aexp is ArgumentException) + { + break; + } + throw aexp; + } + } + } + else + { + using (RMuseumDbContext importJobUpdaterDb = new RMuseumDbContext(Configuration)) + { + job.EndTime = DateTime.Now; + job.Status = ImportJobStatus.Failed; + job.Exception = $"Http result is not ok ({imageResult.StatusCode}) for page {order}, url {imageUrl}"; + importJobUpdaterDb.Update(job); + await importJobUpdaterDb.SaveChangesAsync(); + } + + imageResult.Dispose(); + return; + } + } + + int max_y = -1; + for (int y = 1; ; y++) + { + string imageUrl = $"http://www.bl.uk/manuscripts/Proxy.ashx?view={PageUrls[i]}_files/13/0_{y}.jpg"; + var imageResult = await client.GetAsync(imageUrl); + + int _ImportRetryCount = 5; + int _ImportRetryInitialSleep = 500; + int retryCount = 0; + while (retryCount < _ImportRetryCount && !imageResult.IsSuccessStatusCode && imageResult.StatusCode == HttpStatusCode.ServiceUnavailable) + { + imageResult.Dispose(); + Thread.Sleep(_ImportRetryInitialSleep * (retryCount + 1)); + imageResult = await client.GetAsync(imageUrl); + retryCount++; + } + + if (imageResult.IsSuccessStatusCode) + { + using (Stream imageStream = await imageResult.Content.ReadAsStreamAsync()) + { + if (imageStream.Length <= 248) + break; + imageStream.Position = 0; + try + { + Image tile = Image.FromStream(imageStream); + tiles.Add((0, y), tile); + max_y = y; + } + catch (Exception aexp) + { + if (aexp is ArgumentException) + { + break; + } + throw aexp; + } + } + } + else + { + using (RMuseumDbContext importJobUpdaterDb = new RMuseumDbContext(Configuration)) + { + job.EndTime = DateTime.Now; + job.Status = ImportJobStatus.Failed; + job.Exception = $"Http result is not ok ({imageResult.StatusCode}) for page {order}, url {imageUrl}"; + importJobUpdaterDb.Update(job); + await importJobUpdaterDb.SaveChangesAsync(); + } + + imageResult.Dispose(); + return; + } + } + + for (int x = 0; x <= max_x; x++) + for (int y = 0; y <= max_y; y++) + if (tiles.TryGetValue((x, y), out Image tmp) == false) + { + string imageUrl = $"http://www.bl.uk/manuscripts/Proxy.ashx?view={PageUrls[i]}_files/13/{x}_{y}.jpg"; + var imageResult = await client.GetAsync(imageUrl); + + int _ImportRetryCount = 5; + int _ImportRetryInitialSleep = 500; + int retryCount = 0; + while (retryCount < _ImportRetryCount && !imageResult.IsSuccessStatusCode && imageResult.StatusCode == HttpStatusCode.ServiceUnavailable) + { + imageResult.Dispose(); + Thread.Sleep(_ImportRetryInitialSleep * (retryCount + 1)); + imageResult = await client.GetAsync(imageUrl); + retryCount++; + } + + if (imageResult.IsSuccessStatusCode) + { + using (Stream imageStream = await imageResult.Content.ReadAsStreamAsync()) + { + if (imageStream.Length == 0) + break; + imageStream.Position = 0; + tiles.Add((x, y), Image.FromStream(imageStream)); + } + } + else + { + using (RMuseumDbContext importJobUpdaterDb = new RMuseumDbContext(Configuration)) + { + job.EndTime = DateTime.Now; + job.Status = ImportJobStatus.Failed; + job.Exception = $"Http result is not ok ({imageResult.StatusCode}) for page {order}, url {imageUrl}"; + importJobUpdaterDb.Update(job); + await importJobUpdaterDb.SaveChangesAsync(); + } + + imageResult.Dispose(); + return; + } + } + + int tileWidth = tiles[(0, 0)].Width; + int tileHeight = tiles[(0, 0)].Height; + + int imageWidth = tileWidth * (max_x + 1); + int imageHeight = tileHeight * (max_y + 1); + + using (Image image = new Bitmap(imageWidth, imageHeight)) + { + using (Graphics g = Graphics.FromImage(image)) + { + for (int x = 0; x <= max_x; x++) + for (int y = 0; y < max_y; y++) + { + g.DrawImage(tiles[(x, y)], new Point(x * tileWidth, y * tileHeight)); + tiles[(x, y)].Dispose(); + } + + using (Stream imageStream = new MemoryStream()) + { + ImageCodecInfo jpgEncoder = _pictureFileService.GetEncoder(ImageFormat.Jpeg); + + Encoder myEncoder = + Encoder.Quality; + EncoderParameters jpegParameters = new EncoderParameters(1); + + EncoderParameter qualityParameter = new EncoderParameter(myEncoder, 92L); + jpegParameters.Param[0] = qualityParameter; + + image.Save(imageStream, jpgEncoder, jpegParameters); + RServiceResult picture = await _pictureFileService.Add(page.Name, page.Description, 1, null, $"http://www.bl.uk/manuscripts/Viewer.aspx?ref={PageUrls[i]}", imageStream, $"{order}".PadLeft(4, '0') + ".jpg", friendlyUrl); + if (picture.Result == null) + { + throw new Exception($"_pictureFileService.Add : {picture.ExceptionString}"); + } + + page.Images = new RPictureFile[] { picture.Result }; + page.CoverImageIndex = 0; + + if (book.CoverItemIndex == (order - 1)) + { + book.CoverImage = RPictureFile.Duplicate(picture.Result); + } + + pages.Add(page); + } + } + } + + GC.Collect(); + + + + } + } + + + book.Items = pages.ToArray(); + book.ItemCount = pages.Count; + + if (pages.Count == 0) + { + using (RMuseumDbContext importJobUpdaterDb = new RMuseumDbContext(Configuration)) + { + job.EndTime = DateTime.Now; + job.Status = ImportJobStatus.Failed; + job.Exception = "Pages.Count == 0"; + importJobUpdaterDb.Update(job); + await importJobUpdaterDb.SaveChangesAsync(); + } + return; + } + + await context.Artifacts.AddAsync(book); + await context.SaveChangesAsync(); + + job.ProgressPercent = 100; + job.Status = ImportJobStatus.Succeeded; + job.ArtifactId = book.Id; + job.EndTime = DateTime.Now; + context.Update(job); + await context.SaveChangesAsync(); + + + } + } + else + { + using (RMuseumDbContext importJobUpdaterDb = new RMuseumDbContext(Configuration)) + { + job.EndTime = DateTime.Now; + job.Status = ImportJobStatus.Failed; + job.Exception = $"Http result is not ok ({result.StatusCode}) for {url}"; + importJobUpdaterDb.Update(job); + await importJobUpdaterDb.SaveChangesAsync(); + } + return; + } + + } + } + } + catch (Exception exp) + { + using (RMuseumDbContext importJobUpdaterDb = new RMuseumDbContext(Configuration)) + { + job.EndTime = DateTime.Now; + job.Status = ImportJobStatus.Failed; + job.Exception = exp.ToString(); + importJobUpdaterDb.Update(job); + await importJobUpdaterDb.SaveChangesAsync(); + } + + } + } + ); + + + + return new RServiceResult(true); + } + catch (Exception exp) + { + return new RServiceResult(false, exp.ToString()); + } + } + + } +} diff --git a/RMuseum/Services/Implementation/ArtifactService-Imports/ArtifactService-ImportCambridge.cs b/RMuseum/Services/Implementation/ArtifactService-Imports/ArtifactService-ImportCambridge.cs new file mode 100644 index 00000000..3527a54a --- /dev/null +++ b/RMuseum/Services/Implementation/ArtifactService-Imports/ArtifactService-ImportCambridge.cs @@ -0,0 +1,488 @@ +using Microsoft.AspNetCore.Http; +using Microsoft.EntityFrameworkCore; +using Microsoft.Extensions.Configuration; +using Newtonsoft.Json.Linq; +using RMuseum.DbContext; +using RMuseum.Models.Artifact; +using RMuseum.Models.Artifact.ViewModels; +using RMuseum.Models.Bookmark; +using RMuseum.Models.Bookmark.ViewModels; +using RMuseum.Models.GanjoorIntegration; +using RMuseum.Models.GanjoorIntegration.ViewModels; +using RMuseum.Models.ImportJob; +using RMuseum.Models.Note; +using RMuseum.Models.Note.ViewModels; +using RMuseum.Models.Notification; +using RSecurityBackend.Models.Auth.ViewModels; +using RSecurityBackend.Models.Generic; +using RSecurityBackend.Services; +using RSecurityBackend.Services.Implementation; +using System; +using System.Collections.Generic; +using System.IO; +using System.Linq; +using System.Net; +using System.Net.Http; +using System.Security.Cryptography; +using System.Text.RegularExpressions; +using System.Threading; +using System.Threading.Tasks; +using System.Xml.Linq; +using System.Drawing; +using System.Drawing.Imaging; + +namespace RMuseum.Services.Implementation +{ + + /// + /// IArtifactService implementation + /// + public partial class ArtifactService : IArtifactService + { + /// + /// from http://cudl.lib.cam.ac.uk + /// + /// MS-RAS-00258 + /// + /// + private async Task> StartImportingFromCambridge(string resourceNumber, string friendlyUrl) + { + string url = $"http://cudl.lib.cam.ac.uk/view/{resourceNumber}.json"; + if ( + ( + await _context.ImportJobs + .Where(j => j.JobType == JobType.Cambridge && j.ResourceNumber == resourceNumber && !(j.Status == ImportJobStatus.Failed || j.Status == ImportJobStatus.Aborted)) + .SingleOrDefaultAsync() + ) + != + null + ) + { + return new RServiceResult(false, $"Job is already scheduled or running for importing {url}"); + } + + if (string.IsNullOrEmpty(friendlyUrl)) + { + friendlyUrl = resourceNumber; + } + + if ( + (await _context.Artifacts.Where(a => a.FriendlyUrl == friendlyUrl).SingleOrDefaultAsync()) + != + null + ) + { + return new RServiceResult(false, $"duplicated artifact friendly url '{friendlyUrl}'"); + } + + + + ImportJob job = new ImportJob() + { + JobType = JobType.Cambridge, + ResourceNumber = resourceNumber, + FriendlyUrl = friendlyUrl, + SrcUrl = url, + QueueTime = DateTime.Now, + ProgressPercent = 0, + Status = ImportJobStatus.NotStarted + }; + + + await _context.ImportJobs.AddAsync + ( + job + ); + + await _context.SaveChangesAsync(); + + + try + { + + _backgroundTaskQueue.QueueBackgroundWorkItem + ( + async token => + { + try + { + + + using (var client = new HttpClient()) + { + using (var result = await client.GetAsync(url)) + { + if (result.IsSuccessStatusCode) + { + using (RMuseumDbContext context = new RMuseumDbContext(Configuration)) + { + if ( + (await context.Artifacts.Where(a => a.FriendlyUrl == job.FriendlyUrl).SingleOrDefaultAsync()) + != + null + ) + { + using (RMuseumDbContext importJobUpdaterDb = new RMuseumDbContext(Configuration)) + { + job.EndTime = DateTime.Now; + job.Status = ImportJobStatus.Failed; + job.Exception = "aborted because of duplicated friendly url"; + importJobUpdaterDb.Update(job); + await importJobUpdaterDb.SaveChangesAsync(); + } + return; + } + RArtifactMasterRecord book = new RArtifactMasterRecord($"extracted from {url}", $"extracted from {url}") + { + Status = PublishStatus.Draft, + DateTime = DateTime.Now, + LastModified = DateTime.Now, + CoverItemIndex = 0, + FriendlyUrl = friendlyUrl + }; + + + List meta = new List(); + RTagValue tag; + + string json = await result.Content.ReadAsStringAsync(); + var parsed = JObject.Parse(json); + book.Name = book.NameInEnglish = + parsed.SelectToken("logicalStructures[*].label").Value(); + + book.Description = book.DescriptionInEnglish = + Regex.Replace( + parsed.SelectToken("descriptiveMetadata[*].abstract.displayForm").Value(), + "<.*?>", string.Empty); + + int tagOrder = 1; + foreach (JToken descriptiveMetadata in parsed.SelectTokens("$.descriptiveMetadata[*]").Children()) + { + foreach (JToken child in descriptiveMetadata.Children()) + { + if (child.SelectToken("label") != null && child.SelectToken("display") != null) + { + if (child.SelectToken("display").Value() == "True") + { + string metaName = child.SelectToken("label").Value(); + string metaValue = ""; + if (child.SelectToken("displayForm") != null) + { + metaValue = Regex.Replace( + child.SelectToken("displayForm").Value(), + "<.*?>", string.Empty); + tag = await TagHandler.PrepareAttribute(context, metaName, metaValue, tagOrder++); + meta.Add(tag); + } + else + if (child.SelectToken("value") != null) + { + foreach (JToken value in child.SelectTokens("value").Children()) + { + if (value.SelectToken("displayForm") != null) + { + metaValue = Regex.Replace( + value.SelectToken("displayForm").Value(), + "<.*?>", string.Empty); + tag = await TagHandler.PrepareAttribute(context, metaName, metaValue, tagOrder++); + meta.Add(tag); + } + } + } + } + } + } + + } + + string imageReproPageURL = "https://image01.cudl.lib.cam.ac.uk"; + + + using (RMuseumDbContext importJobUpdaterDb = new RMuseumDbContext(Configuration)) + { + job.StartTime = DateTime.Now; + job.Status = ImportJobStatus.Running; + job.SrcContent = json; + importJobUpdaterDb.Update(job); + await importJobUpdaterDb.SaveChangesAsync(); + } + + + + + tag = await TagHandler.PrepareAttribute(context, "Type", "Book", 1); + meta.Add(tag); + + tag = await TagHandler.PrepareAttribute(context, "Type", "Manuscript", 1); + meta.Add(tag); + + + tag = await TagHandler.PrepareAttribute(context, "Source", "University of Cambridge Digital Library", 1); + string viewerUrl = $"http://cudl.lib.cam.ac.uk/view/{resourceNumber}"; + tag.ValueSupplement = viewerUrl; + + meta.Add(tag); + + book.Tags = meta.ToArray(); + + List pages = new List(); + + int order = 0; + foreach (JToken pageToken in parsed.SelectTokens("$.pages").Children()) + { + order++; + + using (RMuseumDbContext importJobUpdaterDb = new RMuseumDbContext(Configuration)) + { + job.ProgressPercent = order; + importJobUpdaterDb.Update(job); + await importJobUpdaterDb.SaveChangesAsync(); + } + + string imageUrl = imageReproPageURL + pageToken.SelectToken("downloadImageURL").Value(); + + RArtifactItemRecord page = new RArtifactItemRecord() + { + Name = $"تصویر {order}", + NameInEnglish = $"Image {order}", + Description = "", + DescriptionInEnglish = "", + Order = order, + FriendlyUrl = $"p{$"{order}".PadLeft(4, '0')}", + LastModified = DateTime.Now + }; + + List pageMata = new List(); + tag = await TagHandler.PrepareAttribute(context, "Source", "University of Cambridge Digital Library", 1); + tag.ValueSupplement = $"{viewerUrl}/{order}"; + pageMata.Add(tag); + + if (pageToken.SelectToken("label") != null) + { + tag = await TagHandler.PrepareAttribute(context, "Label", pageToken.SelectToken("label").Value(), 1); + pageMata.Add(tag); + } + + + page.Tags = pageMata.ToArray(); + + if (!string.IsNullOrEmpty(imageUrl)) + { + bool recovered = false; + if ( + File.Exists + ( + Path.Combine(Path.Combine(Path.Combine(_pictureFileService.ImageStoragePath, friendlyUrl), "orig"), $"{order}".PadLeft(4, '0') + ".jpg") + ) + && + File.Exists + ( + Path.Combine(Path.Combine(Path.Combine(_pictureFileService.ImageStoragePath, friendlyUrl), "norm"), $"{order}".PadLeft(4, '0') + ".jpg") + ) + && + File.Exists + ( + Path.Combine(Path.Combine(Path.Combine(_pictureFileService.ImageStoragePath, friendlyUrl), "thumb"), $"{order}".PadLeft(4, '0') + ".jpg") + ) + ) + { + RServiceResult picture = await _pictureFileService.RecoverFromeFiles(page.Name, page.Description, 1, + imageUrl, + Path.Combine(Path.Combine(Path.Combine(_pictureFileService.ImageStoragePath, friendlyUrl), "orig"), $"{order}".PadLeft(4, '0') + ".jpg"), + Path.Combine(Path.Combine(Path.Combine(_pictureFileService.ImageStoragePath, friendlyUrl), "norm"), $"{order}".PadLeft(4, '0') + ".jpg"), + Path.Combine(Path.Combine(Path.Combine(_pictureFileService.ImageStoragePath, friendlyUrl), "thumb"), $"{order}".PadLeft(4, '0') + ".jpg"), + $"{order}".PadLeft(4, '0') + ".jpg", friendlyUrl); + if (picture.Result != null) + { + recovered = true; + page.Images = new RPictureFile[] { picture.Result }; + page.CoverImageIndex = 0; + + if (book.CoverItemIndex == (order - 1)) + { + book.CoverImage = RPictureFile.Duplicate(picture.Result); + } + + pages.Add(page); + } + + } + if (!recovered) + { + if ( + File.Exists + ( + Path.Combine(Path.Combine(Path.Combine(_pictureFileService.ImageStoragePath, friendlyUrl), "orig"), $"{order}".PadLeft(4, '0') + ".jpg") + ) + + ) + { + File.Delete + ( + Path.Combine(Path.Combine(Path.Combine(_pictureFileService.ImageStoragePath, friendlyUrl), "orig"), $"{order}".PadLeft(4, '0') + ".jpg") + ); + } + if ( + + File.Exists + ( + Path.Combine(Path.Combine(Path.Combine(_pictureFileService.ImageStoragePath, friendlyUrl), "norm"), $"{order}".PadLeft(4, '0') + ".jpg") + ) + + ) + { + File.Delete + ( + Path.Combine(Path.Combine(Path.Combine(_pictureFileService.ImageStoragePath, friendlyUrl), "norm"), $"{order}".PadLeft(4, '0') + ".jpg") + ); + } + if ( + + File.Exists + ( + Path.Combine(Path.Combine(Path.Combine(_pictureFileService.ImageStoragePath, friendlyUrl), "thumb"), $"{order}".PadLeft(4, '0') + ".jpg") + ) + ) + { + File.Delete + ( + Path.Combine(Path.Combine(Path.Combine(_pictureFileService.ImageStoragePath, friendlyUrl), "thumb"), $"{order}".PadLeft(4, '0') + ".jpg") + ); + } + + var imageResult = await client.GetAsync(imageUrl); + + if (imageResult.StatusCode == HttpStatusCode.NotFound) + { + break;//finished + } + + + int _ImportRetryCount = 200; + int _ImportRetryInitialSleep = 500; + int retryCount = 0; + while (retryCount < _ImportRetryCount && !imageResult.IsSuccessStatusCode && imageResult.StatusCode == HttpStatusCode.ServiceUnavailable) + { + imageResult.Dispose(); + Thread.Sleep(_ImportRetryInitialSleep * (retryCount + 1)); + imageResult = await client.GetAsync(imageUrl); + retryCount++; + } + + if (imageResult.IsSuccessStatusCode) + { + using (Stream imageStream = await imageResult.Content.ReadAsStreamAsync()) + { + RServiceResult picture = await _pictureFileService.Add(page.Name, page.Description, 1, null, imageUrl, imageStream, $"{order}".PadLeft(4, '0') + ".jpg", friendlyUrl); + if (picture.Result == null) + { + throw new Exception($"_pictureFileService.Add : {picture.ExceptionString}"); + } + + + + page.Images = new RPictureFile[] { picture.Result }; + page.CoverImageIndex = 0; + + if (book.CoverItemIndex == (order - 1)) + { + book.CoverImage = RPictureFile.Duplicate(picture.Result); + } + + pages.Add(page); + + } + } + else + { + using (RMuseumDbContext importJobUpdaterDb = new RMuseumDbContext(Configuration)) + { + job.EndTime = DateTime.Now; + job.Status = ImportJobStatus.Failed; + job.Exception = $"Http result is not ok ({imageResult.StatusCode}) for page {order}, url {imageUrl}"; + importJobUpdaterDb.Update(job); + await importJobUpdaterDb.SaveChangesAsync(); + } + + imageResult.Dispose(); + return; + } + + imageResult.Dispose(); + GC.Collect(); + } + } + } + + + book.Items = pages.ToArray(); + book.ItemCount = pages.Count; + + if (pages.Count == 0) + { + using (RMuseumDbContext importJobUpdaterDb = new RMuseumDbContext(Configuration)) + { + job.EndTime = DateTime.Now; + job.Status = ImportJobStatus.Failed; + job.Exception = "Pages.Count == 0"; + importJobUpdaterDb.Update(job); + await importJobUpdaterDb.SaveChangesAsync(); + } + return; + } + + await context.Artifacts.AddAsync(book); + await context.SaveChangesAsync(); + + job.ProgressPercent = 100; + job.Status = ImportJobStatus.Succeeded; + job.ArtifactId = book.Id; + job.EndTime = DateTime.Now; + context.Update(job); + await context.SaveChangesAsync(); + + + } + } + else + { + using (RMuseumDbContext importJobUpdaterDb = new RMuseumDbContext(Configuration)) + { + job.EndTime = DateTime.Now; + job.Status = ImportJobStatus.Failed; + job.Exception = $"Http result is not ok ({result.StatusCode}) for {url}"; + importJobUpdaterDb.Update(job); + await importJobUpdaterDb.SaveChangesAsync(); + } + return; + } + + } + } + } + catch (Exception exp) + { + using (RMuseumDbContext importJobUpdaterDb = new RMuseumDbContext(Configuration)) + { + job.EndTime = DateTime.Now; + job.Status = ImportJobStatus.Failed; + job.Exception = exp.ToString(); + importJobUpdaterDb.Update(job); + await importJobUpdaterDb.SaveChangesAsync(); + } + + } + } + ); + + + + return new RServiceResult(true); + } + catch (Exception exp) + { + return new RServiceResult(false, exp.ToString()); + } + } + } +} diff --git a/RMuseum/Services/Implementation/ArtifactService-Imports/ArtifactService-ImportChesterBeatty.cs b/RMuseum/Services/Implementation/ArtifactService-Imports/ArtifactService-ImportChesterBeatty.cs new file mode 100644 index 00000000..131e1459 --- /dev/null +++ b/RMuseum/Services/Implementation/ArtifactService-Imports/ArtifactService-ImportChesterBeatty.cs @@ -0,0 +1,384 @@ +using Microsoft.AspNetCore.Http; +using Microsoft.EntityFrameworkCore; +using Microsoft.Extensions.Configuration; +using Newtonsoft.Json.Linq; +using RMuseum.DbContext; +using RMuseum.Models.Artifact; +using RMuseum.Models.Artifact.ViewModels; +using RMuseum.Models.Bookmark; +using RMuseum.Models.Bookmark.ViewModels; +using RMuseum.Models.GanjoorIntegration; +using RMuseum.Models.GanjoorIntegration.ViewModels; +using RMuseum.Models.ImportJob; +using RMuseum.Models.Note; +using RMuseum.Models.Note.ViewModels; +using RMuseum.Models.Notification; +using RSecurityBackend.Models.Auth.ViewModels; +using RSecurityBackend.Models.Generic; +using RSecurityBackend.Services; +using RSecurityBackend.Services.Implementation; +using System; +using System.Collections.Generic; +using System.IO; +using System.Linq; +using System.Net; +using System.Net.Http; +using System.Security.Cryptography; +using System.Text.RegularExpressions; +using System.Threading; +using System.Threading.Tasks; +using System.Xml.Linq; +using System.Drawing; +using System.Drawing.Imaging; + +namespace RMuseum.Services.Implementation +{ + + /// + /// IArtifactService implementation + /// + public partial class ArtifactService : IArtifactService + { + /// + /// import from https://viewer.cbl.ie + /// + /// 119 + /// golestan-baysonghori + /// + private async Task> StartImportingFromChesterBeatty(string resourceNumber, string friendlyUrl) + { + try + { + string srcUrl = $"https://viewer.cbl.ie/viewer/object/Per_{resourceNumber}/1/"; + if ( + ( + await _context.ImportJobs + .Where(j => j.JobType == JobType.ChesterBeatty && j.ResourceNumber == resourceNumber && !(j.Status == ImportJobStatus.Failed || j.Status == ImportJobStatus.Aborted)) + .SingleOrDefaultAsync() + ) + != + null + ) + { + return new RServiceResult(false, $"Job is already scheduled or running for importing {srcUrl}"); + } + + if (string.IsNullOrEmpty(friendlyUrl)) + { + return new RServiceResult(false, $"Friendly url is empty, server folder {srcUrl}"); + } + + if ( + (await _context.Artifacts.Where(a => a.FriendlyUrl == friendlyUrl).SingleOrDefaultAsync()) + != + null + ) + { + return new RServiceResult(false, $"duplicated friendly url '{friendlyUrl}'"); + } + + ImportJob job = new ImportJob() + { + JobType = JobType.ChesterBeatty, + ResourceNumber = resourceNumber, + FriendlyUrl = friendlyUrl, + SrcUrl = srcUrl, + QueueTime = DateTime.Now, + ProgressPercent = 0, + Status = ImportJobStatus.NotStarted + }; + + + await _context.ImportJobs.AddAsync + ( + job + ); + + await _context.SaveChangesAsync(); + + _backgroundTaskQueue.QueueBackgroundWorkItem + ( + async token => + { + try + { + using (RMuseumDbContext context = new RMuseumDbContext(Configuration)) + { + RArtifactMasterRecord book = new RArtifactMasterRecord($"extracted from url {job.ResourceNumber}", $"extracted from url {job.ResourceNumber}") + { + Status = PublishStatus.Draft, + DateTime = DateTime.Now, + LastModified = DateTime.Now, + CoverItemIndex = 0, + FriendlyUrl = friendlyUrl, + }; + + + List meta = new List(); + RTagValue tag; + + + tag = await TagHandler.PrepareAttribute(context, "Type", "Book", 1); + meta.Add(tag); + + tag = await TagHandler.PrepareAttribute(context, "Type", "Manuscript", 1); + meta.Add(tag); + + tag = await TagHandler.PrepareAttribute(context, "Source", "Chester Beatty Digital Collections", 1); + tag.ValueSupplement = srcUrl; + + meta.Add(tag); + + + + using (RMuseumDbContext importJobUpdaterDb = new RMuseumDbContext(Configuration)) + { + job.StartTime = DateTime.Now; + job.Status = ImportJobStatus.Running; + job.SrcContent = ""; + importJobUpdaterDb.Update(job); + await importJobUpdaterDb.SaveChangesAsync(); + } + + List pages = new List(); + int order = 0; + using (var client = new HttpClient()) + do + { + + + order++; + + using (RMuseumDbContext importJobUpdaterDb = new RMuseumDbContext(Configuration)) + { + job.ProgressPercent = order; + importJobUpdaterDb.Update(job); + await importJobUpdaterDb.SaveChangesAsync(); + } + + RArtifactItemRecord page = new RArtifactItemRecord() + { + Name = $"تصویر {order}", + NameInEnglish = $"Image {order} of {book.NameInEnglish}", + Description = "", + DescriptionInEnglish = "", + Order = order, + FriendlyUrl = $"p{$"{order}".PadLeft(4, '0')}", + LastModified = DateTime.Now + }; + + string imageUrl = $"https://viewer.cbl.ie/viewer/rest/image/Per_{resourceNumber}/Per{resourceNumber}_{$"{order}".PadLeft(3, '0')}.jpg/full/!10000,10000/0/default.jpg?ignoreWatermark=true"; + + + page.Tags = new RTagValue[] { }; + + bool recovered = false; + + if ( + File.Exists + ( + Path.Combine(Path.Combine(Path.Combine(_pictureFileService.ImageStoragePath, friendlyUrl), "orig"), $"{order}".PadLeft(4, '0') + ".jpg") + ) + && + File.Exists + ( + Path.Combine(Path.Combine(Path.Combine(_pictureFileService.ImageStoragePath, friendlyUrl), "norm"), $"{order}".PadLeft(4, '0') + ".jpg") + ) + && + File.Exists + ( + Path.Combine(Path.Combine(Path.Combine(_pictureFileService.ImageStoragePath, friendlyUrl), "thumb"), $"{order}".PadLeft(4, '0') + ".jpg") + ) + ) + { + RServiceResult picture = await _pictureFileService.RecoverFromeFiles(page.Name, page.Description, 1, + imageUrl, + Path.Combine(Path.Combine(Path.Combine(_pictureFileService.ImageStoragePath, friendlyUrl), "orig"), $"{order}".PadLeft(4, '0') + ".jpg"), + Path.Combine(Path.Combine(Path.Combine(_pictureFileService.ImageStoragePath, friendlyUrl), "norm"), $"{order}".PadLeft(4, '0') + ".jpg"), + Path.Combine(Path.Combine(Path.Combine(_pictureFileService.ImageStoragePath, friendlyUrl), "thumb"), $"{order}".PadLeft(4, '0') + ".jpg"), + $"{order}".PadLeft(4, '0') + ".jpg", friendlyUrl); + if (picture.Result != null) + { + recovered = true; + page.Images = new RPictureFile[] { picture.Result }; + page.CoverImageIndex = 0; + + if (book.CoverItemIndex == (order - 1)) + { + book.CoverImage = RPictureFile.Duplicate(picture.Result); + } + + tag = await TagHandler.PrepareAttribute(context, "Source", "Chester Beatty Digital Collections", 1); + tag.ValueSupplement = $"https://viewer.cbl.ie/viewer/object/Per_{resourceNumber}/{$"{order}".PadLeft(3, '0')}/"; ; + page.Tags = new RTagValue[] { tag }; + + pages.Add(page); + } + + } + + if (!recovered) + { + if ( + File.Exists + ( + Path.Combine(Path.Combine(Path.Combine(_pictureFileService.ImageStoragePath, friendlyUrl), "orig"), $"{order}".PadLeft(4, '0') + ".jpg") + ) + + ) + { + File.Delete + ( + Path.Combine(Path.Combine(Path.Combine(_pictureFileService.ImageStoragePath, friendlyUrl), "orig"), $"{order}".PadLeft(4, '0') + ".jpg") + ); + } + if ( + + File.Exists + ( + Path.Combine(Path.Combine(Path.Combine(_pictureFileService.ImageStoragePath, friendlyUrl), "norm"), $"{order}".PadLeft(4, '0') + ".jpg") + ) + + ) + { + File.Delete + ( + Path.Combine(Path.Combine(Path.Combine(_pictureFileService.ImageStoragePath, friendlyUrl), "norm"), $"{order}".PadLeft(4, '0') + ".jpg") + ); + } + if ( + + File.Exists + ( + Path.Combine(Path.Combine(Path.Combine(_pictureFileService.ImageStoragePath, friendlyUrl), "thumb"), $"{order}".PadLeft(4, '0') + ".jpg") + ) + ) + { + File.Delete + ( + Path.Combine(Path.Combine(Path.Combine(_pictureFileService.ImageStoragePath, friendlyUrl), "thumb"), $"{order}".PadLeft(4, '0') + ".jpg") + ); + } + var imageResult = await client.GetAsync(imageUrl); + + if (imageResult.StatusCode == HttpStatusCode.Forbidden || imageResult.StatusCode == HttpStatusCode.NotFound) + break; + + + int _ImportRetryCount = 5; + int _ImportRetryInitialSleep = 500; + int retryCount = 0; + while (retryCount < _ImportRetryCount && !imageResult.IsSuccessStatusCode && imageResult.StatusCode == HttpStatusCode.ServiceUnavailable) + { + imageResult.Dispose(); + Thread.Sleep(_ImportRetryInitialSleep * (retryCount + 1)); + imageResult = await client.GetAsync(imageUrl); + retryCount++; + } + + if (imageResult.IsSuccessStatusCode) + { + using (Stream imageStream = await imageResult.Content.ReadAsStreamAsync()) + { + RServiceResult picture = await _pictureFileService.Add(page.Name, page.Description, 1, null, imageUrl, imageStream, $"{order}".PadLeft(4, '0') + ".jpg", friendlyUrl); + if (picture.Result == null) + { + throw new Exception($"_pictureFileService.Add : {picture.ExceptionString}"); + } + + page.Images = new RPictureFile[] { picture.Result }; + page.CoverImageIndex = 0; + + if (book.CoverItemIndex == (order - 1)) + { + book.CoverImage = RPictureFile.Duplicate(picture.Result); + } + tag = await TagHandler.PrepareAttribute(context, "Source", "Chester Beatty Digital Collections", 1); + tag.ValueSupplement = $"https://viewer.cbl.ie/viewer/object/Per_{resourceNumber}/{$"{order}".PadLeft(3, '0')}/"; + page.Tags = new RTagValue[] { tag }; + + pages.Add(page); + } + } + else + { + using (RMuseumDbContext importJobUpdaterDb = new RMuseumDbContext(Configuration)) + { + job.EndTime = DateTime.Now; + job.Status = ImportJobStatus.Failed; + job.Exception = $"Http result is not ok ({imageResult.StatusCode}) for page {order}, url {imageUrl}"; + importJobUpdaterDb.Update(job); + await importJobUpdaterDb.SaveChangesAsync(); + } + + imageResult.Dispose(); + return; + } + + + imageResult.Dispose(); + GC.Collect(); + } + + + pages.Add(page); + } + while (true); + + + book.Tags = meta.ToArray(); + + book.Items = pages.ToArray(); + book.ItemCount = pages.Count; + + if (pages.Count == 0) + { + using (RMuseumDbContext importJobUpdaterDb = new RMuseumDbContext(Configuration)) + { + job.EndTime = DateTime.Now; + job.Status = ImportJobStatus.Failed; + job.Exception = "Pages.Count == 0"; + importJobUpdaterDb.Update(job); + await importJobUpdaterDb.SaveChangesAsync(); + } + return; + } + + await context.Artifacts.AddAsync(book); + await context.SaveChangesAsync(); + + job.ProgressPercent = 100; + job.Status = ImportJobStatus.Succeeded; + job.ArtifactId = book.Id; + job.EndTime = DateTime.Now; + context.Update(job); + await context.SaveChangesAsync(); + + + + } + } + catch (Exception exp) + { + using (RMuseumDbContext importJobUpdaterDb = new RMuseumDbContext(Configuration)) + { + job.EndTime = DateTime.Now; + job.Status = ImportJobStatus.Failed; + job.Exception = exp.ToString(); + importJobUpdaterDb.Update(job); + await importJobUpdaterDb.SaveChangesAsync(); + } + + } + } + ); + + return new RServiceResult(true); + } + catch (Exception exp) + { + return new RServiceResult(false, exp.ToString()); + } + } + } +} diff --git a/RMuseum/Services/Implementation/ArtifactService-Imports/ArtifactService-ImportHarvard.cs b/RMuseum/Services/Implementation/ArtifactService-Imports/ArtifactService-ImportHarvard.cs new file mode 100644 index 00000000..5f307855 --- /dev/null +++ b/RMuseum/Services/Implementation/ArtifactService-Imports/ArtifactService-ImportHarvard.cs @@ -0,0 +1,508 @@ +using Microsoft.AspNetCore.Http; +using Microsoft.EntityFrameworkCore; +using Microsoft.Extensions.Configuration; +using Newtonsoft.Json.Linq; +using RMuseum.DbContext; +using RMuseum.Models.Artifact; +using RMuseum.Models.Artifact.ViewModels; +using RMuseum.Models.Bookmark; +using RMuseum.Models.Bookmark.ViewModels; +using RMuseum.Models.GanjoorIntegration; +using RMuseum.Models.GanjoorIntegration.ViewModels; +using RMuseum.Models.ImportJob; +using RMuseum.Models.Note; +using RMuseum.Models.Note.ViewModels; +using RMuseum.Models.Notification; +using RSecurityBackend.Models.Auth.ViewModels; +using RSecurityBackend.Models.Generic; +using RSecurityBackend.Services; +using RSecurityBackend.Services.Implementation; +using System; +using System.Collections.Generic; +using System.IO; +using System.Linq; +using System.Net; +using System.Net.Http; +using System.Security.Cryptography; +using System.Text.RegularExpressions; +using System.Threading; +using System.Threading.Tasks; +using System.Xml.Linq; +using System.Drawing; +using System.Drawing.Imaging; + +namespace RMuseum.Services.Implementation +{ + + /// + /// IArtifactService implementation + /// + public partial class ArtifactService : IArtifactService + { + /// + /// from https://curiosity.lib.harvard.edu + /// + /// example: https://curiosity.lib.harvard.edu/islamic-heritage-project/catalog/40-990114893240203941 + /// + /// + private async Task> StartImportingFromHarvard(string url, string friendlyUrl) + { + try + { + if ( + ( + await _context.ImportJobs + .Where(j => j.JobType == JobType.Harvard && j.ResourceNumber == url && !(j.Status == ImportJobStatus.Failed || j.Status == ImportJobStatus.Aborted)) + .SingleOrDefaultAsync() + ) + != + null + ) + { + return new RServiceResult(false, $"Job is already scheduled or running for importing {url}"); + } + + if (string.IsNullOrEmpty(friendlyUrl)) + { + return new RServiceResult(false, $"Friendly url is empty, url = {url}"); + } + + if ( + (await _context.Artifacts.Where(a => a.FriendlyUrl == friendlyUrl).SingleOrDefaultAsync()) + != + null + ) + { + return new RServiceResult(false, $"duplicated friendly url '{friendlyUrl}'"); + } + + ImportJob job = new ImportJob() + { + JobType = JobType.Harvard, + ResourceNumber = url, + FriendlyUrl = friendlyUrl, + SrcUrl = url, + QueueTime = DateTime.Now, + ProgressPercent = 0, + Status = ImportJobStatus.NotStarted + }; + + + await _context.ImportJobs.AddAsync + ( + job + ); + + await _context.SaveChangesAsync(); + + _backgroundTaskQueue.QueueBackgroundWorkItem + ( + async token => + { + try + { + using (var client = new HttpClient()) + { + + using (var result = await client.GetAsync(url)) + { + if (result.IsSuccessStatusCode) + { + using (RMuseumDbContext context = new RMuseumDbContext(Configuration)) + { + RArtifactMasterRecord book = new RArtifactMasterRecord($"extracted from {url}", $"extracted from {url}") + { + Status = PublishStatus.Draft, + DateTime = DateTime.Now, + LastModified = DateTime.Now, + CoverItemIndex = 0, + FriendlyUrl = friendlyUrl + }; + + + List meta = new List(); + RTagValue tag; + + string html = await result.Content.ReadAsStringAsync(); + + + using (RMuseumDbContext importJobUpdaterDb = new RMuseumDbContext(Configuration)) + { + job.StartTime = DateTime.Now; + job.Status = ImportJobStatus.Running; + job.SrcContent = html; + importJobUpdaterDb.Update(job); + await importJobUpdaterDb.SaveChangesAsync(); + } + + int nStartIndex = html.IndexOf("", nStartIndex); + if (nStartIndex == -1) + break; + nStartIndex++; + string tagName = html.Substring(nStartIndex, html.IndexOf(":", nStartIndex) - nStartIndex); + nStartIndex = html.IndexOf("", nStartIndex); + if (nStartIndex == -1) + break; + nStartIndex++; + string tagValues = html.Substring(nStartIndex, html.IndexOf("", nStartIndex) - nStartIndex); + foreach (string tagValuePart in tagValues.Split("
", StringSplitOptions.RemoveEmptyEntries)) + { + string tagValue = tagValuePart; + bool href = false; + if (tagValue.IndexOf(" pages = (await _InternalHarvardJsonImport(hardvardResourceNumber, job, friendlyUrl, context, book, meta)).Result; + if (pages == null) + { + return; + } + + + book.Tags = meta.ToArray(); + + book.Items = pages.ToArray(); + book.ItemCount = pages.Count; + + if (pages.Count == 0) + { + using (RMuseumDbContext importJobUpdaterDb = new RMuseumDbContext(Configuration)) + { + job.EndTime = DateTime.Now; + job.Status = ImportJobStatus.Failed; + job.Exception = "Pages.Count == 0"; + importJobUpdaterDb.Update(job); + await importJobUpdaterDb.SaveChangesAsync(); + } + return; + } + + await context.Artifacts.AddAsync(book); + await context.SaveChangesAsync(); + + job.ProgressPercent = 100; + job.Status = ImportJobStatus.Succeeded; + job.ArtifactId = book.Id; + job.EndTime = DateTime.Now; + context.Update(job); + await context.SaveChangesAsync(); + + + + } + } + else + { + using (RMuseumDbContext importJobUpdaterDb = new RMuseumDbContext(Configuration)) + { + job.EndTime = DateTime.Now; + job.Status = ImportJobStatus.Failed; + job.Exception = $"Http result is not ok ({result.StatusCode}) for {url}"; + importJobUpdaterDb.Update(job); + await importJobUpdaterDb.SaveChangesAsync(); + } + return; + } + + } + } + } + catch (Exception exp) + { + using (RMuseumDbContext importJobUpdaterDb = new RMuseumDbContext(Configuration)) + { + job.EndTime = DateTime.Now; + job.Status = ImportJobStatus.Failed; + job.Exception = exp.ToString(); + importJobUpdaterDb.Update(job); + await importJobUpdaterDb.SaveChangesAsync(); + } + + } + } + ); + + return new RServiceResult(true); + } + catch (Exception exp) + { + return new RServiceResult(false, exp.ToString()); + } + } + + private async Task>> _InternalHarvardJsonImport(string hardvardResourceNumber, ImportJob job, string friendlyUrl, RMuseumDbContext context, RArtifactMasterRecord book, List meta) + { + List pages = new List(); + + using (var client = new HttpClient()) + { + using (var jsonResult = await client.GetAsync($"https://iiif.lib.harvard.edu/manifests/drs:{hardvardResourceNumber}")) + { + if (jsonResult.IsSuccessStatusCode) + { + string json = await jsonResult.Content.ReadAsStringAsync(); + var parsed = JObject.Parse(json); + book.Name = book.NameInEnglish = book.Description = book.DescriptionInEnglish = + parsed.SelectToken("label").Value(); + + + RTagValue tag; + + tag = await TagHandler.PrepareAttribute(context, "Title", book.Name, 1); + meta.Add(tag); + + tag = await TagHandler.PrepareAttribute(context, "Contributor Names", "تعیین نشده", 1); + meta.Add(tag); + + List labels = new List(); + foreach (JToken structure in parsed.SelectTokens("$.structures[*].label")) + { + labels.Add(structure.Value()); + } + + int order = 0; + var canvases = parsed.SelectToken("sequences").First().SelectToken("canvases").ToArray(); + int pageCount = canvases.Length; + foreach (JToken canvas in canvases) + { + order++; + using (RMuseumDbContext importJobUpdaterDb = new RMuseumDbContext(Configuration)) + { + job.ProgressPercent = order * 100 / (decimal)pageCount; + importJobUpdaterDb.Update(job); + await importJobUpdaterDb.SaveChangesAsync(); + } + string label = canvas.SelectToken("label").Value(); + if (labels.Where(l => l.IndexOf(label) != -1).SingleOrDefault() != null) + label = labels.Where(l => l.IndexOf(label) != -1).SingleOrDefault(); + string imageUrl = canvas.SelectTokens("images[*]").First().SelectToken("resource").SelectToken("@id").Value(); + RArtifactItemRecord page = new RArtifactItemRecord() + { + Name = $"تصویر {order}", + NameInEnglish = label, + Description = "", + DescriptionInEnglish = "", + Order = order, + FriendlyUrl = $"p{$"{order}".PadLeft(4, '0')}", + LastModified = DateTime.Now + }; + + + + tag = await TagHandler.PrepareAttribute(context, "Source", "Harvard University Islamic Heritage Project", 1); + tag.ValueSupplement = imageUrl; + page.Tags = new RTagValue[] { tag }; + + + if (!string.IsNullOrEmpty(imageUrl)) + { + bool recovered = false; + if ( + File.Exists + ( + Path.Combine(Path.Combine(Path.Combine(_pictureFileService.ImageStoragePath, friendlyUrl), "orig"), $"{order}".PadLeft(4, '0') + ".jpg") + ) + && + File.Exists + ( + Path.Combine(Path.Combine(Path.Combine(_pictureFileService.ImageStoragePath, friendlyUrl), "norm"), $"{order}".PadLeft(4, '0') + ".jpg") + ) + && + File.Exists + ( + Path.Combine(Path.Combine(Path.Combine(_pictureFileService.ImageStoragePath, friendlyUrl), "thumb"), $"{order}".PadLeft(4, '0') + ".jpg") + ) + ) + { + RServiceResult picture = await _pictureFileService.RecoverFromeFiles(page.Name, page.Description, 1, + imageUrl, + Path.Combine(Path.Combine(Path.Combine(_pictureFileService.ImageStoragePath, friendlyUrl), "orig"), $"{order}".PadLeft(4, '0') + ".jpg"), + Path.Combine(Path.Combine(Path.Combine(_pictureFileService.ImageStoragePath, friendlyUrl), "norm"), $"{order}".PadLeft(4, '0') + ".jpg"), + Path.Combine(Path.Combine(Path.Combine(_pictureFileService.ImageStoragePath, friendlyUrl), "thumb"), $"{order}".PadLeft(4, '0') + ".jpg"), + $"{order}".PadLeft(4, '0') + ".jpg", friendlyUrl); + if (picture.Result != null) + { + recovered = true; + page.Images = new RPictureFile[] { picture.Result }; + page.CoverImageIndex = 0; + + if (book.CoverItemIndex == (order - 1)) + { + book.CoverImage = RPictureFile.Duplicate(picture.Result); + } + + pages.Add(page); + } + + } + if (!recovered) + { + if ( + File.Exists + ( + Path.Combine(Path.Combine(Path.Combine(_pictureFileService.ImageStoragePath, friendlyUrl), "orig"), $"{order}".PadLeft(4, '0') + ".jpg") + ) + + ) + { + File.Delete + ( + Path.Combine(Path.Combine(Path.Combine(_pictureFileService.ImageStoragePath, friendlyUrl), "orig"), $"{order}".PadLeft(4, '0') + ".jpg") + ); + } + if ( + + File.Exists + ( + Path.Combine(Path.Combine(Path.Combine(_pictureFileService.ImageStoragePath, friendlyUrl), "norm"), $"{order}".PadLeft(4, '0') + ".jpg") + ) + + ) + { + File.Delete + ( + Path.Combine(Path.Combine(Path.Combine(_pictureFileService.ImageStoragePath, friendlyUrl), "norm"), $"{order}".PadLeft(4, '0') + ".jpg") + ); + } + if ( + + File.Exists + ( + Path.Combine(Path.Combine(Path.Combine(_pictureFileService.ImageStoragePath, friendlyUrl), "thumb"), $"{order}".PadLeft(4, '0') + ".jpg") + ) + ) + { + File.Delete + ( + Path.Combine(Path.Combine(Path.Combine(_pictureFileService.ImageStoragePath, friendlyUrl), "thumb"), $"{order}".PadLeft(4, '0') + ".jpg") + ); + } + var imageResult = await client.GetAsync(imageUrl); + + + int _ImportRetryCount = 5; + int _ImportRetryInitialSleep = 500; + int retryCount = 0; + while (retryCount < _ImportRetryCount && !imageResult.IsSuccessStatusCode && imageResult.StatusCode == HttpStatusCode.ServiceUnavailable) + { + imageResult.Dispose(); + Thread.Sleep(_ImportRetryInitialSleep * (retryCount + 1)); + imageResult = await client.GetAsync(imageUrl); + retryCount++; + } + + if (imageResult.IsSuccessStatusCode) + { + using (Stream imageStream = await imageResult.Content.ReadAsStreamAsync()) + { + RServiceResult picture = await _pictureFileService.Add(page.Name, page.Description, 1, null, imageUrl, imageStream, $"{order}".PadLeft(4, '0') + ".jpg", friendlyUrl); + if (picture.Result == null) + { + throw new Exception($"_pictureFileService.Add : {picture.ExceptionString}"); + } + + page.Images = new RPictureFile[] { picture.Result }; + page.CoverImageIndex = 0; + + if (book.CoverItemIndex == (order - 1)) + { + book.CoverImage = RPictureFile.Duplicate(picture.Result); + } + + pages.Add(page); + } + } + else + { + using (RMuseumDbContext importJobUpdaterDb = new RMuseumDbContext(Configuration)) + { + job.EndTime = DateTime.Now; + job.Status = ImportJobStatus.Failed; + job.Exception = $"Http result is not ok ({imageResult.StatusCode}) for page {order}, url {imageUrl}"; + importJobUpdaterDb.Update(job); + await importJobUpdaterDb.SaveChangesAsync(); + } + + imageResult.Dispose(); + return new RServiceResult>(null, "failed"); + } + + imageResult.Dispose(); + GC.Collect(); + } + } + } + } + else + { + using (RMuseumDbContext importJobUpdaterDb = new RMuseumDbContext(Configuration)) + { + job.EndTime = DateTime.Now; + job.Status = ImportJobStatus.Failed; + job.Exception = $"Http result is not ok ({jsonResult.StatusCode}) for https://iiif.lib.harvard.edu/manifests/drs:{hardvardResourceNumber}"; + importJobUpdaterDb.Update(job); + await importJobUpdaterDb.SaveChangesAsync(); + } + return new RServiceResult>(null, "failed"); + } + } + } + + return new RServiceResult>(pages); + + + } + + } +} diff --git a/RMuseum/Services/Implementation/ArtifactService-Imports/ArtifactService-ImportHathiTrust.cs b/RMuseum/Services/Implementation/ArtifactService-Imports/ArtifactService-ImportHathiTrust.cs new file mode 100644 index 00000000..494a50ee --- /dev/null +++ b/RMuseum/Services/Implementation/ArtifactService-Imports/ArtifactService-ImportHathiTrust.cs @@ -0,0 +1,537 @@ +using Microsoft.AspNetCore.Http; +using Microsoft.EntityFrameworkCore; +using Microsoft.Extensions.Configuration; +using Newtonsoft.Json.Linq; +using RMuseum.DbContext; +using RMuseum.Models.Artifact; +using RMuseum.Models.Artifact.ViewModels; +using RMuseum.Models.Bookmark; +using RMuseum.Models.Bookmark.ViewModels; +using RMuseum.Models.GanjoorIntegration; +using RMuseum.Models.GanjoorIntegration.ViewModels; +using RMuseum.Models.ImportJob; +using RMuseum.Models.Note; +using RMuseum.Models.Note.ViewModels; +using RMuseum.Models.Notification; +using RSecurityBackend.Models.Auth.ViewModels; +using RSecurityBackend.Models.Generic; +using RSecurityBackend.Services; +using RSecurityBackend.Services.Implementation; +using System; +using System.Collections.Generic; +using System.IO; +using System.Linq; +using System.Net; +using System.Net.Http; +using System.Security.Cryptography; +using System.Text.RegularExpressions; +using System.Threading; +using System.Threading.Tasks; +using System.Xml.Linq; +using System.Drawing; +using System.Drawing.Imaging; + +namespace RMuseum.Services.Implementation +{ + + /// + /// IArtifactService implementation + /// + public partial class ArtifactService : IArtifactService + { + /// + /// from https://catalog.hathitrust.org + /// + /// 006814127 + /// + /// + private async Task> StartImportingFromHathiTrust(string resourceNumber, string friendlyUrl) + { + string url = $"https://catalog.hathitrust.org/Record/{resourceNumber}.xml"; + if ( + ( + await _context.ImportJobs + .Where(j => j.JobType == JobType.HathiTrust && j.ResourceNumber == resourceNumber && !(j.Status == ImportJobStatus.Failed || j.Status == ImportJobStatus.Aborted)) + .SingleOrDefaultAsync() + ) + != + null + ) + { + return new RServiceResult(false, $"Job is already scheduled or running for importing {url}"); + } + + if (string.IsNullOrEmpty(friendlyUrl)) + { + friendlyUrl = resourceNumber; + } + + if ( + (await _context.Artifacts.Where(a => a.FriendlyUrl == friendlyUrl).SingleOrDefaultAsync()) + != + null + ) + { + return new RServiceResult(false, $"duplicated artifact friendly url '{friendlyUrl}'"); + } + + + + ImportJob job = new ImportJob() + { + JobType = JobType.HathiTrust, + ResourceNumber = resourceNumber, + FriendlyUrl = friendlyUrl, + SrcUrl = url, + QueueTime = DateTime.Now, + ProgressPercent = 0, + Status = ImportJobStatus.NotStarted + }; + + + await _context.ImportJobs.AddAsync + ( + job + ); + + await _context.SaveChangesAsync(); + + + try + { + + _backgroundTaskQueue.QueueBackgroundWorkItem + ( + async token => + { + try + { + + + using (var client = new HttpClient()) + { + + using (var result = await client.GetAsync(url)) + { + if (result.IsSuccessStatusCode) + { + using (RMuseumDbContext context = new RMuseumDbContext(Configuration)) + { + if ( + (await context.Artifacts.Where(a => a.FriendlyUrl == job.FriendlyUrl).SingleOrDefaultAsync()) + != + null + ) + { + using (RMuseumDbContext importJobUpdaterDb = new RMuseumDbContext(Configuration)) + { + job.EndTime = DateTime.Now; + job.Status = ImportJobStatus.Failed; + job.Exception = "aborted because of duplicated friendly url"; + importJobUpdaterDb.Update(job); + await importJobUpdaterDb.SaveChangesAsync(); + } + return; + } + RArtifactMasterRecord book = new RArtifactMasterRecord($"extracted from {url}", $"extracted from {url}") + { + Status = PublishStatus.Draft, + DateTime = DateTime.Now, + LastModified = DateTime.Now, + CoverItemIndex = 0, + FriendlyUrl = friendlyUrl + }; + + + List meta = new List(); + RTagValue tag; + + string xml = await result.Content.ReadAsStringAsync(); + + + using (RMuseumDbContext importJobUpdaterDb = new RMuseumDbContext(Configuration)) + { + job.StartTime = DateTime.Now; + job.Status = ImportJobStatus.Running; + job.SrcContent = xml; + importJobUpdaterDb.Update(job); + await importJobUpdaterDb.SaveChangesAsync(); + } + + string title = ""; + string author = ""; + string pdfResourceNumber = ""; + int tagOrder = 1; + XElement elObject = XDocument.Parse(xml).Root; + foreach (var datafield in elObject.Element("record").Elements("datafield")) + { + tagOrder++; + if (datafield.Attribute("tag") == null) + continue; + string hathiTrustTag = datafield.Attribute("tag").Value; + switch (hathiTrustTag) + { + case "245": + case "246": + foreach (var subfield in datafield.Elements("subfield")) + { + if (subfield.Attribute("code") != null) + { + if (subfield.Attribute("code").Value == "a" || subfield.Attribute("code").Value == "f") + title = (title + " " + subfield.Value).Trim(); + } + } + break; + case "100": + foreach (var subfield in datafield.Elements("subfield")) + { + if (subfield.Attribute("code") != null) + { + if (subfield.Attribute("code").Value == "a" || subfield.Attribute("code").Value == "d") + author = (author + " " + subfield.Value).Trim(); + } + } + break; + case "HOL": + foreach (var subfield in datafield.Elements("subfield")) + { + if (subfield.Attribute("code") != null) + { + if (subfield.Attribute("code").Value == "p") + pdfResourceNumber = subfield.Value; + } + } + break; + + default: + { + if (int.TryParse(hathiTrustTag, out int tmp)) + { + if (tmp >= 100 && tmp <= 900) + { + string note = ""; + foreach (var subfield in datafield.Elements("subfield")) + { + if (subfield.Attribute("code") != null) + { + note = (note + " " + subfield.Value).Trim(); + } + } + + tag = await TagHandler.PrepareAttribute(context, "Notes", note, tagOrder); + meta.Add(tag); + } + } + } + break; + } + + } + + if (string.IsNullOrEmpty(pdfResourceNumber)) + { + using (RMuseumDbContext importJobUpdaterDb = new RMuseumDbContext(Configuration)) + { + job.EndTime = DateTime.Now; + job.Status = ImportJobStatus.Failed; + job.Exception = "pdfResourceNumber not found"; + importJobUpdaterDb.Update(job); + await importJobUpdaterDb.SaveChangesAsync(); + } + return; + } + + tag = await TagHandler.PrepareAttribute(context, "Title", title, 1); + meta.Add(tag); + + tag = await TagHandler.PrepareAttribute(context, "Contributor Names", author, 1); + meta.Add(tag); + + tag = await TagHandler.PrepareAttribute(context, "Type", "Book", 1); + meta.Add(tag); + + tag = await TagHandler.PrepareAttribute(context, "Type", "Manuscript", 1); + meta.Add(tag); + + + tag = await TagHandler.PrepareAttribute(context, "Source", "HathiTrust Digital Library", 1); + string viewerUrl = $"https://babel.hathitrust.org/cgi/pt?id={pdfResourceNumber}"; + tag.ValueSupplement = viewerUrl; + + meta.Add(tag); + + book.Name = book.NameInEnglish = book.Description = book.DescriptionInEnglish = title; + book.Tags = meta.ToArray(); + + List pages = new List(); + + string lastMD5hash = ""; + int order = 0; + while (true) + { + order++; + using (RMuseumDbContext importJobUpdaterDb = new RMuseumDbContext(Configuration)) + { + job.ProgressPercent = order; + importJobUpdaterDb.Update(job); + await importJobUpdaterDb.SaveChangesAsync(); + } + + string imageUrl = $"https://babel.hathitrust.org/cgi/imgsrv/image?id={pdfResourceNumber};seq={order};size=1000;rotation=0"; + RArtifactItemRecord page = new RArtifactItemRecord() + { + Name = $"تصویر {order}", + NameInEnglish = $"Image {order}", + Description = "", + DescriptionInEnglish = "", + Order = order, + FriendlyUrl = $"p{$"{order}".PadLeft(4, '0')}", + LastModified = DateTime.Now + }; + + tag = await TagHandler.PrepareAttribute(context, "Source", "HathiTrust Digital Library", 1); + tag.ValueSupplement = viewerUrl; + page.Tags = new RTagValue[] { tag }; + + if (!string.IsNullOrEmpty(imageUrl)) + { + bool recovered = false; + if ( + File.Exists + ( + Path.Combine(Path.Combine(Path.Combine(_pictureFileService.ImageStoragePath, friendlyUrl), "orig"), $"{order}".PadLeft(4, '0') + ".jpg") + ) + && + File.Exists + ( + Path.Combine(Path.Combine(Path.Combine(_pictureFileService.ImageStoragePath, friendlyUrl), "norm"), $"{order}".PadLeft(4, '0') + ".jpg") + ) + && + File.Exists + ( + Path.Combine(Path.Combine(Path.Combine(_pictureFileService.ImageStoragePath, friendlyUrl), "thumb"), $"{order}".PadLeft(4, '0') + ".jpg") + ) + ) + { + RServiceResult picture = await _pictureFileService.RecoverFromeFiles(page.Name, page.Description, 1, + imageUrl, + Path.Combine(Path.Combine(Path.Combine(_pictureFileService.ImageStoragePath, friendlyUrl), "orig"), $"{order}".PadLeft(4, '0') + ".jpg"), + Path.Combine(Path.Combine(Path.Combine(_pictureFileService.ImageStoragePath, friendlyUrl), "norm"), $"{order}".PadLeft(4, '0') + ".jpg"), + Path.Combine(Path.Combine(Path.Combine(_pictureFileService.ImageStoragePath, friendlyUrl), "thumb"), $"{order}".PadLeft(4, '0') + ".jpg"), + $"{order}".PadLeft(4, '0') + ".jpg", friendlyUrl); + if (picture.Result != null) + { + recovered = true; + page.Images = new RPictureFile[] { picture.Result }; + page.CoverImageIndex = 0; + + if (book.CoverItemIndex == (order - 1)) + { + book.CoverImage = RPictureFile.Duplicate(picture.Result); + } + + pages.Add(page); + } + + } + if (!recovered) + { + if ( + File.Exists + ( + Path.Combine(Path.Combine(Path.Combine(_pictureFileService.ImageStoragePath, friendlyUrl), "orig"), $"{order}".PadLeft(4, '0') + ".jpg") + ) + + ) + { + File.Delete + ( + Path.Combine(Path.Combine(Path.Combine(_pictureFileService.ImageStoragePath, friendlyUrl), "orig"), $"{order}".PadLeft(4, '0') + ".jpg") + ); + } + if ( + + File.Exists + ( + Path.Combine(Path.Combine(Path.Combine(_pictureFileService.ImageStoragePath, friendlyUrl), "norm"), $"{order}".PadLeft(4, '0') + ".jpg") + ) + + ) + { + File.Delete + ( + Path.Combine(Path.Combine(Path.Combine(_pictureFileService.ImageStoragePath, friendlyUrl), "norm"), $"{order}".PadLeft(4, '0') + ".jpg") + ); + } + if ( + + File.Exists + ( + Path.Combine(Path.Combine(Path.Combine(_pictureFileService.ImageStoragePath, friendlyUrl), "thumb"), $"{order}".PadLeft(4, '0') + ".jpg") + ) + ) + { + File.Delete + ( + Path.Combine(Path.Combine(Path.Combine(_pictureFileService.ImageStoragePath, friendlyUrl), "thumb"), $"{order}".PadLeft(4, '0') + ".jpg") + ); + } + var imageResult = await client.GetAsync(imageUrl); + + + int _ImportRetryCount = 200; + int _ImportRetryInitialSleep = 500; + int retryCount = 0; + while (retryCount < _ImportRetryCount && !imageResult.IsSuccessStatusCode && imageResult.StatusCode == HttpStatusCode.ServiceUnavailable) + { + imageResult.Dispose(); + Thread.Sleep(_ImportRetryInitialSleep * (retryCount + 1)); + imageResult = await client.GetAsync(imageUrl); + retryCount++; + } + + if (imageResult.IsSuccessStatusCode) + { + using (Stream imageStream = await imageResult.Content.ReadAsStreamAsync()) + { + RServiceResult picture = await _pictureFileService.Add(page.Name, page.Description, 1, null, imageUrl, imageStream, $"{order}".PadLeft(4, '0') + ".jpg", friendlyUrl); + if (picture.Result == null) + { + throw new Exception($"_pictureFileService.Add : {picture.ExceptionString}"); + } + + bool lastPage = false; + using (var md5 = MD5.Create()) + { + string md5hash = string.Join("", md5.ComputeHash(File.ReadAllBytes(Path.Combine(Path.Combine(Path.Combine(_pictureFileService.ImageStoragePath, friendlyUrl), "thumb"), $"{order}".PadLeft(4, '0') + ".jpg"))).Select(x => x.ToString("X2"))); + if (md5hash == lastMD5hash) + { + File.Delete + ( + Path.Combine(Path.Combine(Path.Combine(_pictureFileService.ImageStoragePath, friendlyUrl), "orig"), $"{order}".PadLeft(4, '0') + ".jpg") + ); + File.Delete + ( + Path.Combine(Path.Combine(Path.Combine(_pictureFileService.ImageStoragePath, friendlyUrl), "norm"), $"{order}".PadLeft(4, '0') + ".jpg") + ); + File.Delete + ( + Path.Combine(Path.Combine(Path.Combine(_pictureFileService.ImageStoragePath, friendlyUrl), "thumb"), $"{order}".PadLeft(4, '0') + ".jpg") + ); + lastPage = true; + } + lastMD5hash = md5hash; + } + + if (!lastPage) + { + page.Images = new RPictureFile[] { picture.Result }; + page.CoverImageIndex = 0; + + if (book.CoverItemIndex == (order - 1)) + { + book.CoverImage = RPictureFile.Duplicate(picture.Result); + } + + pages.Add(page); + } + else + { + break; + } + + } + } + else + { + using (RMuseumDbContext importJobUpdaterDb = new RMuseumDbContext(Configuration)) + { + job.EndTime = DateTime.Now; + job.Status = ImportJobStatus.Failed; + job.Exception = $"Http result is not ok ({imageResult.StatusCode}) for page {order}, url {imageUrl}"; + importJobUpdaterDb.Update(job); + await importJobUpdaterDb.SaveChangesAsync(); + } + + imageResult.Dispose(); + return; + } + + imageResult.Dispose(); + GC.Collect(); + } + } + } + + + book.Items = pages.ToArray(); + book.ItemCount = pages.Count; + + if (pages.Count == 0) + { + using (RMuseumDbContext importJobUpdaterDb = new RMuseumDbContext(Configuration)) + { + job.EndTime = DateTime.Now; + job.Status = ImportJobStatus.Failed; + job.Exception = "ages.Count == 0"; + importJobUpdaterDb.Update(job); + await importJobUpdaterDb.SaveChangesAsync(); + } + return; + } + + await context.Artifacts.AddAsync(book); + await context.SaveChangesAsync(); + + job.ProgressPercent = 100; + job.Status = ImportJobStatus.Succeeded; + job.ArtifactId = book.Id; + job.EndTime = DateTime.Now; + context.Update(job); + await context.SaveChangesAsync(); + + + } + } + else + { + using (RMuseumDbContext importJobUpdaterDb = new RMuseumDbContext(Configuration)) + { + job.EndTime = DateTime.Now; + job.Status = ImportJobStatus.Failed; + job.Exception = $"Http result is not ok ({result.StatusCode}) for {url}"; + importJobUpdaterDb.Update(job); + await importJobUpdaterDb.SaveChangesAsync(); + } + return; + } + + } + } + } + catch (Exception exp) + { + using (RMuseumDbContext importJobUpdaterDb = new RMuseumDbContext(Configuration)) + { + job.EndTime = DateTime.Now; + job.Status = ImportJobStatus.Failed; + job.Exception = exp.ToString(); + importJobUpdaterDb.Update(job); + await importJobUpdaterDb.SaveChangesAsync(); + } + + } + } + ); + + + + return new RServiceResult(true); + } + catch (Exception exp) + { + return new RServiceResult(false, exp.ToString()); + } + } + } +} diff --git a/RMuseum/Services/Implementation/ArtifactService-Imports/ArtifactService-ImportLoc.cs b/RMuseum/Services/Implementation/ArtifactService-Imports/ArtifactService-ImportLoc.cs new file mode 100644 index 00000000..ce793379 --- /dev/null +++ b/RMuseum/Services/Implementation/ArtifactService-Imports/ArtifactService-ImportLoc.cs @@ -0,0 +1,545 @@ +using Microsoft.EntityFrameworkCore; +using Newtonsoft.Json.Linq; +using RMuseum.DbContext; +using RMuseum.Models.Artifact; +using RMuseum.Models.ImportJob; +using RSecurityBackend.Models.Generic; +using System; +using System.Collections.Generic; +using System.IO; +using System.Linq; +using System.Net; +using System.Net.Http; +using System.Threading; +using System.Threading.Tasks; + +namespace RMuseum.Services.Implementation +{ + /// + /// IArtifactService implementation + /// + public partial class ArtifactService : IArtifactService + { + /// + /// from https://www.loc.gov + /// + /// + /// + /// m084 + /// + /// + /// + /// + /// boostan1207 + /// + /// + /// + /// + /// plmp + /// + /// + private async Task> StartImportingFromTheLibraryOfCongress(string resourceNumber, string friendlyUrl, string resourcePrefix) + { + string url = $"https://www.loc.gov/resource/{resourcePrefix}.{resourceNumber}/?fo=json&st=gallery"; + + if ( + ( + await _context.ImportJobs + .Where(j => j.JobType == JobType.Loc && j.ResourceNumber == resourceNumber && !(j.Status == ImportJobStatus.Failed || j.Status == ImportJobStatus.Aborted)) + .SingleOrDefaultAsync() + ) + != + null + ) + { + return new RServiceResult(false, $"Job is already scheduled or running for importing {url}"); + } + + if (string.IsNullOrEmpty(friendlyUrl)) + { + friendlyUrl = resourceNumber; + } + + if ( + (await _context.Artifacts.Where(a => a.FriendlyUrl == friendlyUrl).SingleOrDefaultAsync()) + != + null + ) + { + return new RServiceResult(false, $"duplicated friendly url '{friendlyUrl}'"); + } + + ImportJob job = new ImportJob() + { + JobType = JobType.Loc, + ResourceNumber = resourceNumber, + FriendlyUrl = friendlyUrl, + SrcUrl = url, + QueueTime = DateTime.Now, + ProgressPercent = 0, + Status = ImportJobStatus.NotStarted + }; + + + await _context.ImportJobs.AddAsync + ( + job + ); + + await _context.SaveChangesAsync(); + + try + { + + _backgroundTaskQueue.QueueBackgroundWorkItem + ( + async token => + { + try + { + using (RMuseumDbContext importJobUpdaterDb = new RMuseumDbContext(Configuration)) + { + job.StartTime = DateTime.Now; + job.Status = ImportJobStatus.Running; + importJobUpdaterDb.Update(job); + await importJobUpdaterDb.SaveChangesAsync(); + } + + int pageCount = 0; + int representative_index = 0; + //اول یک صفحه را می‌خوانیم تا تعداد صفحات را مشخص کنیم + using (var client = new HttpClient()) + { + + using (var result = await client.GetAsync(url)) + { + if (result.IsSuccessStatusCode) + { + string json = await result.Content.ReadAsStringAsync(); + var parsed = JObject.Parse(json); + + pageCount = parsed.SelectToken("resource.segment_count").Value(); + representative_index = parsed.SelectToken("resource.representative_index").Value(); + } + else + { + using (RMuseumDbContext importJobUpdaterDb = new RMuseumDbContext(Configuration)) + { + job.EndTime = DateTime.Now; + job.Status = ImportJobStatus.Failed; + job.Exception = $"Http result is not ok ({result.StatusCode}) for {url}"; + importJobUpdaterDb.Update(job); + await importJobUpdaterDb.SaveChangesAsync(); + } + return; + } + + } + } + + //here might be problems: loc json does not return correct answer when number of segments are more than 1000 + /* + if (pageCount > 1000) + { + job.Exception = $"Page count ({pageCount}) was cut to 1000 for this artifact due to loc bug."; + pageCount = 1000; + } + */ + + //حالا که تعداد صفحات را داریم دوباره می‌خوانیم + url = $"https://www.loc.gov/resource/{resourcePrefix}.{resourceNumber}/?c={pageCount}&fo=json&st=gallery"; + using (var client = new HttpClient()) + { + + using (var result = await client.GetAsync(url)) + { + if (result.IsSuccessStatusCode) + { + + //here is a problem, this method could be called from a background service where _context is disposed, so I need to renew it + using (RMuseumDbContext context = new RMuseumDbContext(Configuration)) + { + RArtifactMasterRecord book = new RArtifactMasterRecord($"extracted from {url}", $"extracted from {url}") + { + Status = PublishStatus.Draft, + DateTime = DateTime.Now, + LastModified = DateTime.Now, + CoverItemIndex = representative_index, + FriendlyUrl = friendlyUrl + }; + + string json = await result.Content.ReadAsStringAsync(); + + job.SrcContent = json; + + var parsed = JObject.Parse(json); + + var segmentsArray = parsed.SelectToken("segments").ToArray(); + //here might be problems: loc json does not return correct answer when number of segments are more than 1000 + //I've added some temporary solutions prior + //Here I want to log any paradox I encounter: + if (segmentsArray.Length != pageCount) + { + job.Exception = $"Page count ({pageCount}) is not equal to number of returned resources ({segmentsArray.Length})."; + } + + + + List meta = new List(); + + string string_value = await HandleSimpleValue(context, parsed, meta, "item.title", "Title"); + if (!string.IsNullOrWhiteSpace(string_value)) + { + book.Name = string_value; + book.NameInEnglish = string_value; + } + await HandleSimpleValue(context, parsed, meta, "item.date", "Date"); + string_value = await HandleListValue(context, parsed, meta, "item.other_title", "Other Title"); + if (!string.IsNullOrWhiteSpace(string_value)) + { + book.Name = string_value; + } + await HandleListValue(context, parsed, meta, "item.contributor_names", "Contributor Names"); + await HandleSimpleValue(context, parsed, meta, "item.shelf_id", "Shelf ID"); + await HandleListValue(context, parsed, meta, "item.created_published", "Created / Published"); + await HandleListValue(context, parsed, meta, "item.subject_headings", "Subject Headings"); + await HandleListValue(context, parsed, meta, "item.notes", "Notes"); + await HandleListValue(context, parsed, meta, "item.medium", "Medium"); + await HandleListValue(context, parsed, meta, "item.call_number", "Call Number/Physical Location"); + await HandleListValue(context, parsed, meta, "item.digital_id", "Digital Id"); + await HandleSimpleValue(context, parsed, meta, "item.library_of_congress_control_number", "Library of Congress Control Number"); + await HandleChildrenValue(context, parsed, meta, "item.language", "Language"); + await HandleListValue(context, parsed, meta, "item.online_format", "Online Format"); + await HandleListValue(context, parsed, meta, "item.number_oclc", "OCLC Number"); + string_value = await HandleListValue(context, parsed, meta, "item.description", "Description"); + if (!string.IsNullOrEmpty(string_value)) + { + book.Description = string_value; + book.DescriptionInEnglish = string_value; + } + await HandleSimpleValue(context, parsed, meta, "cite_this.chicago", "Chicago citation style"); + await HandleSimpleValue(context, parsed, meta, "cite_this.apa", "APA citation style"); + await HandleSimpleValue(context, parsed, meta, "cite_this.mla", "MLA citation style"); + await HandleChildrenValue(context, parsed, meta, "item.dates", "Dates"); + await HandleChildrenValue(context, parsed, meta, "item.contributors", "Contributors"); + await HandleChildrenValue(context, parsed, meta, "item.location", "Location"); + await HandleListValue(context, parsed, meta, "item.rights", "Rights & Access"); + + RTagValue tag = await TagHandler.PrepareAttribute(context, "Type", "Book", 1); + meta.Add(tag); + + tag = await TagHandler.PrepareAttribute(context, "Type", "Manuscript", 1); + meta.Add(tag); + + + tag = await TagHandler.PrepareAttribute(context, "Source", "Library of Congress, African and Middle East Division, Near East Section Persian Manuscript Collection", 1); + tag.ValueSupplement = url; + string_value = parsed.SelectToken("item.id").Value(); + if (!string.IsNullOrWhiteSpace(string_value)) + { + tag.ValueSupplement = string_value; + } + + meta.Add(tag); + + + book.Tags = meta.ToArray(); + + + + + int order = 0; + List pages = new List(); + + //due to loc bug for books with more than 1000 pages relying on segmentsArray changed to hard coded image urls and .... + //foreach (JToken segment in segmentsArray) + for (int pageIndex = 1; pageIndex <= pageCount; pageIndex++) + { + /* + using (var fakeClient = new HttpClient()) + { + + using (var fakeRes = await fakeClient.GetAsync("https://ganjgah.ir/api/artifacts/keep-alive")) + { + if (fakeRes.IsSuccessStatusCode) + { + await fakeRes.Content.ReadAsStringAsync(); + } + } + } + */ + + using (RMuseumDbContext importJobUpdaterDb = new RMuseumDbContext(Configuration)) + { + job.ProgressPercent = order * 100 / (decimal)pageCount; + importJobUpdaterDb.Update(job); + await importJobUpdaterDb.SaveChangesAsync(); + } + + order++; + + + + RArtifactItemRecord page = new RArtifactItemRecord() + { + Name = $"تصویر {order}", + NameInEnglish = $"Image {pageIndex} of {book.NameInEnglish}",//segment.SelectToken("title").Value(), + Description = "", + DescriptionInEnglish = "", + Order = order, + FriendlyUrl = $"p{$"{order}".PadLeft(4, '0')}", + LastModified = DateTime.Now + }; + + tag = await TagHandler.PrepareAttribute(context, "Source", "Library of Congress, African and Middle East Division, Near East Section Persian Manuscript Collection", 1); + tag.ValueSupplement = $"http://www.loc.gov/resource/{resourcePrefix}.{resourceNumber}/?sp={pageIndex}";//segment.SelectToken("id").Value(); + page.Tags = new RTagValue[] { tag }; + + string imageUrlPart = $"{pageIndex}".PadLeft(4, '0'); + //string imageUrl = $"https://tile.loc.gov/image-services/iiif/service:amed:{resourcePrefix}:{resourceNumber}:{imageUrlPart}/full/pct:100/0/default.jpg"; + string imageUrl = $"https://tile.loc.gov/image-services/iiif/service:rbc:{resourcePrefix}:2015:{resourceNumber}:{imageUrlPart}/full/pct:100/0/default.jpg"; + /* + List list = segment.SelectToken("image_url").ToObject>(); + if (list != null && list.Count > 0) + { + for (int i = 0; i < list.Count; i++) + { + if (list[i].IndexOf(".jpg") != -1) + { + if (imageUrl == "") + imageUrl = list[i]; + else + { + if (imageUrl.IndexOf("#h=") != -1 && imageUrl.IndexOf("&w=", imageUrl.IndexOf("#h=")) != -1) + { + int h1 = int.Parse(imageUrl.Substring(imageUrl.IndexOf("#h=") + "#h=".Length, imageUrl.IndexOf("&w=") - imageUrl.IndexOf("#h=") - "&w=".Length)); + if (list[i].IndexOf("#h=") != -1 && list[i].IndexOf("&w=", list[i].IndexOf("#h=")) != -1) + { + int h2 = int.Parse(list[i].Substring(list[i].IndexOf("#h=") + "#h=".Length, list[i].IndexOf("&w=") - list[i].IndexOf("#h=") - "&w=".Length)); + + if (h2 > h1) + { + imageUrl = list[i]; + } + } + } + else + imageUrl = list[i]; + + } + } + } + } + */ + + + + if (!string.IsNullOrEmpty(imageUrl)) + { + //imageUrl = "https:" + imageUrl.Substring(0, imageUrl.IndexOf('#')); + bool recovered = false; + if ( + File.Exists + ( + Path.Combine(Path.Combine(Path.Combine(_pictureFileService.ImageStoragePath, friendlyUrl), "orig"), $"{order}".PadLeft(4, '0') + ".jpg") + ) + && + File.Exists + ( + Path.Combine(Path.Combine(Path.Combine(_pictureFileService.ImageStoragePath, friendlyUrl), "norm"), $"{order}".PadLeft(4, '0') + ".jpg") + ) + && + File.Exists + ( + Path.Combine(Path.Combine(Path.Combine(_pictureFileService.ImageStoragePath, friendlyUrl), "thumb"), $"{order}".PadLeft(4, '0') + ".jpg") + ) + ) + { + RServiceResult picture = await _pictureFileService.RecoverFromeFiles(page.Name, page.Description, 1, + imageUrl, + Path.Combine(Path.Combine(Path.Combine(_pictureFileService.ImageStoragePath, friendlyUrl), "orig"), $"{order}".PadLeft(4, '0') + ".jpg"), + Path.Combine(Path.Combine(Path.Combine(_pictureFileService.ImageStoragePath, friendlyUrl), "norm"), $"{order}".PadLeft(4, '0') + ".jpg"), + Path.Combine(Path.Combine(Path.Combine(_pictureFileService.ImageStoragePath, friendlyUrl), "thumb"), $"{order}".PadLeft(4, '0') + ".jpg"), + $"{order}".PadLeft(4, '0') + ".jpg", friendlyUrl); + if (picture.Result != null) + { + recovered = true; + page.Images = new RPictureFile[] { picture.Result }; + page.CoverImageIndex = 0; + + if (book.CoverItemIndex == (order - 1)) + { + book.CoverImage = RPictureFile.Duplicate(picture.Result); + } + + pages.Add(page); + } + + } + if (!recovered) + { + if ( + File.Exists + ( + Path.Combine(Path.Combine(Path.Combine(_pictureFileService.ImageStoragePath, friendlyUrl), "orig"), $"{order}".PadLeft(4, '0') + ".jpg") + ) + + ) + { + File.Delete + ( + Path.Combine(Path.Combine(Path.Combine(_pictureFileService.ImageStoragePath, friendlyUrl), "orig"), $"{order}".PadLeft(4, '0') + ".jpg") + ); + } + if ( + + File.Exists + ( + Path.Combine(Path.Combine(Path.Combine(_pictureFileService.ImageStoragePath, friendlyUrl), "norm"), $"{order}".PadLeft(4, '0') + ".jpg") + ) + + ) + { + File.Delete + ( + Path.Combine(Path.Combine(Path.Combine(_pictureFileService.ImageStoragePath, friendlyUrl), "norm"), $"{order}".PadLeft(4, '0') + ".jpg") + ); + } + if ( + + File.Exists + ( + Path.Combine(Path.Combine(Path.Combine(_pictureFileService.ImageStoragePath, friendlyUrl), "thumb"), $"{order}".PadLeft(4, '0') + ".jpg") + ) + ) + { + File.Delete + ( + Path.Combine(Path.Combine(Path.Combine(_pictureFileService.ImageStoragePath, friendlyUrl), "thumb"), $"{order}".PadLeft(4, '0') + ".jpg") + ); + } + var imageResult = await client.GetAsync(imageUrl); + + + int _ImportRetryCount = 5; + int _ImportRetryInitialSleep = 500; + int retryCount = 0; + while (retryCount < _ImportRetryCount && !imageResult.IsSuccessStatusCode && imageResult.StatusCode == HttpStatusCode.ServiceUnavailable) + { + imageResult.Dispose(); + Thread.Sleep(_ImportRetryInitialSleep * (retryCount + 1)); + imageResult = await client.GetAsync(imageUrl); + retryCount++; + } + + if (imageResult.IsSuccessStatusCode) + { + using (Stream imageStream = await imageResult.Content.ReadAsStreamAsync()) + { + RServiceResult picture = await _pictureFileService.Add(page.Name, page.Description, 1, null, imageUrl, imageStream, $"{order}".PadLeft(4, '0') + ".jpg", friendlyUrl); + if (picture.Result == null) + { + throw new Exception($"_pictureFileService.Add : {picture.ExceptionString}"); + } + + page.Images = new RPictureFile[] { picture.Result }; + page.CoverImageIndex = 0; + + if (book.CoverItemIndex == (order - 1)) + { + book.CoverImage = RPictureFile.Duplicate(picture.Result); + } + + pages.Add(page); + } + } + else + { + using (RMuseumDbContext importJobUpdaterDb = new RMuseumDbContext(Configuration)) + { + job.EndTime = DateTime.Now; + job.Status = ImportJobStatus.Failed; + job.Exception = $"Http result is not ok ({imageResult.StatusCode}) for page {order}, url {imageUrl}"; + importJobUpdaterDb.Update(job); + await importJobUpdaterDb.SaveChangesAsync(); + } + + imageResult.Dispose(); + return; + } + + imageResult.Dispose(); + GC.Collect(); + } + + + + + + + } + } + + book.Items = pages.ToArray(); + book.ItemCount = pages.Count; + + if (book.CoverImage == null && pages.Count > 0) + { + book.CoverImage = RPictureFile.Duplicate(pages[0].Images.First()); + } + + await context.Artifacts.AddAsync(book); + await context.SaveChangesAsync(); + + job.ProgressPercent = 100; + job.Status = ImportJobStatus.Succeeded; + job.ArtifactId = book.Id; + job.EndTime = DateTime.Now; + context.Update(job); + await context.SaveChangesAsync(); + + } + + } + else + { + using (RMuseumDbContext importJobUpdaterDb = new RMuseumDbContext(Configuration)) + { + job.EndTime = DateTime.Now; + job.Status = ImportJobStatus.Failed; + job.Exception = $"Http result is not ok ({result.StatusCode}) for {url}"; + importJobUpdaterDb.Update(job); + await importJobUpdaterDb.SaveChangesAsync(); + } + return; + } + } + + } + } + catch (Exception exp) + { + using (RMuseumDbContext importJobUpdaterDb = new RMuseumDbContext(Configuration)) + { + job.EndTime = DateTime.Now; + job.Status = ImportJobStatus.Failed; + job.Exception = exp.ToString(); + importJobUpdaterDb.Update(job); + await importJobUpdaterDb.SaveChangesAsync(); + } + + } + } + ); + + + + return new RServiceResult(true); + } + catch (Exception exp) + { + return new RServiceResult(false, exp.ToString()); + } + } + } +} diff --git a/RMuseum/Services/Implementation/ArtifactService-Imports/ArtifactService-ImportPenLibraries.cs b/RMuseum/Services/Implementation/ArtifactService-Imports/ArtifactService-ImportPenLibraries.cs new file mode 100644 index 00000000..5192e36b --- /dev/null +++ b/RMuseum/Services/Implementation/ArtifactService-Imports/ArtifactService-ImportPenLibraries.cs @@ -0,0 +1,519 @@ +using Microsoft.AspNetCore.Http; +using Microsoft.EntityFrameworkCore; +using Microsoft.Extensions.Configuration; +using Newtonsoft.Json.Linq; +using RMuseum.DbContext; +using RMuseum.Models.Artifact; +using RMuseum.Models.Artifact.ViewModels; +using RMuseum.Models.Bookmark; +using RMuseum.Models.Bookmark.ViewModels; +using RMuseum.Models.GanjoorIntegration; +using RMuseum.Models.GanjoorIntegration.ViewModels; +using RMuseum.Models.ImportJob; +using RMuseum.Models.Note; +using RMuseum.Models.Note.ViewModels; +using RMuseum.Models.Notification; +using RSecurityBackend.Models.Auth.ViewModels; +using RSecurityBackend.Models.Generic; +using RSecurityBackend.Services; +using RSecurityBackend.Services.Implementation; +using System; +using System.Collections.Generic; +using System.IO; +using System.Linq; +using System.Net; +using System.Net.Http; +using System.Security.Cryptography; +using System.Text.RegularExpressions; +using System.Threading; +using System.Threading.Tasks; +using System.Xml.Linq; +using System.Drawing; +using System.Drawing.Imaging; + +namespace RMuseum.Services.Implementation +{ + + /// + /// IArtifactService implementation + /// + public partial class ArtifactService : IArtifactService + { + /// + /// from http://www.library.upenn.edu/ + /// + /// MEDREN_9949222153503681 + /// + /// + private async Task> StartImportingFromPenLibraries(string resourceNumber, string friendlyUrl) + { + string url = $"http://dla.library.upenn.edu/dla/medren/pageturn.html?id={resourceNumber}&rotation=0&size=0"; + if ( + ( + await _context.ImportJobs + .Where(j => j.JobType == JobType.PennLibraries && j.ResourceNumber == resourceNumber && !(j.Status == ImportJobStatus.Failed || j.Status == ImportJobStatus.Aborted)) + .SingleOrDefaultAsync() + ) + != + null + ) + { + return new RServiceResult(false, $"Job is already scheduled or running for importing {url}"); + } + + if (string.IsNullOrEmpty(friendlyUrl)) + { + friendlyUrl = resourceNumber; + } + + if ( + (await _context.Artifacts.Where(a => a.FriendlyUrl == friendlyUrl).SingleOrDefaultAsync()) + != + null + ) + { + return new RServiceResult(false, $"duplicated artifact friendly url '{friendlyUrl}'"); + } + + + + ImportJob job = new ImportJob() + { + JobType = JobType.PennLibraries, + ResourceNumber = resourceNumber, + FriendlyUrl = friendlyUrl, + SrcUrl = url, + QueueTime = DateTime.Now, + ProgressPercent = 0, + Status = ImportJobStatus.NotStarted + }; + + + await _context.ImportJobs.AddAsync + ( + job + ); + + await _context.SaveChangesAsync(); + + + try + { + + _backgroundTaskQueue.QueueBackgroundWorkItem + ( + async token => + { + try + { + + + using (var client = new HttpClient()) + { + client.Timeout = TimeSpan.FromMinutes(5); + using (var result = await client.GetAsync(url)) + { + if (result.IsSuccessStatusCode) + { + using (RMuseumDbContext context = new RMuseumDbContext(Configuration)) + { + if ( + (await context.Artifacts.Where(a => a.FriendlyUrl == job.FriendlyUrl).SingleOrDefaultAsync()) + != + null + ) + { + using (RMuseumDbContext importJobUpdaterDb = new RMuseumDbContext(Configuration)) + { + job.EndTime = DateTime.Now; + job.Status = ImportJobStatus.Failed; + job.Exception = "aborted because of duplicated friendly url"; + importJobUpdaterDb.Update(job); + await importJobUpdaterDb.SaveChangesAsync(); + } + return; + } + RArtifactMasterRecord book = new RArtifactMasterRecord($"extracted from {url}", $"extracted from {url}") + { + Status = PublishStatus.Draft, + DateTime = DateTime.Now, + LastModified = DateTime.Now, + CoverItemIndex = 0, + FriendlyUrl = friendlyUrl + }; + + + List meta = new List(); + RTagValue tag; + + string html = await result.Content.ReadAsStringAsync(); + + + using (RMuseumDbContext importJobUpdaterDb = new RMuseumDbContext(Configuration)) + { + job.StartTime = DateTime.Now; + job.Status = ImportJobStatus.Running; + job.SrcContent = html; + importJobUpdaterDb.Update(job); + await importJobUpdaterDb.SaveChangesAsync(); + } + + string title = ""; + string author = ""; + int tagOrder = 1; + + int nIdxStart = html.IndexOf("https://repo.library.upenn.edu/djatoka/resolver?"); + if (nIdxStart == -1) + { + using (RMuseumDbContext importJobUpdaterDb = new RMuseumDbContext(Configuration)) + { + job.EndTime = DateTime.Now; + job.Status = ImportJobStatus.Failed; + job.Exception = "https://repo.library.upenn.edu/djatoka/resolver? not found"; + importJobUpdaterDb.Update(job); + await importJobUpdaterDb.SaveChangesAsync(); + } + return; + } + + string firstImageUrl = html.Substring(nIdxStart, html.IndexOf('"', nIdxStart) - nIdxStart).Replace("&", "&"); + + nIdxStart = html.IndexOf("recordinfolabel"); + while (nIdxStart != -1) + { + nIdxStart += "recordinfolabel\">".Length; + int nIdxEnd = html.IndexOf(":", nIdxStart); + string recordinfolabel = html.Substring(nIdxStart, nIdxEnd - nIdxStart); + nIdxStart = html.IndexOf("recordinfotext", nIdxEnd); + nIdxStart += "recordinfotext\">".Length; + nIdxEnd = html.IndexOf("", nIdxStart); + string recordinfotext = html.Substring(nIdxStart, nIdxEnd - nIdxStart).Replace("", "
").Replace("\n", "").Replace("\r", "").Trim(); + + string[] values = recordinfotext.Split("
", StringSplitOptions.RemoveEmptyEntries); + + foreach (string value in values) + { + if (value.Trim().Length == 0) + continue; + if (recordinfolabel == "Title") + { + title = value.Trim(); + tag = await TagHandler.PrepareAttribute(context, "Title", title, 1); + meta.Add(tag); + } + else + if (recordinfolabel == "Author") + { + author = value.Trim(); + tag = await TagHandler.PrepareAttribute(context, "Contributor Names", author, 1); + meta.Add(tag); + } + else + { + tag = await TagHandler.PrepareAttribute(context, recordinfolabel, value.Trim(), tagOrder++); + meta.Add(tag); + } + } + + nIdxStart = html.IndexOf("recordinfolabel", nIdxEnd); + + } + + + + tag = await TagHandler.PrepareAttribute(context, "Type", "Book", 1); + meta.Add(tag); + + tag = await TagHandler.PrepareAttribute(context, "Type", "Manuscript", 1); + meta.Add(tag); + + + tag = await TagHandler.PrepareAttribute(context, "Source", "Penn Libraries", 1); + string viewerUrl = $"http://dla.library.upenn.edu/dla/medren/detail.html?id={resourceNumber}"; + tag.ValueSupplement = viewerUrl; + + meta.Add(tag); + + book.Name = book.NameInEnglish = book.Description = book.DescriptionInEnglish = title; + book.Tags = meta.ToArray(); + + List pages = new List(); + + int order = 0; + while (true) + { + order++; + + using (RMuseumDbContext importJobUpdaterDb = new RMuseumDbContext(Configuration)) + { + job.ProgressPercent = order; + importJobUpdaterDb.Update(job); + await importJobUpdaterDb.SaveChangesAsync(); + } + + string imageUrl = firstImageUrl; + + RArtifactItemRecord page = new RArtifactItemRecord() + { + Name = $"تصویر {order}", + NameInEnglish = $"Image {order}", + Description = "", + DescriptionInEnglish = "", + Order = order, + FriendlyUrl = $"p{$"{order}".PadLeft(4, '0')}", + LastModified = DateTime.Now + }; + + tag = await TagHandler.PrepareAttribute(context, "Source", "Penn Libraries", 1); + tag.ValueSupplement = viewerUrl; + page.Tags = new RTagValue[] { tag }; + + if (!string.IsNullOrEmpty(imageUrl)) + { + bool recovered = false; + if ( + File.Exists + ( + Path.Combine(Path.Combine(Path.Combine(_pictureFileService.ImageStoragePath, friendlyUrl), "orig"), $"{order}".PadLeft(4, '0') + ".jpg") + ) + && + File.Exists + ( + Path.Combine(Path.Combine(Path.Combine(_pictureFileService.ImageStoragePath, friendlyUrl), "norm"), $"{order}".PadLeft(4, '0') + ".jpg") + ) + && + File.Exists + ( + Path.Combine(Path.Combine(Path.Combine(_pictureFileService.ImageStoragePath, friendlyUrl), "thumb"), $"{order}".PadLeft(4, '0') + ".jpg") + ) + ) + { + RServiceResult picture = await _pictureFileService.RecoverFromeFiles(page.Name, page.Description, 1, + imageUrl, + Path.Combine(Path.Combine(Path.Combine(_pictureFileService.ImageStoragePath, friendlyUrl), "orig"), $"{order}".PadLeft(4, '0') + ".jpg"), + Path.Combine(Path.Combine(Path.Combine(_pictureFileService.ImageStoragePath, friendlyUrl), "norm"), $"{order}".PadLeft(4, '0') + ".jpg"), + Path.Combine(Path.Combine(Path.Combine(_pictureFileService.ImageStoragePath, friendlyUrl), "thumb"), $"{order}".PadLeft(4, '0') + ".jpg"), + $"{order}".PadLeft(4, '0') + ".jpg", friendlyUrl); + if (picture.Result != null) + { + recovered = true; + page.Images = new RPictureFile[] { picture.Result }; + page.CoverImageIndex = 0; + + if (book.CoverItemIndex == (order - 1)) + { + book.CoverImage = RPictureFile.Duplicate(picture.Result); + } + + pages.Add(page); + } + + } + if (!recovered) + { + if ( + File.Exists + ( + Path.Combine(Path.Combine(Path.Combine(_pictureFileService.ImageStoragePath, friendlyUrl), "orig"), $"{order}".PadLeft(4, '0') + ".jpg") + ) + + ) + { + File.Delete + ( + Path.Combine(Path.Combine(Path.Combine(_pictureFileService.ImageStoragePath, friendlyUrl), "orig"), $"{order}".PadLeft(4, '0') + ".jpg") + ); + } + if ( + + File.Exists + ( + Path.Combine(Path.Combine(Path.Combine(_pictureFileService.ImageStoragePath, friendlyUrl), "norm"), $"{order}".PadLeft(4, '0') + ".jpg") + ) + + ) + { + File.Delete + ( + Path.Combine(Path.Combine(Path.Combine(_pictureFileService.ImageStoragePath, friendlyUrl), "norm"), $"{order}".PadLeft(4, '0') + ".jpg") + ); + } + if ( + + File.Exists + ( + Path.Combine(Path.Combine(Path.Combine(_pictureFileService.ImageStoragePath, friendlyUrl), "thumb"), $"{order}".PadLeft(4, '0') + ".jpg") + ) + ) + { + File.Delete + ( + Path.Combine(Path.Combine(Path.Combine(_pictureFileService.ImageStoragePath, friendlyUrl), "thumb"), $"{order}".PadLeft(4, '0') + ".jpg") + ); + } + + if (order > 1) + { + string pageUrl = $"http://dla.library.upenn.edu/dla/medren/pageturn.html?id={resourceNumber}&doubleside=0&rotation=0&size=0¤tpage={order}"; + var pageResult = await client.GetAsync(pageUrl); + + if (pageResult.StatusCode == HttpStatusCode.NotFound) + { + break;//finished + } + + string pageHtml = await pageResult.Content.ReadAsStringAsync(); + nIdxStart = pageHtml.IndexOf("https://repo.library.upenn.edu/djatoka/resolver?"); + if (nIdxStart == -1) + { + if (order > 1) + break; //finished + using (RMuseumDbContext importJobUpdaterDb = new RMuseumDbContext(Configuration)) + { + job.EndTime = DateTime.Now; + job.Status = ImportJobStatus.Failed; + job.Exception = $"https://repo.library.upenn.edu/djatoka/resolver? not found on page {order}"; + importJobUpdaterDb.Update(job); + await importJobUpdaterDb.SaveChangesAsync(); + } + return; + } + + imageUrl = pageHtml.Substring(nIdxStart, pageHtml.IndexOf('"', nIdxStart) - nIdxStart).Replace("&", "&"); + + } + var imageResult = await client.GetAsync(imageUrl); + + if (imageResult.StatusCode == HttpStatusCode.NotFound) + { + break;//finished + } + + + int _ImportRetryCount = 200; + int _ImportRetryInitialSleep = 500; + int retryCount = 0; + while (retryCount < _ImportRetryCount && !imageResult.IsSuccessStatusCode && imageResult.StatusCode == HttpStatusCode.ServiceUnavailable) + { + imageResult.Dispose(); + Thread.Sleep(_ImportRetryInitialSleep * (retryCount + 1)); + imageResult = await client.GetAsync(imageUrl); + retryCount++; + } + + if (imageResult.IsSuccessStatusCode) + { + using (Stream imageStream = await imageResult.Content.ReadAsStreamAsync()) + { + RServiceResult picture = await _pictureFileService.Add(page.Name, page.Description, 1, null, imageUrl, imageStream, $"{order}".PadLeft(4, '0') + ".jpg", friendlyUrl); + if (picture.Result == null) + { + throw new Exception($"_pictureFileService.Add : {picture.ExceptionString}"); + } + + + page.Images = new RPictureFile[] { picture.Result }; + page.CoverImageIndex = 0; + + if (book.CoverItemIndex == (order - 1)) + { + book.CoverImage = RPictureFile.Duplicate(picture.Result); + } + + pages.Add(page); + + } + } + else + { + using (RMuseumDbContext importJobUpdaterDb = new RMuseumDbContext(Configuration)) + { + job.EndTime = DateTime.Now; + job.Status = ImportJobStatus.Failed; + job.Exception = $"Http result is not ok ({imageResult.StatusCode}) for page {order}, url {imageUrl}"; + importJobUpdaterDb.Update(job); + await importJobUpdaterDb.SaveChangesAsync(); + } + + imageResult.Dispose(); + return; + } + + imageResult.Dispose(); + GC.Collect(); + } + } + } + + + book.Items = pages.ToArray(); + book.ItemCount = pages.Count; + + if (pages.Count == 0) + { + using (RMuseumDbContext importJobUpdaterDb = new RMuseumDbContext(Configuration)) + { + job.EndTime = DateTime.Now; + job.Status = ImportJobStatus.Failed; + job.Exception = "Pages.Count == 0"; + importJobUpdaterDb.Update(job); + await importJobUpdaterDb.SaveChangesAsync(); + } + return; + } + + await context.Artifacts.AddAsync(book); + await context.SaveChangesAsync(); + + job.ProgressPercent = 100; + job.Status = ImportJobStatus.Succeeded; + job.ArtifactId = book.Id; + job.EndTime = DateTime.Now; + context.Update(job); + await context.SaveChangesAsync(); + + + } + } + else + { + using (RMuseumDbContext importJobUpdaterDb = new RMuseumDbContext(Configuration)) + { + job.EndTime = DateTime.Now; + job.Status = ImportJobStatus.Failed; + job.Exception = $"Http result is not ok ({result.StatusCode}) for {url}"; + importJobUpdaterDb.Update(job); + await importJobUpdaterDb.SaveChangesAsync(); + } + return; + } + + } + } + } + catch (Exception exp) + { + using (RMuseumDbContext importJobUpdaterDb = new RMuseumDbContext(Configuration)) + { + job.EndTime = DateTime.Now; + job.Status = ImportJobStatus.Failed; + job.Exception = exp.ToString(); + importJobUpdaterDb.Update(job); + await importJobUpdaterDb.SaveChangesAsync(); + } + + } + } + ); + + + + return new RServiceResult(true); + } + catch (Exception exp) + { + return new RServiceResult(false, exp.ToString()); + } + } + } +} diff --git a/RMuseum/Services/Implementation/ArtifactService-Imports/ArtifactService-ImportServerFolder.cs b/RMuseum/Services/Implementation/ArtifactService-Imports/ArtifactService-ImportServerFolder.cs new file mode 100644 index 00000000..55af4bec --- /dev/null +++ b/RMuseum/Services/Implementation/ArtifactService-Imports/ArtifactService-ImportServerFolder.cs @@ -0,0 +1,287 @@ +using Microsoft.AspNetCore.Http; +using Microsoft.EntityFrameworkCore; +using Microsoft.Extensions.Configuration; +using Newtonsoft.Json.Linq; +using RMuseum.DbContext; +using RMuseum.Models.Artifact; +using RMuseum.Models.Artifact.ViewModels; +using RMuseum.Models.Bookmark; +using RMuseum.Models.Bookmark.ViewModels; +using RMuseum.Models.GanjoorIntegration; +using RMuseum.Models.GanjoorIntegration.ViewModels; +using RMuseum.Models.ImportJob; +using RMuseum.Models.Note; +using RMuseum.Models.Note.ViewModels; +using RMuseum.Models.Notification; +using RSecurityBackend.Models.Auth.ViewModels; +using RSecurityBackend.Models.Generic; +using RSecurityBackend.Services; +using RSecurityBackend.Services.Implementation; +using System; +using System.Collections.Generic; +using System.IO; +using System.Linq; +using System.Net; +using System.Net.Http; +using System.Security.Cryptography; +using System.Text.RegularExpressions; +using System.Threading; +using System.Threading.Tasks; +using System.Xml.Linq; +using System.Drawing; +using System.Drawing.Imaging; + +namespace RMuseum.Services.Implementation +{ + + /// + /// IArtifactService implementation + /// + public partial class ArtifactService : IArtifactService + { + /// + /// import from server folder + /// + /// C:\Tools\batches\florence + /// shahname-florence + /// https://t.me/dr_khatibi_abolfazl/888 + /// + private async Task> StartImportingFromServerFolder(string folderPath, string friendlyUrl, string srcUrl) + { + try + { + if ( + ( + await _context.ImportJobs + .Where(j => j.JobType == JobType.ServerFolder && j.ResourceNumber == folderPath && !(j.Status == ImportJobStatus.Failed || j.Status == ImportJobStatus.Aborted)) + .SingleOrDefaultAsync() + ) + != + null + ) + { + return new RServiceResult(false, $"Job is already scheduled or running for importing server folder {folderPath}"); + } + + if (string.IsNullOrEmpty(friendlyUrl)) + { + return new RServiceResult(false, $"Friendly url is empty, server folder {folderPath}"); + } + + if ( + (await _context.Artifacts.Where(a => a.FriendlyUrl == friendlyUrl).SingleOrDefaultAsync()) + != + null + ) + { + return new RServiceResult(false, $"duplicated friendly url '{friendlyUrl}'"); + } + + ImportJob job = new ImportJob() + { + JobType = JobType.ServerFolder, + ResourceNumber = folderPath, + FriendlyUrl = friendlyUrl, + SrcUrl = srcUrl, + QueueTime = DateTime.Now, + ProgressPercent = 0, + Status = ImportJobStatus.NotStarted + }; + + + await _context.ImportJobs.AddAsync + ( + job + ); + + await _context.SaveChangesAsync(); + + _backgroundTaskQueue.QueueBackgroundWorkItem + ( + async token => + { + try + { + using (RMuseumDbContext context = new RMuseumDbContext(Configuration)) + { + RArtifactMasterRecord book = new RArtifactMasterRecord($"extracted from server folder {job.ResourceNumber}", $"extracted from server folder {job.ResourceNumber}") + { + Status = PublishStatus.Draft, + DateTime = DateTime.Now, + LastModified = DateTime.Now, + CoverItemIndex = 0, + FriendlyUrl = friendlyUrl + }; + + + List meta = new List(); + RTagValue tag; + + + tag = await TagHandler.PrepareAttribute(context, "Type", "Book", 1); + meta.Add(tag); + + tag = await TagHandler.PrepareAttribute(context, "Type", "Manuscript", 1); + meta.Add(tag); + + + + meta.Add(tag); + + using (RMuseumDbContext importJobUpdaterDb = new RMuseumDbContext(Configuration)) + { + job.StartTime = DateTime.Now; + job.Status = ImportJobStatus.Running; + job.SrcContent = ""; + importJobUpdaterDb.Update(job); + await importJobUpdaterDb.SaveChangesAsync(); + } + + List pages = new List(); + string[] fileNames = Directory.GetFiles(job.ResourceNumber, "*.jpg"); + int order = 0; + foreach (string fileName in fileNames) + { + using (RMuseumDbContext importJobUpdaterDb = new RMuseumDbContext(Configuration)) + { + job.ProgressPercent = order * 100 / (decimal)fileNames.Length; + importJobUpdaterDb.Update(job); + await importJobUpdaterDb.SaveChangesAsync(); + } + + order++; + + RArtifactItemRecord page = new RArtifactItemRecord() + { + Name = $"تصویر {order}", + NameInEnglish = $"Image {order} of {book.NameInEnglish}", + Description = "", + DescriptionInEnglish = "", + Order = order, + FriendlyUrl = $"p{$"{order}".PadLeft(4, '0')}", + LastModified = DateTime.Now + }; + + + page.Tags = new RTagValue[] { }; + + if ( + File.Exists + ( + Path.Combine(Path.Combine(Path.Combine(_pictureFileService.ImageStoragePath, friendlyUrl), "orig"), $"{order}".PadLeft(4, '0') + ".jpg") + ) + + ) + { + File.Delete + ( + Path.Combine(Path.Combine(Path.Combine(_pictureFileService.ImageStoragePath, friendlyUrl), "orig"), $"{order}".PadLeft(4, '0') + ".jpg") + ); + } + if ( + + File.Exists + ( + Path.Combine(Path.Combine(Path.Combine(_pictureFileService.ImageStoragePath, friendlyUrl), "norm"), $"{order}".PadLeft(4, '0') + ".jpg") + ) + + ) + { + File.Delete + ( + Path.Combine(Path.Combine(Path.Combine(_pictureFileService.ImageStoragePath, friendlyUrl), "norm"), $"{order}".PadLeft(4, '0') + ".jpg") + ); + } + if ( + + File.Exists + ( + Path.Combine(Path.Combine(Path.Combine(_pictureFileService.ImageStoragePath, friendlyUrl), "thumb"), $"{order}".PadLeft(4, '0') + ".jpg") + ) + ) + { + File.Delete + ( + Path.Combine(Path.Combine(Path.Combine(_pictureFileService.ImageStoragePath, friendlyUrl), "thumb"), $"{order}".PadLeft(4, '0') + ".jpg") + ); + } + using (FileStream imageStream = new FileStream(fileName, FileMode.Open)) + { + RServiceResult picture = await _pictureFileService.Add(page.Name, page.Description, 1, null, job.SrcUrl, imageStream, $"{order}".PadLeft(4, '0') + ".jpg", friendlyUrl); + if (picture.Result == null) + { + throw new Exception($"_pictureFileService.Add : {picture.ExceptionString}"); + } + + page.Images = new RPictureFile[] { picture.Result }; + + page.CoverImageIndex = 0; + + if (book.CoverItemIndex == (order - 1)) + { + book.CoverImage = RPictureFile.Duplicate(picture.Result); + } + + } + + + pages.Add(page); + } + + + book.Tags = meta.ToArray(); + + book.Items = pages.ToArray(); + book.ItemCount = pages.Count; + + if (pages.Count == 0) + { + using (RMuseumDbContext importJobUpdaterDb = new RMuseumDbContext(Configuration)) + { + job.EndTime = DateTime.Now; + job.Status = ImportJobStatus.Failed; + job.Exception = "Pages.Count == 0"; + importJobUpdaterDb.Update(job); + await importJobUpdaterDb.SaveChangesAsync(); + } + return; + } + + await context.Artifacts.AddAsync(book); + await context.SaveChangesAsync(); + + job.ProgressPercent = 100; + job.Status = ImportJobStatus.Succeeded; + job.ArtifactId = book.Id; + job.EndTime = DateTime.Now; + context.Update(job); + await context.SaveChangesAsync(); + + + + } + } + catch (Exception exp) + { + using (RMuseumDbContext importJobUpdaterDb = new RMuseumDbContext(Configuration)) + { + job.EndTime = DateTime.Now; + job.Status = ImportJobStatus.Failed; + job.Exception = exp.ToString(); + importJobUpdaterDb.Update(job); + await importJobUpdaterDb.SaveChangesAsync(); + } + + } + } + ); + + return new RServiceResult(true); + } + catch (Exception exp) + { + return new RServiceResult(false, exp.ToString()); + } + } + } +} diff --git a/RMuseum/Services/Implementation/ArtifactService-Imports/ArtifactService-ImportWalters.cs b/RMuseum/Services/Implementation/ArtifactService-Imports/ArtifactService-ImportWalters.cs new file mode 100644 index 00000000..3d0f98b2 --- /dev/null +++ b/RMuseum/Services/Implementation/ArtifactService-Imports/ArtifactService-ImportWalters.cs @@ -0,0 +1,506 @@ +using Microsoft.AspNetCore.Http; +using Microsoft.EntityFrameworkCore; +using Microsoft.Extensions.Configuration; +using Newtonsoft.Json.Linq; +using RMuseum.DbContext; +using RMuseum.Models.Artifact; +using RMuseum.Models.Artifact.ViewModels; +using RMuseum.Models.Bookmark; +using RMuseum.Models.Bookmark.ViewModels; +using RMuseum.Models.GanjoorIntegration; +using RMuseum.Models.GanjoorIntegration.ViewModels; +using RMuseum.Models.ImportJob; +using RMuseum.Models.Note; +using RMuseum.Models.Note.ViewModels; +using RMuseum.Models.Notification; +using RSecurityBackend.Models.Auth.ViewModels; +using RSecurityBackend.Models.Generic; +using RSecurityBackend.Services; +using RSecurityBackend.Services.Implementation; +using System; +using System.Collections.Generic; +using System.IO; +using System.Linq; +using System.Net; +using System.Net.Http; +using System.Security.Cryptography; +using System.Text.RegularExpressions; +using System.Threading; +using System.Threading.Tasks; +using System.Xml.Linq; +using System.Drawing; +using System.Drawing.Imaging; + +namespace RMuseum.Services.Implementation +{ + + /// + /// IArtifactService implementation + /// + public partial class ArtifactService : IArtifactService + { + /// + /// from http://www.thedigitalwalters.org/01_ACCESS_WALTERS_MANUSCRIPTS.html + /// + /// W619 + /// golestan-walters-01 + /// + private async Task> StartImportingFromWalters(string resourceNumber, string friendlyUrl) + { + string url = $"http://www.thedigitalwalters.org/Data/WaltersManuscripts/ManuscriptDescriptions/{resourceNumber}_tei.xml"; + if ( + ( + await _context.ImportJobs + .Where(j => j.JobType == JobType.Walters && j.ResourceNumber == resourceNumber && !(j.Status == ImportJobStatus.Failed || j.Status == ImportJobStatus.Aborted)) + .SingleOrDefaultAsync() + ) + != + null + ) + { + return new RServiceResult(false, $"Job is already scheduled or running for importing {url}"); + } + + if (string.IsNullOrEmpty(friendlyUrl)) + { + friendlyUrl = resourceNumber; + } + + if ( + (await _context.Artifacts.Where(a => a.FriendlyUrl == friendlyUrl).SingleOrDefaultAsync()) + != + null + ) + { + return new RServiceResult(false, $"duplicated friendly url '{friendlyUrl}'"); + } + + ImportJob job = new ImportJob() + { + JobType = JobType.Walters, + ResourceNumber = resourceNumber, + FriendlyUrl = friendlyUrl, + SrcUrl = url, + QueueTime = DateTime.Now, + ProgressPercent = 0, + Status = ImportJobStatus.NotStarted + }; + + + await _context.ImportJobs.AddAsync + ( + job + ); + + await _context.SaveChangesAsync(); + + + try + { + + _backgroundTaskQueue.QueueBackgroundWorkItem + ( + async token => + { + try + { + + + using (var client = new HttpClient()) + { + + using (var result = await client.GetAsync(url)) + { + if (result.IsSuccessStatusCode) + { + using (RMuseumDbContext context = new RMuseumDbContext(Configuration)) + { + RArtifactMasterRecord book = new RArtifactMasterRecord($"extracted from {url}", $"extracted from {url}") + { + Status = PublishStatus.Draft, + DateTime = DateTime.Now, + LastModified = DateTime.Now, + CoverItemIndex = 0, + FriendlyUrl = friendlyUrl + }; + + + List meta = new List(); + RTagValue tag; + + string xml = await result.Content.ReadAsStringAsync(); + + + using (RMuseumDbContext importJobUpdaterDb = new RMuseumDbContext(Configuration)) + { + job.StartTime = DateTime.Now; + job.Status = ImportJobStatus.Running; + job.SrcContent = xml; + importJobUpdaterDb.Update(job); + await importJobUpdaterDb.SaveChangesAsync(); + } + + XElement elObject = XDocument.Parse(xml).Root; + + tag = await TagHandler.PrepareAttribute(context, "Type", "Book", 1); + meta.Add(tag); + + tag = await TagHandler.PrepareAttribute(context, "Type", "Manuscript", 1); + meta.Add(tag); + + + + + + + try + { + foreach (var prop in elObject + .Elements("{http://www.tei-c.org/ns/1.0}teiHeader").First() + .Elements("{http://www.tei-c.org/ns/1.0}fileDesc").First() + .Elements("{http://www.tei-c.org/ns/1.0}titleStmt").First() + .Elements("{http://www.tei-c.org/ns/1.0}title")) + { + string label = prop.Value; + book.Name = book.NameInEnglish = + book.Description = book.DescriptionInEnglish = + label; + + tag = await TagHandler.PrepareAttribute(context, "Title", label, 1); + meta.Add(tag); + break; + } + } + catch + { + //ignore non-existing = null tags + } + + try + { + foreach (var prop in elObject + .Elements("{http://www.tei-c.org/ns/1.0}teiHeader").First() + .Elements("{http://www.tei-c.org/ns/1.0}fileDesc").First() + .Elements("{http://www.tei-c.org/ns/1.0}titleStmt").First() + .Elements("{http://www.tei-c.org/ns/1.0}author")) + { + string label = prop.Value; + tag = await TagHandler.PrepareAttribute(context, "Contributor Names", label, 1); + meta.Add(tag); + break; + } + } + catch + { + //ignore non-existing = null tags + } + + try + { + foreach (var prop in elObject + .Elements("{http://www.tei-c.org/ns/1.0}teiHeader").First() + .Elements("{http://www.tei-c.org/ns/1.0}fileDesc").First() + .Elements("{http://www.tei-c.org/ns/1.0}titleStmt").First() + .Elements("{http://www.tei-c.org/ns/1.0}respStmt")) + { + string label = prop.Elements("{http://www.tei-c.org/ns/1.0}name").First().Value; + tag = await TagHandler.PrepareAttribute(context, "Contributor Names", label, 1); + meta.Add(tag); + break; + + } + } + catch + { + //ignore non-existing = null tags + } + + try + { + foreach (var prop in elObject + .Elements("{http://www.tei-c.org/ns/1.0}teiHeader").First() + .Elements("{http://www.tei-c.org/ns/1.0}fileDesc").First() + .Elements("{http://www.tei-c.org/ns/1.0}notesStmt").First() + .Elements("{http://www.tei-c.org/ns/1.0}note")) + { + string label = prop.Value; + tag = await TagHandler.PrepareAttribute(context, "Notes", label, 1); + meta.Add(tag); + } + } + catch + { + //ignore non-existing = null tags + } + + + + + tag = await TagHandler.PrepareAttribute(context, "Source", "Digitized Walters Manuscripts", 1); + tag.ValueSupplement = $"http://www.thedigitalwalters.org/Data/WaltersManuscripts/html/{job.ResourceNumber}/"; + + meta.Add(tag); + + + book.Tags = meta.ToArray(); + List pages = new List(); + + int order = 0; + + foreach (var surface in elObject + .Elements("{http://www.tei-c.org/ns/1.0}facsimile").First() + .Elements("{http://www.tei-c.org/ns/1.0}surface")) + { + foreach (var graphic in surface.Elements("{http://www.tei-c.org/ns/1.0}graphic")) + if (graphic.Attribute("url").Value.Contains("sap.jpg")) + { + + order++; + + + using (RMuseumDbContext importJobUpdaterDb = new RMuseumDbContext(Configuration)) + { + job.ProgressPercent = order; + importJobUpdaterDb.Update(job); + await importJobUpdaterDb.SaveChangesAsync(); + } + + RArtifactItemRecord page = new RArtifactItemRecord() + { + Name = $"تصویر {order}", + NameInEnglish = $"Image {order}", + Description = "", + DescriptionInEnglish = "", + Order = order, + FriendlyUrl = $"p{$"{order}".PadLeft(4, '0')}", + LastModified = DateTime.Now + }; + + + + string imageUrl = $"http://www.thedigitalwalters.org/Data/WaltersManuscripts/{resourceNumber}/data/W.{resourceNumber.Substring(1)}/{graphic.Attribute("url").Value}"; + + tag = await TagHandler.PrepareAttribute(context, "Source", "Digitized Walters Manuscripts", 1); + tag.ValueSupplement = imageUrl; + page.Tags = new RTagValue[] { tag }; + + if (!string.IsNullOrEmpty(imageUrl)) + { + bool recovered = false; + if ( + File.Exists + ( + Path.Combine(Path.Combine(Path.Combine(_pictureFileService.ImageStoragePath, friendlyUrl), "orig"), $"{order}".PadLeft(4, '0') + ".jpg") + ) + && + File.Exists + ( + Path.Combine(Path.Combine(Path.Combine(_pictureFileService.ImageStoragePath, friendlyUrl), "norm"), $"{order}".PadLeft(4, '0') + ".jpg") + ) + && + File.Exists + ( + Path.Combine(Path.Combine(Path.Combine(_pictureFileService.ImageStoragePath, friendlyUrl), "thumb"), $"{order}".PadLeft(4, '0') + ".jpg") + ) + ) + { + RServiceResult picture = await _pictureFileService.RecoverFromeFiles(page.Name, page.Description, 1, + imageUrl, + Path.Combine(Path.Combine(Path.Combine(_pictureFileService.ImageStoragePath, friendlyUrl), "orig"), $"{order}".PadLeft(4, '0') + ".jpg"), + Path.Combine(Path.Combine(Path.Combine(_pictureFileService.ImageStoragePath, friendlyUrl), "norm"), $"{order}".PadLeft(4, '0') + ".jpg"), + Path.Combine(Path.Combine(Path.Combine(_pictureFileService.ImageStoragePath, friendlyUrl), "thumb"), $"{order}".PadLeft(4, '0') + ".jpg"), + $"{order}".PadLeft(4, '0') + ".jpg", friendlyUrl); + if (picture.Result != null) + { + recovered = true; + page.Images = new RPictureFile[] { picture.Result }; + page.CoverImageIndex = 0; + + if (book.CoverItemIndex == (order - 1)) + { + book.CoverImage = RPictureFile.Duplicate(picture.Result); + } + + pages.Add(page); + } + + } + if (!recovered) + { + if ( + File.Exists + ( + Path.Combine(Path.Combine(Path.Combine(_pictureFileService.ImageStoragePath, friendlyUrl), "orig"), $"{order}".PadLeft(4, '0') + ".jpg") + ) + + ) + { + File.Delete + ( + Path.Combine(Path.Combine(Path.Combine(_pictureFileService.ImageStoragePath, friendlyUrl), "orig"), $"{order}".PadLeft(4, '0') + ".jpg") + ); + } + if ( + + File.Exists + ( + Path.Combine(Path.Combine(Path.Combine(_pictureFileService.ImageStoragePath, friendlyUrl), "norm"), $"{order}".PadLeft(4, '0') + ".jpg") + ) + + ) + { + File.Delete + ( + Path.Combine(Path.Combine(Path.Combine(_pictureFileService.ImageStoragePath, friendlyUrl), "norm"), $"{order}".PadLeft(4, '0') + ".jpg") + ); + } + if ( + + File.Exists + ( + Path.Combine(Path.Combine(Path.Combine(_pictureFileService.ImageStoragePath, friendlyUrl), "thumb"), $"{order}".PadLeft(4, '0') + ".jpg") + ) + ) + { + File.Delete + ( + Path.Combine(Path.Combine(Path.Combine(_pictureFileService.ImageStoragePath, friendlyUrl), "thumb"), $"{order}".PadLeft(4, '0') + ".jpg") + ); + } + var imageResult = await client.GetAsync(imageUrl); + + + int _ImportRetryCount = 5; + int _ImportRetryInitialSleep = 500; + int retryCount = 0; + while (retryCount < _ImportRetryCount && !imageResult.IsSuccessStatusCode && imageResult.StatusCode == HttpStatusCode.ServiceUnavailable) + { + imageResult.Dispose(); + Thread.Sleep(_ImportRetryInitialSleep * (retryCount + 1)); + imageResult = await client.GetAsync(imageUrl); + retryCount++; + } + + if (imageResult.IsSuccessStatusCode) + { + using (Stream imageStream = await imageResult.Content.ReadAsStreamAsync()) + { + RServiceResult picture = await _pictureFileService.Add(page.Name, page.Description, 1, null, imageUrl, imageStream, $"{order}".PadLeft(4, '0') + ".jpg", friendlyUrl); + if (picture.Result == null) + { + throw new Exception($"_pictureFileService.Add : {picture.ExceptionString}"); + } + + page.Images = new RPictureFile[] { picture.Result }; + page.CoverImageIndex = 0; + + if (book.CoverItemIndex == (order - 1)) + { + book.CoverImage = RPictureFile.Duplicate(picture.Result); + } + + pages.Add(page); + } + } + else + { + using (RMuseumDbContext importJobUpdaterDb = new RMuseumDbContext(Configuration)) + { + job.EndTime = DateTime.Now; + job.Status = ImportJobStatus.Failed; + job.Exception = $"Http result is not ok ({imageResult.StatusCode}) for page {order}, url {imageUrl}"; + importJobUpdaterDb.Update(job); + await importJobUpdaterDb.SaveChangesAsync(); + } + + imageResult.Dispose(); + return; + } + + imageResult.Dispose(); + GC.Collect(); + } + + } + + + + + } + } + + + + book.Items = pages.ToArray(); + book.ItemCount = pages.Count; + + if (pages.Count == 0) + { + using (RMuseumDbContext importJobUpdaterDb = new RMuseumDbContext(Configuration)) + { + job.EndTime = DateTime.Now; + job.Status = ImportJobStatus.Failed; + job.Exception = "Pages.Count == 0"; + importJobUpdaterDb.Update(job); + await importJobUpdaterDb.SaveChangesAsync(); + } + return; + } + + await context.Artifacts.AddAsync(book); + await context.SaveChangesAsync(); + + job.ProgressPercent = 100; + job.Status = ImportJobStatus.Succeeded; + job.ArtifactId = book.Id; + job.EndTime = DateTime.Now; + context.Update(job); + await context.SaveChangesAsync(); + + + + } + } + else + { + using (RMuseumDbContext importJobUpdaterDb = new RMuseumDbContext(Configuration)) + { + job.EndTime = DateTime.Now; + job.Status = ImportJobStatus.Failed; + job.Exception = $"Http result is not ok ({result.StatusCode}) for {url}"; + importJobUpdaterDb.Update(job); + await importJobUpdaterDb.SaveChangesAsync(); + } + return; + } + + } + } + } + catch (Exception exp) + { + using (RMuseumDbContext importJobUpdaterDb = new RMuseumDbContext(Configuration)) + { + job.EndTime = DateTime.Now; + job.Status = ImportJobStatus.Failed; + job.Exception = exp.ToString(); + importJobUpdaterDb.Update(job); + await importJobUpdaterDb.SaveChangesAsync(); + } + + } + } + ); + + + + return new RServiceResult(true); + } + catch (Exception exp) + { + return new RServiceResult(false, exp.ToString()); + } + } + + } +} diff --git a/RMuseum/Services/Implementation/ArtifactService-Imports/ArtificatService-ImportPrinceton.cs b/RMuseum/Services/Implementation/ArtifactService-Imports/ArtificatService-ImportPrinceton.cs new file mode 100644 index 00000000..10d7e26e --- /dev/null +++ b/RMuseum/Services/Implementation/ArtifactService-Imports/ArtificatService-ImportPrinceton.cs @@ -0,0 +1,478 @@ +using Microsoft.EntityFrameworkCore; +using RMuseum.DbContext; +using RMuseum.Models.Artifact; +using RMuseum.Models.ImportJob; +using RSecurityBackend.Models.Generic; +using System; +using System.Collections.Generic; +using System.IO; +using System.Linq; +using System.Net; +using System.Net.Http; +using System.Threading; +using System.Threading.Tasks; +using System.Xml.Linq; + +namespace RMuseum.Services.Implementation +{ + + /// + /// IArtifactService implementation + /// + public partial class ArtifactService : IArtifactService + { + /// + /// from http://pudl.princeton.edu/ + /// + /// dj52w476m + /// + /// + private async Task> StartImportingFromPrinceton(string resourceNumber, string friendlyUrl) + { + string url = $"http://pudl.princeton.edu/mdCompiler2.php?obj={resourceNumber}"; + if ( + ( + await _context.ImportJobs + .Where(j => j.JobType == JobType.Princeton && j.ResourceNumber == resourceNumber && !(j.Status == ImportJobStatus.Failed || j.Status == ImportJobStatus.Aborted)) + .SingleOrDefaultAsync() + ) + != + null + ) + { + return new RServiceResult(false, $"Job is already scheduled or running for importing {url}"); + } + + if (string.IsNullOrEmpty(friendlyUrl)) + { + friendlyUrl = resourceNumber; + } + + if ( + (await _context.Artifacts.Where(a => a.FriendlyUrl == friendlyUrl).SingleOrDefaultAsync()) + != + null + ) + { + return new RServiceResult(false, $"duplicated friendly url '{friendlyUrl}'"); + } + + ImportJob job = new ImportJob() + { + JobType = JobType.Princeton, + ResourceNumber = resourceNumber, + FriendlyUrl = friendlyUrl, + SrcUrl = url, + QueueTime = DateTime.Now, + ProgressPercent = 0, + Status = ImportJobStatus.NotStarted + }; + + + await _context.ImportJobs.AddAsync + ( + job + ); + + await _context.SaveChangesAsync(); + + + try + { + + _backgroundTaskQueue.QueueBackgroundWorkItem + ( + async token => + { + try + { + + + using (var client = new HttpClient()) + { + + using (var result = await client.GetAsync(url)) + { + if (result.IsSuccessStatusCode) + { + using (RMuseumDbContext context = new RMuseumDbContext(Configuration)) + { + RArtifactMasterRecord book = new RArtifactMasterRecord($"extracted from {url}", $"extracted from {url}") + { + Status = PublishStatus.Draft, + DateTime = DateTime.Now, + LastModified = DateTime.Now, + CoverItemIndex = 0, + FriendlyUrl = friendlyUrl + }; + + + List meta = new List(); + RTagValue tag; + + string xml = await result.Content.ReadAsStringAsync(); + + + using (RMuseumDbContext importJobUpdaterDb = new RMuseumDbContext(Configuration)) + { + job.StartTime = DateTime.Now; + job.Status = ImportJobStatus.Running; + job.SrcContent = xml; + importJobUpdaterDb.Update(job); + await importJobUpdaterDb.SaveChangesAsync(); + } + + XElement elObject = XDocument.Parse(xml).Root; + foreach (var prop in elObject.Element("dmd").Element("properties").Elements("property")) + { + if (prop.Element("label") == null) + continue; + string label = prop.Element("label").Value.Replace(":", ""); + int order = 1; + foreach (var value in prop.Elements("valueGrp").Elements("value")) + { + tag = await TagHandler.PrepareAttribute(context, label, value.Value, order); + if (value.Attribute("href") != null) + { + if (value.Attribute("href").Value.IndexOf("http://localhost") != 0) + { + tag.ValueSupplement = value.Attribute("href").Value; + } + } + meta.Add(tag); + + if (label == "Title") + { + book.Name = book.NameInEnglish = + book.Description = book.DescriptionInEnglish = + value.Value; + } + order++; + } + } + + tag = await TagHandler.PrepareAttribute(context, "Type", "Book", 1); + meta.Add(tag); + + tag = await TagHandler.PrepareAttribute(context, "Type", "Manuscript", 1); + meta.Add(tag); + + + tag = await TagHandler.PrepareAttribute(context, "Source", "Princeton Digital Library of Islamic Manuscripts", 1); + tag.ValueSupplement = $"http://pudl.princeton.edu/objects/{job.ResourceNumber}"; + + meta.Add(tag); + + + book.Tags = meta.ToArray(); + List pages = new List(); + + foreach (var structure in elObject.Elements("structure")) + { + if (structure.Attribute("type") != null && structure.Attribute("type").Value == "RelatedObjects") + { + if (structure.Element("div") == null || structure.Element("div").Element("OrderedList") == null) + { + using (RMuseumDbContext importJobUpdaterDb = new RMuseumDbContext(Configuration)) + { + job.EndTime = DateTime.Now; + job.Status = ImportJobStatus.Failed; + job.Exception = "structure[RelatedObjects].div.OrderedList is null"; + importJobUpdaterDb.Update(job); + await importJobUpdaterDb.SaveChangesAsync(); + return; + } + } + + + int pageCount = structure.Element("div").Element("OrderedList").Elements("div").Count(); + int inlineOrder = 0; + + foreach (var div in structure.Element("div").Element("OrderedList").Elements("div")) + { + inlineOrder++; + using (RMuseumDbContext importJobUpdaterDb = new RMuseumDbContext(Configuration)) + { + job.ProgressPercent = inlineOrder * 100 / (decimal)pageCount; + importJobUpdaterDb.Update(job); + await importJobUpdaterDb.SaveChangesAsync(); + } + + int order = int.Parse(div.Attribute("order").Value); + RArtifactItemRecord page = new RArtifactItemRecord() + { + Name = $"تصویر {order}", + NameInEnglish = div.Attribute("label").Value, + Description = "", + DescriptionInEnglish = "", + Order = order, + FriendlyUrl = $"p{$"{order}".PadLeft(4, '0')}", + LastModified = DateTime.Now + }; + + string imageUrl = div.Attribute("img").Value; + imageUrl = "https://libimages.princeton.edu/loris/" + imageUrl.Substring(imageUrl.LastIndexOf(":") + 1); + imageUrl += $"/full/,{div.Attribute("h").Value}/0/default.jpg"; + + tag = await TagHandler.PrepareAttribute(context, "Source", "Princeton Digital Library of Islamic Manuscripts", 1); + tag.ValueSupplement = imageUrl; + page.Tags = new RTagValue[] { tag }; + + if (!string.IsNullOrEmpty(imageUrl)) + { + bool recovered = false; + if ( + File.Exists + ( + Path.Combine(Path.Combine(Path.Combine(_pictureFileService.ImageStoragePath, friendlyUrl), "orig"), $"{order}".PadLeft(4, '0') + ".jpg") + ) + && + File.Exists + ( + Path.Combine(Path.Combine(Path.Combine(_pictureFileService.ImageStoragePath, friendlyUrl), "norm"), $"{order}".PadLeft(4, '0') + ".jpg") + ) + && + File.Exists + ( + Path.Combine(Path.Combine(Path.Combine(_pictureFileService.ImageStoragePath, friendlyUrl), "thumb"), $"{order}".PadLeft(4, '0') + ".jpg") + ) + ) + { + RServiceResult picture = await _pictureFileService.RecoverFromeFiles(page.Name, page.Description, 1, + imageUrl, + Path.Combine(Path.Combine(Path.Combine(_pictureFileService.ImageStoragePath, friendlyUrl), "orig"), $"{order}".PadLeft(4, '0') + ".jpg"), + Path.Combine(Path.Combine(Path.Combine(_pictureFileService.ImageStoragePath, friendlyUrl), "norm"), $"{order}".PadLeft(4, '0') + ".jpg"), + Path.Combine(Path.Combine(Path.Combine(_pictureFileService.ImageStoragePath, friendlyUrl), "thumb"), $"{order}".PadLeft(4, '0') + ".jpg"), + $"{order}".PadLeft(4, '0') + ".jpg", friendlyUrl); + if (picture.Result != null) + { + recovered = true; + page.Images = new RPictureFile[] { picture.Result }; + page.CoverImageIndex = 0; + + if (book.CoverItemIndex == (order - 1)) + { + book.CoverImage = RPictureFile.Duplicate(picture.Result); + } + + pages.Add(page); + } + + } + if (!recovered) + { + if ( + File.Exists + ( + Path.Combine(Path.Combine(Path.Combine(_pictureFileService.ImageStoragePath, friendlyUrl), "orig"), $"{order}".PadLeft(4, '0') + ".jpg") + ) + + ) + { + File.Delete + ( + Path.Combine(Path.Combine(Path.Combine(_pictureFileService.ImageStoragePath, friendlyUrl), "orig"), $"{order}".PadLeft(4, '0') + ".jpg") + ); + } + if ( + + File.Exists + ( + Path.Combine(Path.Combine(Path.Combine(_pictureFileService.ImageStoragePath, friendlyUrl), "norm"), $"{order}".PadLeft(4, '0') + ".jpg") + ) + + ) + { + File.Delete + ( + Path.Combine(Path.Combine(Path.Combine(_pictureFileService.ImageStoragePath, friendlyUrl), "norm"), $"{order}".PadLeft(4, '0') + ".jpg") + ); + } + if ( + + File.Exists + ( + Path.Combine(Path.Combine(Path.Combine(_pictureFileService.ImageStoragePath, friendlyUrl), "thumb"), $"{order}".PadLeft(4, '0') + ".jpg") + ) + ) + { + File.Delete + ( + Path.Combine(Path.Combine(Path.Combine(_pictureFileService.ImageStoragePath, friendlyUrl), "thumb"), $"{order}".PadLeft(4, '0') + ".jpg") + ); + } + var imageResult = await client.GetAsync(imageUrl); + + + int _ImportRetryCount = 5; + int _ImportRetryInitialSleep = 500; + int retryCount = 0; + while (retryCount < _ImportRetryCount && !imageResult.IsSuccessStatusCode && imageResult.StatusCode == HttpStatusCode.ServiceUnavailable) + { + imageResult.Dispose(); + Thread.Sleep(_ImportRetryInitialSleep * (retryCount + 1)); + imageResult = await client.GetAsync(imageUrl); + retryCount++; + } + + if (imageResult.IsSuccessStatusCode) + { + using (Stream imageStream = await imageResult.Content.ReadAsStreamAsync()) + { + RServiceResult picture = await _pictureFileService.Add(page.Name, page.Description, 1, null, imageUrl, imageStream, $"{order}".PadLeft(4, '0') + ".jpg", friendlyUrl); + if (picture.Result == null) + { + throw new Exception($"_pictureFileService.Add : {picture.ExceptionString}"); + } + + page.Images = new RPictureFile[] { picture.Result }; + page.CoverImageIndex = 0; + + if (book.CoverItemIndex == (order - 1)) + { + book.CoverImage = RPictureFile.Duplicate(picture.Result); + } + + pages.Add(page); + } + } + else + { + using (RMuseumDbContext importJobUpdaterDb = new RMuseumDbContext(Configuration)) + { + job.EndTime = DateTime.Now; + job.Status = ImportJobStatus.Failed; + job.Exception = $"Http result is not ok ({imageResult.StatusCode}) for page {order}, url {imageUrl}"; + importJobUpdaterDb.Update(job); + await importJobUpdaterDb.SaveChangesAsync(); + } + + imageResult.Dispose(); + return; + } + + imageResult.Dispose(); + GC.Collect(); + } + + } + + + + } + + } + } + + foreach (var structure in elObject.Elements("structure")) + { + if (structure.Attribute("type") != null && structure.Attribute("type").Value == "Physical") + { + if (structure.Element("RTLBoundManuscript") != null) + { + foreach (var leaf in structure.Element("RTLBoundManuscript").Elements("Leaf")) + { + foreach (var side in leaf.Elements("Side")) + { + int pageOrder = int.Parse(side.Attribute("order").Value); + tag = await TagHandler.PrepareAttribute(context, "Leaf Side", side.Attribute("label").Value, 100); + RArtifactItemRecord page = pages.Where(p => p.Order == pageOrder).SingleOrDefault(); + if (page != null) + { + List tags = new List(page.Tags); + tags.Add(tag); + page.Tags = tags; + } + } + } + foreach (var folio in structure.Element("RTLBoundManuscript").Elements("Folio")) + { + foreach (var side in folio.Elements("Side")) + { + int pageOrder = int.Parse(side.Attribute("order").Value); + tag = await TagHandler.PrepareAttribute(context, "Folio Side", folio.Attribute("label").Value + ":" + side.Attribute("label").Value, 101); + RArtifactItemRecord page = pages.Where(p => p.Order == pageOrder).SingleOrDefault(); + if (page != null) + { + List tags = new List(page.Tags); + tags.Add(tag); + page.Tags = tags; + } + } + } + } + } + } + + book.Items = pages.ToArray(); + book.ItemCount = pages.Count; + + if (pages.Count == 0) + { + using (RMuseumDbContext importJobUpdaterDb = new RMuseumDbContext(Configuration)) + { + job.EndTime = DateTime.Now; + job.Status = ImportJobStatus.Failed; + job.Exception = "ages.Count == 0"; + importJobUpdaterDb.Update(job); + await importJobUpdaterDb.SaveChangesAsync(); + } + return; + } + + await context.Artifacts.AddAsync(book); + await context.SaveChangesAsync(); + + job.ProgressPercent = 100; + job.Status = ImportJobStatus.Succeeded; + job.ArtifactId = book.Id; + job.EndTime = DateTime.Now; + context.Update(job); + await context.SaveChangesAsync(); + + + + } + } + else + { + using (RMuseumDbContext importJobUpdaterDb = new RMuseumDbContext(Configuration)) + { + job.EndTime = DateTime.Now; + job.Status = ImportJobStatus.Failed; + job.Exception = $"Http result is not ok ({result.StatusCode}) for {url}"; + importJobUpdaterDb.Update(job); + await importJobUpdaterDb.SaveChangesAsync(); + } + return; + } + + } + } + } + catch (Exception exp) + { + using (RMuseumDbContext importJobUpdaterDb = new RMuseumDbContext(Configuration)) + { + job.EndTime = DateTime.Now; + job.Status = ImportJobStatus.Failed; + job.Exception = exp.ToString(); + importJobUpdaterDb.Update(job); + await importJobUpdaterDb.SaveChangesAsync(); + } + + } + } + ); + + + + return new RServiceResult(true); + } + catch (Exception exp) + { + return new RServiceResult(false, exp.ToString()); + } + } + } +} diff --git a/RMuseum/Services/Implementation/ArtifactService.cs b/RMuseum/Services/Implementation/ArtifactService.cs index f22de644..68149cd1 100644 --- a/RMuseum/Services/Implementation/ArtifactService.cs +++ b/RMuseum/Services/Implementation/ArtifactService.cs @@ -23,20 +23,15 @@ using System.IO; using System.Linq; using System.Net; using System.Net.Http; -using System.Security.Cryptography; -using System.Text.RegularExpressions; using System.Threading; using System.Threading.Tasks; -using System.Xml.Linq; -using System.Drawing; -using System.Drawing.Imaging; namespace RMuseum.Services.Implementation { /// /// IArtifactService implementation /// - public class ArtifactService : IArtifactService + public partial class ArtifactService : IArtifactService { /// @@ -1382,4772 +1377,7 @@ namespace RMuseum.Services.Implementation : await StartImportingFromTheLibraryOfCongress(resourceNumber, friendlyUrl, resourcePrefix); } - - - - /// - /// from https://www.loc.gov - /// - /// - /// - /// m084 - /// - /// - /// - /// - /// boostan1207 - /// - /// - /// - /// - /// plmp - /// - /// - private async Task> StartImportingFromTheLibraryOfCongress(string resourceNumber, string friendlyUrl, string resourcePrefix) - { - string url = $"https://www.loc.gov/resource/{resourcePrefix}.{resourceNumber}/?fo=json&st=gallery"; - - if ( - ( - await _context.ImportJobs - .Where(j => j.JobType == JobType.Loc && j.ResourceNumber == resourceNumber && !(j.Status == ImportJobStatus.Failed || j.Status == ImportJobStatus.Aborted)) - .SingleOrDefaultAsync() - ) - != - null - ) - { - return new RServiceResult(false, $"Job is already scheduled or running for importing {url}"); - } - - if(string.IsNullOrEmpty(friendlyUrl)) - { - friendlyUrl = resourceNumber; - } - - if( - (await _context.Artifacts.Where(a => a.FriendlyUrl == friendlyUrl).SingleOrDefaultAsync()) - != - null - ) - { - return new RServiceResult(false, $"duplicated friendly url '{friendlyUrl}'"); - } - - ImportJob job = new ImportJob() - { - JobType = JobType.Loc, - ResourceNumber = resourceNumber, - FriendlyUrl = friendlyUrl, - SrcUrl = url, - QueueTime = DateTime.Now, - ProgressPercent = 0, - Status = ImportJobStatus.NotStarted - }; - - - await _context.ImportJobs.AddAsync - ( - job - ); - - await _context.SaveChangesAsync(); - - try - { - - _backgroundTaskQueue.QueueBackgroundWorkItem - ( - async token => - { - try - { - using(RMuseumDbContext importJobUpdaterDb = new RMuseumDbContext(Configuration) ) - { - job.StartTime = DateTime.Now; - job.Status = ImportJobStatus.Running; - importJobUpdaterDb.Update(job); - await importJobUpdaterDb.SaveChangesAsync(); - } - - int pageCount = 0; - int representative_index = 0; - //اول یک صفحه را می‌خوانیم تا تعداد صفحات را مشخص کنیم - using (var client = new HttpClient()) - { - - using (var result = await client.GetAsync(url)) - { - if (result.IsSuccessStatusCode) - { - string json = await result.Content.ReadAsStringAsync(); - var parsed = JObject.Parse(json); - - pageCount = parsed.SelectToken("resource.segment_count").Value(); - representative_index = parsed.SelectToken("resource.representative_index").Value(); - } - else - { - using (RMuseumDbContext importJobUpdaterDb = new RMuseumDbContext(Configuration)) - { - job.EndTime = DateTime.Now; - job.Status = ImportJobStatus.Failed; - job.Exception = $"Http result is not ok ({result.StatusCode}) for {url}"; - importJobUpdaterDb.Update(job); - await importJobUpdaterDb.SaveChangesAsync(); - } - return; - } - - } - } - - //here might be problems: loc json does not return correct answer when number of segments are more than 1000 - /* - if (pageCount > 1000) - { - job.Exception = $"Page count ({pageCount}) was cut to 1000 for this artifact due to loc bug."; - pageCount = 1000; - } - */ - - //حالا که تعداد صفحات را داریم دوباره می‌خوانیم - url = $"https://www.loc.gov/resource/{resourcePrefix}.{resourceNumber}/?c={pageCount}&fo=json&st=gallery"; - using (var client = new HttpClient()) - { - - using (var result = await client.GetAsync(url)) - { - if (result.IsSuccessStatusCode) - { - - //here is a problem, this method could be called from a background service where _context is disposed, so I need to renew it - using(RMuseumDbContext context = new RMuseumDbContext(Configuration)) - { - RArtifactMasterRecord book = new RArtifactMasterRecord($"extracted from {url}", $"extracted from {url}") - { - Status = PublishStatus.Draft, - DateTime = DateTime.Now, - LastModified = DateTime.Now, - CoverItemIndex = representative_index, - FriendlyUrl = friendlyUrl - }; - - string json = await result.Content.ReadAsStringAsync(); - - job.SrcContent = json; - - var parsed = JObject.Parse(json); - - var segmentsArray = parsed.SelectToken("segments").ToArray(); - //here might be problems: loc json does not return correct answer when number of segments are more than 1000 - //I've added some temporary solutions prior - //Here I want to log any paradox I encounter: - if(segmentsArray.Length != pageCount) - { - job.Exception = $"Page count ({pageCount}) is not equal to number of returned resources ({segmentsArray.Length})."; - } - - - - List meta = new List(); - - string string_value = await HandleSimpleValue(context, parsed, meta, "item.title", "Title"); - if (!string.IsNullOrWhiteSpace(string_value)) - { - book.Name = string_value; - book.NameInEnglish = string_value; - } - await HandleSimpleValue(context, parsed, meta, "item.date", "Date"); - string_value = await HandleListValue(context, parsed, meta, "item.other_title", "Other Title"); - if (!string.IsNullOrWhiteSpace(string_value)) - { - book.Name = string_value; - } - await HandleListValue(context, parsed, meta, "item.contributor_names", "Contributor Names"); - await HandleSimpleValue(context, parsed, meta, "item.shelf_id", "Shelf ID"); - await HandleListValue(context, parsed, meta, "item.created_published", "Created / Published"); - await HandleListValue(context, parsed, meta, "item.subject_headings", "Subject Headings"); - await HandleListValue(context, parsed, meta, "item.notes", "Notes"); - await HandleListValue(context, parsed, meta, "item.medium", "Medium"); - await HandleListValue(context, parsed, meta, "item.call_number", "Call Number/Physical Location"); - await HandleListValue(context, parsed, meta, "item.digital_id", "Digital Id"); - await HandleSimpleValue(context, parsed, meta, "item.library_of_congress_control_number", "Library of Congress Control Number"); - await HandleChildrenValue(context, parsed, meta, "item.language", "Language"); - await HandleListValue(context, parsed, meta, "item.online_format", "Online Format"); - await HandleListValue(context, parsed, meta, "item.number_oclc", "OCLC Number"); - string_value = await HandleListValue(context, parsed, meta, "item.description", "Description"); - if(!string.IsNullOrEmpty(string_value)) - { - book.Description = string_value; - book.DescriptionInEnglish = string_value; - } - await HandleSimpleValue(context, parsed, meta, "cite_this.chicago", "Chicago citation style"); - await HandleSimpleValue(context, parsed, meta, "cite_this.apa", "APA citation style"); - await HandleSimpleValue(context, parsed, meta, "cite_this.mla", "MLA citation style"); - await HandleChildrenValue(context, parsed, meta, "item.dates", "Dates"); - await HandleChildrenValue(context, parsed, meta, "item.contributors", "Contributors"); - await HandleChildrenValue(context, parsed, meta, "item.location", "Location"); - await HandleListValue(context, parsed, meta, "item.rights", "Rights & Access"); - - RTagValue tag = await TagHandler.PrepareAttribute(context, "Type", "Book", 1); - meta.Add(tag); - - tag = await TagHandler.PrepareAttribute(context, "Type", "Manuscript", 1); - meta.Add(tag); - - - tag = await TagHandler.PrepareAttribute(context, "Source", "Library of Congress, African and Middle East Division, Near East Section Persian Manuscript Collection", 1); - tag.ValueSupplement = url; - string_value = parsed.SelectToken("item.id").Value(); - if (!string.IsNullOrWhiteSpace(string_value)) - { - tag.ValueSupplement = string_value; - } - - meta.Add(tag); - - - book.Tags = meta.ToArray(); - - - - - int order = 0; - List pages = new List(); - - //due to loc bug for books with more than 1000 pages relying on segmentsArray changed to hard coded image urls and .... - //foreach (JToken segment in segmentsArray) - for(int pageIndex = 1; pageIndex <= pageCount; pageIndex++) - { - /* - using (var fakeClient = new HttpClient()) - { - - using (var fakeRes = await fakeClient.GetAsync("https://ganjgah.ir/api/artifacts/keep-alive")) - { - if (fakeRes.IsSuccessStatusCode) - { - await fakeRes.Content.ReadAsStringAsync(); - } - } - } - */ - - using (RMuseumDbContext importJobUpdaterDb = new RMuseumDbContext(Configuration)) - { - job.ProgressPercent = order * 100 / (decimal)pageCount; - importJobUpdaterDb.Update(job); - await importJobUpdaterDb.SaveChangesAsync(); - } - - order++; - - - - RArtifactItemRecord page = new RArtifactItemRecord() - { - Name = $"تصویر {order}", - NameInEnglish = $"Image {pageIndex} of {book.NameInEnglish}",//segment.SelectToken("title").Value(), - Description = "", - DescriptionInEnglish = "", - Order = order, - FriendlyUrl = $"p{$"{order}".PadLeft(4, '0')}", - LastModified = DateTime.Now - }; - - tag = await TagHandler.PrepareAttribute(context, "Source", "Library of Congress, African and Middle East Division, Near East Section Persian Manuscript Collection", 1); - tag.ValueSupplement = $"http://www.loc.gov/resource/{resourcePrefix}.{resourceNumber}/?sp={pageIndex}";//segment.SelectToken("id").Value(); - page.Tags = new RTagValue[] { tag }; - - string imageUrlPart = $"{pageIndex}".PadLeft(4, '0'); - //string imageUrl = $"https://tile.loc.gov/image-services/iiif/service:amed:{resourcePrefix}:{resourceNumber}:{imageUrlPart}/full/pct:100/0/default.jpg"; - string imageUrl = $"https://tile.loc.gov/image-services/iiif/service:rbc:{resourcePrefix}:2015:{resourceNumber}:{imageUrlPart}/full/pct:100/0/default.jpg"; - /* - List list = segment.SelectToken("image_url").ToObject>(); - if (list != null && list.Count > 0) - { - for (int i = 0; i < list.Count; i++) - { - if (list[i].IndexOf(".jpg") != -1) - { - if (imageUrl == "") - imageUrl = list[i]; - else - { - if (imageUrl.IndexOf("#h=") != -1 && imageUrl.IndexOf("&w=", imageUrl.IndexOf("#h=")) != -1) - { - int h1 = int.Parse(imageUrl.Substring(imageUrl.IndexOf("#h=") + "#h=".Length, imageUrl.IndexOf("&w=") - imageUrl.IndexOf("#h=") - "&w=".Length)); - if (list[i].IndexOf("#h=") != -1 && list[i].IndexOf("&w=", list[i].IndexOf("#h=")) != -1) - { - int h2 = int.Parse(list[i].Substring(list[i].IndexOf("#h=") + "#h=".Length, list[i].IndexOf("&w=") - list[i].IndexOf("#h=") - "&w=".Length)); - - if (h2 > h1) - { - imageUrl = list[i]; - } - } - } - else - imageUrl = list[i]; - - } - } - } - } - */ - - - - if (!string.IsNullOrEmpty(imageUrl)) - { - //imageUrl = "https:" + imageUrl.Substring(0, imageUrl.IndexOf('#')); - bool recovered = false; - if ( - File.Exists - ( - Path.Combine(Path.Combine(Path.Combine(_pictureFileService.ImageStoragePath, friendlyUrl), "orig"), $"{order}".PadLeft(4, '0') + ".jpg") - ) - && - File.Exists - ( - Path.Combine(Path.Combine(Path.Combine(_pictureFileService.ImageStoragePath, friendlyUrl), "norm"), $"{order}".PadLeft(4, '0') + ".jpg") - ) - && - File.Exists - ( - Path.Combine(Path.Combine(Path.Combine(_pictureFileService.ImageStoragePath, friendlyUrl), "thumb"), $"{order}".PadLeft(4, '0') + ".jpg") - ) - ) - { - RServiceResult picture = await _pictureFileService.RecoverFromeFiles(page.Name, page.Description, 1, - imageUrl, - Path.Combine(Path.Combine(Path.Combine(_pictureFileService.ImageStoragePath, friendlyUrl), "orig"), $"{order}".PadLeft(4, '0') + ".jpg"), - Path.Combine(Path.Combine(Path.Combine(_pictureFileService.ImageStoragePath, friendlyUrl), "norm"), $"{order}".PadLeft(4, '0') + ".jpg"), - Path.Combine(Path.Combine(Path.Combine(_pictureFileService.ImageStoragePath, friendlyUrl), "thumb"), $"{order}".PadLeft(4, '0') + ".jpg"), - $"{order}".PadLeft(4, '0') + ".jpg", friendlyUrl); - if (picture.Result != null) - { - recovered = true; - page.Images = new RPictureFile[] { picture.Result }; - page.CoverImageIndex = 0; - - if (book.CoverItemIndex == (order - 1)) - { - book.CoverImage = RPictureFile.Duplicate(picture.Result); - } - - pages.Add(page); - } - - } - if(!recovered) - { - if ( - File.Exists - ( - Path.Combine(Path.Combine(Path.Combine(_pictureFileService.ImageStoragePath, friendlyUrl), "orig"), $"{order}".PadLeft(4, '0') + ".jpg") - ) - - ) - { - File.Delete - ( - Path.Combine(Path.Combine(Path.Combine(_pictureFileService.ImageStoragePath, friendlyUrl), "orig"), $"{order}".PadLeft(4, '0') + ".jpg") - ); - } - if ( - - File.Exists - ( - Path.Combine(Path.Combine(Path.Combine(_pictureFileService.ImageStoragePath, friendlyUrl), "norm"), $"{order}".PadLeft(4, '0') + ".jpg") - ) - - ) - { - File.Delete - ( - Path.Combine(Path.Combine(Path.Combine(_pictureFileService.ImageStoragePath, friendlyUrl), "norm"), $"{order}".PadLeft(4, '0') + ".jpg") - ); -} - if ( - - File.Exists - ( - Path.Combine(Path.Combine(Path.Combine(_pictureFileService.ImageStoragePath, friendlyUrl), "thumb"), $"{order}".PadLeft(4, '0') + ".jpg") - ) - ) - { - File.Delete - ( - Path.Combine(Path.Combine(Path.Combine(_pictureFileService.ImageStoragePath, friendlyUrl), "thumb"), $"{order}".PadLeft(4, '0') + ".jpg") - ); - } - var imageResult = await client.GetAsync(imageUrl); - - - int _ImportRetryCount = 5; - int _ImportRetryInitialSleep = 500; - int retryCount = 0; - while (retryCount < _ImportRetryCount && !imageResult.IsSuccessStatusCode && imageResult.StatusCode == HttpStatusCode.ServiceUnavailable) - { - imageResult.Dispose(); - Thread.Sleep(_ImportRetryInitialSleep * (retryCount + 1)); - imageResult = await client.GetAsync(imageUrl); - retryCount++; - } - - if (imageResult.IsSuccessStatusCode) - { - using (Stream imageStream = await imageResult.Content.ReadAsStreamAsync()) - { - RServiceResult picture = await _pictureFileService.Add(page.Name, page.Description, 1, null, imageUrl, imageStream, $"{order}".PadLeft(4, '0') + ".jpg", friendlyUrl); - if (picture.Result == null) - { - throw new Exception($"_pictureFileService.Add : {picture.ExceptionString}"); - } - - page.Images = new RPictureFile[] { picture.Result }; - page.CoverImageIndex = 0; - - if (book.CoverItemIndex == (order - 1)) - { - book.CoverImage = RPictureFile.Duplicate(picture.Result); - } - - pages.Add(page); - } - } - else - { - using (RMuseumDbContext importJobUpdaterDb = new RMuseumDbContext(Configuration)) - { - job.EndTime = DateTime.Now; - job.Status = ImportJobStatus.Failed; - job.Exception = $"Http result is not ok ({imageResult.StatusCode}) for page {order}, url {imageUrl}"; - importJobUpdaterDb.Update(job); - await importJobUpdaterDb.SaveChangesAsync(); - } - - imageResult.Dispose(); - return; - } - - imageResult.Dispose(); - GC.Collect(); - } - - - - - - - } - } - - book.Items = pages.ToArray(); - book.ItemCount = pages.Count; - - if (book.CoverImage == null && pages.Count > 0) - { - book.CoverImage = RPictureFile.Duplicate(pages[0].Images.First()); - } - - await context.Artifacts.AddAsync(book); - await context.SaveChangesAsync(); - - job.ProgressPercent = 100; - job.Status = ImportJobStatus.Succeeded; - job.ArtifactId = book.Id; - job.EndTime = DateTime.Now; - context.Update(job); - await context.SaveChangesAsync(); - - } - - } - else - { - using (RMuseumDbContext importJobUpdaterDb = new RMuseumDbContext(Configuration)) - { - job.EndTime = DateTime.Now; - job.Status = ImportJobStatus.Failed; - job.Exception = $"Http result is not ok ({result.StatusCode}) for {url}"; - importJobUpdaterDb.Update(job); - await importJobUpdaterDb.SaveChangesAsync(); - } - return; - } - } - - } - } - catch (Exception exp) - { - using (RMuseumDbContext importJobUpdaterDb = new RMuseumDbContext(Configuration)) - { - job.EndTime = DateTime.Now; - job.Status = ImportJobStatus.Failed; - job.Exception = exp.ToString(); - importJobUpdaterDb.Update(job); - await importJobUpdaterDb.SaveChangesAsync(); - } - - } - } - ); - - - - return new RServiceResult(true); - } - catch(Exception exp) - { - return new RServiceResult(false, exp.ToString()); - } - } - - /// - /// from http://pudl.princeton.edu/ - /// - /// dj52w476m - /// - /// - private async Task> StartImportingFromPrinceton(string resourceNumber, string friendlyUrl) - { - string url = $"http://pudl.princeton.edu/mdCompiler2.php?obj={resourceNumber}"; - if ( - ( - await _context.ImportJobs - .Where(j => j.JobType == JobType.Princeton && j.ResourceNumber == resourceNumber && !(j.Status == ImportJobStatus.Failed || j.Status == ImportJobStatus.Aborted)) - .SingleOrDefaultAsync() - ) - != - null - ) - { - return new RServiceResult(false, $"Job is already scheduled or running for importing {url}"); - } - - if (string.IsNullOrEmpty(friendlyUrl)) - { - friendlyUrl = resourceNumber; - } - - if ( - (await _context.Artifacts.Where(a => a.FriendlyUrl == friendlyUrl).SingleOrDefaultAsync()) - != - null - ) - { - return new RServiceResult(false, $"duplicated friendly url '{friendlyUrl}'"); - } - - ImportJob job = new ImportJob() - { - JobType = JobType.Princeton, - ResourceNumber = resourceNumber, - FriendlyUrl = friendlyUrl, - SrcUrl = url, - QueueTime = DateTime.Now, - ProgressPercent = 0, - Status = ImportJobStatus.NotStarted - }; - - - await _context.ImportJobs.AddAsync - ( - job - ); - - await _context.SaveChangesAsync(); - - - try - { - - _backgroundTaskQueue.QueueBackgroundWorkItem - ( - async token => - { - try - { - - - using (var client = new HttpClient()) - { - - using (var result = await client.GetAsync(url)) - { - if (result.IsSuccessStatusCode) - { - using (RMuseumDbContext context = new RMuseumDbContext(Configuration)) - { - RArtifactMasterRecord book = new RArtifactMasterRecord($"extracted from {url}", $"extracted from {url}") - { - Status = PublishStatus.Draft, - DateTime = DateTime.Now, - LastModified = DateTime.Now, - CoverItemIndex = 0, - FriendlyUrl = friendlyUrl - }; - - - List meta = new List(); - RTagValue tag; - - string xml = await result.Content.ReadAsStringAsync(); - - - using (RMuseumDbContext importJobUpdaterDb = new RMuseumDbContext(Configuration)) - { - job.StartTime = DateTime.Now; - job.Status = ImportJobStatus.Running; - job.SrcContent = xml; - importJobUpdaterDb.Update(job); - await importJobUpdaterDb.SaveChangesAsync(); - } - - XElement elObject = XDocument.Parse(xml).Root; - foreach(var prop in elObject.Element("dmd").Element("properties").Elements("property")) - { - if (prop.Element("label") == null) - continue; - string label = prop.Element("label").Value.Replace(":", ""); - int order = 1; - foreach (var value in prop.Elements("valueGrp").Elements("value")) - { - tag = await TagHandler.PrepareAttribute(context, label, value.Value, order); - if(value.Attribute("href") != null) - { - if(value.Attribute("href").Value.IndexOf("http://localhost") != 0) - { - tag.ValueSupplement = value.Attribute("href").Value; - } - } - meta.Add(tag); - - if(label == "Title") - { - book.Name = book.NameInEnglish = - book.Description = book.DescriptionInEnglish = - value.Value; - } - order++; - } - } - - tag = await TagHandler.PrepareAttribute(context, "Type", "Book", 1); - meta.Add(tag); - - tag = await TagHandler.PrepareAttribute(context, "Type", "Manuscript", 1); - meta.Add(tag); - - - tag = await TagHandler.PrepareAttribute(context, "Source", "Princeton Digital Library of Islamic Manuscripts", 1); - tag.ValueSupplement = $"http://pudl.princeton.edu/objects/{job.ResourceNumber}"; - - meta.Add(tag); - - - book.Tags = meta.ToArray(); - List pages = new List(); - - foreach (var structure in elObject.Elements("structure")) - { - if (structure.Attribute("type") != null && structure.Attribute("type").Value == "RelatedObjects") - { - if (structure.Element("div") == null || structure.Element("div").Element("OrderedList") == null) - { - using (RMuseumDbContext importJobUpdaterDb = new RMuseumDbContext(Configuration)) - { - job.EndTime = DateTime.Now; - job.Status = ImportJobStatus.Failed; - job.Exception = "structure[RelatedObjects].div.OrderedList is null"; - importJobUpdaterDb.Update(job); - await importJobUpdaterDb.SaveChangesAsync(); - return; - } - } - - - int pageCount = structure.Element("div").Element("OrderedList").Elements("div").Count(); - int inlineOrder = 0; - - foreach (var div in structure.Element("div").Element("OrderedList").Elements("div")) - { - inlineOrder++; - using (RMuseumDbContext importJobUpdaterDb = new RMuseumDbContext(Configuration)) - { - job.ProgressPercent = inlineOrder * 100 / (decimal)pageCount; - importJobUpdaterDb.Update(job); - await importJobUpdaterDb.SaveChangesAsync(); - } - - int order = int.Parse(div.Attribute("order").Value); - RArtifactItemRecord page = new RArtifactItemRecord() - { - Name = $"تصویر {order}", - NameInEnglish = div.Attribute("label").Value, - Description = "", - DescriptionInEnglish = "", - Order = order, - FriendlyUrl = $"p{$"{order}".PadLeft(4, '0')}", - LastModified = DateTime.Now - }; - - string imageUrl = div.Attribute("img").Value; - imageUrl = "https://libimages.princeton.edu/loris/" + imageUrl.Substring(imageUrl.LastIndexOf(":") + 1); - imageUrl += $"/full/,{div.Attribute("h").Value}/0/default.jpg"; - - tag = await TagHandler.PrepareAttribute(context, "Source", "Princeton Digital Library of Islamic Manuscripts", 1); - tag.ValueSupplement = imageUrl; - page.Tags = new RTagValue[] { tag }; - - if (!string.IsNullOrEmpty(imageUrl)) - { - bool recovered = false; - if ( - File.Exists - ( - Path.Combine(Path.Combine(Path.Combine(_pictureFileService.ImageStoragePath, friendlyUrl), "orig"), $"{order}".PadLeft(4, '0') + ".jpg") - ) - && - File.Exists - ( - Path.Combine(Path.Combine(Path.Combine(_pictureFileService.ImageStoragePath, friendlyUrl), "norm"), $"{order}".PadLeft(4, '0') + ".jpg") - ) - && - File.Exists - ( - Path.Combine(Path.Combine(Path.Combine(_pictureFileService.ImageStoragePath, friendlyUrl), "thumb"), $"{order}".PadLeft(4, '0') + ".jpg") - ) - ) - { - RServiceResult picture = await _pictureFileService.RecoverFromeFiles(page.Name, page.Description, 1, - imageUrl, - Path.Combine(Path.Combine(Path.Combine(_pictureFileService.ImageStoragePath, friendlyUrl), "orig"), $"{order}".PadLeft(4, '0') + ".jpg"), - Path.Combine(Path.Combine(Path.Combine(_pictureFileService.ImageStoragePath, friendlyUrl), "norm"), $"{order}".PadLeft(4, '0') + ".jpg"), - Path.Combine(Path.Combine(Path.Combine(_pictureFileService.ImageStoragePath, friendlyUrl), "thumb"), $"{order}".PadLeft(4, '0') + ".jpg"), - $"{order}".PadLeft(4, '0') + ".jpg", friendlyUrl); - if (picture.Result != null) - { - recovered = true; - page.Images = new RPictureFile[] { picture.Result }; - page.CoverImageIndex = 0; - - if (book.CoverItemIndex == (order - 1)) - { - book.CoverImage = RPictureFile.Duplicate(picture.Result); - } - - pages.Add(page); - } - - } - if (!recovered) - { - if ( - File.Exists - ( - Path.Combine(Path.Combine(Path.Combine(_pictureFileService.ImageStoragePath, friendlyUrl), "orig"), $"{order}".PadLeft(4, '0') + ".jpg") - ) - - ) - { - File.Delete - ( - Path.Combine(Path.Combine(Path.Combine(_pictureFileService.ImageStoragePath, friendlyUrl), "orig"), $"{order}".PadLeft(4, '0') + ".jpg") - ); - } - if ( - - File.Exists - ( - Path.Combine(Path.Combine(Path.Combine(_pictureFileService.ImageStoragePath, friendlyUrl), "norm"), $"{order}".PadLeft(4, '0') + ".jpg") - ) - - ) - { - File.Delete - ( - Path.Combine(Path.Combine(Path.Combine(_pictureFileService.ImageStoragePath, friendlyUrl), "norm"), $"{order}".PadLeft(4, '0') + ".jpg") - ); - } - if ( - - File.Exists - ( - Path.Combine(Path.Combine(Path.Combine(_pictureFileService.ImageStoragePath, friendlyUrl), "thumb"), $"{order}".PadLeft(4, '0') + ".jpg") - ) - ) - { - File.Delete - ( - Path.Combine(Path.Combine(Path.Combine(_pictureFileService.ImageStoragePath, friendlyUrl), "thumb"), $"{order}".PadLeft(4, '0') + ".jpg") - ); - } - var imageResult = await client.GetAsync(imageUrl); - - - int _ImportRetryCount = 5; - int _ImportRetryInitialSleep = 500; - int retryCount = 0; - while (retryCount < _ImportRetryCount && !imageResult.IsSuccessStatusCode && imageResult.StatusCode == HttpStatusCode.ServiceUnavailable) - { - imageResult.Dispose(); - Thread.Sleep(_ImportRetryInitialSleep * (retryCount + 1)); - imageResult = await client.GetAsync(imageUrl); - retryCount++; - } - - if (imageResult.IsSuccessStatusCode) - { - using (Stream imageStream = await imageResult.Content.ReadAsStreamAsync()) - { - RServiceResult picture = await _pictureFileService.Add(page.Name, page.Description, 1, null, imageUrl, imageStream, $"{order}".PadLeft(4, '0') + ".jpg", friendlyUrl); - if (picture.Result == null) - { - throw new Exception($"_pictureFileService.Add : {picture.ExceptionString}"); - } - - page.Images = new RPictureFile[] { picture.Result }; - page.CoverImageIndex = 0; - - if (book.CoverItemIndex == (order - 1)) - { - book.CoverImage = RPictureFile.Duplicate(picture.Result); - } - - pages.Add(page); - } - } - else - { - using (RMuseumDbContext importJobUpdaterDb = new RMuseumDbContext(Configuration)) - { - job.EndTime = DateTime.Now; - job.Status = ImportJobStatus.Failed; - job.Exception = $"Http result is not ok ({imageResult.StatusCode}) for page {order}, url {imageUrl}"; - importJobUpdaterDb.Update(job); - await importJobUpdaterDb.SaveChangesAsync(); - } - - imageResult.Dispose(); - return; - } - - imageResult.Dispose(); - GC.Collect(); - } - - } - - - - } - - } - } - - foreach (var structure in elObject.Elements("structure")) - { - if (structure.Attribute("type") != null && structure.Attribute("type").Value == "Physical") - { - if(structure.Element("RTLBoundManuscript") != null) - { - foreach (var leaf in structure.Element("RTLBoundManuscript").Elements("Leaf")) - { - foreach (var side in leaf.Elements("Side")) - { - int pageOrder = int.Parse(side.Attribute("order").Value); - tag = await TagHandler.PrepareAttribute(context, "Leaf Side", side.Attribute("label").Value, 100); - RArtifactItemRecord page = pages.Where(p => p.Order == pageOrder).SingleOrDefault(); - if (page != null) - { - List tags = new List(page.Tags); - tags.Add(tag); - page.Tags = tags; - } - } - } - foreach (var folio in structure.Element("RTLBoundManuscript").Elements("Folio")) - { - foreach (var side in folio.Elements("Side")) - { - int pageOrder = int.Parse(side.Attribute("order").Value); - tag = await TagHandler.PrepareAttribute(context, "Folio Side", folio.Attribute("label").Value + ":" + side.Attribute("label").Value, 101); - RArtifactItemRecord page = pages.Where(p => p.Order == pageOrder).SingleOrDefault(); - if (page != null) - { - List tags = new List(page.Tags); - tags.Add(tag); - page.Tags = tags; - } - } - } - } - } - } - - book.Items = pages.ToArray(); - book.ItemCount = pages.Count; - - if(pages.Count == 0) - { - using (RMuseumDbContext importJobUpdaterDb = new RMuseumDbContext(Configuration)) - { - job.EndTime = DateTime.Now; - job.Status = ImportJobStatus.Failed; - job.Exception = "ages.Count == 0"; - importJobUpdaterDb.Update(job); - await importJobUpdaterDb.SaveChangesAsync(); - } - return; - } - - await context.Artifacts.AddAsync(book); - await context.SaveChangesAsync(); - - job.ProgressPercent = 100; - job.Status = ImportJobStatus.Succeeded; - job.ArtifactId = book.Id; - job.EndTime = DateTime.Now; - context.Update(job); - await context.SaveChangesAsync(); - - - - } - } - else - { - using (RMuseumDbContext importJobUpdaterDb = new RMuseumDbContext(Configuration)) - { - job.EndTime = DateTime.Now; - job.Status = ImportJobStatus.Failed; - job.Exception = $"Http result is not ok ({result.StatusCode}) for {url}"; - importJobUpdaterDb.Update(job); - await importJobUpdaterDb.SaveChangesAsync(); - } - return; - } - - } - } - } - catch (Exception exp) - { - using (RMuseumDbContext importJobUpdaterDb = new RMuseumDbContext(Configuration)) - { - job.EndTime = DateTime.Now; - job.Status = ImportJobStatus.Failed; - job.Exception = exp.ToString(); - importJobUpdaterDb.Update(job); - await importJobUpdaterDb.SaveChangesAsync(); - } - - } - } - ); - - - - return new RServiceResult(true); - } - catch (Exception exp) - { - return new RServiceResult(false, exp.ToString()); - } - } - - /// - /// from https://curiosity.lib.harvard.edu - /// - /// example: https://curiosity.lib.harvard.edu/islamic-heritage-project/catalog/40-990114893240203941 - /// - /// - private async Task> StartImportingFromHarvard(string url, string friendlyUrl) - { - try - { - if ( - ( - await _context.ImportJobs - .Where(j => j.JobType == JobType.Harvard && j.ResourceNumber == url && !(j.Status == ImportJobStatus.Failed || j.Status == ImportJobStatus.Aborted)) - .SingleOrDefaultAsync() - ) - != - null - ) - { - return new RServiceResult(false, $"Job is already scheduled or running for importing {url}"); - } - - if (string.IsNullOrEmpty(friendlyUrl)) - { - return new RServiceResult(false, $"Friendly url is empty, url = {url}"); - } - - if ( - (await _context.Artifacts.Where(a => a.FriendlyUrl == friendlyUrl).SingleOrDefaultAsync()) - != - null - ) - { - return new RServiceResult(false, $"duplicated friendly url '{friendlyUrl}'"); - } - - ImportJob job = new ImportJob() - { - JobType = JobType.Harvard, - ResourceNumber = url, - FriendlyUrl = friendlyUrl, - SrcUrl = url, - QueueTime = DateTime.Now, - ProgressPercent = 0, - Status = ImportJobStatus.NotStarted - }; - - - await _context.ImportJobs.AddAsync - ( - job - ); - - await _context.SaveChangesAsync(); - - _backgroundTaskQueue.QueueBackgroundWorkItem - ( - async token => - { - try - { - using (var client = new HttpClient()) - { - - using (var result = await client.GetAsync(url)) - { - if (result.IsSuccessStatusCode) - { - using (RMuseumDbContext context = new RMuseumDbContext(Configuration)) - { - RArtifactMasterRecord book = new RArtifactMasterRecord($"extracted from {url}", $"extracted from {url}") - { - Status = PublishStatus.Draft, - DateTime = DateTime.Now, - LastModified = DateTime.Now, - CoverItemIndex = 0, - FriendlyUrl = friendlyUrl - }; - - - List meta = new List(); - RTagValue tag; - - string html = await result.Content.ReadAsStringAsync(); - - - using (RMuseumDbContext importJobUpdaterDb = new RMuseumDbContext(Configuration)) - { - job.StartTime = DateTime.Now; - job.Status = ImportJobStatus.Running; - job.SrcContent = html; - importJobUpdaterDb.Update(job); - await importJobUpdaterDb.SaveChangesAsync(); - } - - int nStartIndex = html.IndexOf("", nStartIndex); - if (nStartIndex == -1) - break; - nStartIndex++; - string tagName = html.Substring(nStartIndex, html.IndexOf(":", nStartIndex) - nStartIndex); - nStartIndex = html.IndexOf("", nStartIndex); - if (nStartIndex == -1) - break; - nStartIndex++; - string tagValues = html.Substring(nStartIndex, html.IndexOf("", nStartIndex) - nStartIndex); - foreach(string tagValuePart in tagValues.Split("
", StringSplitOptions.RemoveEmptyEntries)) - { - string tagValue = tagValuePart; - bool href = false; - if(tagValue.IndexOf(" pages = (await _InternalHarvardJsonImport(hardvardResourceNumber, job, friendlyUrl, context, book, meta)).Result; - if(pages == null) - { - return; - } - - - book.Tags = meta.ToArray(); - - book.Items = pages.ToArray(); - book.ItemCount = pages.Count; - - if (pages.Count == 0) - { - using (RMuseumDbContext importJobUpdaterDb = new RMuseumDbContext(Configuration)) - { - job.EndTime = DateTime.Now; - job.Status = ImportJobStatus.Failed; - job.Exception = "Pages.Count == 0"; - importJobUpdaterDb.Update(job); - await importJobUpdaterDb.SaveChangesAsync(); - } - return; - } - - await context.Artifacts.AddAsync(book); - await context.SaveChangesAsync(); - - job.ProgressPercent = 100; - job.Status = ImportJobStatus.Succeeded; - job.ArtifactId = book.Id; - job.EndTime = DateTime.Now; - context.Update(job); - await context.SaveChangesAsync(); - - - - } - } - else - { - using (RMuseumDbContext importJobUpdaterDb = new RMuseumDbContext(Configuration)) - { - job.EndTime = DateTime.Now; - job.Status = ImportJobStatus.Failed; - job.Exception = $"Http result is not ok ({result.StatusCode}) for {url}"; - importJobUpdaterDb.Update(job); - await importJobUpdaterDb.SaveChangesAsync(); - } - return; - } - - } - } - } - catch (Exception exp) - { - using (RMuseumDbContext importJobUpdaterDb = new RMuseumDbContext(Configuration)) - { - job.EndTime = DateTime.Now; - job.Status = ImportJobStatus.Failed; - job.Exception = exp.ToString(); - importJobUpdaterDb.Update(job); - await importJobUpdaterDb.SaveChangesAsync(); - } - - } - } - ); - - return new RServiceResult(true); - } - catch (Exception exp) - { - return new RServiceResult(false, exp.ToString()); - } - } - - private async Task>> _InternalHarvardJsonImport(string hardvardResourceNumber, ImportJob job, string friendlyUrl, RMuseumDbContext context, RArtifactMasterRecord book, List meta) - { - List pages = new List(); - - using (var client = new HttpClient()) - { - using (var jsonResult = await client.GetAsync($"https://iiif.lib.harvard.edu/manifests/drs:{hardvardResourceNumber}")) - { - if (jsonResult.IsSuccessStatusCode) - { - string json = await jsonResult.Content.ReadAsStringAsync(); - var parsed = JObject.Parse(json); - book.Name = book.NameInEnglish = book.Description = book.DescriptionInEnglish = - parsed.SelectToken("label").Value(); - - - RTagValue tag; - - tag = await TagHandler.PrepareAttribute(context, "Title", book.Name, 1); - meta.Add(tag); - - tag = await TagHandler.PrepareAttribute(context, "Contributor Names", "تعیین نشده", 1); - meta.Add(tag); - - List labels = new List(); - foreach (JToken structure in parsed.SelectTokens("$.structures[*].label")) - { - labels.Add(structure.Value()); - } - - int order = 0; - var canvases = parsed.SelectToken("sequences").First().SelectToken("canvases").ToArray(); - int pageCount = canvases.Length; - foreach (JToken canvas in canvases) - { - order++; - using (RMuseumDbContext importJobUpdaterDb = new RMuseumDbContext(Configuration)) - { - job.ProgressPercent = order * 100 / (decimal)pageCount; - importJobUpdaterDb.Update(job); - await importJobUpdaterDb.SaveChangesAsync(); - } - string label = canvas.SelectToken("label").Value(); - if (labels.Where(l => l.IndexOf(label) != -1).SingleOrDefault() != null) - label = labels.Where(l => l.IndexOf(label) != -1).SingleOrDefault(); - string imageUrl = canvas.SelectTokens("images[*]").First().SelectToken("resource").SelectToken("@id").Value(); - RArtifactItemRecord page = new RArtifactItemRecord() - { - Name = $"تصویر {order}", - NameInEnglish = label, - Description = "", - DescriptionInEnglish = "", - Order = order, - FriendlyUrl = $"p{$"{order}".PadLeft(4, '0')}", - LastModified = DateTime.Now - }; - - - - tag = await TagHandler.PrepareAttribute(context, "Source", "Harvard University Islamic Heritage Project", 1); - tag.ValueSupplement = imageUrl; - page.Tags = new RTagValue[] { tag }; - - - if (!string.IsNullOrEmpty(imageUrl)) - { - bool recovered = false; - if ( - File.Exists - ( - Path.Combine(Path.Combine(Path.Combine(_pictureFileService.ImageStoragePath, friendlyUrl), "orig"), $"{order}".PadLeft(4, '0') + ".jpg") - ) - && - File.Exists - ( - Path.Combine(Path.Combine(Path.Combine(_pictureFileService.ImageStoragePath, friendlyUrl), "norm"), $"{order}".PadLeft(4, '0') + ".jpg") - ) - && - File.Exists - ( - Path.Combine(Path.Combine(Path.Combine(_pictureFileService.ImageStoragePath, friendlyUrl), "thumb"), $"{order}".PadLeft(4, '0') + ".jpg") - ) - ) - { - RServiceResult picture = await _pictureFileService.RecoverFromeFiles(page.Name, page.Description, 1, - imageUrl, - Path.Combine(Path.Combine(Path.Combine(_pictureFileService.ImageStoragePath, friendlyUrl), "orig"), $"{order}".PadLeft(4, '0') + ".jpg"), - Path.Combine(Path.Combine(Path.Combine(_pictureFileService.ImageStoragePath, friendlyUrl), "norm"), $"{order}".PadLeft(4, '0') + ".jpg"), - Path.Combine(Path.Combine(Path.Combine(_pictureFileService.ImageStoragePath, friendlyUrl), "thumb"), $"{order}".PadLeft(4, '0') + ".jpg"), - $"{order}".PadLeft(4, '0') + ".jpg", friendlyUrl); - if (picture.Result != null) - { - recovered = true; - page.Images = new RPictureFile[] { picture.Result }; - page.CoverImageIndex = 0; - - if (book.CoverItemIndex == (order - 1)) - { - book.CoverImage = RPictureFile.Duplicate(picture.Result); - } - - pages.Add(page); - } - - } - if (!recovered) - { - if ( - File.Exists - ( - Path.Combine(Path.Combine(Path.Combine(_pictureFileService.ImageStoragePath, friendlyUrl), "orig"), $"{order}".PadLeft(4, '0') + ".jpg") - ) - - ) - { - File.Delete - ( - Path.Combine(Path.Combine(Path.Combine(_pictureFileService.ImageStoragePath, friendlyUrl), "orig"), $"{order}".PadLeft(4, '0') + ".jpg") - ); - } - if ( - - File.Exists - ( - Path.Combine(Path.Combine(Path.Combine(_pictureFileService.ImageStoragePath, friendlyUrl), "norm"), $"{order}".PadLeft(4, '0') + ".jpg") - ) - - ) - { - File.Delete - ( - Path.Combine(Path.Combine(Path.Combine(_pictureFileService.ImageStoragePath, friendlyUrl), "norm"), $"{order}".PadLeft(4, '0') + ".jpg") - ); - } - if ( - - File.Exists - ( - Path.Combine(Path.Combine(Path.Combine(_pictureFileService.ImageStoragePath, friendlyUrl), "thumb"), $"{order}".PadLeft(4, '0') + ".jpg") - ) - ) - { - File.Delete - ( - Path.Combine(Path.Combine(Path.Combine(_pictureFileService.ImageStoragePath, friendlyUrl), "thumb"), $"{order}".PadLeft(4, '0') + ".jpg") - ); - } - var imageResult = await client.GetAsync(imageUrl); - - - int _ImportRetryCount = 5; - int _ImportRetryInitialSleep = 500; - int retryCount = 0; - while (retryCount < _ImportRetryCount && !imageResult.IsSuccessStatusCode && imageResult.StatusCode == HttpStatusCode.ServiceUnavailable) - { - imageResult.Dispose(); - Thread.Sleep(_ImportRetryInitialSleep * (retryCount + 1)); - imageResult = await client.GetAsync(imageUrl); - retryCount++; - } - - if (imageResult.IsSuccessStatusCode) - { - using (Stream imageStream = await imageResult.Content.ReadAsStreamAsync()) - { - RServiceResult picture = await _pictureFileService.Add(page.Name, page.Description, 1, null, imageUrl, imageStream, $"{order}".PadLeft(4, '0') + ".jpg", friendlyUrl); - if (picture.Result == null) - { - throw new Exception($"_pictureFileService.Add : {picture.ExceptionString}"); - } - - page.Images = new RPictureFile[] { picture.Result }; - page.CoverImageIndex = 0; - - if (book.CoverItemIndex == (order - 1)) - { - book.CoverImage = RPictureFile.Duplicate(picture.Result); - } - - pages.Add(page); - } - } - else - { - using (RMuseumDbContext importJobUpdaterDb = new RMuseumDbContext(Configuration)) - { - job.EndTime = DateTime.Now; - job.Status = ImportJobStatus.Failed; - job.Exception = $"Http result is not ok ({imageResult.StatusCode}) for page {order}, url {imageUrl}"; - importJobUpdaterDb.Update(job); - await importJobUpdaterDb.SaveChangesAsync(); - } - - imageResult.Dispose(); - return new RServiceResult>(null, "failed"); - } - - imageResult.Dispose(); - GC.Collect(); - } - } - } - } - else - { - using (RMuseumDbContext importJobUpdaterDb = new RMuseumDbContext(Configuration)) - { - job.EndTime = DateTime.Now; - job.Status = ImportJobStatus.Failed; - job.Exception = $"Http result is not ok ({jsonResult.StatusCode}) for https://iiif.lib.harvard.edu/manifests/drs:{hardvardResourceNumber}"; - importJobUpdaterDb.Update(job); - await importJobUpdaterDb.SaveChangesAsync(); - } - return new RServiceResult>(null, "failed"); - } - } - } - - return new RServiceResult>(pages); - - - } - - /// - /// import from http://www.qajarwomen.org - /// - /// 43117279 - /// atame - /// http://www.qajarwomen.org/fa/items/1018A10.html - /// - private async Task> StartImportingFromHarvardDirectly(string hardvardResourceNumber, string friendlyUrl, string srcUrl) - { - try - { - if ( - ( - await _context.ImportJobs - .Where(j => j.JobType == JobType.HarvardDirect && j.ResourceNumber == hardvardResourceNumber && !(j.Status == ImportJobStatus.Failed || j.Status == ImportJobStatus.Aborted)) - .SingleOrDefaultAsync() - ) - != - null - ) - { - return new RServiceResult(false, $"Job is already scheduled or running for importing harvard direct resource number {hardvardResourceNumber}"); - } - - if (string.IsNullOrEmpty(friendlyUrl)) - { - return new RServiceResult(false, $"Friendly url is empty, harvard direct resource number {hardvardResourceNumber}"); - } - - if ( - (await _context.Artifacts.Where(a => a.FriendlyUrl == friendlyUrl).SingleOrDefaultAsync()) - != - null - ) - { - return new RServiceResult(false, $"duplicated friendly url '{friendlyUrl}'"); - } - - ImportJob job = new ImportJob() - { - JobType = JobType.HarvardDirect, - ResourceNumber = hardvardResourceNumber, - FriendlyUrl = friendlyUrl, - SrcUrl = srcUrl, - QueueTime = DateTime.Now, - ProgressPercent = 0, - Status = ImportJobStatus.NotStarted - }; - - - await _context.ImportJobs.AddAsync - ( - job - ); - - await _context.SaveChangesAsync(); - - _backgroundTaskQueue.QueueBackgroundWorkItem - ( - async token => - { - try - { - using (RMuseumDbContext context = new RMuseumDbContext(Configuration)) - { - RArtifactMasterRecord book = new RArtifactMasterRecord($"extracted from harvard resource number {job.ResourceNumber}", $"extracted from harvard resource number {job.ResourceNumber}") - { - Status = PublishStatus.Draft, - DateTime = DateTime.Now, - LastModified = DateTime.Now, - CoverItemIndex = 0, - FriendlyUrl = friendlyUrl - }; - - - List meta = new List(); - RTagValue tag; - - - - - - tag = await TagHandler.PrepareAttribute(context, "Notes", "وارد شده از سایت دنیای زنان در عصر قاجار", 1); - meta.Add(tag); - - - - tag = await TagHandler.PrepareAttribute(context, "Type", "Book", 1); - meta.Add(tag); - - tag = await TagHandler.PrepareAttribute(context, "Type", "Manuscript", 1); - meta.Add(tag); - - - tag = await TagHandler.PrepareAttribute(context, "Source", "دنیای زنان در عصر قاجار", 1); - tag.ValueSupplement = $"{job.SrcUrl}"; - - meta.Add(tag); - - using (RMuseumDbContext importJobUpdaterDb = new RMuseumDbContext(Configuration)) - { - job.StartTime = DateTime.Now; - job.Status = ImportJobStatus.Running; - job.SrcContent = $"https://iiif.lib.harvard.edu/manifests/drs:{hardvardResourceNumber}"; - importJobUpdaterDb.Update(job); - await importJobUpdaterDb.SaveChangesAsync(); - } - - List pages = (await _InternalHarvardJsonImport(hardvardResourceNumber, job, friendlyUrl, context, book, meta)).Result; - if (pages == null) - { - return; - } - - - book.Tags = meta.ToArray(); - - book.Items = pages.ToArray(); - book.ItemCount = pages.Count; - - if (pages.Count == 0) - { - using (RMuseumDbContext importJobUpdaterDb = new RMuseumDbContext(Configuration)) - { - job.EndTime = DateTime.Now; - job.Status = ImportJobStatus.Failed; - job.Exception = "Pages.Count == 0"; - importJobUpdaterDb.Update(job); - await importJobUpdaterDb.SaveChangesAsync(); - } - return; - } - - await context.Artifacts.AddAsync(book); - await context.SaveChangesAsync(); - - job.ProgressPercent = 100; - job.Status = ImportJobStatus.Succeeded; - job.ArtifactId = book.Id; - job.EndTime = DateTime.Now; - context.Update(job); - await context.SaveChangesAsync(); - - - - } - } - catch (Exception exp) - { - using (RMuseumDbContext importJobUpdaterDb = new RMuseumDbContext(Configuration)) - { - job.EndTime = DateTime.Now; - job.Status = ImportJobStatus.Failed; - job.Exception = exp.ToString(); - importJobUpdaterDb.Update(job); - await importJobUpdaterDb.SaveChangesAsync(); - } - - } - } - ); - - return new RServiceResult(true); - } - catch(Exception exp) - { - return new RServiceResult(false, exp.ToString()); - } - } - - - /// - /// from https://catalog.hathitrust.org - /// - /// 006814127 - /// - /// - private async Task> StartImportingFromHathiTrust(string resourceNumber, string friendlyUrl) - { - string url = $"https://catalog.hathitrust.org/Record/{resourceNumber}.xml"; - if ( - ( - await _context.ImportJobs - .Where(j => j.JobType == JobType.HathiTrust && j.ResourceNumber == resourceNumber && !(j.Status == ImportJobStatus.Failed || j.Status == ImportJobStatus.Aborted)) - .SingleOrDefaultAsync() - ) - != - null - ) - { - return new RServiceResult(false, $"Job is already scheduled or running for importing {url}"); - } - - if (string.IsNullOrEmpty(friendlyUrl)) - { - friendlyUrl = resourceNumber; - } - - if ( - (await _context.Artifacts.Where(a => a.FriendlyUrl == friendlyUrl).SingleOrDefaultAsync()) - != - null - ) - { - return new RServiceResult(false, $"duplicated artifact friendly url '{friendlyUrl}'"); - } - - - - ImportJob job = new ImportJob() - { - JobType = JobType.HathiTrust, - ResourceNumber = resourceNumber, - FriendlyUrl = friendlyUrl, - SrcUrl = url, - QueueTime = DateTime.Now, - ProgressPercent = 0, - Status = ImportJobStatus.NotStarted - }; - - - await _context.ImportJobs.AddAsync - ( - job - ); - - await _context.SaveChangesAsync(); - - - try - { - - _backgroundTaskQueue.QueueBackgroundWorkItem - ( - async token => - { - try - { - - - using (var client = new HttpClient()) - { - - using (var result = await client.GetAsync(url)) - { - if (result.IsSuccessStatusCode) - { - using (RMuseumDbContext context = new RMuseumDbContext(Configuration)) - { - if ( - (await context.Artifacts.Where(a => a.FriendlyUrl == job.FriendlyUrl).SingleOrDefaultAsync()) - != - null - ) - { - using (RMuseumDbContext importJobUpdaterDb = new RMuseumDbContext(Configuration)) - { - job.EndTime = DateTime.Now; - job.Status = ImportJobStatus.Failed; - job.Exception = "aborted because of duplicated friendly url"; - importJobUpdaterDb.Update(job); - await importJobUpdaterDb.SaveChangesAsync(); - } - return; - } - RArtifactMasterRecord book = new RArtifactMasterRecord($"extracted from {url}", $"extracted from {url}") - { - Status = PublishStatus.Draft, - DateTime = DateTime.Now, - LastModified = DateTime.Now, - CoverItemIndex = 0, - FriendlyUrl = friendlyUrl - }; - - - List meta = new List(); - RTagValue tag; - - string xml = await result.Content.ReadAsStringAsync(); - - - using (RMuseumDbContext importJobUpdaterDb = new RMuseumDbContext(Configuration)) - { - job.StartTime = DateTime.Now; - job.Status = ImportJobStatus.Running; - job.SrcContent = xml; - importJobUpdaterDb.Update(job); - await importJobUpdaterDb.SaveChangesAsync(); - } - - string title = ""; - string author = ""; - string pdfResourceNumber = ""; - int tagOrder = 1; - XElement elObject = XDocument.Parse(xml).Root; - foreach (var datafield in elObject.Element("record").Elements("datafield")) - { - tagOrder++; - if (datafield.Attribute("tag") == null) - continue; - string hathiTrustTag = datafield.Attribute("tag").Value; - switch(hathiTrustTag) - { - case "245": - case "246": - foreach (var subfield in datafield.Elements("subfield")) - { - if(subfield.Attribute("code") != null) - { - if (subfield.Attribute("code").Value == "a" || subfield.Attribute("code").Value == "f") - title = (title + " " + subfield.Value).Trim(); - } - } - break; - case "100": - foreach (var subfield in datafield.Elements("subfield")) - { - if (subfield.Attribute("code") != null) - { - if (subfield.Attribute("code").Value == "a" || subfield.Attribute("code").Value == "d") - author = (author + " " + subfield.Value).Trim(); - } - } - break; - case "HOL": - foreach (var subfield in datafield.Elements("subfield")) - { - if (subfield.Attribute("code") != null) - { - if (subfield.Attribute("code").Value == "p") - pdfResourceNumber = subfield.Value; - } - } - break; - - default: - { - if(int.TryParse(hathiTrustTag, out int tmp)) - { - if(tmp >= 100 && tmp <= 900) - { - string note = ""; - foreach (var subfield in datafield.Elements("subfield")) - { - if (subfield.Attribute("code") != null) - { - note = (note + " " + subfield.Value).Trim(); - } - } - - tag = await TagHandler.PrepareAttribute(context, "Notes", note, tagOrder); - meta.Add(tag); - } - } - } - break; - } - - } - - if(string.IsNullOrEmpty(pdfResourceNumber)) - { - using (RMuseumDbContext importJobUpdaterDb = new RMuseumDbContext(Configuration)) - { - job.EndTime = DateTime.Now; - job.Status = ImportJobStatus.Failed; - job.Exception = "pdfResourceNumber not found"; - importJobUpdaterDb.Update(job); - await importJobUpdaterDb.SaveChangesAsync(); - } - return; - } - - tag = await TagHandler.PrepareAttribute(context, "Title", title, 1); - meta.Add(tag); - - tag = await TagHandler.PrepareAttribute(context, "Contributor Names", author, 1); - meta.Add(tag); - - tag = await TagHandler.PrepareAttribute(context, "Type", "Book", 1); - meta.Add(tag); - - tag = await TagHandler.PrepareAttribute(context, "Type", "Manuscript", 1); - meta.Add(tag); - - - tag = await TagHandler.PrepareAttribute(context, "Source", "HathiTrust Digital Library", 1); - string viewerUrl = $"https://babel.hathitrust.org/cgi/pt?id={pdfResourceNumber}"; - tag.ValueSupplement = viewerUrl; - - meta.Add(tag); - - book.Name = book.NameInEnglish = book.Description = book.DescriptionInEnglish = title; - book.Tags = meta.ToArray(); - - List pages = new List(); - - string lastMD5hash = ""; - int order = 0; - while(true) - { - order++; - using (RMuseumDbContext importJobUpdaterDb = new RMuseumDbContext(Configuration)) - { - job.ProgressPercent = order; - importJobUpdaterDb.Update(job); - await importJobUpdaterDb.SaveChangesAsync(); - } - - string imageUrl = $"https://babel.hathitrust.org/cgi/imgsrv/image?id={pdfResourceNumber};seq={order};size=1000;rotation=0"; - RArtifactItemRecord page = new RArtifactItemRecord() - { - Name = $"تصویر {order}", - NameInEnglish = $"Image {order}", - Description = "", - DescriptionInEnglish = "", - Order = order, - FriendlyUrl = $"p{$"{order}".PadLeft(4, '0')}", - LastModified = DateTime.Now - }; - - tag = await TagHandler.PrepareAttribute(context, "Source", "HathiTrust Digital Library", 1); - tag.ValueSupplement = viewerUrl; - page.Tags = new RTagValue[] { tag }; - - if (!string.IsNullOrEmpty(imageUrl)) - { - bool recovered = false; - if ( - File.Exists - ( - Path.Combine(Path.Combine(Path.Combine(_pictureFileService.ImageStoragePath, friendlyUrl), "orig"), $"{order}".PadLeft(4, '0') + ".jpg") - ) - && - File.Exists - ( - Path.Combine(Path.Combine(Path.Combine(_pictureFileService.ImageStoragePath, friendlyUrl), "norm"), $"{order}".PadLeft(4, '0') + ".jpg") - ) - && - File.Exists - ( - Path.Combine(Path.Combine(Path.Combine(_pictureFileService.ImageStoragePath, friendlyUrl), "thumb"), $"{order}".PadLeft(4, '0') + ".jpg") - ) - ) - { - RServiceResult picture = await _pictureFileService.RecoverFromeFiles(page.Name, page.Description, 1, - imageUrl, - Path.Combine(Path.Combine(Path.Combine(_pictureFileService.ImageStoragePath, friendlyUrl), "orig"), $"{order}".PadLeft(4, '0') + ".jpg"), - Path.Combine(Path.Combine(Path.Combine(_pictureFileService.ImageStoragePath, friendlyUrl), "norm"), $"{order}".PadLeft(4, '0') + ".jpg"), - Path.Combine(Path.Combine(Path.Combine(_pictureFileService.ImageStoragePath, friendlyUrl), "thumb"), $"{order}".PadLeft(4, '0') + ".jpg"), - $"{order}".PadLeft(4, '0') + ".jpg", friendlyUrl); - if (picture.Result != null) - { - recovered = true; - page.Images = new RPictureFile[] { picture.Result }; - page.CoverImageIndex = 0; - - if (book.CoverItemIndex == (order - 1)) - { - book.CoverImage = RPictureFile.Duplicate(picture.Result); - } - - pages.Add(page); - } - - } - if (!recovered) - { - if ( - File.Exists - ( - Path.Combine(Path.Combine(Path.Combine(_pictureFileService.ImageStoragePath, friendlyUrl), "orig"), $"{order}".PadLeft(4, '0') + ".jpg") - ) - - ) - { - File.Delete - ( - Path.Combine(Path.Combine(Path.Combine(_pictureFileService.ImageStoragePath, friendlyUrl), "orig"), $"{order}".PadLeft(4, '0') + ".jpg") - ); - } - if ( - - File.Exists - ( - Path.Combine(Path.Combine(Path.Combine(_pictureFileService.ImageStoragePath, friendlyUrl), "norm"), $"{order}".PadLeft(4, '0') + ".jpg") - ) - - ) - { - File.Delete - ( - Path.Combine(Path.Combine(Path.Combine(_pictureFileService.ImageStoragePath, friendlyUrl), "norm"), $"{order}".PadLeft(4, '0') + ".jpg") - ); - } - if ( - - File.Exists - ( - Path.Combine(Path.Combine(Path.Combine(_pictureFileService.ImageStoragePath, friendlyUrl), "thumb"), $"{order}".PadLeft(4, '0') + ".jpg") - ) - ) - { - File.Delete - ( - Path.Combine(Path.Combine(Path.Combine(_pictureFileService.ImageStoragePath, friendlyUrl), "thumb"), $"{order}".PadLeft(4, '0') + ".jpg") - ); - } - var imageResult = await client.GetAsync(imageUrl); - - - int _ImportRetryCount = 200; - int _ImportRetryInitialSleep = 500; - int retryCount = 0; - while (retryCount < _ImportRetryCount && !imageResult.IsSuccessStatusCode && imageResult.StatusCode == HttpStatusCode.ServiceUnavailable) - { - imageResult.Dispose(); - Thread.Sleep(_ImportRetryInitialSleep * (retryCount + 1)); - imageResult = await client.GetAsync(imageUrl); - retryCount++; - } - - if (imageResult.IsSuccessStatusCode) - { - using (Stream imageStream = await imageResult.Content.ReadAsStreamAsync()) - { - RServiceResult picture = await _pictureFileService.Add(page.Name, page.Description, 1, null, imageUrl, imageStream, $"{order}".PadLeft(4, '0') + ".jpg", friendlyUrl); - if (picture.Result == null) - { - throw new Exception($"_pictureFileService.Add : {picture.ExceptionString}"); - } - - bool lastPage = false; - using (var md5 = MD5.Create()) - { - string md5hash = string.Join("", md5.ComputeHash(File.ReadAllBytes(Path.Combine(Path.Combine(Path.Combine(_pictureFileService.ImageStoragePath, friendlyUrl), "thumb"), $"{order}".PadLeft(4, '0') + ".jpg"))).Select(x => x.ToString("X2"))); - if (md5hash == lastMD5hash) - { - File.Delete - ( - Path.Combine(Path.Combine(Path.Combine(_pictureFileService.ImageStoragePath, friendlyUrl), "orig"), $"{order}".PadLeft(4, '0') + ".jpg") - ); - File.Delete - ( - Path.Combine(Path.Combine(Path.Combine(_pictureFileService.ImageStoragePath, friendlyUrl), "norm"), $"{order}".PadLeft(4, '0') + ".jpg") - ); - File.Delete - ( - Path.Combine(Path.Combine(Path.Combine(_pictureFileService.ImageStoragePath, friendlyUrl), "thumb"), $"{order}".PadLeft(4, '0') + ".jpg") - ); - lastPage = true; - } - lastMD5hash = md5hash; - } - - if(!lastPage) - { - page.Images = new RPictureFile[] { picture.Result }; - page.CoverImageIndex = 0; - - if (book.CoverItemIndex == (order - 1)) - { - book.CoverImage = RPictureFile.Duplicate(picture.Result); - } - - pages.Add(page); - } - else - { - break; - } - - } - } - else - { - using (RMuseumDbContext importJobUpdaterDb = new RMuseumDbContext(Configuration)) - { - job.EndTime = DateTime.Now; - job.Status = ImportJobStatus.Failed; - job.Exception = $"Http result is not ok ({imageResult.StatusCode}) for page {order}, url {imageUrl}"; - importJobUpdaterDb.Update(job); - await importJobUpdaterDb.SaveChangesAsync(); - } - - imageResult.Dispose(); - return; - } - - imageResult.Dispose(); - GC.Collect(); - } - } - } - - - book.Items = pages.ToArray(); - book.ItemCount = pages.Count; - - if (pages.Count == 0) - { - using (RMuseumDbContext importJobUpdaterDb = new RMuseumDbContext(Configuration)) - { - job.EndTime = DateTime.Now; - job.Status = ImportJobStatus.Failed; - job.Exception = "ages.Count == 0"; - importJobUpdaterDb.Update(job); - await importJobUpdaterDb.SaveChangesAsync(); - } - return; - } - - await context.Artifacts.AddAsync(book); - await context.SaveChangesAsync(); - - job.ProgressPercent = 100; - job.Status = ImportJobStatus.Succeeded; - job.ArtifactId = book.Id; - job.EndTime = DateTime.Now; - context.Update(job); - await context.SaveChangesAsync(); - - - } - } - else - { - using (RMuseumDbContext importJobUpdaterDb = new RMuseumDbContext(Configuration)) - { - job.EndTime = DateTime.Now; - job.Status = ImportJobStatus.Failed; - job.Exception = $"Http result is not ok ({result.StatusCode}) for {url}"; - importJobUpdaterDb.Update(job); - await importJobUpdaterDb.SaveChangesAsync(); - } - return; - } - - } - } - } - catch (Exception exp) - { - using (RMuseumDbContext importJobUpdaterDb = new RMuseumDbContext(Configuration)) - { - job.EndTime = DateTime.Now; - job.Status = ImportJobStatus.Failed; - job.Exception = exp.ToString(); - importJobUpdaterDb.Update(job); - await importJobUpdaterDb.SaveChangesAsync(); - } - - } - } - ); - - - - return new RServiceResult(true); - } - catch (Exception exp) - { - return new RServiceResult(false, exp.ToString()); - } - } - - /// - /// from http://www.library.upenn.edu/ - /// - /// MEDREN_9949222153503681 - /// - /// - private async Task> StartImportingFromPenLibraries(string resourceNumber, string friendlyUrl) - { - string url = $"http://dla.library.upenn.edu/dla/medren/pageturn.html?id={resourceNumber}&rotation=0&size=0"; - if ( - ( - await _context.ImportJobs - .Where(j => j.JobType == JobType.PennLibraries && j.ResourceNumber == resourceNumber && !(j.Status == ImportJobStatus.Failed || j.Status == ImportJobStatus.Aborted)) - .SingleOrDefaultAsync() - ) - != - null - ) - { - return new RServiceResult(false, $"Job is already scheduled or running for importing {url}"); - } - - if (string.IsNullOrEmpty(friendlyUrl)) - { - friendlyUrl = resourceNumber; - } - - if ( - (await _context.Artifacts.Where(a => a.FriendlyUrl == friendlyUrl).SingleOrDefaultAsync()) - != - null - ) - { - return new RServiceResult(false, $"duplicated artifact friendly url '{friendlyUrl}'"); - } - - - - ImportJob job = new ImportJob() - { - JobType = JobType.PennLibraries, - ResourceNumber = resourceNumber, - FriendlyUrl = friendlyUrl, - SrcUrl = url, - QueueTime = DateTime.Now, - ProgressPercent = 0, - Status = ImportJobStatus.NotStarted - }; - - - await _context.ImportJobs.AddAsync - ( - job - ); - - await _context.SaveChangesAsync(); - - - try - { - - _backgroundTaskQueue.QueueBackgroundWorkItem - ( - async token => - { - try - { - - - using (var client = new HttpClient()) - { - client.Timeout = TimeSpan.FromMinutes(5); - using (var result = await client.GetAsync(url)) - { - if (result.IsSuccessStatusCode) - { - using (RMuseumDbContext context = new RMuseumDbContext(Configuration)) - { - if ( - (await context.Artifacts.Where(a => a.FriendlyUrl == job.FriendlyUrl).SingleOrDefaultAsync()) - != - null - ) - { - using (RMuseumDbContext importJobUpdaterDb = new RMuseumDbContext(Configuration)) - { - job.EndTime = DateTime.Now; - job.Status = ImportJobStatus.Failed; - job.Exception = "aborted because of duplicated friendly url"; - importJobUpdaterDb.Update(job); - await importJobUpdaterDb.SaveChangesAsync(); - } - return; - } - RArtifactMasterRecord book = new RArtifactMasterRecord($"extracted from {url}", $"extracted from {url}") - { - Status = PublishStatus.Draft, - DateTime = DateTime.Now, - LastModified = DateTime.Now, - CoverItemIndex = 0, - FriendlyUrl = friendlyUrl - }; - - - List meta = new List(); - RTagValue tag; - - string html = await result.Content.ReadAsStringAsync(); - - - using (RMuseumDbContext importJobUpdaterDb = new RMuseumDbContext(Configuration)) - { - job.StartTime = DateTime.Now; - job.Status = ImportJobStatus.Running; - job.SrcContent = html; - importJobUpdaterDb.Update(job); - await importJobUpdaterDb.SaveChangesAsync(); - } - - string title = ""; - string author = ""; - int tagOrder = 1; - - int nIdxStart = html.IndexOf("https://repo.library.upenn.edu/djatoka/resolver?"); - if(nIdxStart == -1) - { - using (RMuseumDbContext importJobUpdaterDb = new RMuseumDbContext(Configuration)) - { - job.EndTime = DateTime.Now; - job.Status = ImportJobStatus.Failed; - job.Exception = "https://repo.library.upenn.edu/djatoka/resolver? not found"; - importJobUpdaterDb.Update(job); - await importJobUpdaterDb.SaveChangesAsync(); - } - return; - } - - string firstImageUrl = html.Substring(nIdxStart, html.IndexOf('"', nIdxStart) - nIdxStart).Replace("&", "&"); - - nIdxStart = html.IndexOf("recordinfolabel"); - while(nIdxStart != -1) - { - nIdxStart += "recordinfolabel\">".Length; - int nIdxEnd = html.IndexOf(":", nIdxStart); - string recordinfolabel = html.Substring(nIdxStart, nIdxEnd - nIdxStart); - nIdxStart = html.IndexOf("recordinfotext", nIdxEnd); - nIdxStart += "recordinfotext\">".Length; - nIdxEnd = html.IndexOf("", nIdxStart); - string recordinfotext = html.Substring(nIdxStart, nIdxEnd - nIdxStart).Replace("
", "
").Replace("\n", "").Replace("\r", "").Trim(); - - string[] values = recordinfotext.Split("
", StringSplitOptions.RemoveEmptyEntries); - - foreach(string value in values) - { - if (value.Trim().Length == 0) - continue; - if (recordinfolabel == "Title") - { - title = value.Trim(); - tag = await TagHandler.PrepareAttribute(context, "Title", title, 1); - meta.Add(tag); - } - else - if (recordinfolabel == "Author") - { - author = value.Trim(); - tag = await TagHandler.PrepareAttribute(context, "Contributor Names", author, 1); - meta.Add(tag); - } - else - { - tag = await TagHandler.PrepareAttribute(context, recordinfolabel, value.Trim(), tagOrder++); - meta.Add(tag); - } - } - - nIdxStart = html.IndexOf("recordinfolabel", nIdxEnd); - - } - - - - tag = await TagHandler.PrepareAttribute(context, "Type", "Book", 1); - meta.Add(tag); - - tag = await TagHandler.PrepareAttribute(context, "Type", "Manuscript", 1); - meta.Add(tag); - - - tag = await TagHandler.PrepareAttribute(context, "Source", "Penn Libraries", 1); - string viewerUrl = $"http://dla.library.upenn.edu/dla/medren/detail.html?id={resourceNumber}"; - tag.ValueSupplement = viewerUrl; - - meta.Add(tag); - - book.Name = book.NameInEnglish = book.Description = book.DescriptionInEnglish = title; - book.Tags = meta.ToArray(); - - List pages = new List(); - - int order = 0; - while (true) - { - order++; - - using (RMuseumDbContext importJobUpdaterDb = new RMuseumDbContext(Configuration)) - { - job.ProgressPercent = order; - importJobUpdaterDb.Update(job); - await importJobUpdaterDb.SaveChangesAsync(); - } - - string imageUrl = firstImageUrl; - - RArtifactItemRecord page = new RArtifactItemRecord() - { - Name = $"تصویر {order}", - NameInEnglish = $"Image {order}", - Description = "", - DescriptionInEnglish = "", - Order = order, - FriendlyUrl = $"p{$"{order}".PadLeft(4, '0')}", - LastModified = DateTime.Now - }; - - tag = await TagHandler.PrepareAttribute(context, "Source", "Penn Libraries", 1); - tag.ValueSupplement = viewerUrl; - page.Tags = new RTagValue[] { tag }; - - if (!string.IsNullOrEmpty(imageUrl)) - { - bool recovered = false; - if ( - File.Exists - ( - Path.Combine(Path.Combine(Path.Combine(_pictureFileService.ImageStoragePath, friendlyUrl), "orig"), $"{order}".PadLeft(4, '0') + ".jpg") - ) - && - File.Exists - ( - Path.Combine(Path.Combine(Path.Combine(_pictureFileService.ImageStoragePath, friendlyUrl), "norm"), $"{order}".PadLeft(4, '0') + ".jpg") - ) - && - File.Exists - ( - Path.Combine(Path.Combine(Path.Combine(_pictureFileService.ImageStoragePath, friendlyUrl), "thumb"), $"{order}".PadLeft(4, '0') + ".jpg") - ) - ) - { - RServiceResult picture = await _pictureFileService.RecoverFromeFiles(page.Name, page.Description, 1, - imageUrl, - Path.Combine(Path.Combine(Path.Combine(_pictureFileService.ImageStoragePath, friendlyUrl), "orig"), $"{order}".PadLeft(4, '0') + ".jpg"), - Path.Combine(Path.Combine(Path.Combine(_pictureFileService.ImageStoragePath, friendlyUrl), "norm"), $"{order}".PadLeft(4, '0') + ".jpg"), - Path.Combine(Path.Combine(Path.Combine(_pictureFileService.ImageStoragePath, friendlyUrl), "thumb"), $"{order}".PadLeft(4, '0') + ".jpg"), - $"{order}".PadLeft(4, '0') + ".jpg", friendlyUrl); - if (picture.Result != null) - { - recovered = true; - page.Images = new RPictureFile[] { picture.Result }; - page.CoverImageIndex = 0; - - if (book.CoverItemIndex == (order - 1)) - { - book.CoverImage = RPictureFile.Duplicate(picture.Result); - } - - pages.Add(page); - } - - } - if (!recovered) - { - if ( - File.Exists - ( - Path.Combine(Path.Combine(Path.Combine(_pictureFileService.ImageStoragePath, friendlyUrl), "orig"), $"{order}".PadLeft(4, '0') + ".jpg") - ) - - ) - { - File.Delete - ( - Path.Combine(Path.Combine(Path.Combine(_pictureFileService.ImageStoragePath, friendlyUrl), "orig"), $"{order}".PadLeft(4, '0') + ".jpg") - ); - } - if ( - - File.Exists - ( - Path.Combine(Path.Combine(Path.Combine(_pictureFileService.ImageStoragePath, friendlyUrl), "norm"), $"{order}".PadLeft(4, '0') + ".jpg") - ) - - ) - { - File.Delete - ( - Path.Combine(Path.Combine(Path.Combine(_pictureFileService.ImageStoragePath, friendlyUrl), "norm"), $"{order}".PadLeft(4, '0') + ".jpg") - ); - } - if ( - - File.Exists - ( - Path.Combine(Path.Combine(Path.Combine(_pictureFileService.ImageStoragePath, friendlyUrl), "thumb"), $"{order}".PadLeft(4, '0') + ".jpg") - ) - ) - { - File.Delete - ( - Path.Combine(Path.Combine(Path.Combine(_pictureFileService.ImageStoragePath, friendlyUrl), "thumb"), $"{order}".PadLeft(4, '0') + ".jpg") - ); - } - - if (order > 1) - { - string pageUrl = $"http://dla.library.upenn.edu/dla/medren/pageturn.html?id={resourceNumber}&doubleside=0&rotation=0&size=0¤tpage={order}"; - var pageResult = await client.GetAsync(pageUrl); - - if (pageResult.StatusCode == HttpStatusCode.NotFound) - { - break;//finished - } - - string pageHtml = await pageResult.Content.ReadAsStringAsync(); - nIdxStart = pageHtml.IndexOf("https://repo.library.upenn.edu/djatoka/resolver?"); - if (nIdxStart == -1) - { - if (order > 1) - break; //finished - using (RMuseumDbContext importJobUpdaterDb = new RMuseumDbContext(Configuration)) - { - job.EndTime = DateTime.Now; - job.Status = ImportJobStatus.Failed; - job.Exception = $"https://repo.library.upenn.edu/djatoka/resolver? not found on page {order}"; - importJobUpdaterDb.Update(job); - await importJobUpdaterDb.SaveChangesAsync(); - } - return; - } - - imageUrl = pageHtml.Substring(nIdxStart, pageHtml.IndexOf('"', nIdxStart) - nIdxStart).Replace("&", "&"); - - } - var imageResult = await client.GetAsync(imageUrl); - - if(imageResult.StatusCode == HttpStatusCode.NotFound) - { - break;//finished - } - - - int _ImportRetryCount = 200; - int _ImportRetryInitialSleep = 500; - int retryCount = 0; - while (retryCount < _ImportRetryCount && !imageResult.IsSuccessStatusCode && imageResult.StatusCode == HttpStatusCode.ServiceUnavailable) - { - imageResult.Dispose(); - Thread.Sleep(_ImportRetryInitialSleep * (retryCount + 1)); - imageResult = await client.GetAsync(imageUrl); - retryCount++; - } - - if (imageResult.IsSuccessStatusCode) - { - using (Stream imageStream = await imageResult.Content.ReadAsStreamAsync()) - { - RServiceResult picture = await _pictureFileService.Add(page.Name, page.Description, 1, null, imageUrl, imageStream, $"{order}".PadLeft(4, '0') + ".jpg", friendlyUrl); - if (picture.Result == null) - { - throw new Exception($"_pictureFileService.Add : {picture.ExceptionString}"); - } - - - page.Images = new RPictureFile[] { picture.Result }; - page.CoverImageIndex = 0; - - if (book.CoverItemIndex == (order - 1)) - { - book.CoverImage = RPictureFile.Duplicate(picture.Result); - } - - pages.Add(page); - - } - } - else - { - using (RMuseumDbContext importJobUpdaterDb = new RMuseumDbContext(Configuration)) - { - job.EndTime = DateTime.Now; - job.Status = ImportJobStatus.Failed; - job.Exception = $"Http result is not ok ({imageResult.StatusCode}) for page {order}, url {imageUrl}"; - importJobUpdaterDb.Update(job); - await importJobUpdaterDb.SaveChangesAsync(); - } - - imageResult.Dispose(); - return; - } - - imageResult.Dispose(); - GC.Collect(); - } - } - } - - - book.Items = pages.ToArray(); - book.ItemCount = pages.Count; - - if (pages.Count == 0) - { - using (RMuseumDbContext importJobUpdaterDb = new RMuseumDbContext(Configuration)) - { - job.EndTime = DateTime.Now; - job.Status = ImportJobStatus.Failed; - job.Exception = "Pages.Count == 0"; - importJobUpdaterDb.Update(job); - await importJobUpdaterDb.SaveChangesAsync(); - } - return; - } - - await context.Artifacts.AddAsync(book); - await context.SaveChangesAsync(); - - job.ProgressPercent = 100; - job.Status = ImportJobStatus.Succeeded; - job.ArtifactId = book.Id; - job.EndTime = DateTime.Now; - context.Update(job); - await context.SaveChangesAsync(); - - - } - } - else - { - using (RMuseumDbContext importJobUpdaterDb = new RMuseumDbContext(Configuration)) - { - job.EndTime = DateTime.Now; - job.Status = ImportJobStatus.Failed; - job.Exception = $"Http result is not ok ({result.StatusCode}) for {url}"; - importJobUpdaterDb.Update(job); - await importJobUpdaterDb.SaveChangesAsync(); - } - return; - } - - } - } - } - catch (Exception exp) - { - using (RMuseumDbContext importJobUpdaterDb = new RMuseumDbContext(Configuration)) - { - job.EndTime = DateTime.Now; - job.Status = ImportJobStatus.Failed; - job.Exception = exp.ToString(); - importJobUpdaterDb.Update(job); - await importJobUpdaterDb.SaveChangesAsync(); - } - - } - } - ); - - - - return new RServiceResult(true); - } - catch (Exception exp) - { - return new RServiceResult(false, exp.ToString()); - } - } - - - /// - /// from http://cudl.lib.cam.ac.uk - /// - /// MS-RAS-00258 - /// - /// - private async Task> StartImportingFromCambridge(string resourceNumber, string friendlyUrl) - { - string url = $"http://cudl.lib.cam.ac.uk/view/{resourceNumber}.json"; - if ( - ( - await _context.ImportJobs - .Where(j => j.JobType == JobType.Cambridge && j.ResourceNumber == resourceNumber && !(j.Status == ImportJobStatus.Failed || j.Status == ImportJobStatus.Aborted)) - .SingleOrDefaultAsync() - ) - != - null - ) - { - return new RServiceResult(false, $"Job is already scheduled or running for importing {url}"); - } - - if (string.IsNullOrEmpty(friendlyUrl)) - { - friendlyUrl = resourceNumber; - } - - if ( - (await _context.Artifacts.Where(a => a.FriendlyUrl == friendlyUrl).SingleOrDefaultAsync()) - != - null - ) - { - return new RServiceResult(false, $"duplicated artifact friendly url '{friendlyUrl}'"); - } - - - - ImportJob job = new ImportJob() - { - JobType = JobType.Cambridge, - ResourceNumber = resourceNumber, - FriendlyUrl = friendlyUrl, - SrcUrl = url, - QueueTime = DateTime.Now, - ProgressPercent = 0, - Status = ImportJobStatus.NotStarted - }; - - - await _context.ImportJobs.AddAsync - ( - job - ); - - await _context.SaveChangesAsync(); - - - try - { - - _backgroundTaskQueue.QueueBackgroundWorkItem - ( - async token => - { - try - { - - - using (var client = new HttpClient()) - { - using (var result = await client.GetAsync(url)) - { - if (result.IsSuccessStatusCode) - { - using (RMuseumDbContext context = new RMuseumDbContext(Configuration)) - { - if ( - (await context.Artifacts.Where(a => a.FriendlyUrl == job.FriendlyUrl).SingleOrDefaultAsync()) - != - null - ) - { - using (RMuseumDbContext importJobUpdaterDb = new RMuseumDbContext(Configuration)) - { - job.EndTime = DateTime.Now; - job.Status = ImportJobStatus.Failed; - job.Exception = "aborted because of duplicated friendly url"; - importJobUpdaterDb.Update(job); - await importJobUpdaterDb.SaveChangesAsync(); - } - return; - } - RArtifactMasterRecord book = new RArtifactMasterRecord($"extracted from {url}", $"extracted from {url}") - { - Status = PublishStatus.Draft, - DateTime = DateTime.Now, - LastModified = DateTime.Now, - CoverItemIndex = 0, - FriendlyUrl = friendlyUrl - }; - - - List meta = new List(); - RTagValue tag; - - string json = await result.Content.ReadAsStringAsync(); - var parsed = JObject.Parse(json); - book.Name = book.NameInEnglish = - parsed.SelectToken("logicalStructures[*].label").Value(); - - book.Description = book.DescriptionInEnglish = - Regex.Replace( - parsed.SelectToken("descriptiveMetadata[*].abstract.displayForm").Value(), - "<.*?>", string.Empty); - - int tagOrder = 1; - foreach (JToken descriptiveMetadata in parsed.SelectTokens("$.descriptiveMetadata[*]").Children()) - { - foreach(JToken child in descriptiveMetadata.Children()) - { - if (child.SelectToken("label") != null && child.SelectToken("display") != null) - { - if (child.SelectToken("display").Value() == "True") - { - string metaName = child.SelectToken("label").Value(); - string metaValue = ""; - if (child.SelectToken("displayForm") != null) - { - metaValue = Regex.Replace( - child.SelectToken("displayForm").Value(), - "<.*?>", string.Empty); - tag = await TagHandler.PrepareAttribute(context, metaName, metaValue, tagOrder++); - meta.Add(tag); - } - else - if (child.SelectToken("value") != null) - { - foreach (JToken value in child.SelectTokens("value").Children()) - { - if (value.SelectToken("displayForm") != null) - { - metaValue = Regex.Replace( - value.SelectToken("displayForm").Value(), - "<.*?>", string.Empty); - tag = await TagHandler.PrepareAttribute(context, metaName, metaValue, tagOrder++); - meta.Add(tag); - } - } - } - } - } - } - - } - - string imageReproPageURL = "https://image01.cudl.lib.cam.ac.uk"; - - - using (RMuseumDbContext importJobUpdaterDb = new RMuseumDbContext(Configuration)) - { - job.StartTime = DateTime.Now; - job.Status = ImportJobStatus.Running; - job.SrcContent = json; - importJobUpdaterDb.Update(job); - await importJobUpdaterDb.SaveChangesAsync(); - } - - - - - tag = await TagHandler.PrepareAttribute(context, "Type", "Book", 1); - meta.Add(tag); - - tag = await TagHandler.PrepareAttribute(context, "Type", "Manuscript", 1); - meta.Add(tag); - - - tag = await TagHandler.PrepareAttribute(context, "Source", "University of Cambridge Digital Library", 1); - string viewerUrl = $"http://cudl.lib.cam.ac.uk/view/{resourceNumber}"; - tag.ValueSupplement = viewerUrl; - - meta.Add(tag); - - book.Tags = meta.ToArray(); - - List pages = new List(); - - int order = 0; - foreach (JToken pageToken in parsed.SelectTokens("$.pages").Children()) - { - order++; - - using (RMuseumDbContext importJobUpdaterDb = new RMuseumDbContext(Configuration)) - { - job.ProgressPercent = order; - importJobUpdaterDb.Update(job); - await importJobUpdaterDb.SaveChangesAsync(); - } - - string imageUrl = imageReproPageURL + pageToken.SelectToken("downloadImageURL").Value(); - - RArtifactItemRecord page = new RArtifactItemRecord() - { - Name = $"تصویر {order}", - NameInEnglish = $"Image {order}", - Description = "", - DescriptionInEnglish = "", - Order = order, - FriendlyUrl = $"p{$"{order}".PadLeft(4, '0')}", - LastModified = DateTime.Now - }; - - List pageMata = new List(); - tag = await TagHandler.PrepareAttribute(context, "Source", "University of Cambridge Digital Library", 1); - tag.ValueSupplement = $"{viewerUrl}/{order}"; - pageMata.Add(tag); - - if(pageToken.SelectToken("label") != null) - { - tag = await TagHandler.PrepareAttribute(context, "Label", pageToken.SelectToken("label").Value(), 1); - pageMata.Add(tag); - } - - - page.Tags = pageMata.ToArray(); - - if (!string.IsNullOrEmpty(imageUrl)) - { - bool recovered = false; - if ( - File.Exists - ( - Path.Combine(Path.Combine(Path.Combine(_pictureFileService.ImageStoragePath, friendlyUrl), "orig"), $"{order}".PadLeft(4, '0') + ".jpg") - ) - && - File.Exists - ( - Path.Combine(Path.Combine(Path.Combine(_pictureFileService.ImageStoragePath, friendlyUrl), "norm"), $"{order}".PadLeft(4, '0') + ".jpg") - ) - && - File.Exists - ( - Path.Combine(Path.Combine(Path.Combine(_pictureFileService.ImageStoragePath, friendlyUrl), "thumb"), $"{order}".PadLeft(4, '0') + ".jpg") - ) - ) - { - RServiceResult picture = await _pictureFileService.RecoverFromeFiles(page.Name, page.Description, 1, - imageUrl, - Path.Combine(Path.Combine(Path.Combine(_pictureFileService.ImageStoragePath, friendlyUrl), "orig"), $"{order}".PadLeft(4, '0') + ".jpg"), - Path.Combine(Path.Combine(Path.Combine(_pictureFileService.ImageStoragePath, friendlyUrl), "norm"), $"{order}".PadLeft(4, '0') + ".jpg"), - Path.Combine(Path.Combine(Path.Combine(_pictureFileService.ImageStoragePath, friendlyUrl), "thumb"), $"{order}".PadLeft(4, '0') + ".jpg"), - $"{order}".PadLeft(4, '0') + ".jpg", friendlyUrl); - if (picture.Result != null) - { - recovered = true; - page.Images = new RPictureFile[] { picture.Result }; - page.CoverImageIndex = 0; - - if (book.CoverItemIndex == (order - 1)) - { - book.CoverImage = RPictureFile.Duplicate(picture.Result); - } - - pages.Add(page); - } - - } - if (!recovered) - { - if ( - File.Exists - ( - Path.Combine(Path.Combine(Path.Combine(_pictureFileService.ImageStoragePath, friendlyUrl), "orig"), $"{order}".PadLeft(4, '0') + ".jpg") - ) - - ) - { - File.Delete - ( - Path.Combine(Path.Combine(Path.Combine(_pictureFileService.ImageStoragePath, friendlyUrl), "orig"), $"{order}".PadLeft(4, '0') + ".jpg") - ); - } - if ( - - File.Exists - ( - Path.Combine(Path.Combine(Path.Combine(_pictureFileService.ImageStoragePath, friendlyUrl), "norm"), $"{order}".PadLeft(4, '0') + ".jpg") - ) - - ) - { - File.Delete - ( - Path.Combine(Path.Combine(Path.Combine(_pictureFileService.ImageStoragePath, friendlyUrl), "norm"), $"{order}".PadLeft(4, '0') + ".jpg") - ); - } - if ( - - File.Exists - ( - Path.Combine(Path.Combine(Path.Combine(_pictureFileService.ImageStoragePath, friendlyUrl), "thumb"), $"{order}".PadLeft(4, '0') + ".jpg") - ) - ) - { - File.Delete - ( - Path.Combine(Path.Combine(Path.Combine(_pictureFileService.ImageStoragePath, friendlyUrl), "thumb"), $"{order}".PadLeft(4, '0') + ".jpg") - ); - } - - var imageResult = await client.GetAsync(imageUrl); - - if (imageResult.StatusCode == HttpStatusCode.NotFound) - { - break;//finished - } - - - int _ImportRetryCount = 200; - int _ImportRetryInitialSleep = 500; - int retryCount = 0; - while (retryCount < _ImportRetryCount && !imageResult.IsSuccessStatusCode && imageResult.StatusCode == HttpStatusCode.ServiceUnavailable) - { - imageResult.Dispose(); - Thread.Sleep(_ImportRetryInitialSleep * (retryCount + 1)); - imageResult = await client.GetAsync(imageUrl); - retryCount++; - } - - if (imageResult.IsSuccessStatusCode) - { - using (Stream imageStream = await imageResult.Content.ReadAsStreamAsync()) - { - RServiceResult picture = await _pictureFileService.Add(page.Name, page.Description, 1, null, imageUrl, imageStream, $"{order}".PadLeft(4, '0') + ".jpg", friendlyUrl); - if (picture.Result == null) - { - throw new Exception($"_pictureFileService.Add : {picture.ExceptionString}"); - } - - - - page.Images = new RPictureFile[] { picture.Result }; - page.CoverImageIndex = 0; - - if (book.CoverItemIndex == (order - 1)) - { - book.CoverImage = RPictureFile.Duplicate(picture.Result); - } - - pages.Add(page); - - } - } - else - { - using (RMuseumDbContext importJobUpdaterDb = new RMuseumDbContext(Configuration)) - { - job.EndTime = DateTime.Now; - job.Status = ImportJobStatus.Failed; - job.Exception = $"Http result is not ok ({imageResult.StatusCode}) for page {order}, url {imageUrl}"; - importJobUpdaterDb.Update(job); - await importJobUpdaterDb.SaveChangesAsync(); - } - - imageResult.Dispose(); - return; - } - - imageResult.Dispose(); - GC.Collect(); - } - } - } - - - book.Items = pages.ToArray(); - book.ItemCount = pages.Count; - - if (pages.Count == 0) - { - using (RMuseumDbContext importJobUpdaterDb = new RMuseumDbContext(Configuration)) - { - job.EndTime = DateTime.Now; - job.Status = ImportJobStatus.Failed; - job.Exception = "Pages.Count == 0"; - importJobUpdaterDb.Update(job); - await importJobUpdaterDb.SaveChangesAsync(); - } - return; - } - - await context.Artifacts.AddAsync(book); - await context.SaveChangesAsync(); - - job.ProgressPercent = 100; - job.Status = ImportJobStatus.Succeeded; - job.ArtifactId = book.Id; - job.EndTime = DateTime.Now; - context.Update(job); - await context.SaveChangesAsync(); - - - } - } - else - { - using (RMuseumDbContext importJobUpdaterDb = new RMuseumDbContext(Configuration)) - { - job.EndTime = DateTime.Now; - job.Status = ImportJobStatus.Failed; - job.Exception = $"Http result is not ok ({result.StatusCode}) for {url}"; - importJobUpdaterDb.Update(job); - await importJobUpdaterDb.SaveChangesAsync(); - } - return; - } - - } - } - } - catch (Exception exp) - { - using (RMuseumDbContext importJobUpdaterDb = new RMuseumDbContext(Configuration)) - { - job.EndTime = DateTime.Now; - job.Status = ImportJobStatus.Failed; - job.Exception = exp.ToString(); - importJobUpdaterDb.Update(job); - await importJobUpdaterDb.SaveChangesAsync(); - } - - } - } - ); - - - - return new RServiceResult(true); - } - catch (Exception exp) - { - return new RServiceResult(false, exp.ToString()); - } - } - - - /// - /// from http://www.bl.uk - /// - /// grenville_xli_f001r - /// - /// - private async Task> StartImportingFromBritishLibrary(string resourceNumber, string friendlyUrl) - { - string url = $"http://www.bl.uk/manuscripts/Viewer.aspx?ref={resourceNumber}"; - if ( - ( - await _context.ImportJobs - .Where(j => j.JobType == JobType.BritishLibrary && j.ResourceNumber == resourceNumber && !(j.Status == ImportJobStatus.Failed || j.Status == ImportJobStatus.Aborted)) - .SingleOrDefaultAsync() - ) - != - null - ) - { - return new RServiceResult(false, $"Job is already scheduled or running for importing {url}"); - } - - if (string.IsNullOrEmpty(friendlyUrl)) - { - friendlyUrl = resourceNumber; - } - - if ( - (await _context.Artifacts.Where(a => a.FriendlyUrl == friendlyUrl).SingleOrDefaultAsync()) - != - null - ) - { - return new RServiceResult(false, $"duplicated artifact friendly url '{friendlyUrl}'"); - } - - - - ImportJob job = new ImportJob() - { - JobType = JobType.BritishLibrary, - ResourceNumber = resourceNumber, - FriendlyUrl = friendlyUrl, - SrcUrl = url, - QueueTime = DateTime.Now, - ProgressPercent = 0, - Status = ImportJobStatus.NotStarted - }; - - - await _context.ImportJobs.AddAsync - ( - job - ); - - await _context.SaveChangesAsync(); - - - try - { - - _backgroundTaskQueue.QueueBackgroundWorkItem - ( - async token => - { - try - { - - - using (var client = new HttpClient()) - { - client.Timeout = TimeSpan.FromMinutes(5); - using (var result = await client.GetAsync(url)) - { - if (result.IsSuccessStatusCode) - { - using (RMuseumDbContext context = new RMuseumDbContext(Configuration)) - { - if ( - (await context.Artifacts.Where(a => a.FriendlyUrl == job.FriendlyUrl).SingleOrDefaultAsync()) - != - null - ) - { - using (RMuseumDbContext importJobUpdaterDb = new RMuseumDbContext(Configuration)) - { - job.EndTime = DateTime.Now; - job.Status = ImportJobStatus.Failed; - job.Exception = "aborted because of duplicated friendly url"; - importJobUpdaterDb.Update(job); - await importJobUpdaterDb.SaveChangesAsync(); - } - return; - } - RArtifactMasterRecord book = new RArtifactMasterRecord($"extracted from {url}", $"extracted from {url}") - { - Status = PublishStatus.Draft, - DateTime = DateTime.Now, - LastModified = DateTime.Now, - CoverItemIndex = 0, - FriendlyUrl = friendlyUrl - }; - - - List meta = new List(); - RTagValue tag; - - string html = await result.Content.ReadAsStringAsync(); - - - using (RMuseumDbContext importJobUpdaterDb = new RMuseumDbContext(Configuration)) - { - job.StartTime = DateTime.Now; - job.Status = ImportJobStatus.Running; - job.SrcContent = html; - importJobUpdaterDb.Update(job); - await importJobUpdaterDb.SaveChangesAsync(); - } - - - int nIdxStart = html.IndexOf("PageList"); - if (nIdxStart == -1) - { - using (RMuseumDbContext importJobUpdaterDb = new RMuseumDbContext(Configuration)) - { - job.EndTime = DateTime.Now; - job.Status = ImportJobStatus.Failed; - job.Exception = "PageList not found"; - importJobUpdaterDb.Update(job); - await importJobUpdaterDb.SaveChangesAsync(); - } - return; - } - - nIdxStart = html.IndexOf("value=\"", nIdxStart); - if (nIdxStart == -1) - { - using (RMuseumDbContext importJobUpdaterDb = new RMuseumDbContext(Configuration)) - { - job.EndTime = DateTime.Now; - job.Status = ImportJobStatus.Failed; - job.Exception = "value after PageList not found"; - importJobUpdaterDb.Update(job); - await importJobUpdaterDb.SaveChangesAsync(); - } - return; - } - - nIdxStart += "value=\"".Length; - - string strPageList = html.Substring(nIdxStart, html.IndexOf('"', nIdxStart) - nIdxStart); - - nIdxStart = html.IndexOf("TextList"); - if (nIdxStart == -1) - { - using (RMuseumDbContext importJobUpdaterDb = new RMuseumDbContext(Configuration)) - { - job.EndTime = DateTime.Now; - job.Status = ImportJobStatus.Failed; - job.Exception = "TextList not found"; - importJobUpdaterDb.Update(job); - await importJobUpdaterDb.SaveChangesAsync(); - } - return; - } - - nIdxStart = html.IndexOf("value=\"", nIdxStart); - if (nIdxStart == -1) - { - using (RMuseumDbContext importJobUpdaterDb = new RMuseumDbContext(Configuration)) - { - job.EndTime = DateTime.Now; - job.Status = ImportJobStatus.Failed; - job.Exception = "value after TextList not found"; - importJobUpdaterDb.Update(job); - await importJobUpdaterDb.SaveChangesAsync(); - } - return; - } - - nIdxStart += "value=\"".Length; - - string strTextList = html.Substring(nIdxStart, html.IndexOf('"', nIdxStart) - nIdxStart); - - nIdxStart = html.IndexOf("TitleList"); - if (nIdxStart == -1) - { - using (RMuseumDbContext importJobUpdaterDb = new RMuseumDbContext(Configuration)) - { - job.EndTime = DateTime.Now; - job.Status = ImportJobStatus.Failed; - job.Exception = "TitleList not found"; - importJobUpdaterDb.Update(job); - await importJobUpdaterDb.SaveChangesAsync(); - } - return; - } - - nIdxStart = html.IndexOf("value=\"", nIdxStart); - if (nIdxStart == -1) - { - using (RMuseumDbContext importJobUpdaterDb = new RMuseumDbContext(Configuration)) - { - job.EndTime = DateTime.Now; - job.Status = ImportJobStatus.Failed; - job.Exception = "value after TitleList not found"; - importJobUpdaterDb.Update(job); - await importJobUpdaterDb.SaveChangesAsync(); - } - return; - } - - nIdxStart += "value=\"".Length; - - string strTitleList = html.Substring(nIdxStart, html.IndexOf('"', nIdxStart) - nIdxStart); - - string[] PageUrls = strPageList.Split("||", StringSplitOptions.None); - string[] PageTexts = strTextList.Split("||", StringSplitOptions.None); - string[] PageTitles = strTitleList.Split("||", StringSplitOptions.None); - - if(PageUrls.Length != PageTexts.Length || PageTexts.Length != PageTitles.Length) - { - using (RMuseumDbContext importJobUpdaterDb = new RMuseumDbContext(Configuration)) - { - job.EndTime = DateTime.Now; - job.Status = ImportJobStatus.Failed; - job.Exception = "PageUrls.Length != PageTexts.Length || PageTexts.Length != PageTitles.Length"; - importJobUpdaterDb.Update(job); - await importJobUpdaterDb.SaveChangesAsync(); - } - return; - } - - tag = await TagHandler.PrepareAttribute(context, "Title", "Untitled", 1); - meta.Add(tag); - - tag = await TagHandler.PrepareAttribute(context, "Contributor Names", "Unknown", 1); - meta.Add(tag); - - tag = await TagHandler.PrepareAttribute(context, "Type", "Book", 1); - meta.Add(tag); - - tag = await TagHandler.PrepareAttribute(context, "Type", "Manuscript", 1); - meta.Add(tag); - - book.Tags = meta.ToArray(); - - - tag = await TagHandler.PrepareAttribute(context, "Source", "British Library", 1); - string viewerUrl = $"http://www.bl.uk/manuscripts/FullDisplay.aspx?ref={resourceNumber.Substring(0, resourceNumber.LastIndexOf('_'))}"; - tag.ValueSupplement = viewerUrl; - - meta.Add(tag); - - List pages = new List(); - - int order = 0; - for(int i = 0; i < PageUrls.Length; i++) - { - if (PageUrls[i] == "##") - continue; - order++; - - using (RMuseumDbContext importJobUpdaterDb = new RMuseumDbContext(Configuration)) - { - job.ProgressPercent = order; - importJobUpdaterDb.Update(job); - await importJobUpdaterDb.SaveChangesAsync(); - } - - - RArtifactItemRecord page = new RArtifactItemRecord() - { - Name = $"تصویر {order}", - NameInEnglish = $"Image {order}", - Description = "", - DescriptionInEnglish = "", - Order = order, - FriendlyUrl = $"p{$"{order}".PadLeft(4, '0')}", - LastModified = DateTime.Now - }; - - - List pageTags = new List(); - tag = await TagHandler.PrepareAttribute(context, "Source", "British Library", 1); - tag.ValueSupplement = $"http://www.bl.uk/manuscripts/Viewer.aspx?ref={PageUrls[i]}"; - pageTags.Add(tag); - - - if (!string.IsNullOrEmpty(PageTitles[i])) - { - RTagValue toc = await TagHandler.PrepareAttribute(context, "Title in TOC", PageTitles[i], 1); - toc.ValueSupplement = "1";//font size - pageTags.Add(toc); - } - - if (!string.IsNullOrEmpty(PageTexts[i])) - { - tag = await TagHandler.PrepareAttribute(context, "Label", PageTexts[i], 1); - pageTags.Add(tag); - - } - - page.Tags = pageTags.ToArray(); - - - bool recovered = false; - if ( - File.Exists - ( - Path.Combine(Path.Combine(Path.Combine(_pictureFileService.ImageStoragePath, friendlyUrl), "orig"), $"{order}".PadLeft(4, '0') + ".jpg") - ) - && - File.Exists - ( - Path.Combine(Path.Combine(Path.Combine(_pictureFileService.ImageStoragePath, friendlyUrl), "norm"), $"{order}".PadLeft(4, '0') + ".jpg") - ) - && - File.Exists - ( - Path.Combine(Path.Combine(Path.Combine(_pictureFileService.ImageStoragePath, friendlyUrl), "thumb"), $"{order}".PadLeft(4, '0') + ".jpg") - ) - ) - { - RServiceResult picture = await _pictureFileService.RecoverFromeFiles(page.Name, page.Description, 1, - viewerUrl, - Path.Combine(Path.Combine(Path.Combine(_pictureFileService.ImageStoragePath, friendlyUrl), "orig"), $"{order}".PadLeft(4, '0') + ".jpg"), - Path.Combine(Path.Combine(Path.Combine(_pictureFileService.ImageStoragePath, friendlyUrl), "norm"), $"{order}".PadLeft(4, '0') + ".jpg"), - Path.Combine(Path.Combine(Path.Combine(_pictureFileService.ImageStoragePath, friendlyUrl), "thumb"), $"{order}".PadLeft(4, '0') + ".jpg"), - $"{order}".PadLeft(4, '0') + ".jpg", friendlyUrl); - if (picture.Result != null) - { - recovered = true; - page.Images = new RPictureFile[] { picture.Result }; - page.CoverImageIndex = 0; - - if (book.CoverItemIndex == (order - 1)) - { - book.CoverImage = RPictureFile.Duplicate(picture.Result); - } - - pages.Add(page); - } - - } - if (!recovered) - { - if ( - File.Exists - ( - Path.Combine(Path.Combine(Path.Combine(_pictureFileService.ImageStoragePath, friendlyUrl), "orig"), $"{order}".PadLeft(4, '0') + ".jpg") - ) - - ) - { - File.Delete - ( - Path.Combine(Path.Combine(Path.Combine(_pictureFileService.ImageStoragePath, friendlyUrl), "orig"), $"{order}".PadLeft(4, '0') + ".jpg") - ); - } - if ( - - File.Exists - ( - Path.Combine(Path.Combine(Path.Combine(_pictureFileService.ImageStoragePath, friendlyUrl), "norm"), $"{order}".PadLeft(4, '0') + ".jpg") - ) - - ) - { - File.Delete - ( - Path.Combine(Path.Combine(Path.Combine(_pictureFileService.ImageStoragePath, friendlyUrl), "norm"), $"{order}".PadLeft(4, '0') + ".jpg") - ); - } - if ( - - File.Exists - ( - Path.Combine(Path.Combine(Path.Combine(_pictureFileService.ImageStoragePath, friendlyUrl), "thumb"), $"{order}".PadLeft(4, '0') + ".jpg") - ) - ) - { - File.Delete - ( - Path.Combine(Path.Combine(Path.Combine(_pictureFileService.ImageStoragePath, friendlyUrl), "thumb"), $"{order}".PadLeft(4, '0') + ".jpg") - ); - } - - /* - failed multithread attempt: - - BLTileMixer mixer = new BLTileMixer(); - RServiceResult blResult = await mixer.DownloadMix(PageUrls[i], order); - */ - - - Dictionary<(int x, int y), Image> tiles = new Dictionary<(int x, int y), Image>(); - int max_x = -1; - for(int x = 0; ;x++) - { - string imageUrl = $"http://www.bl.uk/manuscripts/Proxy.ashx?view={PageUrls[i]}_files/13/{x}_0.jpg"; - var imageResult = await client.GetAsync(imageUrl); - - int _ImportRetryCount = 5; - int _ImportRetryInitialSleep = 500; - int retryCount = 0; - while (retryCount < _ImportRetryCount && !imageResult.IsSuccessStatusCode && imageResult.StatusCode == HttpStatusCode.ServiceUnavailable) - { - imageResult.Dispose(); - Thread.Sleep(_ImportRetryInitialSleep * (retryCount + 1)); - imageResult = await client.GetAsync(imageUrl); - retryCount++; - } - - if (imageResult.IsSuccessStatusCode) - { - using (Stream imageStream = await imageResult.Content.ReadAsStreamAsync()) - { - - imageStream.Position = 0; - try - { - Image tile = Image.FromStream(imageStream); - tiles.Add((x, 0), tile); - max_x = x; - } - catch(Exception aexp) - { - if (aexp is ArgumentException) - { - break; - } - throw aexp; - } - } - } - else - { - using (RMuseumDbContext importJobUpdaterDb = new RMuseumDbContext(Configuration)) - { - job.EndTime = DateTime.Now; - job.Status = ImportJobStatus.Failed; - job.Exception = $"Http result is not ok ({imageResult.StatusCode}) for page {order}, url {imageUrl}"; - importJobUpdaterDb.Update(job); - await importJobUpdaterDb.SaveChangesAsync(); - } - - imageResult.Dispose(); - return; - } - } - - int max_y = -1; - for (int y = 1; ; y++) - { - string imageUrl = $"http://www.bl.uk/manuscripts/Proxy.ashx?view={PageUrls[i]}_files/13/0_{y}.jpg"; - var imageResult = await client.GetAsync(imageUrl); - - int _ImportRetryCount = 5; - int _ImportRetryInitialSleep = 500; - int retryCount = 0; - while (retryCount < _ImportRetryCount && !imageResult.IsSuccessStatusCode && imageResult.StatusCode == HttpStatusCode.ServiceUnavailable) - { - imageResult.Dispose(); - Thread.Sleep(_ImportRetryInitialSleep * (retryCount + 1)); - imageResult = await client.GetAsync(imageUrl); - retryCount++; - } - - if (imageResult.IsSuccessStatusCode) - { - using (Stream imageStream = await imageResult.Content.ReadAsStreamAsync()) - { - if (imageStream.Length <= 248) - break; - imageStream.Position = 0; - try - { - Image tile = Image.FromStream(imageStream); - tiles.Add((0, y), tile); - max_y = y; - } - catch (Exception aexp) - { - if (aexp is ArgumentException) - { - break; - } - throw aexp; - } - } - } - else - { - using (RMuseumDbContext importJobUpdaterDb = new RMuseumDbContext(Configuration)) - { - job.EndTime = DateTime.Now; - job.Status = ImportJobStatus.Failed; - job.Exception = $"Http result is not ok ({imageResult.StatusCode}) for page {order}, url {imageUrl}"; - importJobUpdaterDb.Update(job); - await importJobUpdaterDb.SaveChangesAsync(); - } - - imageResult.Dispose(); - return; - } - } - - for(int x = 0; x <= max_x; x++) - for(int y = 0; y <= max_y; y++) - if(tiles.TryGetValue((x, y), out Image tmp) == false) - { - string imageUrl = $"http://www.bl.uk/manuscripts/Proxy.ashx?view={PageUrls[i]}_files/13/{x}_{y}.jpg"; - var imageResult = await client.GetAsync(imageUrl); - - int _ImportRetryCount = 5; - int _ImportRetryInitialSleep = 500; - int retryCount = 0; - while (retryCount < _ImportRetryCount && !imageResult.IsSuccessStatusCode && imageResult.StatusCode == HttpStatusCode.ServiceUnavailable) - { - imageResult.Dispose(); - Thread.Sleep(_ImportRetryInitialSleep * (retryCount + 1)); - imageResult = await client.GetAsync(imageUrl); - retryCount++; - } - - if (imageResult.IsSuccessStatusCode) - { - using (Stream imageStream = await imageResult.Content.ReadAsStreamAsync()) - { - if (imageStream.Length == 0) - break; - imageStream.Position = 0; - tiles.Add((x, y), Image.FromStream(imageStream)); - } - } - else - { - using (RMuseumDbContext importJobUpdaterDb = new RMuseumDbContext(Configuration)) - { - job.EndTime = DateTime.Now; - job.Status = ImportJobStatus.Failed; - job.Exception = $"Http result is not ok ({imageResult.StatusCode}) for page {order}, url {imageUrl}"; - importJobUpdaterDb.Update(job); - await importJobUpdaterDb.SaveChangesAsync(); - } - - imageResult.Dispose(); - return; - } - } - - int tileWidth = tiles[(0, 0)].Width; - int tileHeight = tiles[(0, 0)].Height; - - int imageWidth = tileWidth * (max_x + 1); - int imageHeight = tileHeight * (max_y + 1); - - using(Image image = new Bitmap(imageWidth, imageHeight)) - { - using(Graphics g = Graphics.FromImage(image)) - { - for(int x = 0; x <= max_x; x++) - for(int y = 0; y < max_y; y++) - { - g.DrawImage(tiles[(x, y)], new Point(x * tileWidth, y * tileHeight)); - tiles[(x, y)].Dispose(); - } - - using (Stream imageStream = new MemoryStream()) - { - ImageCodecInfo jpgEncoder = _pictureFileService.GetEncoder(ImageFormat.Jpeg); - - Encoder myEncoder = - Encoder.Quality; - EncoderParameters jpegParameters = new EncoderParameters(1); - - EncoderParameter qualityParameter = new EncoderParameter(myEncoder, 92L); - jpegParameters.Param[0] = qualityParameter; - - image.Save(imageStream, jpgEncoder, jpegParameters); - RServiceResult picture = await _pictureFileService.Add(page.Name, page.Description, 1, null, $"http://www.bl.uk/manuscripts/Viewer.aspx?ref={PageUrls[i]}", imageStream, $"{order}".PadLeft(4, '0') + ".jpg", friendlyUrl); - if (picture.Result == null) - { - throw new Exception($"_pictureFileService.Add : {picture.ExceptionString}"); - } - - page.Images = new RPictureFile[] { picture.Result }; - page.CoverImageIndex = 0; - - if (book.CoverItemIndex == (order - 1)) - { - book.CoverImage = RPictureFile.Duplicate(picture.Result); - } - - pages.Add(page); - } - } - } - - GC.Collect(); - - - - } - } - - - book.Items = pages.ToArray(); - book.ItemCount = pages.Count; - - if (pages.Count == 0) - { - using (RMuseumDbContext importJobUpdaterDb = new RMuseumDbContext(Configuration)) - { - job.EndTime = DateTime.Now; - job.Status = ImportJobStatus.Failed; - job.Exception = "Pages.Count == 0"; - importJobUpdaterDb.Update(job); - await importJobUpdaterDb.SaveChangesAsync(); - } - return; - } - - await context.Artifacts.AddAsync(book); - await context.SaveChangesAsync(); - - job.ProgressPercent = 100; - job.Status = ImportJobStatus.Succeeded; - job.ArtifactId = book.Id; - job.EndTime = DateTime.Now; - context.Update(job); - await context.SaveChangesAsync(); - - - } - } - else - { - using (RMuseumDbContext importJobUpdaterDb = new RMuseumDbContext(Configuration)) - { - job.EndTime = DateTime.Now; - job.Status = ImportJobStatus.Failed; - job.Exception = $"Http result is not ok ({result.StatusCode}) for {url}"; - importJobUpdaterDb.Update(job); - await importJobUpdaterDb.SaveChangesAsync(); - } - return; - } - - } - } - } - catch (Exception exp) - { - using (RMuseumDbContext importJobUpdaterDb = new RMuseumDbContext(Configuration)) - { - job.EndTime = DateTime.Now; - job.Status = ImportJobStatus.Failed; - job.Exception = exp.ToString(); - importJobUpdaterDb.Update(job); - await importJobUpdaterDb.SaveChangesAsync(); - } - - } - } - ); - - - - return new RServiceResult(true); - } - catch (Exception exp) - { - return new RServiceResult(false, exp.ToString()); - } - } - - - - /// - /// import from server folder - /// - /// C:\Tools\batches\florence - /// shahname-florence - /// https://t.me/dr_khatibi_abolfazl/888 - /// - private async Task> StartImportingFromServerFolder(string folderPath, string friendlyUrl, string srcUrl) - { - try - { - if ( - ( - await _context.ImportJobs - .Where(j => j.JobType == JobType.ServerFolder && j.ResourceNumber == folderPath && !(j.Status == ImportJobStatus.Failed || j.Status == ImportJobStatus.Aborted)) - .SingleOrDefaultAsync() - ) - != - null - ) - { - return new RServiceResult(false, $"Job is already scheduled or running for importing server folder {folderPath}"); - } - - if (string.IsNullOrEmpty(friendlyUrl)) - { - return new RServiceResult(false, $"Friendly url is empty, server folder {folderPath}"); - } - - if ( - (await _context.Artifacts.Where(a => a.FriendlyUrl == friendlyUrl).SingleOrDefaultAsync()) - != - null - ) - { - return new RServiceResult(false, $"duplicated friendly url '{friendlyUrl}'"); - } - - ImportJob job = new ImportJob() - { - JobType = JobType.ServerFolder, - ResourceNumber = folderPath, - FriendlyUrl = friendlyUrl, - SrcUrl = srcUrl, - QueueTime = DateTime.Now, - ProgressPercent = 0, - Status = ImportJobStatus.NotStarted - }; - - - await _context.ImportJobs.AddAsync - ( - job - ); - - await _context.SaveChangesAsync(); - - _backgroundTaskQueue.QueueBackgroundWorkItem - ( - async token => - { - try - { - using (RMuseumDbContext context = new RMuseumDbContext(Configuration)) - { - RArtifactMasterRecord book = new RArtifactMasterRecord($"extracted from server folder {job.ResourceNumber}", $"extracted from server folder {job.ResourceNumber}") - { - Status = PublishStatus.Draft, - DateTime = DateTime.Now, - LastModified = DateTime.Now, - CoverItemIndex = 0, - FriendlyUrl = friendlyUrl - }; - - - List meta = new List(); - RTagValue tag; - - - tag = await TagHandler.PrepareAttribute(context, "Type", "Book", 1); - meta.Add(tag); - - tag = await TagHandler.PrepareAttribute(context, "Type", "Manuscript", 1); - meta.Add(tag); - - - - meta.Add(tag); - - using (RMuseumDbContext importJobUpdaterDb = new RMuseumDbContext(Configuration)) - { - job.StartTime = DateTime.Now; - job.Status = ImportJobStatus.Running; - job.SrcContent = ""; - importJobUpdaterDb.Update(job); - await importJobUpdaterDb.SaveChangesAsync(); - } - - List pages = new List(); - string[] fileNames = Directory.GetFiles(job.ResourceNumber, "*.jpg"); - int order = 0; - foreach (string fileName in fileNames) - { - using (RMuseumDbContext importJobUpdaterDb = new RMuseumDbContext(Configuration)) - { - job.ProgressPercent = order * 100 / (decimal)fileNames.Length; - importJobUpdaterDb.Update(job); - await importJobUpdaterDb.SaveChangesAsync(); - } - - order++; - - RArtifactItemRecord page = new RArtifactItemRecord() - { - Name = $"تصویر {order}", - NameInEnglish = $"Image {order} of {book.NameInEnglish}", - Description = "", - DescriptionInEnglish = "", - Order = order, - FriendlyUrl = $"p{$"{order}".PadLeft(4, '0')}", - LastModified = DateTime.Now - }; - - - page.Tags = new RTagValue[] { }; - - if ( - File.Exists - ( - Path.Combine(Path.Combine(Path.Combine(_pictureFileService.ImageStoragePath, friendlyUrl), "orig"), $"{order}".PadLeft(4, '0') + ".jpg") - ) - - ) - { - File.Delete - ( - Path.Combine(Path.Combine(Path.Combine(_pictureFileService.ImageStoragePath, friendlyUrl), "orig"), $"{order}".PadLeft(4, '0') + ".jpg") - ); - } - if ( - - File.Exists - ( - Path.Combine(Path.Combine(Path.Combine(_pictureFileService.ImageStoragePath, friendlyUrl), "norm"), $"{order}".PadLeft(4, '0') + ".jpg") - ) - - ) - { - File.Delete - ( - Path.Combine(Path.Combine(Path.Combine(_pictureFileService.ImageStoragePath, friendlyUrl), "norm"), $"{order}".PadLeft(4, '0') + ".jpg") - ); - } - if ( - - File.Exists - ( - Path.Combine(Path.Combine(Path.Combine(_pictureFileService.ImageStoragePath, friendlyUrl), "thumb"), $"{order}".PadLeft(4, '0') + ".jpg") - ) - ) - { - File.Delete - ( - Path.Combine(Path.Combine(Path.Combine(_pictureFileService.ImageStoragePath, friendlyUrl), "thumb"), $"{order}".PadLeft(4, '0') + ".jpg") - ); - } - using (FileStream imageStream = new FileStream(fileName, FileMode.Open)) - { - RServiceResult picture = await _pictureFileService.Add(page.Name, page.Description, 1, null, job.SrcUrl, imageStream, $"{order}".PadLeft(4, '0') + ".jpg", friendlyUrl); - if (picture.Result == null) - { - throw new Exception($"_pictureFileService.Add : {picture.ExceptionString}"); - } - - page.Images = new RPictureFile[] { picture.Result }; - - page.CoverImageIndex = 0; - - if (book.CoverItemIndex == (order - 1)) - { - book.CoverImage = RPictureFile.Duplicate(picture.Result); - } - - } - - - pages.Add(page); - } - - - book.Tags = meta.ToArray(); - - book.Items = pages.ToArray(); - book.ItemCount = pages.Count; - - if (pages.Count == 0) - { - using (RMuseumDbContext importJobUpdaterDb = new RMuseumDbContext(Configuration)) - { - job.EndTime = DateTime.Now; - job.Status = ImportJobStatus.Failed; - job.Exception = "Pages.Count == 0"; - importJobUpdaterDb.Update(job); - await importJobUpdaterDb.SaveChangesAsync(); - } - return; - } - - await context.Artifacts.AddAsync(book); - await context.SaveChangesAsync(); - - job.ProgressPercent = 100; - job.Status = ImportJobStatus.Succeeded; - job.ArtifactId = book.Id; - job.EndTime = DateTime.Now; - context.Update(job); - await context.SaveChangesAsync(); - - - - } - } - catch (Exception exp) - { - using (RMuseumDbContext importJobUpdaterDb = new RMuseumDbContext(Configuration)) - { - job.EndTime = DateTime.Now; - job.Status = ImportJobStatus.Failed; - job.Exception = exp.ToString(); - importJobUpdaterDb.Update(job); - await importJobUpdaterDb.SaveChangesAsync(); - } - - } - } - ); - - return new RServiceResult(true); - } - catch (Exception exp) - { - return new RServiceResult(false, exp.ToString()); - } - } - - - /// - /// from http://www.thedigitalwalters.org/01_ACCESS_WALTERS_MANUSCRIPTS.html - /// - /// W619 - /// golestan-walters-01 - /// - private async Task> StartImportingFromWalters(string resourceNumber, string friendlyUrl) - { - string url = $"http://www.thedigitalwalters.org/Data/WaltersManuscripts/ManuscriptDescriptions/{resourceNumber}_tei.xml"; - if ( - ( - await _context.ImportJobs - .Where(j => j.JobType == JobType.Walters && j.ResourceNumber == resourceNumber && !(j.Status == ImportJobStatus.Failed || j.Status == ImportJobStatus.Aborted)) - .SingleOrDefaultAsync() - ) - != - null - ) - { - return new RServiceResult(false, $"Job is already scheduled or running for importing {url}"); - } - - if (string.IsNullOrEmpty(friendlyUrl)) - { - friendlyUrl = resourceNumber; - } - - if ( - (await _context.Artifacts.Where(a => a.FriendlyUrl == friendlyUrl).SingleOrDefaultAsync()) - != - null - ) - { - return new RServiceResult(false, $"duplicated friendly url '{friendlyUrl}'"); - } - - ImportJob job = new ImportJob() - { - JobType = JobType.Walters, - ResourceNumber = resourceNumber, - FriendlyUrl = friendlyUrl, - SrcUrl = url, - QueueTime = DateTime.Now, - ProgressPercent = 0, - Status = ImportJobStatus.NotStarted - }; - - - await _context.ImportJobs.AddAsync - ( - job - ); - - await _context.SaveChangesAsync(); - - - try - { - - _backgroundTaskQueue.QueueBackgroundWorkItem - ( - async token => - { - try - { - - - using (var client = new HttpClient()) - { - - using (var result = await client.GetAsync(url)) - { - if (result.IsSuccessStatusCode) - { - using (RMuseumDbContext context = new RMuseumDbContext(Configuration)) - { - RArtifactMasterRecord book = new RArtifactMasterRecord($"extracted from {url}", $"extracted from {url}") - { - Status = PublishStatus.Draft, - DateTime = DateTime.Now, - LastModified = DateTime.Now, - CoverItemIndex = 0, - FriendlyUrl = friendlyUrl - }; - - - List meta = new List(); - RTagValue tag; - - string xml = await result.Content.ReadAsStringAsync(); - - - using (RMuseumDbContext importJobUpdaterDb = new RMuseumDbContext(Configuration)) - { - job.StartTime = DateTime.Now; - job.Status = ImportJobStatus.Running; - job.SrcContent = xml; - importJobUpdaterDb.Update(job); - await importJobUpdaterDb.SaveChangesAsync(); - } - - XElement elObject = XDocument.Parse(xml).Root; - - tag = await TagHandler.PrepareAttribute(context, "Type", "Book", 1); - meta.Add(tag); - - tag = await TagHandler.PrepareAttribute(context, "Type", "Manuscript", 1); - meta.Add(tag); - - - - - - - try - { - foreach (var prop in elObject - .Elements("{http://www.tei-c.org/ns/1.0}teiHeader").First() - .Elements("{http://www.tei-c.org/ns/1.0}fileDesc").First() - .Elements("{http://www.tei-c.org/ns/1.0}titleStmt").First() - .Elements("{http://www.tei-c.org/ns/1.0}title")) - { - string label = prop.Value; - book.Name = book.NameInEnglish = - book.Description = book.DescriptionInEnglish = - label; - - tag = await TagHandler.PrepareAttribute(context, "Title", label, 1); - meta.Add(tag); - break; - } - } - catch - { - //ignore non-existing = null tags - } - - try - { - foreach (var prop in elObject - .Elements("{http://www.tei-c.org/ns/1.0}teiHeader").First() - .Elements("{http://www.tei-c.org/ns/1.0}fileDesc").First() - .Elements("{http://www.tei-c.org/ns/1.0}titleStmt").First() - .Elements("{http://www.tei-c.org/ns/1.0}author")) - { - string label = prop.Value; - tag = await TagHandler.PrepareAttribute(context, "Contributor Names", label, 1); - meta.Add(tag); - break; - } - } - catch - { - //ignore non-existing = null tags - } - - try - { - foreach (var prop in elObject - .Elements("{http://www.tei-c.org/ns/1.0}teiHeader").First() - .Elements("{http://www.tei-c.org/ns/1.0}fileDesc").First() - .Elements("{http://www.tei-c.org/ns/1.0}titleStmt").First() - .Elements("{http://www.tei-c.org/ns/1.0}respStmt")) - { - string label = prop.Elements("{http://www.tei-c.org/ns/1.0}name").First().Value; - tag = await TagHandler.PrepareAttribute(context, "Contributor Names", label, 1); - meta.Add(tag); - break; - - } - } - catch - { - //ignore non-existing = null tags - } - - try - { - foreach (var prop in elObject - .Elements("{http://www.tei-c.org/ns/1.0}teiHeader").First() - .Elements("{http://www.tei-c.org/ns/1.0}fileDesc").First() - .Elements("{http://www.tei-c.org/ns/1.0}notesStmt").First() - .Elements("{http://www.tei-c.org/ns/1.0}note")) - { - string label = prop.Value; - tag = await TagHandler.PrepareAttribute(context, "Notes", label, 1); - meta.Add(tag); - } - } - catch - { - //ignore non-existing = null tags - } - - - - - tag = await TagHandler.PrepareAttribute(context, "Source", "Digitized Walters Manuscripts", 1); - tag.ValueSupplement = $"http://www.thedigitalwalters.org/Data/WaltersManuscripts/html/{job.ResourceNumber}/"; - - meta.Add(tag); - - - book.Tags = meta.ToArray(); - List pages = new List(); - - int order = 0; - - foreach (var surface in elObject - .Elements("{http://www.tei-c.org/ns/1.0}facsimile").First() - .Elements("{http://www.tei-c.org/ns/1.0}surface")) - { - foreach(var graphic in surface.Elements("{http://www.tei-c.org/ns/1.0}graphic")) - if (graphic.Attribute("url").Value.Contains("sap.jpg")) - { - - order++; - - - using (RMuseumDbContext importJobUpdaterDb = new RMuseumDbContext(Configuration)) - { - job.ProgressPercent = order; - importJobUpdaterDb.Update(job); - await importJobUpdaterDb.SaveChangesAsync(); - } - - RArtifactItemRecord page = new RArtifactItemRecord() - { - Name = $"تصویر {order}", - NameInEnglish = $"Image {order}", - Description = "", - DescriptionInEnglish = "", - Order = order, - FriendlyUrl = $"p{$"{order}".PadLeft(4, '0')}", - LastModified = DateTime.Now - }; - - - - string imageUrl = $"http://www.thedigitalwalters.org/Data/WaltersManuscripts/{resourceNumber}/data/W.{resourceNumber.Substring(1)}/{graphic.Attribute("url").Value}"; - - tag = await TagHandler.PrepareAttribute(context, "Source", "Digitized Walters Manuscripts", 1); - tag.ValueSupplement = imageUrl; - page.Tags = new RTagValue[] { tag }; - - if (!string.IsNullOrEmpty(imageUrl)) - { - bool recovered = false; - if ( - File.Exists - ( - Path.Combine(Path.Combine(Path.Combine(_pictureFileService.ImageStoragePath, friendlyUrl), "orig"), $"{order}".PadLeft(4, '0') + ".jpg") - ) - && - File.Exists - ( - Path.Combine(Path.Combine(Path.Combine(_pictureFileService.ImageStoragePath, friendlyUrl), "norm"), $"{order}".PadLeft(4, '0') + ".jpg") - ) - && - File.Exists - ( - Path.Combine(Path.Combine(Path.Combine(_pictureFileService.ImageStoragePath, friendlyUrl), "thumb"), $"{order}".PadLeft(4, '0') + ".jpg") - ) - ) - { - RServiceResult picture = await _pictureFileService.RecoverFromeFiles(page.Name, page.Description, 1, - imageUrl, - Path.Combine(Path.Combine(Path.Combine(_pictureFileService.ImageStoragePath, friendlyUrl), "orig"), $"{order}".PadLeft(4, '0') + ".jpg"), - Path.Combine(Path.Combine(Path.Combine(_pictureFileService.ImageStoragePath, friendlyUrl), "norm"), $"{order}".PadLeft(4, '0') + ".jpg"), - Path.Combine(Path.Combine(Path.Combine(_pictureFileService.ImageStoragePath, friendlyUrl), "thumb"), $"{order}".PadLeft(4, '0') + ".jpg"), - $"{order}".PadLeft(4, '0') + ".jpg", friendlyUrl); - if (picture.Result != null) - { - recovered = true; - page.Images = new RPictureFile[] { picture.Result }; - page.CoverImageIndex = 0; - - if (book.CoverItemIndex == (order - 1)) - { - book.CoverImage = RPictureFile.Duplicate(picture.Result); - } - - pages.Add(page); - } - - } - if (!recovered) - { - if ( - File.Exists - ( - Path.Combine(Path.Combine(Path.Combine(_pictureFileService.ImageStoragePath, friendlyUrl), "orig"), $"{order}".PadLeft(4, '0') + ".jpg") - ) - - ) - { - File.Delete - ( - Path.Combine(Path.Combine(Path.Combine(_pictureFileService.ImageStoragePath, friendlyUrl), "orig"), $"{order}".PadLeft(4, '0') + ".jpg") - ); - } - if ( - - File.Exists - ( - Path.Combine(Path.Combine(Path.Combine(_pictureFileService.ImageStoragePath, friendlyUrl), "norm"), $"{order}".PadLeft(4, '0') + ".jpg") - ) - - ) - { - File.Delete - ( - Path.Combine(Path.Combine(Path.Combine(_pictureFileService.ImageStoragePath, friendlyUrl), "norm"), $"{order}".PadLeft(4, '0') + ".jpg") - ); - } - if ( - - File.Exists - ( - Path.Combine(Path.Combine(Path.Combine(_pictureFileService.ImageStoragePath, friendlyUrl), "thumb"), $"{order}".PadLeft(4, '0') + ".jpg") - ) - ) - { - File.Delete - ( - Path.Combine(Path.Combine(Path.Combine(_pictureFileService.ImageStoragePath, friendlyUrl), "thumb"), $"{order}".PadLeft(4, '0') + ".jpg") - ); - } - var imageResult = await client.GetAsync(imageUrl); - - - int _ImportRetryCount = 5; - int _ImportRetryInitialSleep = 500; - int retryCount = 0; - while (retryCount < _ImportRetryCount && !imageResult.IsSuccessStatusCode && imageResult.StatusCode == HttpStatusCode.ServiceUnavailable) - { - imageResult.Dispose(); - Thread.Sleep(_ImportRetryInitialSleep * (retryCount + 1)); - imageResult = await client.GetAsync(imageUrl); - retryCount++; - } - - if (imageResult.IsSuccessStatusCode) - { - using (Stream imageStream = await imageResult.Content.ReadAsStreamAsync()) - { - RServiceResult picture = await _pictureFileService.Add(page.Name, page.Description, 1, null, imageUrl, imageStream, $"{order}".PadLeft(4, '0') + ".jpg", friendlyUrl); - if (picture.Result == null) - { - throw new Exception($"_pictureFileService.Add : {picture.ExceptionString}"); - } - - page.Images = new RPictureFile[] { picture.Result }; - page.CoverImageIndex = 0; - - if (book.CoverItemIndex == (order - 1)) - { - book.CoverImage = RPictureFile.Duplicate(picture.Result); - } - - pages.Add(page); - } - } - else - { - using (RMuseumDbContext importJobUpdaterDb = new RMuseumDbContext(Configuration)) - { - job.EndTime = DateTime.Now; - job.Status = ImportJobStatus.Failed; - job.Exception = $"Http result is not ok ({imageResult.StatusCode}) for page {order}, url {imageUrl}"; - importJobUpdaterDb.Update(job); - await importJobUpdaterDb.SaveChangesAsync(); - } - - imageResult.Dispose(); - return; - } - - imageResult.Dispose(); - GC.Collect(); - } - - } - - - - - } - } - - - - book.Items = pages.ToArray(); - book.ItemCount = pages.Count; - - if (pages.Count == 0) - { - using (RMuseumDbContext importJobUpdaterDb = new RMuseumDbContext(Configuration)) - { - job.EndTime = DateTime.Now; - job.Status = ImportJobStatus.Failed; - job.Exception = "Pages.Count == 0"; - importJobUpdaterDb.Update(job); - await importJobUpdaterDb.SaveChangesAsync(); - } - return; - } - - await context.Artifacts.AddAsync(book); - await context.SaveChangesAsync(); - - job.ProgressPercent = 100; - job.Status = ImportJobStatus.Succeeded; - job.ArtifactId = book.Id; - job.EndTime = DateTime.Now; - context.Update(job); - await context.SaveChangesAsync(); - - - - } - } - else - { - using (RMuseumDbContext importJobUpdaterDb = new RMuseumDbContext(Configuration)) - { - job.EndTime = DateTime.Now; - job.Status = ImportJobStatus.Failed; - job.Exception = $"Http result is not ok ({result.StatusCode}) for {url}"; - importJobUpdaterDb.Update(job); - await importJobUpdaterDb.SaveChangesAsync(); - } - return; - } - - } - } - } - catch (Exception exp) - { - using (RMuseumDbContext importJobUpdaterDb = new RMuseumDbContext(Configuration)) - { - job.EndTime = DateTime.Now; - job.Status = ImportJobStatus.Failed; - job.Exception = exp.ToString(); - importJobUpdaterDb.Update(job); - await importJobUpdaterDb.SaveChangesAsync(); - } - - } - } - ); - - - - return new RServiceResult(true); - } - catch (Exception exp) - { - return new RServiceResult(false, exp.ToString()); - } - } - - /// - /// import from https://viewer.cbl.ie - /// - /// 119 - /// golestan-baysonghori - /// - private async Task> StartImportingFromChesterBeatty(string resourceNumber, string friendlyUrl) - { - try - { - string srcUrl = $"https://viewer.cbl.ie/viewer/object/Per_{resourceNumber}/1/"; - if ( - ( - await _context.ImportJobs - .Where(j => j.JobType == JobType.ChesterBeatty && j.ResourceNumber == resourceNumber && !(j.Status == ImportJobStatus.Failed || j.Status == ImportJobStatus.Aborted)) - .SingleOrDefaultAsync() - ) - != - null - ) - { - return new RServiceResult(false, $"Job is already scheduled or running for importing {srcUrl}"); - } - - if (string.IsNullOrEmpty(friendlyUrl)) - { - return new RServiceResult(false, $"Friendly url is empty, server folder {srcUrl}"); - } - - if ( - (await _context.Artifacts.Where(a => a.FriendlyUrl == friendlyUrl).SingleOrDefaultAsync()) - != - null - ) - { - return new RServiceResult(false, $"duplicated friendly url '{friendlyUrl}'"); - } - - ImportJob job = new ImportJob() - { - JobType = JobType.ChesterBeatty, - ResourceNumber = resourceNumber, - FriendlyUrl = friendlyUrl, - SrcUrl = srcUrl, - QueueTime = DateTime.Now, - ProgressPercent = 0, - Status = ImportJobStatus.NotStarted - }; - - - await _context.ImportJobs.AddAsync - ( - job - ); - - await _context.SaveChangesAsync(); - - _backgroundTaskQueue.QueueBackgroundWorkItem - ( - async token => - { - try - { - using (RMuseumDbContext context = new RMuseumDbContext(Configuration)) - { - RArtifactMasterRecord book = new RArtifactMasterRecord($"extracted from url {job.ResourceNumber}", $"extracted from url {job.ResourceNumber}") - { - Status = PublishStatus.Draft, - DateTime = DateTime.Now, - LastModified = DateTime.Now, - CoverItemIndex = 0, - FriendlyUrl = friendlyUrl, - }; - - - List meta = new List(); - RTagValue tag; - - - tag = await TagHandler.PrepareAttribute(context, "Type", "Book", 1); - meta.Add(tag); - - tag = await TagHandler.PrepareAttribute(context, "Type", "Manuscript", 1); - meta.Add(tag); - - tag = await TagHandler.PrepareAttribute(context, "Source", "Chester Beatty Digital Collections", 1); - tag.ValueSupplement = srcUrl; - - meta.Add(tag); - - - - using (RMuseumDbContext importJobUpdaterDb = new RMuseumDbContext(Configuration)) - { - job.StartTime = DateTime.Now; - job.Status = ImportJobStatus.Running; - job.SrcContent = ""; - importJobUpdaterDb.Update(job); - await importJobUpdaterDb.SaveChangesAsync(); - } - - List pages = new List(); - int order = 0; - using (var client = new HttpClient()) - do - { - - - order++; - - using (RMuseumDbContext importJobUpdaterDb = new RMuseumDbContext(Configuration)) - { - job.ProgressPercent = order; - importJobUpdaterDb.Update(job); - await importJobUpdaterDb.SaveChangesAsync(); - } - - RArtifactItemRecord page = new RArtifactItemRecord() - { - Name = $"تصویر {order}", - NameInEnglish = $"Image {order} of {book.NameInEnglish}", - Description = "", - DescriptionInEnglish = "", - Order = order, - FriendlyUrl = $"p{$"{order}".PadLeft(4, '0')}", - LastModified = DateTime.Now - }; - - string imageUrl = $"https://viewer.cbl.ie/viewer/rest/image/Per_{resourceNumber}/Per{resourceNumber}_{$"{order}".PadLeft(3, '0')}.jpg/full/!10000,10000/0/default.jpg?ignoreWatermark=true"; - - - page.Tags = new RTagValue[] { }; - - bool recovered = false; - - if ( - File.Exists - ( - Path.Combine(Path.Combine(Path.Combine(_pictureFileService.ImageStoragePath, friendlyUrl), "orig"), $"{order}".PadLeft(4, '0') + ".jpg") - ) - && - File.Exists - ( - Path.Combine(Path.Combine(Path.Combine(_pictureFileService.ImageStoragePath, friendlyUrl), "norm"), $"{order}".PadLeft(4, '0') + ".jpg") - ) - && - File.Exists - ( - Path.Combine(Path.Combine(Path.Combine(_pictureFileService.ImageStoragePath, friendlyUrl), "thumb"), $"{order}".PadLeft(4, '0') + ".jpg") - ) - ) - { - RServiceResult picture = await _pictureFileService.RecoverFromeFiles(page.Name, page.Description, 1, - imageUrl, - Path.Combine(Path.Combine(Path.Combine(_pictureFileService.ImageStoragePath, friendlyUrl), "orig"), $"{order}".PadLeft(4, '0') + ".jpg"), - Path.Combine(Path.Combine(Path.Combine(_pictureFileService.ImageStoragePath, friendlyUrl), "norm"), $"{order}".PadLeft(4, '0') + ".jpg"), - Path.Combine(Path.Combine(Path.Combine(_pictureFileService.ImageStoragePath, friendlyUrl), "thumb"), $"{order}".PadLeft(4, '0') + ".jpg"), - $"{order}".PadLeft(4, '0') + ".jpg", friendlyUrl); - if (picture.Result != null) - { - recovered = true; - page.Images = new RPictureFile[] { picture.Result }; - page.CoverImageIndex = 0; - - if (book.CoverItemIndex == (order - 1)) - { - book.CoverImage = RPictureFile.Duplicate(picture.Result); - } - - tag = await TagHandler.PrepareAttribute(context, "Source", "Chester Beatty Digital Collections", 1); - tag.ValueSupplement = $"https://viewer.cbl.ie/viewer/object/Per_{resourceNumber}/{$"{order}".PadLeft(3, '0')}/"; ; - page.Tags = new RTagValue[] { tag }; - - pages.Add(page); - } - - } - - if (!recovered) - { - if ( - File.Exists - ( - Path.Combine(Path.Combine(Path.Combine(_pictureFileService.ImageStoragePath, friendlyUrl), "orig"), $"{order}".PadLeft(4, '0') + ".jpg") - ) - - ) - { - File.Delete - ( - Path.Combine(Path.Combine(Path.Combine(_pictureFileService.ImageStoragePath, friendlyUrl), "orig"), $"{order}".PadLeft(4, '0') + ".jpg") - ); - } - if ( - - File.Exists - ( - Path.Combine(Path.Combine(Path.Combine(_pictureFileService.ImageStoragePath, friendlyUrl), "norm"), $"{order}".PadLeft(4, '0') + ".jpg") - ) - - ) - { - File.Delete - ( - Path.Combine(Path.Combine(Path.Combine(_pictureFileService.ImageStoragePath, friendlyUrl), "norm"), $"{order}".PadLeft(4, '0') + ".jpg") - ); - } - if ( - - File.Exists - ( - Path.Combine(Path.Combine(Path.Combine(_pictureFileService.ImageStoragePath, friendlyUrl), "thumb"), $"{order}".PadLeft(4, '0') + ".jpg") - ) - ) - { - File.Delete - ( - Path.Combine(Path.Combine(Path.Combine(_pictureFileService.ImageStoragePath, friendlyUrl), "thumb"), $"{order}".PadLeft(4, '0') + ".jpg") - ); - } - var imageResult = await client.GetAsync(imageUrl); - - if (imageResult.StatusCode == HttpStatusCode.Forbidden || imageResult.StatusCode == HttpStatusCode.NotFound) - break; - - - int _ImportRetryCount = 5; - int _ImportRetryInitialSleep = 500; - int retryCount = 0; - while (retryCount < _ImportRetryCount && !imageResult.IsSuccessStatusCode && imageResult.StatusCode == HttpStatusCode.ServiceUnavailable) - { - imageResult.Dispose(); - Thread.Sleep(_ImportRetryInitialSleep * (retryCount + 1)); - imageResult = await client.GetAsync(imageUrl); - retryCount++; - } - - if (imageResult.IsSuccessStatusCode) - { - using (Stream imageStream = await imageResult.Content.ReadAsStreamAsync()) - { - RServiceResult picture = await _pictureFileService.Add(page.Name, page.Description, 1, null, imageUrl, imageStream, $"{order}".PadLeft(4, '0') + ".jpg", friendlyUrl); - if (picture.Result == null) - { - throw new Exception($"_pictureFileService.Add : {picture.ExceptionString}"); - } - - page.Images = new RPictureFile[] { picture.Result }; - page.CoverImageIndex = 0; - - if (book.CoverItemIndex == (order - 1)) - { - book.CoverImage = RPictureFile.Duplicate(picture.Result); - } - tag = await TagHandler.PrepareAttribute(context, "Source", "Chester Beatty Digital Collections", 1); - tag.ValueSupplement = $"https://viewer.cbl.ie/viewer/object/Per_{resourceNumber}/{$"{order}".PadLeft(3, '0')}/"; - page.Tags = new RTagValue[] { tag }; - - pages.Add(page); - } - } - else - { - using (RMuseumDbContext importJobUpdaterDb = new RMuseumDbContext(Configuration)) - { - job.EndTime = DateTime.Now; - job.Status = ImportJobStatus.Failed; - job.Exception = $"Http result is not ok ({imageResult.StatusCode}) for page {order}, url {imageUrl}"; - importJobUpdaterDb.Update(job); - await importJobUpdaterDb.SaveChangesAsync(); - } - - imageResult.Dispose(); - return; - } - - - imageResult.Dispose(); - GC.Collect(); - } - - - pages.Add(page); - } - while (true); - - - book.Tags = meta.ToArray(); - - book.Items = pages.ToArray(); - book.ItemCount = pages.Count; - - if (pages.Count == 0) - { - using (RMuseumDbContext importJobUpdaterDb = new RMuseumDbContext(Configuration)) - { - job.EndTime = DateTime.Now; - job.Status = ImportJobStatus.Failed; - job.Exception = "Pages.Count == 0"; - importJobUpdaterDb.Update(job); - await importJobUpdaterDb.SaveChangesAsync(); - } - return; - } - - await context.Artifacts.AddAsync(book); - await context.SaveChangesAsync(); - - job.ProgressPercent = 100; - job.Status = ImportJobStatus.Succeeded; - job.ArtifactId = book.Id; - job.EndTime = DateTime.Now; - context.Update(job); - await context.SaveChangesAsync(); - - - - } - } - catch (Exception exp) - { - using (RMuseumDbContext importJobUpdaterDb = new RMuseumDbContext(Configuration)) - { - job.EndTime = DateTime.Now; - job.Status = ImportJobStatus.Failed; - job.Exception = exp.ToString(); - importJobUpdaterDb.Update(job); - await importJobUpdaterDb.SaveChangesAsync(); - } - - } - } - ); - - return new RServiceResult(true); - } - catch (Exception exp) - { - return new RServiceResult(false, exp.ToString()); - } - } - + /// /// reschedule jobs