#401 code for parsing soha library pages
This commit is contained in:
Hamid Reza Mohammadi
parent
9ad3be2c43
commit
a66ec0f8c6
@ -1,4 +1,6 @@
|
||||
using DNTPersianUtils.Core;
|
||||
using ganjoor;
|
||||
using Microsoft.AspNetCore.Mvc.RazorPages;
|
||||
using Microsoft.EntityFrameworkCore;
|
||||
using Microsoft.Extensions.Primitives;
|
||||
using Newtonsoft.Json.Linq;
|
||||
@ -10,6 +12,7 @@ using RMuseum.Models.PDFLibrary.ViewModels;
|
||||
using RSecurityBackend.Models.Generic;
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.IO;
|
||||
using System.Linq;
|
||||
using System.Net.Http;
|
||||
using System.Security.Policy;
|
||||
@ -69,6 +72,25 @@ namespace RMuseum.Services.Implementation
|
||||
|
||||
try
|
||||
{
|
||||
var pdfSource = await context.PDFSources.Where(s => s.Name == "سها").SingleOrDefaultAsync();
|
||||
if(pdfSource == null)
|
||||
{
|
||||
PDFSource newSource = new PDFSource()
|
||||
{
|
||||
Name = "سها",
|
||||
Url = "https://sohalibrary.com",
|
||||
Description = "سامانهٔ جستجوی یکپارچهٔ نرمافزار سنا"
|
||||
};
|
||||
context.PDFSources.Add(newSource);
|
||||
await context.SaveChangesAsync();
|
||||
pdfSource = await context.PDFSources.Where(s => s.Name == "سها").SingleAsync();
|
||||
}
|
||||
NewPDFBookViewModel model = new NewPDFBookViewModel();
|
||||
model.PDFSourceId = pdfSource.Id;
|
||||
model.OriginalSourceName = pdfSource.Name;
|
||||
model.OriginalSourceUrl = srcUrl;
|
||||
model.BookScriptType = BookScriptType.Printed;
|
||||
|
||||
string html = "";
|
||||
using (var client = new HttpClient())
|
||||
{
|
||||
@ -103,14 +125,14 @@ namespace RMuseum.Services.Implementation
|
||||
List<RTagValue> meta = new List<RTagValue>();
|
||||
int idxStart;
|
||||
int idx = html.IndexOf("branch-link");
|
||||
if(idx != -1)
|
||||
if (idx != -1)
|
||||
{
|
||||
idxStart = html.IndexOf(">", idx);
|
||||
if(idxStart != -1)
|
||||
if (idxStart != -1)
|
||||
{
|
||||
int idxEnd = html.IndexOf("<", idxStart);
|
||||
|
||||
if(idxEnd != -1)
|
||||
if (idxEnd != -1)
|
||||
{
|
||||
meta.Add
|
||||
(
|
||||
@ -120,10 +142,10 @@ namespace RMuseum.Services.Implementation
|
||||
}
|
||||
}
|
||||
|
||||
NewPDFBookViewModel model = new NewPDFBookViewModel();
|
||||
|
||||
|
||||
idx = html.IndexOf("title-normal-for-book-name");
|
||||
if(idx == -1)
|
||||
if (idx == -1)
|
||||
{
|
||||
job.EndTime = DateTime.Now;
|
||||
job.Status = ImportJobStatus.Failed;
|
||||
@ -146,7 +168,7 @@ namespace RMuseum.Services.Implementation
|
||||
}
|
||||
|
||||
idxStart = html.IndexOf("width-150");
|
||||
while(idxStart != -1)
|
||||
while (idxStart != -1)
|
||||
{
|
||||
idxStart = html.IndexOf(">", idxStart);
|
||||
if (idxStart == -1) break;
|
||||
@ -154,7 +176,7 @@ namespace RMuseum.Services.Implementation
|
||||
if (idxEnd == -1) break;
|
||||
|
||||
string tagName = html.Substring(idxStart + 1, idxEnd - idxStart - 1).ToPersianNumbers().ApplyCorrectYeKe();
|
||||
|
||||
|
||||
idxStart = html.IndexOf("value-name", idxEnd);
|
||||
if (idxStart == -1) break;
|
||||
idxStart = html.IndexOf(">", idxStart);
|
||||
@ -169,14 +191,66 @@ namespace RMuseum.Services.Implementation
|
||||
{
|
||||
model.AuthorsLine = tagValue;
|
||||
tagName = "Author";
|
||||
|
||||
var existingAuthor = await context.Authors.AsNoTracking().Where(a => a.Name == tagValue).FirstOrDefaultAsync();
|
||||
if(existingAuthor != null)
|
||||
{
|
||||
model.WriterId = existingAuthor.Id;
|
||||
}
|
||||
else
|
||||
{
|
||||
var newAuthor = new Author()
|
||||
{
|
||||
Name = tagValue
|
||||
};
|
||||
context.Authors.Add(newAuthor);
|
||||
await context.SaveChangesAsync();
|
||||
model.WriterId = newAuthor.Id;
|
||||
}
|
||||
}
|
||||
if(tagName == "مترجم")
|
||||
if (tagName == "مترجم")
|
||||
{
|
||||
model.TranslatorsLine = tagValue;
|
||||
model.IsTranslation = true;
|
||||
tagName = "Translator";
|
||||
|
||||
var existingAuthor = await context.Authors.AsNoTracking().Where(a => a.Name == tagValue).FirstOrDefaultAsync();
|
||||
if (existingAuthor != null)
|
||||
{
|
||||
model.TranslatorId = existingAuthor.Id;
|
||||
}
|
||||
else
|
||||
{
|
||||
var newAuthor = new Author()
|
||||
{
|
||||
Name = tagValue
|
||||
};
|
||||
context.Authors.Add(newAuthor);
|
||||
await context.SaveChangesAsync();
|
||||
model.TranslatorId = newAuthor.Id;
|
||||
}
|
||||
}
|
||||
if(tagName == "زبان")
|
||||
if (tagName == "مصحح")
|
||||
{
|
||||
tagName = "Collector";
|
||||
|
||||
var existingAuthor = await context.Authors.AsNoTracking().Where(a => a.Name == tagValue).FirstOrDefaultAsync();
|
||||
if (existingAuthor != null)
|
||||
{
|
||||
model.CollectorId = existingAuthor.Id;
|
||||
}
|
||||
else
|
||||
{
|
||||
var newAuthor = new Author()
|
||||
{
|
||||
Name = tagValue
|
||||
};
|
||||
context.Authors.Add(newAuthor);
|
||||
await context.SaveChangesAsync();
|
||||
model.CollectorId = newAuthor.Id;
|
||||
}
|
||||
}
|
||||
if (tagName == "زبان")
|
||||
{
|
||||
model.Language = tagValue;
|
||||
tagName = "Language";
|
||||
@ -189,6 +263,38 @@ namespace RMuseum.Services.Implementation
|
||||
}
|
||||
tagName = "Volume";
|
||||
}
|
||||
if(tagName == "ناشر")
|
||||
{
|
||||
model.PublisherLine = tagValue;
|
||||
tagName = "Publisher";
|
||||
}
|
||||
if(tagName == "محل نشر")
|
||||
{
|
||||
model.PublishingLocation = tagValue;
|
||||
tagName = "Publishing Location";
|
||||
}
|
||||
if(tagName == "تاریخ انتشار")
|
||||
{
|
||||
model.PublishingDate = tagValue;
|
||||
tagName = "Publishing Date";
|
||||
}
|
||||
if(tagName == "موضوع")
|
||||
{
|
||||
tagName = "Subject";
|
||||
}
|
||||
if (tagName == "تعداد صفحات")
|
||||
{
|
||||
if(tagValue.Contains("ـ"))
|
||||
{
|
||||
tagValue = tagValue.Substring(tagValue.IndexOf("ـ")).Trim();
|
||||
if(int.TryParse(tagValue, out int v))
|
||||
{
|
||||
model.ClaimedPageCount = v;
|
||||
}
|
||||
}
|
||||
tagName = "Page Count";
|
||||
}
|
||||
|
||||
meta.Add
|
||||
(
|
||||
await TagHandler.PrepareAttribute(context, tagName, tagValue, 1)
|
||||
@ -203,6 +309,76 @@ namespace RMuseum.Services.Implementation
|
||||
downloadUrl = "https://sohalibrary.com" + downloadUrl;
|
||||
|
||||
|
||||
model.OriginalFileUrl = downloadUrl;
|
||||
|
||||
using (var client = new HttpClient())
|
||||
{
|
||||
using (var result = await client.GetAsync(downloadUrl))
|
||||
{
|
||||
if (result.IsSuccessStatusCode)
|
||||
{
|
||||
string fileName = string.Empty;
|
||||
if (result.Headers.TryGetValues("content-disposition", out IEnumerable<string> values))
|
||||
{
|
||||
string contentDisposition = values.ElementAt(0);
|
||||
if (!string.IsNullOrEmpty(contentDisposition))
|
||||
{
|
||||
string lookFor = "filename=";
|
||||
int index = contentDisposition.IndexOf(lookFor, StringComparison.CurrentCultureIgnoreCase);
|
||||
if (index >= 0)
|
||||
fileName = contentDisposition.Substring(index + lookFor.Length);
|
||||
}
|
||||
}
|
||||
|
||||
if(string.IsNullOrEmpty(fileName))
|
||||
{
|
||||
fileName = downloadUrl.Substring(downloadUrl.LastIndexOf('/') + 1) + ".pdf";
|
||||
}
|
||||
|
||||
model.LocalImportingPDFFilePath = Path.Combine(_imageFileService.ImageStoragePath, fileName);
|
||||
|
||||
using (Stream pdfStream = await result.Content.ReadAsStreamAsync())
|
||||
{
|
||||
pdfStream.Seek(0, SeekOrigin.Begin);
|
||||
using (FileStream fs = File.OpenWrite(model.LocalImportingPDFFilePath))
|
||||
{
|
||||
pdfStream.CopyTo(fs);
|
||||
}
|
||||
}
|
||||
job.EndTime = DateTime.Now;
|
||||
job.Status = ImportJobStatus.Succeeded;
|
||||
context.Update(job);
|
||||
await context.SaveChangesAsync();
|
||||
|
||||
string fileChecksum = PoemAudio.ComputeCheckSum(model.LocalImportingPDFFilePath);
|
||||
|
||||
var res = await ImportLocalPDFFileAsync(context, model, fileChecksum);
|
||||
File.Delete(model.LocalImportingPDFFilePath);
|
||||
if(res.Result != null)
|
||||
{
|
||||
var pdf = res.Result;
|
||||
foreach (var tag in meta)
|
||||
{
|
||||
pdf.Tags.Add(tag);
|
||||
}
|
||||
context.Update(pdf);
|
||||
await context.SaveChangesAsync();
|
||||
}
|
||||
|
||||
}
|
||||
else
|
||||
{
|
||||
job.EndTime = DateTime.Now;
|
||||
job.Status = ImportJobStatus.Failed;
|
||||
job.Exception = $"Http result is not ok ({result.StatusCode}) for {downloadUrl}";
|
||||
context.Update(job);
|
||||
await context.SaveChangesAsync();
|
||||
return;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
|
||||
}
|
||||
catch (Exception e)
|
||||
@ -214,7 +390,7 @@ namespace RMuseum.Services.Implementation
|
||||
context.Update(job);
|
||||
await context.SaveChangesAsync();
|
||||
}
|
||||
|
||||
|
||||
}
|
||||
}
|
||||
);
|
||||
|
||||
Loading…
Reference in New Issue
Block a user