anas/divan
1
0
Fork 0
Code Issues 12 Pull Requests Actions Packages Projects Releases Wiki Activity

#391 make new artifact upload to ftp optional and controllable by the import api

Browse Source
This commit is contained in:
Hamid Reza Mohammadi 2023-06-30 12:13:56 +03:30
parent fd1a585267
commit 6d7b2c7e33
15 changed files with 163 additions and 98 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

19
RMuseum/Controllers/ArtifactController.cs
View File

@ -1082,16 +1082,17 @@ namespace RMuseum.Controllers
/// <param name="resourceNumber">119/foldername</param>
/// <param name="friendlyUrl">golestan-baysonghori/artifact id</param>
/// <param name="resourcePrefix"></param>
/// <param name="skipUpload"></param>
/// <returns></returns>
[HttpPost]
[Route("import")]
[Authorize(Policy = RMuseumSecurableItem.ArtifactEntityShortName + ":" + RMuseumSecurableItem.ImportOperationShortName)]
[ProducesResponseType((int)HttpStatusCode.OK, Type = typeof(bool))]
[ProducesResponseType((int)HttpStatusCode.BadRequest, Type = typeof(string))]
public async Task<IActionResult> Import(string srcType, string resourceNumber, string friendlyUrl, string resourcePrefix)
public async Task<IActionResult> Import(string srcType, string resourceNumber, string friendlyUrl, string resourcePrefix, bool skipUpload)
{
RServiceResult<bool> res =
await _artifactService.Import(srcType, resourceNumber, friendlyUrl, resourcePrefix);
await _artifactService.Import(srcType, resourceNumber, friendlyUrl, resourcePrefix, skipUpload);
if (res.Result)
return Ok();
return BadRequest(res.ExceptionString);
@ -1121,9 +1122,9 @@ namespace RMuseum.Controllers
[Authorize(Policy = RMuseumSecurableItem.ArtifactEntityShortName + ":" + RMuseumSecurableItem.ImportOperationShortName)]
[ProducesResponseType((int)HttpStatusCode.OK)]
[ProducesResponseType((int)HttpStatusCode.BadRequest, Type = typeof(string))]
public async Task<IActionResult> RetryImport()
public async Task<IActionResult> RetryImport(JobType jobType = JobType.BritishLibrary, bool skipUpload = false)
{
RServiceResult<bool> res = await _artifactService.RescheduleJobs(JobType.BritishLibrary);
RServiceResult<bool> res = await _artifactService.RescheduleJobs(jobType, skipUpload);
if (res.Result)
return Ok();
return BadRequest(res.ExceptionString);
@ -1133,17 +1134,18 @@ namespace RMuseum.Controllers
/// due to a bug in loc json outputs some artifacts with more than 1000 pages were downloaded incompletely
/// </summary>
/// <param name="pass">123456</param>
/// <param name="skipUpload"></param>
/// <returns></returns>
[HttpPost]
[Route("reexamineimport")]
[Authorize(Policy = RMuseumSecurableItem.ArtifactEntityShortName + ":" + RMuseumSecurableItem.ImportOperationShortName)]
[ProducesResponseType((int)HttpStatusCode.OK, Type = typeof(string[]))]
[ProducesResponseType((int)HttpStatusCode.BadRequest, Type = typeof(string))]
public async Task<IActionResult> ReExamineLocDownloads(string pass)
public async Task<IActionResult> ReExamineLocDownloads(string pass, bool skipUpload)
{
if (pass != "123456") //this is a heavy processor consuming call, so prevent mistaken call of it from swagger ui
return BadRequest("invalid password");
RServiceResult<string[]> res = await _artifactService.ReExamineLocDownloads();
RServiceResult<string[]> res = await _artifactService.ReExamineLocDownloads(skipUpload);
if (!string.IsNullOrEmpty(res.ExceptionString))
return BadRequest(res.ExceptionString);
return Ok(res.Result);
@ -2079,6 +2081,7 @@ namespace RMuseum.Controllers
/// upload artifact to external server
/// </summary>
/// <param name="artifactId"></param>
/// <param name="skipUpload"></param>
/// <returns></returns>
[HttpPut("upload/external/{artifactId}")]
@ -2086,11 +2089,11 @@ namespace RMuseum.Controllers
[ProducesResponseType((int)HttpStatusCode.OK)]
[ProducesResponseType((int)HttpStatusCode.BadRequest, Type = typeof(string))]
[ProducesResponseType((int)HttpStatusCode.Unauthorized)]
public IActionResult StartUploadingArtifactToExternalServer(Guid artifactId)
public IActionResult StartUploadingArtifactToExternalServer(Guid artifactId, bool skipUpload)
{
try
{
var res = _artifactService.StartUploadingArtifactToExternalServer(artifactId);
var res = _artifactService.StartUploadingArtifactToExternalServer(artifactId, skipUpload);
if (!string.IsNullOrEmpty(res.ExceptionString))
return BadRequest(res.ExceptionString);
return Ok();

74
RMuseum/RMuseum.xml
View File

@ -313,7 +313,7 @@
</summary>
<returns></returns>
</member>
<member name="M:RMuseum.Controllers.ArtifactController.Import(System.String,System.String,System.String,System.String)">
<member name="M:RMuseum.Controllers.ArtifactController.Import(System.String,System.String,System.String,System.String,System.Boolean)">
<summary>
import from external resources
</summary>
@ -321,6 +321,7 @@
<param name="resourceNumber">119/foldername</param>
<param name="friendlyUrl">golestan-baysonghori/artifact id</param>
<param name="resourcePrefix"></param>
<param name="skipUpload"></param>
<returns></returns>
</member>
<member name="M:RMuseum.Controllers.ArtifactController.KeepAlive">
@ -329,17 +330,18 @@
</summary>
<returns>true</returns>
</member>
<member name="M:RMuseum.Controllers.ArtifactController.RetryImport">
<member name="M:RMuseum.Controllers.ArtifactController.RetryImport(RMuseum.Models.ImportJob.JobType,System.Boolean)">
<summary>
retry import
</summary>
<returns></returns>
</member>
<member name="M:RMuseum.Controllers.ArtifactController.ReExamineLocDownloads(System.String)">
<member name="M:RMuseum.Controllers.ArtifactController.ReExamineLocDownloads(System.String,System.Boolean)">
<summary>
due to a bug in loc json outputs some artifacts with more than 1000 pages were downloaded incompletely
</summary>
<param name="pass">123456</param>
<param name="skipUpload"></param>
<returns></returns>
</member>
<member name="M:RMuseum.Controllers.ArtifactController.BookmarkArtifact(System.Guid)">
@ -626,11 +628,12 @@
<param name="artifactId"></param>
<returns></returns>
</member>
<member name="M:RMuseum.Controllers.ArtifactController.StartUploadingArtifactToExternalServer(System.Guid)">
<member name="M:RMuseum.Controllers.ArtifactController.StartUploadingArtifactToExternalServer(System.Guid,System.Boolean)">
<summary>
upload artifact to external server
</summary>
<param name="artifactId"></param>
<param name="skipUpload"></param>
<returns></returns>
</member>
<member name="P:RMuseum.Controllers.ArtifactController.ReadOnlyMode">
@ -12001,7 +12004,7 @@
<param name="stream"></param>
<returns></returns>
</member>
<member name="M:RMuseum.Services.IArtifactService.Import(System.String,System.String,System.String,System.String)">
<member name="M:RMuseum.Services.IArtifactService.Import(System.String,System.String,System.String,System.String,System.Boolean)">
<summary>
import from external resources
</summary>
@ -12009,19 +12012,22 @@
<param name="resourceNumber">119</param>
<param name="friendlyUrl">golestan-baysonghori</param>
<param name="resourcePrefix"></param>
<param name="skipUpload"></param>
<returns></returns>
</member>
<member name="M:RMuseum.Services.IArtifactService.RescheduleJobs(RMuseum.Models.ImportJob.JobType)">
<member name="M:RMuseum.Services.IArtifactService.RescheduleJobs(RMuseum.Models.ImportJob.JobType,System.Boolean)">
<summary>
reschedule jobs
</summary>
<param name="jobType"></param>
<param name="skipUpload"></param>
<returns></returns>
</member>
<member name="M:RMuseum.Services.IArtifactService.ReExamineLocDownloads">
<member name="M:RMuseum.Services.IArtifactService.ReExamineLocDownloads(System.Boolean)">
<summary>
due to a bug in loc json outputs some artifacts with more than 1000 pages were downloaded incompletely
</summary>
<param name="skipUpload"></param>
<returns></returns>
</member>
<member name="M:RMuseum.Services.IArtifactService.GetImportJobs(RSecurityBackend.Models.Generic.PagingParameterModel)">
@ -12337,11 +12343,12 @@
<param name="artifactId"></param>
<returns></returns>
</member>
<member name="M:RMuseum.Services.IArtifactService.StartUploadingArtifactToExternalServer(System.Guid)">
<member name="M:RMuseum.Services.IArtifactService.StartUploadingArtifactToExternalServer(System.Guid,System.Boolean)">
<summary>
upload artifact to external server
</summary>
<param name="artifactId"></param>
<param name="skipUpload"></param>
<returns></returns>
</member>
<member name="T:RMuseum.Services.IDonationService">
@ -13925,56 +13932,62 @@
</summary>
<returns></returns>
</member>
<member name="M:RMuseum.Services.Implementation.ArtifactService.StartImportingFromHarvardDirectly(System.String,System.String,System.String)">
<member name="M:RMuseum.Services.Implementation.ArtifactService.StartImportingFromHarvardDirectly(System.String,System.String,System.String,System.Boolean)">
<summary>
import from http://www.qajarwomen.org
</summary>
<param name="hardvardResourceNumber">43117279</param>
<param name="friendlyUrl">atame</param>
<param name="srcUrl">http://www.qajarwomen.org/fa/items/1018A10.html</param>
<param name="skipUpload"></param>
<returns></returns>
</member>
<member name="M:RMuseum.Services.Implementation.ArtifactService.StartImportingFromBritishLibrary(System.String,System.String)">
<member name="M:RMuseum.Services.Implementation.ArtifactService.StartImportingFromBritishLibrary(System.String,System.String,System.Boolean)">
<summary>
from http://www.bl.uk
</summary>
<param name="resourceNumber">grenville_xli_f001r</param>
<param name="friendlyUrl"></param>
<param name="skipUpload"></param>
<returns></returns>
</member>
<member name="M:RMuseum.Services.Implementation.ArtifactService.StartImportingFromCambridge(System.String,System.String)">
<member name="M:RMuseum.Services.Implementation.ArtifactService.StartImportingFromCambridge(System.String,System.String,System.Boolean)">
<summary>
from http://cudl.lib.cam.ac.uk
</summary>
<param name="resourceNumber">MS-RAS-00258</param>
<param name="friendlyUrl"></param>
<param name="skipUpload"></param>
<returns></returns>
</member>
<member name="M:RMuseum.Services.Implementation.ArtifactService.StartImportingFromChesterBeatty(System.String,System.String)">
<member name="M:RMuseum.Services.Implementation.ArtifactService.StartImportingFromChesterBeatty(System.String,System.String,System.Boolean)">
<summary>
import from https://viewer.cbl.ie
</summary>
<param name="resourceNumber">119</param>
<param name="friendlyUrl">golestan-baysonghori</param>
<param name="skipUpload"></param>
<returns></returns>
</member>
<member name="M:RMuseum.Services.Implementation.ArtifactService.StartImportingFromHarvard(System.String,System.String)">
<member name="M:RMuseum.Services.Implementation.ArtifactService.StartImportingFromHarvard(System.String,System.String,System.Boolean)">
<summary>
from https://curiosity.lib.harvard.edu
</summary>
<param name="url">example: https://curiosity.lib.harvard.edu/islamic-heritage-project/catalog/40-990114893240203941</param>
<param name="friendlyUrl"></param>
<param name="skipUpload"></param>
<returns></returns>
</member>
<member name="M:RMuseum.Services.Implementation.ArtifactService.StartImportingFromHathiTrust(System.String,System.String)">
<member name="M:RMuseum.Services.Implementation.ArtifactService.StartImportingFromHathiTrust(System.String,System.String,System.Boolean)">
<summary>
from https://catalog.hathitrust.org
</summary>
<param name="resourceNumber">006814127</param>
<param name="friendlyUrl"></param>
<param name="skipUpload"></param>
<returns></returns>
</member>
<member name="M:RMuseum.Services.Implementation.ArtifactService.StartImportingFromTheLibraryOfCongress(System.String,System.String,System.String)">
<member name="M:RMuseum.Services.Implementation.ArtifactService.StartImportingFromTheLibraryOfCongress(System.String,System.String,System.String,System.Boolean)">
<summary>
from https://www.loc.gov
</summary>
@ -13992,45 +14005,51 @@
<example>
plmp
</example>
<param name="skipUpload"></param>
<returns></returns>
</member>
<member name="M:RMuseum.Services.Implementation.ArtifactService.ReExamineLocDownloads">
<member name="M:RMuseum.Services.Implementation.ArtifactService.ReExamineLocDownloads(System.Boolean)">
<summary>
due to a bug in loc json outputs some artifacts with more than 1000 pages were downloaded incompletely
</summary>
<param name="skipUpload"></param>
<returns></returns>
</member>
<member name="M:RMuseum.Services.Implementation.ArtifactService.StartImportingFromPenLibraries(System.String,System.String)">
<member name="M:RMuseum.Services.Implementation.ArtifactService.StartImportingFromPenLibraries(System.String,System.String,System.Boolean)">
<summary>
from http://www.library.upenn.edu/
</summary>
<param name="resourceNumber">MEDREN_9949222153503681</param>
<param name="friendlyUrl"></param>
<param name="skipUpload"></param>
<returns></returns>
</member>
<member name="M:RMuseum.Services.Implementation.ArtifactService.StartImportingFromServerFolder(System.String,System.String,System.String)">
<member name="M:RMuseum.Services.Implementation.ArtifactService.StartImportingFromServerFolder(System.String,System.String,System.String,System.Boolean)">
<summary>
import from server folder
</summary>
<param name="folderPath">C:\Tools\batches\florence</param>
<param name="friendlyUrl">shahname-florence</param>
<param name="srcUrl">https://t.me/dr_khatibi_abolfazl/888</param>
<param name="skipUpload"></param>
<returns></returns>
</member>
<member name="M:RMuseum.Services.Implementation.ArtifactService.StartAppendingFromServerFolder(System.String,System.Guid)">
<member name="M:RMuseum.Services.Implementation.ArtifactService.StartAppendingFromServerFolder(System.String,System.Guid,System.Boolean)">
<summary>
start appending from server folder
</summary>
<param name="folderPath"></param>
<param name="artifactId"></param>
<param name="skipUpload"></param>
<returns></returns>
</member>
<member name="M:RMuseum.Services.Implementation.ArtifactService.StartImportingFromWalters(System.String,System.String)">
<member name="M:RMuseum.Services.Implementation.ArtifactService.StartImportingFromWalters(System.String,System.String,System.Boolean)">
<summary>
from http://www.thedigitalwalters.org/01_ACCESS_WALTERS_MANUSCRIPTS.html
</summary>
<param name="resourceNumber">W619</param>
<param name="friendlyUrl">golestan-walters-01</param>
<param name="skipUpload"></param>
<returns></returns>
</member>
<member name="M:RMuseum.Services.Implementation.ArtifactService.StartSettingArtifactAsTextOriginalSource(System.Int32,System.Guid)">
@ -14047,12 +14066,13 @@
</summary>
<returns></returns>
</member>
<member name="M:RMuseum.Services.Implementation.ArtifactService.StartImportingFromPrinceton(System.String,System.String)">
<member name="M:RMuseum.Services.Implementation.ArtifactService.StartImportingFromPrinceton(System.String,System.String,System.Boolean)">
<summary>
from http://pudl.princeton.edu/
</summary>
<param name="resourceNumber">dj52w476m</param>
<param name="friendlyUrl"></param>
<param name="skipUpload"></param>
<returns></returns>
</member>
<member name="M:RMuseum.Services.Implementation.ArtifactService.GetAll(RSecurityBackend.Models.Generic.PagingParameterModel,RMuseum.Models.Artifact.PublishStatus[])">
@ -14281,7 +14301,7 @@
<param name="stream"></param>
<returns></returns>
</member>
<member name="M:RMuseum.Services.Implementation.ArtifactService.Import(System.String,System.String,System.String,System.String)">
<member name="M:RMuseum.Services.Implementation.ArtifactService.Import(System.String,System.String,System.String,System.String,System.Boolean)">
<summary>
import from external resources
</summary>
@ -14289,28 +14309,32 @@
<param name="resourceNumber">119/foldername</param>
<param name="friendlyUrl">golestan-baysonghori/artifact id</param>
<param name="resourcePrefix"></param>
<param name="skipUpload"></param>
<returns></returns>
</member>
<member name="M:RMuseum.Services.Implementation.ArtifactService.StartUploadingArtifactToExternalServer(System.Guid)">
<member name="M:RMuseum.Services.Implementation.ArtifactService.StartUploadingArtifactToExternalServer(System.Guid,System.Boolean)">
<summary>
upload artifact to external server
</summary>
<param name="artifactId"></param>
<param name="skipUpload"></param>
<returns></returns>
</member>
<member name="M:RMuseum.Services.Implementation.ArtifactService._UploadArtifactToExternalServer(RMuseum.Models.Artifact.RArtifactMasterRecord,RMuseum.DbContext.RMuseumDbContext)">
<member name="M:RMuseum.Services.Implementation.ArtifactService._UploadArtifactToExternalServer(RMuseum.Models.Artifact.RArtifactMasterRecord,RMuseum.DbContext.RMuseumDbContext,System.Boolean)">
<summary>
upload artifact to external server
</summary>
<param name="book"></param>
<param name="context"></param>
<param name="skipUpload"></param>
<returns></returns>
</member>
<member name="M:RMuseum.Services.Implementation.ArtifactService.RescheduleJobs(RMuseum.Models.ImportJob.JobType)">
<member name="M:RMuseum.Services.Implementation.ArtifactService.RescheduleJobs(RMuseum.Models.ImportJob.JobType,System.Boolean)">
<summary>
reschedule jobs
</summary>
<param name="jobType"></param>
<param name="skipUpload"></param>
<returns></returns>
</member>
<member name="M:RMuseum.Services.Implementation.ArtifactService.RemoveArtifact(System.Guid,System.Boolean)">

12
RMuseum/Services/IArtifactService.cs
View File

@ -257,21 +257,24 @@ namespace RMuseum.Services
/// <param name="resourceNumber">119</param>
/// <param name="friendlyUrl">golestan-baysonghori</param>
/// <param name="resourcePrefix"></param>
/// <param name="skipUpload"></param>
/// <returns></returns>
Task<RServiceResult<bool>> Import(string srcType, string resourceNumber, string friendlyUrl, string resourcePrefix);
Task<RServiceResult<bool>> Import(string srcType, string resourceNumber, string friendlyUrl, string resourcePrefix, bool skipUpload);
/// <summary>
/// reschedule jobs
/// </summary>
/// <param name="jobType"></param>
/// <param name="skipUpload"></param>
/// <returns></returns>
Task<RServiceResult<bool>> RescheduleJobs(JobType jobType);
Task<RServiceResult<bool>> RescheduleJobs(JobType jobType, bool skipUpload);
/// <summary>
/// due to a bug in loc json outputs some artifacts with more than 1000 pages were downloaded incompletely
/// </summary>
/// <param name="skipUpload"></param>
/// <returns></returns>
Task<RServiceResult<string[]>> ReExamineLocDownloads();
Task<RServiceResult<string[]>> ReExamineLocDownloads(bool skipUpload);
/// <summary>
/// import jobs
@ -592,8 +595,9 @@ namespace RMuseum.Services
/// upload artifact to external server
/// </summary>
/// <param name="artifactId"></param>
/// <param name="skipUpload"></param>
/// <returns></returns>
RServiceResult<bool> StartUploadingArtifactToExternalServer(Guid artifactId);
RServiceResult<bool> StartUploadingArtifactToExternalServer(Guid artifactId, bool skipUpload);
}
}

5
RMuseum/Services/Implementation/ArtifactService-Partials/ArtifactService-HarvardDirectly.cs
View File

@ -22,8 +22,9 @@ namespace RMuseum.Services.Implementation
/// <param name="hardvardResourceNumber">43117279</param>
/// <param name="friendlyUrl">atame</param>
/// <param name="srcUrl">http://www.qajarwomen.org/fa/items/1018A10.html</param>
/// <param name="skipUpload"></param>
/// <returns></returns>
private async Task<RServiceResult<bool>> StartImportingFromHarvardDirectly(string hardvardResourceNumber, string friendlyUrl, string srcUrl)
private async Task<RServiceResult<bool>> StartImportingFromHarvardDirectly(string hardvardResourceNumber, string friendlyUrl, string srcUrl, bool skipUpload)
{
try
{
@ -152,7 +153,7 @@ namespace RMuseum.Services.Implementation
await context.Artifacts.AddAsync(book);
await context.SaveChangesAsync();
var resFTPUpload = await _UploadArtifactToExternalServer(book, context);
var resFTPUpload = await _UploadArtifactToExternalServer(book, context, skipUpload);
if (!string.IsNullOrEmpty(resFTPUpload.ExceptionString))
{
job.EndTime = DateTime.Now;

5
RMuseum/Services/Implementation/ArtifactService-Partials/ArtifactService-ImportBritishLibrary.cs
View File

@ -27,8 +27,9 @@ namespace RMuseum.Services.Implementation
/// </summary>
/// <param name="resourceNumber">grenville_xli_f001r</param>
/// <param name="friendlyUrl"></param>
/// <param name="skipUpload"></param>
/// <returns></returns>
private async Task<RServiceResult<bool>> StartImportingFromBritishLibrary(string resourceNumber, string friendlyUrl)
private async Task<RServiceResult<bool>> StartImportingFromBritishLibrary(string resourceNumber, string friendlyUrl, bool skipUpload)
{
string url = $"http://www.bl.uk/manuscripts/Viewer.aspx?ref={resourceNumber}";
if (
@ -641,7 +642,7 @@ namespace RMuseum.Services.Implementation
await context.Artifacts.AddAsync(book);
await context.SaveChangesAsync();
var resFTPUpload = await _UploadArtifactToExternalServer(book, context);
var resFTPUpload = await _UploadArtifactToExternalServer(book, context, skipUpload);
if (!string.IsNullOrEmpty(resFTPUpload.ExceptionString))
{
job.EndTime = DateTime.Now;

5
RMuseum/Services/Implementation/ArtifactService-Partials/ArtifactService-ImportCambridge.cs
View File

@ -43,8 +43,9 @@ namespace RMuseum.Services.Implementation
/// </summary>
/// <param name="resourceNumber">MS-RAS-00258</param>
/// <param name="friendlyUrl"></param>
/// <param name="skipUpload"></param>
/// <returns></returns>
private async Task<RServiceResult<bool>> StartImportingFromCambridge(string resourceNumber, string friendlyUrl)
private async Task<RServiceResult<bool>> StartImportingFromCambridge(string resourceNumber, string friendlyUrl, bool skipUpload)
{
string url = $"http://cudl.lib.cam.ac.uk/view/{resourceNumber}.json";
if (
@ -433,7 +434,7 @@ namespace RMuseum.Services.Implementation
await context.Artifacts.AddAsync(book);
await context.SaveChangesAsync();
var resFTPUpload = await _UploadArtifactToExternalServer(book, context);
var resFTPUpload = await _UploadArtifactToExternalServer(book, context, skipUpload);
if (!string.IsNullOrEmpty(resFTPUpload.ExceptionString))
{
job.EndTime = DateTime.Now;

5
RMuseum/Services/Implementation/ArtifactService-Partials/ArtifactService-ImportChesterBeatty.cs
View File

@ -25,8 +25,9 @@ namespace RMuseum.Services.Implementation
/// </summary>
/// <param name="resourceNumber">119</param>
/// <param name="friendlyUrl">golestan-baysonghori</param>
/// <param name="skipUpload"></param>
/// <returns></returns>
private async Task<RServiceResult<bool>> StartImportingFromChesterBeatty(string resourceNumber, string friendlyUrl)
private async Task<RServiceResult<bool>> StartImportingFromChesterBeatty(string resourceNumber, string friendlyUrl, bool skipUpload)
{
try
{
@ -328,7 +329,7 @@ namespace RMuseum.Services.Implementation
await context.Artifacts.AddAsync(book);
await context.SaveChangesAsync();
var resFTPUpload = await _UploadArtifactToExternalServer(book, context);
var resFTPUpload = await _UploadArtifactToExternalServer(book, context, skipUpload);
if (!string.IsNullOrEmpty(resFTPUpload.ExceptionString))
{
job.EndTime = DateTime.Now;

5
RMuseum/Services/Implementation/ArtifactService-Partials/ArtifactService-ImportHarvard.cs
View File

@ -26,8 +26,9 @@ namespace RMuseum.Services.Implementation
/// </summary>
/// <param name="url">example: https://curiosity.lib.harvard.edu/islamic-heritage-project/catalog/40-990114893240203941</param>
/// <param name="friendlyUrl"></param>
/// <param name="skipUpload"></param>
/// <returns></returns>
private async Task<RServiceResult<bool>> StartImportingFromHarvard(string url, string friendlyUrl)
private async Task<RServiceResult<bool>> StartImportingFromHarvard(string url, string friendlyUrl, bool skipUpload)
{
try
{
@ -214,7 +215,7 @@ namespace RMuseum.Services.Implementation
await context.SaveChangesAsync();
var resFTPUpload = await _UploadArtifactToExternalServer(book, context);
var resFTPUpload = await _UploadArtifactToExternalServer(book, context, skipUpload);
if (!string.IsNullOrEmpty(resFTPUpload.ExceptionString))
{
job.EndTime = DateTime.Now;

5
RMuseum/Services/Implementation/ArtifactService-Partials/ArtifactService-ImportHathiTrust.cs
View File

@ -27,8 +27,9 @@ namespace RMuseum.Services.Implementation
/// </summary>
/// <param name="resourceNumber">006814127</param>
/// <param name="friendlyUrl"></param>
/// <param name="skipUpload"></param>
/// <returns></returns>
private async Task<RServiceResult<bool>> StartImportingFromHathiTrust(string resourceNumber, string friendlyUrl)
private async Task<RServiceResult<bool>> StartImportingFromHathiTrust(string resourceNumber, string friendlyUrl, bool skipUpload)
{
string url = $"https://catalog.hathitrust.org/Record/{resourceNumber}.xml";
if (
@ -466,7 +467,7 @@ namespace RMuseum.Services.Implementation
await context.Artifacts.AddAsync(book);
await context.SaveChangesAsync();
var resFTPUpload = await _UploadArtifactToExternalServer(book, context);
var resFTPUpload = await _UploadArtifactToExternalServer(book, context, skipUpload);
if (!string.IsNullOrEmpty(resFTPUpload.ExceptionString))
{
job.EndTime = DateTime.Now;

10
RMuseum/Services/Implementation/ArtifactService-Partials/ArtifactService-ImportLoc.cs
View File

@ -37,8 +37,9 @@ namespace RMuseum.Services.Implementation
/// <example>
/// plmp
/// </example>
/// <param name="skipUpload"></param>
/// <returns></returns>
private async Task<RServiceResult<bool>> StartImportingFromTheLibraryOfCongress(string resourceNumber, string friendlyUrl, string resourcePrefix)
private async Task<RServiceResult<bool>> StartImportingFromTheLibraryOfCongress(string resourceNumber, string friendlyUrl, string resourcePrefix, bool skipUpload)
{
string url = $"https://www.loc.gov/resource/{resourcePrefix}.{resourceNumber}/?fo=json&st=gallery";
@ -473,7 +474,7 @@ namespace RMuseum.Services.Implementation
await context.Artifacts.AddAsync(book);
await context.SaveChangesAsync();
var resFTPUpload = await _UploadArtifactToExternalServer(book, context);
var resFTPUpload = await _UploadArtifactToExternalServer(book, context, skipUpload);
if (!string.IsNullOrEmpty(resFTPUpload.ExceptionString))
{
job.EndTime = DateTime.Now;
@ -539,8 +540,9 @@ namespace RMuseum.Services.Implementation
/// <summary>
/// due to a bug in loc json outputs some artifacts with more than 1000 pages were downloaded incompletely
/// </summary>
/// <param name="skipUpload"></param>
/// <returns></returns>
public async Task<RServiceResult<string[]>> ReExamineLocDownloads()
public async Task<RServiceResult<string[]>> ReExamineLocDownloads(bool skipUpload)
{
try
{
@ -609,7 +611,7 @@ namespace RMuseum.Services.Implementation
await RemoveArtifact((Guid)job.ArtifactId, false);
_context.ImportJobs.Remove(job);
await _context.SaveChangesAsync();
RServiceResult<bool> rescheduled = await StartImportingFromTheLibraryOfCongress(job.ResourceNumber, job.FriendlyUrl, "rbc0001");//plmp
RServiceResult<bool> rescheduled = await StartImportingFromTheLibraryOfCongress(job.ResourceNumber, job.FriendlyUrl, "rbc0001", skipUpload);//plmp
if (rescheduled.Result)
{

5
RMuseum/Services/Implementation/ArtifactService-Partials/ArtifactService-ImportPenLibraries.cs
View File

@ -25,8 +25,9 @@ namespace RMuseum.Services.Implementation
/// </summary>
/// <param name="resourceNumber">MEDREN_9949222153503681</param>
/// <param name="friendlyUrl"></param>
/// <param name="skipUpload"></param>
/// <returns></returns>
private async Task<RServiceResult<bool>> StartImportingFromPenLibraries(string resourceNumber, string friendlyUrl)
private async Task<RServiceResult<bool>> StartImportingFromPenLibraries(string resourceNumber, string friendlyUrl, bool skipUpload)
{
string url = $"http://dla.library.upenn.edu/dla/medren/pageturn.html?id={resourceNumber}&rotation=0&size=0";
if (
@ -446,7 +447,7 @@ namespace RMuseum.Services.Implementation
await context.Artifacts.AddAsync(book);
await context.SaveChangesAsync();
var resFTPUpload = await _UploadArtifactToExternalServer(book, context);
var resFTPUpload = await _UploadArtifactToExternalServer(book, context, skipUpload);
if (!string.IsNullOrEmpty(resFTPUpload.ExceptionString))
{
job.EndTime = DateTime.Now;

10
RMuseum/Services/Implementation/ArtifactService-Partials/ArtifactService-ImportServerFolder.cs
View File

@ -23,8 +23,9 @@ namespace RMuseum.Services.Implementation
/// <param name="folderPath">C:\Tools\batches\florence</param>
/// <param name="friendlyUrl">shahname-florence</param>
/// <param name="srcUrl">https://t.me/dr_khatibi_abolfazl/888</param>
/// <param name="skipUpload"></param>
/// <returns></returns>
private async Task<RServiceResult<bool>> StartImportingFromServerFolder(string folderPath, string friendlyUrl, string srcUrl)
private async Task<RServiceResult<bool>> StartImportingFromServerFolder(string folderPath, string friendlyUrl, string srcUrl, bool skipUpload)
{
try
{
@ -138,7 +139,7 @@ namespace RMuseum.Services.Implementation
await context.Artifacts.AddAsync(book);
await context.SaveChangesAsync();
var resFTPUpload = await _UploadArtifactToExternalServer(book, context);
var resFTPUpload = await _UploadArtifactToExternalServer(book, context, skipUpload);
if (!string.IsNullOrEmpty(resFTPUpload.ExceptionString))
{
job.EndTime = DateTime.Now;
@ -191,8 +192,9 @@ namespace RMuseum.Services.Implementation
/// </summary>
/// <param name="folderPath"></param>
/// <param name="artifactId"></param>
/// <param name="skipUpload"></param>
/// <returns></returns>
private async Task<RServiceResult<bool>> StartAppendingFromServerFolder(string folderPath, Guid artifactId)
private async Task<RServiceResult<bool>> StartAppendingFromServerFolder(string folderPath, Guid artifactId, bool skipUpload)
{
try
{
@ -249,7 +251,7 @@ namespace RMuseum.Services.Implementation
context.Update(book);
await context.SaveChangesAsync();
var resFTPUpload = await _UploadArtifactToExternalServer(book, context);
var resFTPUpload = await _UploadArtifactToExternalServer(book, context, skipUpload);
if (!string.IsNullOrEmpty(resFTPUpload.ExceptionString))
{
job.EndTime = DateTime.Now;

5
RMuseum/Services/Implementation/ArtifactService-Partials/ArtifactService-ImportWalters.cs
View File

@ -26,8 +26,9 @@ namespace RMuseum.Services.Implementation
/// </summary>
/// <param name="resourceNumber">W619</param>
/// <param name="friendlyUrl">golestan-walters-01</param>
/// <param name="skipUpload"></param>
/// <returns></returns>
private async Task<RServiceResult<bool>> StartImportingFromWalters(string resourceNumber, string friendlyUrl)
private async Task<RServiceResult<bool>> StartImportingFromWalters(string resourceNumber, string friendlyUrl, bool skipUpload)
{
string url = $"http://www.thedigitalwalters.org/Data/WaltersManuscripts/ManuscriptDescriptions/{resourceNumber}_tei.xml";
if (
@ -432,7 +433,7 @@ namespace RMuseum.Services.Implementation
await context.Artifacts.AddAsync(book);
await context.SaveChangesAsync();
var resFTPUpload = await _UploadArtifactToExternalServer(book, context);
var resFTPUpload = await _UploadArtifactToExternalServer(book, context, skipUpload);
if (!string.IsNullOrEmpty(resFTPUpload.ExceptionString))
{
job.EndTime = DateTime.Now;

5
RMuseum/Services/Implementation/ArtifactService-Partials/ArtificatService-ImportPrinceton.cs
View File

@ -26,8 +26,9 @@ namespace RMuseum.Services.Implementation
/// </summary>
/// <param name="resourceNumber">dj52w476m</param>
/// <param name="friendlyUrl"></param>
/// <param name="skipUpload"></param>
/// <returns></returns>
private async Task<RServiceResult<bool>> StartImportingFromPrinceton(string resourceNumber, string friendlyUrl)
private async Task<RServiceResult<bool>> StartImportingFromPrinceton(string resourceNumber, string friendlyUrl, bool skipUpload)
{
string url = $"http://pudl.princeton.edu/mdCompiler2.php?obj={resourceNumber}";
if (
@ -423,7 +424,7 @@ namespace RMuseum.Services.Implementation
await context.Artifacts.AddAsync(book);
await context.SaveChangesAsync();
var resFTPUpload = await _UploadArtifactToExternalServer(book, context);
var resFTPUpload = await _UploadArtifactToExternalServer(book, context, skipUpload);
if(!string.IsNullOrEmpty(resFTPUpload.ExceptionString))
{
job.EndTime = DateTime.Now;

91
RMuseum/Services/Implementation/ArtifactService.cs
View File

@ -1165,52 +1165,54 @@ namespace RMuseum.Services.Implementation
/// <param name="resourceNumber">119/foldername</param>
/// <param name="friendlyUrl">golestan-baysonghori/artifact id</param>
/// <param name="resourcePrefix"></param>
/// <param name="skipUpload"></param>
/// <returns></returns>
public async Task<RServiceResult<bool>> Import(string srcType, string resourceNumber, string friendlyUrl, string resourcePrefix)
public async Task<RServiceResult<bool>> Import(string srcType, string resourceNumber, string friendlyUrl, string resourcePrefix, bool skipUpload)
{
return
srcType == "princeton" ?
await StartImportingFromPrinceton(resourceNumber, friendlyUrl)
await StartImportingFromPrinceton(resourceNumber, friendlyUrl, skipUpload)
:
srcType == "harvard" ?
await StartImportingFromHarvard(resourceNumber, friendlyUrl)
await StartImportingFromHarvard(resourceNumber, friendlyUrl, skipUpload)
:
srcType == "qajarwomen" ?
await StartImportingFromHarvardDirectly(resourceNumber, friendlyUrl, resourcePrefix)
await StartImportingFromHarvardDirectly(resourceNumber, friendlyUrl, resourcePrefix, skipUpload)
:
srcType == "hathitrust" ?
await StartImportingFromHathiTrust(resourceNumber, friendlyUrl)
await StartImportingFromHathiTrust(resourceNumber, friendlyUrl, skipUpload)
:
srcType == "penn" ?
await StartImportingFromPenLibraries(resourceNumber, friendlyUrl)
await StartImportingFromPenLibraries(resourceNumber, friendlyUrl, skipUpload)
:
srcType == "cam" ?
await StartImportingFromCambridge(resourceNumber, friendlyUrl)
await StartImportingFromCambridge(resourceNumber, friendlyUrl, skipUpload)
:
srcType == "bl" ?
await StartImportingFromBritishLibrary(resourceNumber, friendlyUrl)
await StartImportingFromBritishLibrary(resourceNumber, friendlyUrl, skipUpload)
:
srcType == "folder" ?
await StartImportingFromServerFolder(resourceNumber, friendlyUrl, resourcePrefix)
await StartImportingFromServerFolder(resourceNumber, friendlyUrl, resourcePrefix, skipUpload)
:
srcType == "append" ?
await StartAppendingFromServerFolder(resourceNumber, Guid.Parse(friendlyUrl))
await StartAppendingFromServerFolder(resourceNumber, Guid.Parse(friendlyUrl), skipUpload)
:
srcType == "walters" ?
await StartImportingFromWalters(resourceNumber, friendlyUrl)
await StartImportingFromWalters(resourceNumber, friendlyUrl, skipUpload)
:
srcType == "cbl" ?
await StartImportingFromChesterBeatty(resourceNumber, friendlyUrl)
await StartImportingFromChesterBeatty(resourceNumber, friendlyUrl, skipUpload)
:
await StartImportingFromTheLibraryOfCongress(resourceNumber, friendlyUrl, resourcePrefix);
await StartImportingFromTheLibraryOfCongress(resourceNumber, friendlyUrl, resourcePrefix, skipUpload);
}
/// <summary>
/// upload artifact to external server
/// </summary>
/// <param name="artifactId"></param>
/// <param name="skipUpload"></param>
/// <returns></returns>
public RServiceResult<bool> StartUploadingArtifactToExternalServer(Guid artifactId)
public RServiceResult<bool> StartUploadingArtifactToExternalServer(Guid artifactId, bool skipUpload)
{
try
{
@ -1227,7 +1229,7 @@ namespace RMuseum.Services.Implementation
{
var book = await context.Artifacts.Include(a => a.Items).ThenInclude(i => i.Images).Include(a => a.CoverImage).SingleAsync();
await jobProgressServiceEF.UpdateJob(job.Id, 2, $"بارگذاری {book.Name}");
var resUpload = await _UploadArtifactToExternalServer(book, context);
var resUpload = await _UploadArtifactToExternalServer(book, context, skipUpload);
if(!string.IsNullOrEmpty(resUpload.ExceptionString))
{
await jobProgressServiceEF.UpdateJob(job.Id, 100, "", false, resUpload.ExceptionString);
@ -1259,8 +1261,9 @@ namespace RMuseum.Services.Implementation
/// </summary>
/// <param name="book"></param>
/// <param name="context"></param>
/// <param name="skipUpload"></param>
/// <returns></returns>
private async Task<RServiceResult<bool>> _UploadArtifactToExternalServer(RArtifactMasterRecord book, RMuseumDbContext context)
private async Task<RServiceResult<bool>> _UploadArtifactToExternalServer(RArtifactMasterRecord book, RMuseumDbContext context, bool skipUpload)
{
try
{
@ -1272,9 +1275,14 @@ namespace RMuseum.Services.Implementation
Configuration.GetSection("ExternalFTPServer")["Username"],
Configuration.GetSection("ExternalFTPServer")["Password"]
);
ftpClient.ValidateCertificate += FtpClient_ValidateCertificate;
await ftpClient.AutoConnect();
ftpClient.Config.RetryAttempts = 3;
if(!skipUpload)
{
ftpClient.ValidateCertificate += FtpClient_ValidateCertificate;
await ftpClient.AutoConnect();
ftpClient.Config.RetryAttempts = 3;
}
foreach (var imageSizeString in new string[] { "orig", "norm", "thumb" })
{
@ -1284,8 +1292,12 @@ namespace RMuseum.Services.Implementation
book.CoverImage.ExternalNormalSizeImageUrl = $"{Configuration.GetSection("ExternalFTPServer")["RootUrl"]}/{book.CoverImage.FolderName}/orig/{Path.GetFileName(localFilePath)}";
context.Update(book.CoverImage);
}
var remoteFilePath = $"{Configuration.GetSection("ExternalFTPServer")["RootPath"]}/images/{book.CoverImage.FolderName}/{imageSizeString}/{Path.GetFileName(localFilePath)}";
await ftpClient.UploadFile(localFilePath, remoteFilePath, createRemoteDir: true);
if(!skipUpload)
{
var remoteFilePath = $"{Configuration.GetSection("ExternalFTPServer")["RootPath"]}/images/{book.CoverImage.FolderName}/{imageSizeString}/{Path.GetFileName(localFilePath)}";
await ftpClient.UploadFile(localFilePath, remoteFilePath, createRemoteDir: true);
}
}
@ -1301,13 +1313,21 @@ namespace RMuseum.Services.Implementation
image.ExternalNormalSizeImageUrl = $"{Configuration.GetSection("ExternalFTPServer")["RootUrl"]}/{image.FolderName}/orig/{Path.GetFileName(localFilePath)}";
context.Update(image);
}
var remoteFilePath = $"{Configuration.GetSection("ExternalFTPServer")["RootPath"]}/images/{book.CoverImage.FolderName}/{imageSizeString}/{Path.GetFileName(localFilePath)}";
await ftpClient.UploadFile(localFilePath, remoteFilePath, createRemoteDir: true);
if(!skipUpload)
{
var remoteFilePath = $"{Configuration.GetSection("ExternalFTPServer")["RootPath"]}/images/{book.CoverImage.FolderName}/{imageSizeString}/{Path.GetFileName(localFilePath)}";
await ftpClient.UploadFile(localFilePath, remoteFilePath, createRemoteDir: true);
}
}
}
}
if(!skipUpload)
{
await ftpClient.Disconnect();
}
await ftpClient.Disconnect();
await context.SaveChangesAsync();
}
@ -1324,8 +1344,9 @@ namespace RMuseum.Services.Implementation
/// reschedule jobs
/// </summary>
/// <param name="jobType"></param>
/// <param name="skipUpload"></param>
/// <returns></returns>
public async Task<RServiceResult<bool>> RescheduleJobs(JobType jobType)
public async Task<RServiceResult<bool>> RescheduleJobs(JobType jobType, bool skipUpload)
{
ImportJob[] jobs = await _context.ImportJobs.Where(j => j.Status != ImportJobStatus.Succeeded && j.JobType == jobType).OrderByDescending(j => j.ProgressPercent).ToArrayAsync();
@ -1356,34 +1377,34 @@ namespace RMuseum.Services.Implementation
RServiceResult<bool> rescheduled =
job.JobType == JobType.Princeton ?
await StartImportingFromPrinceton(job.ResourceNumber, job.FriendlyUrl)
await StartImportingFromPrinceton(job.ResourceNumber, job.FriendlyUrl, skipUpload)
:
job.JobType == JobType.Harvard ?
await StartImportingFromHarvard(job.ResourceNumber, job.FriendlyUrl)
await StartImportingFromHarvard(job.ResourceNumber, job.FriendlyUrl, skipUpload)
:
job.JobType == JobType.HarvardDirect ?
await StartImportingFromHarvardDirectly(job.ResourceNumber, job.FriendlyUrl, job.SrcUrl)
await StartImportingFromHarvardDirectly(job.ResourceNumber, job.FriendlyUrl, job.SrcUrl, skipUpload)
:
job.JobType == JobType.HathiTrust ?
await StartImportingFromHathiTrust(job.ResourceNumber, job.FriendlyUrl)
await StartImportingFromHathiTrust(job.ResourceNumber, job.FriendlyUrl, skipUpload)
:
job.JobType == JobType.PennLibraries ?
await StartImportingFromPenLibraries(job.ResourceNumber, job.FriendlyUrl)
await StartImportingFromPenLibraries(job.ResourceNumber, job.FriendlyUrl, skipUpload)
:
job.JobType == JobType.Cambridge ?
await StartImportingFromCambridge(job.ResourceNumber, job.FriendlyUrl)
await StartImportingFromCambridge(job.ResourceNumber, job.FriendlyUrl, skipUpload)
:
job.JobType == JobType.BritishLibrary ?
await StartImportingFromBritishLibrary(job.ResourceNumber, job.FriendlyUrl)
await StartImportingFromBritishLibrary(job.ResourceNumber, job.FriendlyUrl, skipUpload)
:
job.JobType == JobType.ServerFolder ?
await StartImportingFromServerFolder(job.ResourceNumber, job.FriendlyUrl, job.SrcUrl)
await StartImportingFromServerFolder(job.ResourceNumber, job.FriendlyUrl, job.SrcUrl, skipUpload)
:
job.JobType == JobType.Walters ?
await StartImportingFromWalters(job.ResourceNumber, job.FriendlyUrl)
await StartImportingFromWalters(job.ResourceNumber, job.FriendlyUrl, skipUpload)
:
job.JobType == JobType.ChesterBeatty ?
await StartImportingFromChesterBeatty(job.ResourceNumber, job.FriendlyUrl)
await StartImportingFromChesterBeatty(job.ResourceNumber, job.FriendlyUrl, skipUpload)
:
new RServiceResult<bool>(false, "StartImportingFromTheLibraryOfCongress NOT SUPPORTED");